From e220f9ddd6745a9b3ae10b2dff2b16a4b72d1626 Mon Sep 17 00:00:00 2001 From: "Lennart Kats (databricks)" Date: Mon, 16 Sep 2024 20:35:07 +0200 Subject: [PATCH 01/34] Use the friendly name of service principals when shortening their name (#1770) ## Summary Use the friendly name of service principals when shortening their name. This change is helpful for the prefix in development mode. Instead of adding a prefix like `[dev 1706906c-c0a2-4c25-9f57-3a7aa3cb8123]`, we'll prefix like `[dev my_principal]`. --- .../config/mutator/populate_current_user.go | 2 +- internal/init_test.go | 2 +- libs/auth/user.go | 9 ++- libs/auth/user_test.go | 71 +++++++++++++++---- libs/template/helpers.go | 2 +- 5 files changed, 66 insertions(+), 20 deletions(-) diff --git a/bundle/config/mutator/populate_current_user.go b/bundle/config/mutator/populate_current_user.go index b5e0bd437..1e99b327c 100644 --- a/bundle/config/mutator/populate_current_user.go +++ b/bundle/config/mutator/populate_current_user.go @@ -33,7 +33,7 @@ func (m *populateCurrentUser) Apply(ctx context.Context, b *bundle.Bundle) diag. } b.Config.Workspace.CurrentUser = &config.User{ - ShortName: auth.GetShortUserName(me.UserName), + ShortName: auth.GetShortUserName(me), User: me, } diff --git a/internal/init_test.go b/internal/init_test.go index c3cb0127e..d1a89f7b7 100644 --- a/internal/init_test.go +++ b/internal/init_test.go @@ -126,7 +126,7 @@ func TestAccBundleInitHelpers(t *testing.T) { }{ { funcName: "{{short_name}}", - expected: auth.GetShortUserName(me.UserName), + expected: auth.GetShortUserName(me), }, { funcName: "{{user_name}}", diff --git a/libs/auth/user.go b/libs/auth/user.go index 8eaa87633..c6aa974f3 100644 --- a/libs/auth/user.go +++ b/libs/auth/user.go @@ -4,12 +4,17 @@ import ( "strings" "github.com/databricks/cli/libs/textutil" + "github.com/databricks/databricks-sdk-go/service/iam" ) // Get a short-form username, based on the user's primary email address. // We leave the full range of unicode letters in tact, but remove all "special" characters, // including dots, which are not supported in e.g. experiment names. -func GetShortUserName(emailAddress string) string { - local, _, _ := strings.Cut(emailAddress, "@") +func GetShortUserName(user *iam.User) string { + name := user.UserName + if IsServicePrincipal(user.UserName) && user.DisplayName != "" { + name = user.DisplayName + } + local, _, _ := strings.Cut(name, "@") return textutil.NormalizeString(local) } diff --git a/libs/auth/user_test.go b/libs/auth/user_test.go index 62b2d29ac..24b61464b 100644 --- a/libs/auth/user_test.go +++ b/libs/auth/user_test.go @@ -3,70 +3,111 @@ package auth import ( "testing" + "github.com/databricks/databricks-sdk-go/service/iam" "github.com/stretchr/testify/assert" ) func TestGetShortUserName(t *testing.T) { tests := []struct { name string - email string + user *iam.User expected string }{ { - email: "test.user.1234@example.com", + user: &iam.User{ + UserName: "test.user.1234@example.com", + }, expected: "test_user_1234", }, { - email: "tést.üser@example.com", + user: &iam.User{ + UserName: "tést.üser@example.com", + }, expected: "tést_üser", }, { - email: "test$.user@example.com", + user: &iam.User{ + UserName: "test$.user@example.com", + }, expected: "test_user", }, { - email: `jöhn.dœ@domain.com`, // Using non-ASCII characters. + user: &iam.User{ + UserName: `jöhn.dœ@domain.com`, // Using non-ASCII characters. + }, expected: "jöhn_dœ", }, { - email: `first+tag@email.com`, // The plus (+) sign is used for "sub-addressing" in some email services. + user: &iam.User{ + UserName: `first+tag@email.com`, // The plus (+) sign is used for "sub-addressing" in some email services. + }, expected: "first_tag", }, { - email: `email@sub.domain.com`, // Using a sub-domain. + user: &iam.User{ + UserName: `email@sub.domain.com`, // Using a sub-domain. + }, expected: "email", }, { - email: `"_quoted"@domain.com`, // Quoted strings can be part of the local-part. + user: &iam.User{ + UserName: `"_quoted"@domain.com`, // Quoted strings can be part of the local-part. + }, expected: "quoted", }, { - email: `name-o'mally@website.org`, // Single quote in the local-part. + user: &iam.User{ + UserName: `name-o'mally@website.org`, // Single quote in the local-part. + }, expected: "name_o_mally", }, { - email: `user%domain@external.com`, // Percent sign can be used for email routing in legacy systems. + user: &iam.User{ + UserName: `user%domain@external.com`, // Percent sign can be used for email routing in legacy systems. + }, expected: "user_domain", }, { - email: `long.name.with.dots@domain.net`, // Multiple dots in the local-part. + user: &iam.User{ + UserName: `long.name.with.dots@domain.net`, // Multiple dots in the local-part. + }, expected: "long_name_with_dots", }, { - email: `me&you@together.com`, // Using an ampersand (&) in the local-part. + user: &iam.User{ + UserName: `me&you@together.com`, // Using an ampersand (&) in the local-part. + }, expected: "me_you", }, { - email: `user!def!xyz@domain.org`, // The exclamation mark can be valid in some legacy systems. + user: &iam.User{ + UserName: `user!def!xyz@domain.org`, // The exclamation mark can be valid in some legacy systems. + }, expected: "user_def_xyz", }, { - email: `admin@ιντερνετ.com`, // Domain in non-ASCII characters (IDN or Internationalized Domain Name). + user: &iam.User{ + UserName: `admin@ιντερνετ.com`, // Domain in non-ASCII characters (IDN or Internationalized Domain Name). + }, expected: "admin", }, + { + user: &iam.User{ + UserName: `1706906c-c0a2-4c25-9f57-3a7aa3cb8123`, + DisplayName: "my-service-principal", + }, + expected: "my_service_principal", + }, + { + user: &iam.User{ + UserName: `1706906c-c0a2-4c25-9f57-3a7aa3cb8123`, + // This service princpal has DisplayName (it's an optional property) + }, + expected: "1706906c_c0a2_4c25_9f57_3a7aa3cb8123", + }, } for _, tt := range tests { - assert.Equal(t, tt.expected, GetShortUserName(tt.email)) + assert.Equal(t, tt.expected, GetShortUserName(tt.user)) } } diff --git a/libs/template/helpers.go b/libs/template/helpers.go index 1dfe74d73..88c73cc47 100644 --- a/libs/template/helpers.go +++ b/libs/template/helpers.go @@ -119,7 +119,7 @@ func loadHelpers(ctx context.Context) template.FuncMap { return "", err } } - return auth.GetShortUserName(cachedUser.UserName), nil + return auth.GetShortUserName(cachedUser), nil }, // Get the default workspace catalog. If there is no default, or if // Unity Catalog is not enabled, return an empty string. From bcab6ca37b27c71156cdb3a9119db9becef4f869 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 18 Sep 2024 12:23:07 +0200 Subject: [PATCH 02/34] Fixed detecting full syntax variable override which includes type field (#1775) ## Changes Fixes #1773 ## Tests Confirmed manually --- bundle/config/root.go | 21 ++++++++++++++++--- bundle/tests/complex_variables_test.go | 18 ++++++++++++++++ bundle/tests/variables/complex/databricks.yml | 13 ++++++++++++ 3 files changed, 49 insertions(+), 3 deletions(-) diff --git a/bundle/config/root.go b/bundle/config/root.go index 46578769c..884c2e1ca 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -409,18 +409,33 @@ func (r *Root) MergeTargetOverrides(name string) error { var variableKeywords = []string{"default", "lookup"} // isFullVariableOverrideDef checks if the given value is a full syntax varaible override. -// A full syntax variable override is a map with only one of the following -// keys: "default", "lookup". +// A full syntax variable override is a map with either 1 of 2 keys. +// If it's 2 keys, the keys should be "default" and "type". +// If it's 1 key, the key should be one of the following keys: "default", "lookup". func isFullVariableOverrideDef(v dyn.Value) bool { mv, ok := v.AsMap() if !ok { return false } - if mv.Len() != 1 { + // If the map has more than 2 keys, it is not a full variable override. + if mv.Len() > 2 { return false } + // If the map has 2 keys, one of them should be "default" and the other is "type" + if mv.Len() == 2 { + if _, ok := mv.GetByString("type"); !ok { + return false + } + + if _, ok := mv.GetByString("default"); !ok { + return false + } + + return true + } + for _, keyword := range variableKeywords { if _, ok := mv.GetByString(keyword); ok { return true diff --git a/bundle/tests/complex_variables_test.go b/bundle/tests/complex_variables_test.go index 6371071ce..7a9a53a76 100644 --- a/bundle/tests/complex_variables_test.go +++ b/bundle/tests/complex_variables_test.go @@ -88,3 +88,21 @@ func TestComplexVariablesOverrideWithMultipleFiles(t *testing.T) { require.Equalf(t, "false", cluster.NewCluster.SparkConf["spark.speculation"], "cluster: %v", cluster.JobClusterKey) } } + +func TestComplexVariablesOverrideWithFullSyntax(t *testing.T) { + b, diags := loadTargetWithDiags("variables/complex", "dev") + require.Empty(t, diags) + + diags = bundle.Apply(context.Background(), b, bundle.Seq( + mutator.SetVariables(), + mutator.ResolveVariableReferencesInComplexVariables(), + mutator.ResolveVariableReferences( + "variables", + ), + )) + require.NoError(t, diags.Error()) + require.Empty(t, diags) + + complexvar := b.Config.Variables["complexvar"].Value + require.Equal(t, map[string]interface{}{"key1": "1", "key2": "2", "key3": "3"}, complexvar) +} diff --git a/bundle/tests/variables/complex/databricks.yml b/bundle/tests/variables/complex/databricks.yml index ca27f606d..3b32a7c8e 100644 --- a/bundle/tests/variables/complex/databricks.yml +++ b/bundle/tests/variables/complex/databricks.yml @@ -35,6 +35,13 @@ variables: - jar: "/path/to/jar" - egg: "/path/to/egg" - whl: "/path/to/whl" + complexvar: + type: complex + description: "A complex variable" + default: + key1: "value1" + key2: "value2" + key3: "value3" targets: @@ -49,3 +56,9 @@ targets: spark_conf: spark.speculation: false spark.databricks.delta.retentionDurationCheck.enabled: false + complexvar: + type: complex + default: + key1: "1" + key2: "2" + key3: "3" From e2c1d51d8437963bec84c857b74bb210b78b26b0 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 18 Sep 2024 13:26:16 +0200 Subject: [PATCH 03/34] [Release] Release v0.228.1 (#1778) Bundles: * Added listing cluster filtering for cluster lookups ([#1754](https://github.com/databricks/cli/pull/1754)). * Expand library globs relative to the sync root ([#1756](https://github.com/databricks/cli/pull/1756)). * Fixed generated YAML missing 'default' for empty values ([#1765](https://github.com/databricks/cli/pull/1765)). * Use periodic triggers in all templates ([#1739](https://github.com/databricks/cli/pull/1739)). * Use the friendly name of service principals when shortening their name ([#1770](https://github.com/databricks/cli/pull/1770)). * Fixed detecting full syntax variable override which includes type field ([#1775](https://github.com/databricks/cli/pull/1775)). Internal: * Pass copy of `dyn.Path` to callback function ([#1747](https://github.com/databricks/cli/pull/1747)). * Make bundle JSON schema modular with `$defs` ([#1700](https://github.com/databricks/cli/pull/1700)). * Alias variables block in the `Target` struct ([#1748](https://github.com/databricks/cli/pull/1748)). * Add end to end integration tests for bundle JSON schema ([#1726](https://github.com/databricks/cli/pull/1726)). * Fix artifact upload integration tests ([#1767](https://github.com/databricks/cli/pull/1767)). API Changes: * Added `databricks quality-monitors regenerate-dashboard` command. OpenAPI commit d05898328669a3f8ab0c2ecee37db2673d3ea3f7 (2024-09-04) Dependency updates: * Bump golang.org/x/term from 0.23.0 to 0.24.0 ([#1757](https://github.com/databricks/cli/pull/1757)). * Bump golang.org/x/oauth2 from 0.22.0 to 0.23.0 ([#1761](https://github.com/databricks/cli/pull/1761)). * Bump golang.org/x/text from 0.17.0 to 0.18.0 ([#1759](https://github.com/databricks/cli/pull/1759)). * Bump github.com/databricks/databricks-sdk-go from 0.45.0 to 0.46.0 ([#1760](https://github.com/databricks/cli/pull/1760)). --- CHANGELOG.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d63831253..32a7e5cfa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,32 @@ # Version changelog +## [Release] Release v0.228.1 + +Bundles: + * Added listing cluster filtering for cluster lookups ([#1754](https://github.com/databricks/cli/pull/1754)). + * Expand library globs relative to the sync root ([#1756](https://github.com/databricks/cli/pull/1756)). + * Fixed generated YAML missing 'default' for empty values ([#1765](https://github.com/databricks/cli/pull/1765)). + * Use periodic triggers in all templates ([#1739](https://github.com/databricks/cli/pull/1739)). + * Use the friendly name of service principals when shortening their name ([#1770](https://github.com/databricks/cli/pull/1770)). + * Fixed detecting full syntax variable override which includes type field ([#1775](https://github.com/databricks/cli/pull/1775)). + +Internal: + * Pass copy of `dyn.Path` to callback function ([#1747](https://github.com/databricks/cli/pull/1747)). + * Make bundle JSON schema modular with `` ([#1700](https://github.com/databricks/cli/pull/1700)). + * Alias variables block in the `Target` struct ([#1748](https://github.com/databricks/cli/pull/1748)). + * Add end to end integration tests for bundle JSON schema ([#1726](https://github.com/databricks/cli/pull/1726)). + * Fix artifact upload integration tests ([#1767](https://github.com/databricks/cli/pull/1767)). + +API Changes: + * Added `databricks quality-monitors regenerate-dashboard` command. + +OpenAPI commit d05898328669a3f8ab0c2ecee37db2673d3ea3f7 (2024-09-04) +Dependency updates: + * Bump golang.org/x/term from 0.23.0 to 0.24.0 ([#1757](https://github.com/databricks/cli/pull/1757)). + * Bump golang.org/x/oauth2 from 0.22.0 to 0.23.0 ([#1761](https://github.com/databricks/cli/pull/1761)). + * Bump golang.org/x/text from 0.17.0 to 0.18.0 ([#1759](https://github.com/databricks/cli/pull/1759)). + * Bump github.com/databricks/databricks-sdk-go from 0.45.0 to 0.46.0 ([#1760](https://github.com/databricks/cli/pull/1760)). + ## [Release] Release v0.228.0 CLI: From cf989a7e10e56f0b021eb4ffc5a7b793da25b540 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 19 Sep 2024 13:21:32 +0200 Subject: [PATCH 04/34] Upgrade to TF provider 1.52 (#1781) ## Changes Upgrade to TF provider 1.52 We also temporarily skip generating plugin framework structs to unblock upgrade as generation does not work yet and need to be fixed separately --- .../tf/codegen/generator/generator.go | 14 +++++++++++++- bundle/internal/tf/codegen/schema/version.go | 2 +- .../tf/schema/data_source_clusters.go | 16 ++++++++++++---- .../schema/data_source_external_location.go | 1 + .../internal/tf/schema/data_source_share.go | 2 ++ ...omatic_cluster_update_workspace_setting.go | 18 ++++++------------ bundle/internal/tf/schema/resource_cluster.go | 1 + ...ance_security_profile_workspace_setting.go | 4 ++-- ...d_security_monitoring_workspace_setting.go | 2 +- .../tf/schema/resource_model_serving.go | 18 ++++++++++-------- bundle/internal/tf/schema/resource_share.go | 19 +++++++++++++------ .../internal/tf/schema/resource_sql_table.go | 1 + bundle/internal/tf/schema/root.go | 2 +- 13 files changed, 64 insertions(+), 36 deletions(-) diff --git a/bundle/internal/tf/codegen/generator/generator.go b/bundle/internal/tf/codegen/generator/generator.go index 86d762439..b31fdf153 100644 --- a/bundle/internal/tf/codegen/generator/generator.go +++ b/bundle/internal/tf/codegen/generator/generator.go @@ -51,9 +51,15 @@ func (r *root) Generate(path string) error { } func Run(ctx context.Context, schema *tfjson.ProviderSchema, path string) error { - // Generate types for resources. + // Generate types for resources var resources []*namedBlock for _, k := range sortKeys(schema.ResourceSchemas) { + // Skipping all plugin framework struct generation. + // TODO: This is a temporary fix, generation should be fixed in the future. + if strings.HasSuffix(k, "_pluginframework") { + continue + } + v := schema.ResourceSchemas[k] b := &namedBlock{ filePattern: "resource_%s.go", @@ -71,6 +77,12 @@ func Run(ctx context.Context, schema *tfjson.ProviderSchema, path string) error // Generate types for data sources. var dataSources []*namedBlock for _, k := range sortKeys(schema.DataSourceSchemas) { + // Skipping all plugin framework struct generation. + // TODO: This is a temporary fix, generation should be fixed in the future. + if strings.HasSuffix(k, "_pluginframework") { + continue + } + v := schema.DataSourceSchemas[k] b := &namedBlock{ filePattern: "data_source_%s.go", diff --git a/bundle/internal/tf/codegen/schema/version.go b/bundle/internal/tf/codegen/schema/version.go index efb297243..b71ea7d1c 100644 --- a/bundle/internal/tf/codegen/schema/version.go +++ b/bundle/internal/tf/codegen/schema/version.go @@ -1,3 +1,3 @@ package schema -const ProviderVersion = "1.50.0" +const ProviderVersion = "1.52.0" diff --git a/bundle/internal/tf/schema/data_source_clusters.go b/bundle/internal/tf/schema/data_source_clusters.go index 7a5f3053d..8c5f9578e 100644 --- a/bundle/internal/tf/schema/data_source_clusters.go +++ b/bundle/internal/tf/schema/data_source_clusters.go @@ -2,8 +2,16 @@ package schema -type DataSourceClusters struct { - ClusterNameContains string `json:"cluster_name_contains,omitempty"` - Id string `json:"id,omitempty"` - Ids []string `json:"ids,omitempty"` +type DataSourceClustersFilterBy struct { + ClusterSources []string `json:"cluster_sources,omitempty"` + ClusterStates []string `json:"cluster_states,omitempty"` + IsPinned bool `json:"is_pinned,omitempty"` + PolicyId string `json:"policy_id,omitempty"` +} + +type DataSourceClusters struct { + ClusterNameContains string `json:"cluster_name_contains,omitempty"` + Id string `json:"id,omitempty"` + Ids []string `json:"ids,omitempty"` + FilterBy *DataSourceClustersFilterBy `json:"filter_by,omitempty"` } diff --git a/bundle/internal/tf/schema/data_source_external_location.go b/bundle/internal/tf/schema/data_source_external_location.go index a3e78cbd3..e1ad9dc3d 100644 --- a/bundle/internal/tf/schema/data_source_external_location.go +++ b/bundle/internal/tf/schema/data_source_external_location.go @@ -19,6 +19,7 @@ type DataSourceExternalLocationExternalLocationInfo struct { CreatedBy string `json:"created_by,omitempty"` CredentialId string `json:"credential_id,omitempty"` CredentialName string `json:"credential_name,omitempty"` + Fallback bool `json:"fallback,omitempty"` IsolationMode string `json:"isolation_mode,omitempty"` MetastoreId string `json:"metastore_id,omitempty"` Name string `json:"name,omitempty"` diff --git a/bundle/internal/tf/schema/data_source_share.go b/bundle/internal/tf/schema/data_source_share.go index 3b40fbb51..da9afaaef 100644 --- a/bundle/internal/tf/schema/data_source_share.go +++ b/bundle/internal/tf/schema/data_source_share.go @@ -18,12 +18,14 @@ type DataSourceShareObject struct { AddedBy string `json:"added_by,omitempty"` CdfEnabled bool `json:"cdf_enabled,omitempty"` Comment string `json:"comment,omitempty"` + Content string `json:"content,omitempty"` DataObjectType string `json:"data_object_type"` HistoryDataSharingStatus string `json:"history_data_sharing_status,omitempty"` Name string `json:"name"` SharedAs string `json:"shared_as,omitempty"` StartVersion int `json:"start_version,omitempty"` Status string `json:"status,omitempty"` + StringSharedAs string `json:"string_shared_as,omitempty"` Partition []DataSourceShareObjectPartition `json:"partition,omitempty"` } diff --git a/bundle/internal/tf/schema/resource_automatic_cluster_update_workspace_setting.go b/bundle/internal/tf/schema/resource_automatic_cluster_update_workspace_setting.go index e95639de8..5d7f6a140 100644 --- a/bundle/internal/tf/schema/resource_automatic_cluster_update_workspace_setting.go +++ b/bundle/internal/tf/schema/resource_automatic_cluster_update_workspace_setting.go @@ -2,20 +2,14 @@ package schema -type ResourceAutomaticClusterUpdateWorkspaceSettingAutomaticClusterUpdateWorkspaceEnablementDetails struct { - ForcedForComplianceMode bool `json:"forced_for_compliance_mode,omitempty"` - UnavailableForDisabledEntitlement bool `json:"unavailable_for_disabled_entitlement,omitempty"` - UnavailableForNonEnterpriseTier bool `json:"unavailable_for_non_enterprise_tier,omitempty"` -} - type ResourceAutomaticClusterUpdateWorkspaceSettingAutomaticClusterUpdateWorkspaceMaintenanceWindowWeekDayBasedScheduleWindowStartTime struct { - Hours int `json:"hours,omitempty"` - Minutes int `json:"minutes,omitempty"` + Hours int `json:"hours"` + Minutes int `json:"minutes"` } type ResourceAutomaticClusterUpdateWorkspaceSettingAutomaticClusterUpdateWorkspaceMaintenanceWindowWeekDayBasedSchedule struct { - DayOfWeek string `json:"day_of_week,omitempty"` - Frequency string `json:"frequency,omitempty"` + DayOfWeek string `json:"day_of_week"` + Frequency string `json:"frequency"` WindowStartTime *ResourceAutomaticClusterUpdateWorkspaceSettingAutomaticClusterUpdateWorkspaceMaintenanceWindowWeekDayBasedScheduleWindowStartTime `json:"window_start_time,omitempty"` } @@ -25,9 +19,9 @@ type ResourceAutomaticClusterUpdateWorkspaceSettingAutomaticClusterUpdateWorkspa type ResourceAutomaticClusterUpdateWorkspaceSettingAutomaticClusterUpdateWorkspace struct { CanToggle bool `json:"can_toggle,omitempty"` - Enabled bool `json:"enabled,omitempty"` + Enabled bool `json:"enabled"` + EnablementDetails []any `json:"enablement_details,omitempty"` RestartEvenIfNoUpdatesAvailable bool `json:"restart_even_if_no_updates_available,omitempty"` - EnablementDetails *ResourceAutomaticClusterUpdateWorkspaceSettingAutomaticClusterUpdateWorkspaceEnablementDetails `json:"enablement_details,omitempty"` MaintenanceWindow *ResourceAutomaticClusterUpdateWorkspaceSettingAutomaticClusterUpdateWorkspaceMaintenanceWindow `json:"maintenance_window,omitempty"` } diff --git a/bundle/internal/tf/schema/resource_cluster.go b/bundle/internal/tf/schema/resource_cluster.go index e4106d049..4ae063c89 100644 --- a/bundle/internal/tf/schema/resource_cluster.go +++ b/bundle/internal/tf/schema/resource_cluster.go @@ -176,6 +176,7 @@ type ResourceCluster struct { IdempotencyToken string `json:"idempotency_token,omitempty"` InstancePoolId string `json:"instance_pool_id,omitempty"` IsPinned bool `json:"is_pinned,omitempty"` + NoWait bool `json:"no_wait,omitempty"` NodeTypeId string `json:"node_type_id,omitempty"` NumWorkers int `json:"num_workers,omitempty"` PolicyId string `json:"policy_id,omitempty"` diff --git a/bundle/internal/tf/schema/resource_compliance_security_profile_workspace_setting.go b/bundle/internal/tf/schema/resource_compliance_security_profile_workspace_setting.go index 50815f753..8265adaed 100644 --- a/bundle/internal/tf/schema/resource_compliance_security_profile_workspace_setting.go +++ b/bundle/internal/tf/schema/resource_compliance_security_profile_workspace_setting.go @@ -3,8 +3,8 @@ package schema type ResourceComplianceSecurityProfileWorkspaceSettingComplianceSecurityProfileWorkspace struct { - ComplianceStandards []string `json:"compliance_standards,omitempty"` - IsEnabled bool `json:"is_enabled,omitempty"` + ComplianceStandards []string `json:"compliance_standards"` + IsEnabled bool `json:"is_enabled"` } type ResourceComplianceSecurityProfileWorkspaceSetting struct { diff --git a/bundle/internal/tf/schema/resource_enhanced_security_monitoring_workspace_setting.go b/bundle/internal/tf/schema/resource_enhanced_security_monitoring_workspace_setting.go index 2f552402a..e9c3b0abb 100644 --- a/bundle/internal/tf/schema/resource_enhanced_security_monitoring_workspace_setting.go +++ b/bundle/internal/tf/schema/resource_enhanced_security_monitoring_workspace_setting.go @@ -3,7 +3,7 @@ package schema type ResourceEnhancedSecurityMonitoringWorkspaceSettingEnhancedSecurityMonitoringWorkspace struct { - IsEnabled bool `json:"is_enabled,omitempty"` + IsEnabled bool `json:"is_enabled"` } type ResourceEnhancedSecurityMonitoringWorkspaceSetting struct { diff --git a/bundle/internal/tf/schema/resource_model_serving.go b/bundle/internal/tf/schema/resource_model_serving.go index 379807a5d..29d55cd5f 100644 --- a/bundle/internal/tf/schema/resource_model_serving.go +++ b/bundle/internal/tf/schema/resource_model_serving.go @@ -95,14 +95,16 @@ type ResourceModelServingConfigServedEntities struct { } type ResourceModelServingConfigServedModels struct { - EnvironmentVars map[string]string `json:"environment_vars,omitempty"` - InstanceProfileArn string `json:"instance_profile_arn,omitempty"` - ModelName string `json:"model_name"` - ModelVersion string `json:"model_version"` - Name string `json:"name,omitempty"` - ScaleToZeroEnabled bool `json:"scale_to_zero_enabled,omitempty"` - WorkloadSize string `json:"workload_size"` - WorkloadType string `json:"workload_type,omitempty"` + EnvironmentVars map[string]string `json:"environment_vars,omitempty"` + InstanceProfileArn string `json:"instance_profile_arn,omitempty"` + MaxProvisionedThroughput int `json:"max_provisioned_throughput,omitempty"` + MinProvisionedThroughput int `json:"min_provisioned_throughput,omitempty"` + ModelName string `json:"model_name"` + ModelVersion string `json:"model_version"` + Name string `json:"name,omitempty"` + ScaleToZeroEnabled bool `json:"scale_to_zero_enabled,omitempty"` + WorkloadSize string `json:"workload_size,omitempty"` + WorkloadType string `json:"workload_type,omitempty"` } type ResourceModelServingConfigTrafficConfigRoutes struct { diff --git a/bundle/internal/tf/schema/resource_share.go b/bundle/internal/tf/schema/resource_share.go index e531e7770..37f4d4546 100644 --- a/bundle/internal/tf/schema/resource_share.go +++ b/bundle/internal/tf/schema/resource_share.go @@ -18,20 +18,27 @@ type ResourceShareObject struct { AddedBy string `json:"added_by,omitempty"` CdfEnabled bool `json:"cdf_enabled,omitempty"` Comment string `json:"comment,omitempty"` + Content string `json:"content,omitempty"` DataObjectType string `json:"data_object_type"` HistoryDataSharingStatus string `json:"history_data_sharing_status,omitempty"` Name string `json:"name"` SharedAs string `json:"shared_as,omitempty"` StartVersion int `json:"start_version,omitempty"` Status string `json:"status,omitempty"` + StringSharedAs string `json:"string_shared_as,omitempty"` Partition []ResourceShareObjectPartition `json:"partition,omitempty"` } type ResourceShare struct { - CreatedAt int `json:"created_at,omitempty"` - CreatedBy string `json:"created_by,omitempty"` - Id string `json:"id,omitempty"` - Name string `json:"name"` - Owner string `json:"owner,omitempty"` - Object []ResourceShareObject `json:"object,omitempty"` + Comment string `json:"comment,omitempty"` + CreatedAt int `json:"created_at,omitempty"` + CreatedBy string `json:"created_by,omitempty"` + Id string `json:"id,omitempty"` + Name string `json:"name"` + Owner string `json:"owner,omitempty"` + StorageLocation string `json:"storage_location,omitempty"` + StorageRoot string `json:"storage_root,omitempty"` + UpdatedAt int `json:"updated_at,omitempty"` + UpdatedBy string `json:"updated_by,omitempty"` + Object []ResourceShareObject `json:"object,omitempty"` } diff --git a/bundle/internal/tf/schema/resource_sql_table.go b/bundle/internal/tf/schema/resource_sql_table.go index 51fb3bc0d..4f305c52e 100644 --- a/bundle/internal/tf/schema/resource_sql_table.go +++ b/bundle/internal/tf/schema/resource_sql_table.go @@ -15,6 +15,7 @@ type ResourceSqlTable struct { ClusterKeys []string `json:"cluster_keys,omitempty"` Comment string `json:"comment,omitempty"` DataSourceFormat string `json:"data_source_format,omitempty"` + EffectiveProperties map[string]string `json:"effective_properties,omitempty"` Id string `json:"id,omitempty"` Name string `json:"name"` Options map[string]string `json:"options,omitempty"` diff --git a/bundle/internal/tf/schema/root.go b/bundle/internal/tf/schema/root.go index ebdb7f095..5fc34d6b4 100644 --- a/bundle/internal/tf/schema/root.go +++ b/bundle/internal/tf/schema/root.go @@ -21,7 +21,7 @@ type Root struct { const ProviderHost = "registry.terraform.io" const ProviderSource = "databricks/databricks" -const ProviderVersion = "1.50.0" +const ProviderVersion = "1.52.0" func NewRoot() *Root { return &Root{ From 6c57683dc6077282dd95e03b19396f602dd5d635 Mon Sep 17 00:00:00 2001 From: "Lennart Kats (databricks)" Date: Sat, 21 Sep 2024 08:36:47 +0200 Subject: [PATCH 05/34] Reduce time until the prompt is shown for bundle run (#1727) ## Summary Makes the `databricks bundle run` command use local state before showing the menu prompt, which makes it show more quickly. For large/busy workspaces this means the prompt can show 2-3 seconds earlier. --- cmd/bundle/run.go | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/cmd/bundle/run.go b/cmd/bundle/run.go index 63458f85c..9ef5eb8ff 100644 --- a/cmd/bundle/run.go +++ b/cmd/bundle/run.go @@ -55,13 +55,7 @@ task or a Python wheel task, the second example applies. return diags.Error() } - diags = bundle.Apply(ctx, b, bundle.Seq( - phases.Initialize(), - terraform.Interpolate(), - terraform.Write(), - terraform.StatePull(), - terraform.Load(terraform.ErrorOnEmptyState), - )) + diags = bundle.Apply(ctx, b, phases.Initialize()) if err := diags.Error(); err != nil { return err } @@ -84,6 +78,16 @@ task or a Python wheel task, the second example applies. return fmt.Errorf("expected a KEY of the resource to run") } + diags = bundle.Apply(ctx, b, bundle.Seq( + terraform.Interpolate(), + terraform.Write(), + terraform.StatePull(), + terraform.Load(terraform.ErrorOnEmptyState), + )) + if err := diags.Error(); err != nil { + return err + } + runner, err := run.Find(b, args[0]) if err != nil { return err From 7665c639bd34392f5d95c177b520d48b9ffa40f4 Mon Sep 17 00:00:00 2001 From: "Lennart Kats (databricks)" Date: Mon, 23 Sep 2024 11:52:04 +0200 Subject: [PATCH 06/34] Use Unity Catalog for pipelines in the default-python template (#1766) ## Summary Enables Unity Catalog for pipelines in the default template. Pipelines will default to non-Unity Catalog pipelines if a catalog is not specified. *Small caveat*: there are cases where admins lock down the default catalog of a workspace and don't allow the creation of a new schema there. If that happens, the pipeline would fail at runtime with a clear error indicating what happened. ("PERMISSION_DENIED: User does not have CREATE SCHEMA on Catalog 'main'."). I've seen this with an internal Databricks workspace, where creating new non-UC schemas wasn't locked down, but creation in the `main` was. ## Testing - Validated on a non-UC + UC workspace. The catalog selection logic here is the same as applied for the SQL templates. --- .../resources/{{.project_name}}_pipeline.yml.tmpl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_pipeline.yml.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_pipeline.yml.tmpl index 4b8f74d17..bf4690461 100644 --- a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_pipeline.yml.tmpl +++ b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_pipeline.yml.tmpl @@ -3,6 +3,12 @@ resources: pipelines: {{.project_name}}_pipeline: name: {{.project_name}}_pipeline + {{- if eq default_catalog ""}} + ## Specify the 'catalog' field to configure this pipeline to make use of Unity Catalog: + # catalog: catalog_name + {{- else}} + catalog: {{default_catalog}} + {{- end}} target: {{.project_name}}_${bundle.environment} libraries: - notebook: From ac80d3dfcb648c26c131ad21aec45449ac5c3b31 Mon Sep 17 00:00:00 2001 From: Ilia Babanov Date: Mon, 23 Sep 2024 12:09:11 +0200 Subject: [PATCH 07/34] Add verbose flag to the "bundle deploy" command (#1774) ## Changes - Extract sync output logic from `cmd/sync` into `lib/sync` - Add hidden `verbose` flag to the `bundle deploy` command, it's false by default and hidden from the `--help` output - Pass output handler to the `deploy/files/upload` mutator if the verbose option is true The was an idea to use in-place output overriding each past file sync event in the output, bit that wont work for the extension, since it doesn't display deploy logs in the terminal. Example output: ``` ~/tmp/defpy: ~/cli/cli bundle deploy --sync-progress Building defpy... Uploading defpy-0.0.1+20240917.112755-py3-none-any.whl... Uploading bundle files to /Users/ilia.babanov@databricks.com/.bundle/defpy/dev/files... Action: PUT: requirements-dev.txt, resources/defpy_pipeline.yml, pytest.ini, src/defpy/main.py, src/defpy/__init__.py, src/dlt_pipeline.ipynb, tests/main_test.py, src/notebook.ipynb, setup.py, resources/defpy_job.yml, .vscode/extensions.json, .vscode/settings.json, fixtures/.gitkeep, .vscode/__builtins__.pyi, README.md, .gitignore, databricks.yml Uploaded tests Uploaded resources Uploaded fixtures Uploaded .vscode Uploaded src/defpy Uploaded requirements-dev.txt Uploaded .gitignore Uploaded fixtures/.gitkeep Uploaded src/defpy/__init__.py Uploaded databricks.yml Uploaded README.md Uploaded setup.py Uploaded .vscode/__builtins__.pyi Uploaded .vscode/extensions.json Uploaded src/dlt_pipeline.ipynb Uploaded .vscode/settings.json Uploaded resources/defpy_job.yml Uploaded pytest.ini Uploaded src/defpy/main.py Uploaded tests/main_test.py Uploaded resources/defpy_pipeline.yml Uploaded src/notebook.ipynb Initial Sync Complete Deploying resources... Updating deployment state... Deployment complete! ``` Output example in the extension: Screenshot 2024-09-19 at 11 07 48 ## Tests Manually for the `sync` and `bundle deploy` commands + vscode extension sync and deploy flows --- bundle/deploy/files/upload.go | 18 ++++++++++---- bundle/phases/deploy.go | 5 ++-- cmd/bundle/deploy.go | 14 ++++++++++- cmd/sync/sync.go | 38 ++++++++++++++--------------- {cmd => libs}/sync/output.go | 6 ++--- libs/sync/sync.go | 45 +++++++++++++++++++++++++---------- 6 files changed, 82 insertions(+), 44 deletions(-) rename {cmd => libs}/sync/output.go (83%) diff --git a/bundle/deploy/files/upload.go b/bundle/deploy/files/upload.go index 2c126623e..77b83611b 100644 --- a/bundle/deploy/files/upload.go +++ b/bundle/deploy/files/upload.go @@ -8,9 +8,12 @@ import ( "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/sync" ) -type upload struct{} +type upload struct { + outputHandler sync.OutputHandler +} func (m *upload) Name() string { return "files.Upload" @@ -18,11 +21,18 @@ func (m *upload) Name() string { func (m *upload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { cmdio.LogString(ctx, fmt.Sprintf("Uploading bundle files to %s...", b.Config.Workspace.FilePath)) - sync, err := GetSync(ctx, bundle.ReadOnly(b)) + opts, err := GetSyncOptions(ctx, bundle.ReadOnly(b)) if err != nil { return diag.FromErr(err) } + opts.OutputHandler = m.outputHandler + sync, err := sync.New(ctx, *opts) + if err != nil { + return diag.FromErr(err) + } + defer sync.Close() + b.Files, err = sync.RunOnce(ctx) if err != nil { return diag.FromErr(err) @@ -32,6 +42,6 @@ func (m *upload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { return nil } -func Upload() bundle.Mutator { - return &upload{} +func Upload(outputHandler sync.OutputHandler) bundle.Mutator { + return &upload{outputHandler} } diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 49544227e..097c561eb 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -18,6 +18,7 @@ import ( "github.com/databricks/cli/bundle/python" "github.com/databricks/cli/bundle/scripts" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/sync" terraformlib "github.com/databricks/cli/libs/terraform" tfjson "github.com/hashicorp/terraform-json" ) @@ -128,7 +129,7 @@ properties such as the 'catalog' or 'storage' are changed:` } // The deploy phase deploys artifacts and resources. -func Deploy() bundle.Mutator { +func Deploy(outputHandler sync.OutputHandler) bundle.Mutator { // Core mutators that CRUD resources and modify deployment state. These // mutators need informed consent if they are potentially destructive. deployCore := bundle.Defer( @@ -157,7 +158,7 @@ func Deploy() bundle.Mutator { libraries.ExpandGlobReferences(), libraries.Upload(), python.TransformWheelTask(), - files.Upload(), + files.Upload(outputHandler), deploy.StateUpdate(), deploy.StatePush(), permissions.ApplyWorkspaceRootPermissions(), diff --git a/cmd/bundle/deploy.go b/cmd/bundle/deploy.go index 1166875ab..492317347 100644 --- a/cmd/bundle/deploy.go +++ b/cmd/bundle/deploy.go @@ -10,6 +10,7 @@ import ( "github.com/databricks/cli/cmd/bundle/utils" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/sync" "github.com/spf13/cobra" ) @@ -25,11 +26,15 @@ func newDeployCommand() *cobra.Command { var failOnActiveRuns bool var computeID string var autoApprove bool + var verbose bool cmd.Flags().BoolVar(&force, "force", false, "Force-override Git branch validation.") cmd.Flags().BoolVar(&forceLock, "force-lock", false, "Force acquisition of deployment lock.") cmd.Flags().BoolVar(&failOnActiveRuns, "fail-on-active-runs", false, "Fail if there are running jobs or pipelines in the deployment.") cmd.Flags().StringVarP(&computeID, "compute-id", "c", "", "Override compute in the deployment with the given compute ID.") cmd.Flags().BoolVar(&autoApprove, "auto-approve", false, "Skip interactive approvals that might be required for deployment.") + cmd.Flags().BoolVar(&verbose, "verbose", false, "Enable verbose output.") + // Verbose flag currently only affects file sync output, it's used by the vscode extension + cmd.Flags().MarkHidden("verbose") cmd.RunE = func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() @@ -51,11 +56,18 @@ func newDeployCommand() *cobra.Command { return nil }) + var outputHandler sync.OutputHandler + if verbose { + outputHandler = func(ctx context.Context, c <-chan sync.Event) { + sync.TextOutput(ctx, c, cmd.OutOrStdout()) + } + } + diags = diags.Extend( bundle.Apply(ctx, b, bundle.Seq( phases.Initialize(), phases.Build(), - phases.Deploy(), + phases.Deploy(outputHandler), )), ) } diff --git a/cmd/sync/sync.go b/cmd/sync/sync.go index 23a4c018f..2092d9e33 100644 --- a/cmd/sync/sync.go +++ b/cmd/sync/sync.go @@ -6,7 +6,6 @@ import ( "fmt" "io" "path/filepath" - stdsync "sync" "time" "github.com/databricks/cli/bundle" @@ -46,6 +45,21 @@ func (f *syncFlags) syncOptionsFromArgs(cmd *cobra.Command, args []string) (*syn return nil, flag.ErrHelp } + var outputFunc func(context.Context, <-chan sync.Event, io.Writer) + switch f.output { + case flags.OutputText: + outputFunc = sync.TextOutput + case flags.OutputJSON: + outputFunc = sync.JsonOutput + } + + var outputHandler sync.OutputHandler + if outputFunc != nil { + outputHandler = func(ctx context.Context, events <-chan sync.Event) { + outputFunc(ctx, events, cmd.OutOrStdout()) + } + } + opts := sync.SyncOptions{ LocalRoot: vfs.MustNew(args[0]), Paths: []string{"."}, @@ -62,6 +76,8 @@ func (f *syncFlags) syncOptionsFromArgs(cmd *cobra.Command, args []string) (*syn // exist and add it to the `.gitignore` file in the root. SnapshotBasePath: filepath.Join(args[0], ".databricks"), WorkspaceClient: root.WorkspaceClient(cmd.Context()), + + OutputHandler: outputHandler, } return &opts, nil } @@ -118,23 +134,7 @@ func New() *cobra.Command { if err != nil { return err } - - var outputFunc func(context.Context, <-chan sync.Event, io.Writer) - switch f.output { - case flags.OutputText: - outputFunc = textOutput - case flags.OutputJSON: - outputFunc = jsonOutput - } - - var wg stdsync.WaitGroup - if outputFunc != nil { - wg.Add(1) - go func() { - defer wg.Done() - outputFunc(ctx, s.Events(), cmd.OutOrStdout()) - }() - } + defer s.Close() if f.watch { err = s.RunContinuous(ctx) @@ -142,8 +142,6 @@ func New() *cobra.Command { _, err = s.RunOnce(ctx) } - s.Close() - wg.Wait() return err } diff --git a/cmd/sync/output.go b/libs/sync/output.go similarity index 83% rename from cmd/sync/output.go rename to libs/sync/output.go index 2785343f9..c01b25ef6 100644 --- a/cmd/sync/output.go +++ b/libs/sync/output.go @@ -5,12 +5,10 @@ import ( "context" "encoding/json" "io" - - "github.com/databricks/cli/libs/sync" ) // Read synchronization events and write them as JSON to the specified writer (typically stdout). -func jsonOutput(ctx context.Context, ch <-chan sync.Event, w io.Writer) { +func JsonOutput(ctx context.Context, ch <-chan Event, w io.Writer) { enc := json.NewEncoder(w) for { select { @@ -31,7 +29,7 @@ func jsonOutput(ctx context.Context, ch <-chan sync.Event, w io.Writer) { } // Read synchronization events and write them as text to the specified writer (typically stdout). -func textOutput(ctx context.Context, ch <-chan sync.Event, w io.Writer) { +func TextOutput(ctx context.Context, ch <-chan Event, w io.Writer) { bw := bufio.NewWriter(w) for { diff --git a/libs/sync/sync.go b/libs/sync/sync.go index 9eaebf2ad..cc9c73944 100644 --- a/libs/sync/sync.go +++ b/libs/sync/sync.go @@ -3,6 +3,7 @@ package sync import ( "context" "fmt" + stdsync "sync" "time" "github.com/databricks/cli/libs/filer" @@ -15,6 +16,8 @@ import ( "github.com/databricks/databricks-sdk-go/service/iam" ) +type OutputHandler func(context.Context, <-chan Event) + type SyncOptions struct { LocalRoot vfs.Path Paths []string @@ -34,6 +37,8 @@ type SyncOptions struct { CurrentUser *iam.User Host string + + OutputHandler OutputHandler } type Sync struct { @@ -49,6 +54,10 @@ type Sync struct { // Synchronization progress events are sent to this event notifier. notifier EventNotifier seq int + + // WaitGroup is automatically created when an output handler is provided in the SyncOptions. + // Close call is required to ensure the output handler goroutine handles all events in time. + outputWaitGroup *stdsync.WaitGroup } // New initializes and returns a new [Sync] instance. @@ -106,31 +115,41 @@ func New(ctx context.Context, opts SyncOptions) (*Sync, error) { return nil, err } + var notifier EventNotifier + var outputWaitGroup = &stdsync.WaitGroup{} + if opts.OutputHandler != nil { + ch := make(chan Event, MaxRequestsInFlight) + notifier = &ChannelNotifier{ch} + outputWaitGroup.Add(1) + go func() { + defer outputWaitGroup.Done() + opts.OutputHandler(ctx, ch) + }() + } else { + notifier = &NopNotifier{} + } + return &Sync{ SyncOptions: &opts, - fileSet: fileSet, - includeFileSet: includeFileSet, - excludeFileSet: excludeFileSet, - snapshot: snapshot, - filer: filer, - notifier: &NopNotifier{}, - seq: 0, + fileSet: fileSet, + includeFileSet: includeFileSet, + excludeFileSet: excludeFileSet, + snapshot: snapshot, + filer: filer, + notifier: notifier, + outputWaitGroup: outputWaitGroup, + seq: 0, }, nil } -func (s *Sync) Events() <-chan Event { - ch := make(chan Event, MaxRequestsInFlight) - s.notifier = &ChannelNotifier{ch} - return ch -} - func (s *Sync) Close() { if s.notifier == nil { return } s.notifier.Close() s.notifier = nil + s.outputWaitGroup.Wait() } func (s *Sync) notifyStart(ctx context.Context, d diff) { From 56ed9bebf39b2ef6430e42dbb840684285217436 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Mon, 23 Sep 2024 12:42:34 +0200 Subject: [PATCH 08/34] Added support for creating all-purpose clusters (#1698) ## Changes Added support for creating all-purpose clusters Example of configuration ``` bundle: name: clusters resources: clusters: test_cluster: cluster_name: "Test Cluster" num_workers: 2 node_type_id: "i3.xlarge" autoscale: min_workers: 2 max_workers: 7 spark_version: "13.3.x-scala2.12" spark_conf: "spark.executor.memory": "2g" jobs: test_job: name: "Test Job" tasks: - task_key: test_task existing_cluster_id: ${resources.clusters.test_cluster.id} notebook_task: notebook_path: "./src/test.py" targets: development: mode: development compute_id: ${resources.clusters.test_cluster.id} ``` ## Tests Added unit, config and E2E tests --- bundle/config/bundle.go | 7 +- bundle/config/mutator/apply_presets.go | 15 +++ bundle/config/mutator/compute_id_compat.go | 87 +++++++++++++++++ .../config/mutator/compute_id_compate_test.go | 57 +++++++++++ bundle/config/mutator/mutator.go | 1 + bundle/config/mutator/override_compute.go | 8 +- .../config/mutator/override_compute_test.go | 4 +- .../mutator/process_target_mode_test.go | 10 ++ bundle/config/mutator/run_as_test.go | 2 + bundle/config/resources.go | 1 + bundle/config/resources/clusters.go | 39 ++++++++ bundle/config/root.go | 6 +- bundle/config/target.go | 7 +- bundle/deploy/terraform/convert.go | 22 +++++ bundle/deploy/terraform/convert_test.go | 56 +++++++++++ bundle/deploy/terraform/interpolate.go | 2 + bundle/deploy/terraform/interpolate_test.go | 2 + .../deploy/terraform/tfdyn/convert_cluster.go | 52 ++++++++++ .../terraform/tfdyn/convert_cluster_test.go | 97 +++++++++++++++++++ bundle/tests/clusters/databricks.yml | 36 +++++++ bundle/tests/clusters_test.go | 36 +++++++ cmd/bundle/deploy.go | 11 ++- .../clusters/databricks_template_schema.json | 16 +++ .../clusters/template/databricks.yml.tmpl | 24 +++++ .../bundles/clusters/template/hello_world.py | 1 + internal/bundle/clusters_test.go | 56 +++++++++++ internal/testutil/cloud.go | 4 + 27 files changed, 643 insertions(+), 16 deletions(-) create mode 100644 bundle/config/mutator/compute_id_compat.go create mode 100644 bundle/config/mutator/compute_id_compate_test.go create mode 100644 bundle/config/resources/clusters.go create mode 100644 bundle/deploy/terraform/tfdyn/convert_cluster.go create mode 100644 bundle/deploy/terraform/tfdyn/convert_cluster_test.go create mode 100644 bundle/tests/clusters/databricks.yml create mode 100644 bundle/tests/clusters_test.go create mode 100644 internal/bundle/bundles/clusters/databricks_template_schema.json create mode 100644 internal/bundle/bundles/clusters/template/databricks.yml.tmpl create mode 100644 internal/bundle/bundles/clusters/template/hello_world.py create mode 100644 internal/bundle/clusters_test.go diff --git a/bundle/config/bundle.go b/bundle/config/bundle.go index 78648dfd7..f533c4d18 100644 --- a/bundle/config/bundle.go +++ b/bundle/config/bundle.go @@ -38,8 +38,11 @@ type Bundle struct { // Annotated readonly as this should be set at the target level. Mode Mode `json:"mode,omitempty" bundle:"readonly"` - // Overrides the compute used for jobs and other supported assets. - ComputeID string `json:"compute_id,omitempty"` + // DEPRECATED: Overrides the compute used for jobs and other supported assets. + ComputeId string `json:"compute_id,omitempty"` + + // Overrides the cluster used for jobs and other supported assets. + ClusterId string `json:"cluster_id,omitempty"` // Deployment section specifies deployment related configuration for bundle Deployment Deployment `json:"deployment,omitempty"` diff --git a/bundle/config/mutator/apply_presets.go b/bundle/config/mutator/apply_presets.go index 28d015c10..27af82e54 100644 --- a/bundle/config/mutator/apply_presets.go +++ b/bundle/config/mutator/apply_presets.go @@ -160,6 +160,21 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos // the Databricks UI and via the SQL API. } + // Clusters: Prefix, Tags + for _, c := range r.Clusters { + c.ClusterName = prefix + c.ClusterName + if c.CustomTags == nil { + c.CustomTags = make(map[string]string) + } + for _, tag := range tags { + normalisedKey := b.Tagging.NormalizeKey(tag.Key) + normalisedValue := b.Tagging.NormalizeValue(tag.Value) + if _, ok := c.CustomTags[normalisedKey]; !ok { + c.CustomTags[normalisedKey] = normalisedValue + } + } + } + return nil } diff --git a/bundle/config/mutator/compute_id_compat.go b/bundle/config/mutator/compute_id_compat.go new file mode 100644 index 000000000..3afe02e9e --- /dev/null +++ b/bundle/config/mutator/compute_id_compat.go @@ -0,0 +1,87 @@ +package mutator + +import ( + "context" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +type computeIdToClusterId struct{} + +func ComputeIdToClusterId() bundle.Mutator { + return &computeIdToClusterId{} +} + +func (m *computeIdToClusterId) Name() string { + return "ComputeIdToClusterId" +} + +func (m *computeIdToClusterId) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + var diags diag.Diagnostics + + // The "compute_id" key is set; rewrite it to "cluster_id". + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + v, d := rewriteComputeIdToClusterId(v, dyn.NewPath(dyn.Key("bundle"))) + diags = diags.Extend(d) + + // Check if the "compute_id" key is set in any target overrides. + return dyn.MapByPattern(v, dyn.NewPattern(dyn.Key("targets"), dyn.AnyKey()), func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + v, d := rewriteComputeIdToClusterId(v, dyn.Path{}) + diags = diags.Extend(d) + return v, nil + }) + }) + + diags = diags.Extend(diag.FromErr(err)) + return diags +} + +func rewriteComputeIdToClusterId(v dyn.Value, p dyn.Path) (dyn.Value, diag.Diagnostics) { + var diags diag.Diagnostics + computeIdPath := p.Append(dyn.Key("compute_id")) + computeId, err := dyn.GetByPath(v, computeIdPath) + + // If the "compute_id" key is not set, we don't need to do anything. + if err != nil { + return v, nil + } + + if computeId.Kind() == dyn.KindInvalid { + return v, nil + } + + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Warning, + Summary: "compute_id is deprecated, please use cluster_id instead", + Locations: computeId.Locations(), + Paths: []dyn.Path{computeIdPath}, + }) + + clusterIdPath := p.Append(dyn.Key("cluster_id")) + nv, err := dyn.SetByPath(v, clusterIdPath, computeId) + if err != nil { + return dyn.InvalidValue, diag.FromErr(err) + } + // Drop the "compute_id" key. + vout, err := dyn.Walk(nv, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + switch len(p) { + case 0: + return v, nil + case 1: + if p[0] == dyn.Key("compute_id") { + return v, dyn.ErrDrop + } + return v, nil + case 2: + if p[1] == dyn.Key("compute_id") { + return v, dyn.ErrDrop + } + } + return v, dyn.ErrSkip + }) + + diags = diags.Extend(diag.FromErr(err)) + return vout, diags +} diff --git a/bundle/config/mutator/compute_id_compate_test.go b/bundle/config/mutator/compute_id_compate_test.go new file mode 100644 index 000000000..e59d37e39 --- /dev/null +++ b/bundle/config/mutator/compute_id_compate_test.go @@ -0,0 +1,57 @@ +package mutator_test + +import ( + "context" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/mutator" + "github.com/databricks/cli/libs/diag" + "github.com/stretchr/testify/assert" +) + +func TestComputeIdToClusterId(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Bundle: config.Bundle{ + ComputeId: "compute-id", + }, + }, + } + + diags := bundle.Apply(context.Background(), b, mutator.ComputeIdToClusterId()) + assert.NoError(t, diags.Error()) + assert.Equal(t, "compute-id", b.Config.Bundle.ClusterId) + assert.Empty(t, b.Config.Bundle.ComputeId) + + assert.Len(t, diags, 1) + assert.Equal(t, "compute_id is deprecated, please use cluster_id instead", diags[0].Summary) + assert.Equal(t, diag.Warning, diags[0].Severity) +} + +func TestComputeIdToClusterIdInTargetOverride(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Targets: map[string]*config.Target{ + "dev": { + ComputeId: "compute-id-dev", + }, + }, + }, + } + + diags := bundle.Apply(context.Background(), b, mutator.ComputeIdToClusterId()) + assert.NoError(t, diags.Error()) + assert.Empty(t, b.Config.Targets["dev"].ComputeId) + + diags = diags.Extend(bundle.Apply(context.Background(), b, mutator.SelectTarget("dev"))) + assert.NoError(t, diags.Error()) + + assert.Equal(t, "compute-id-dev", b.Config.Bundle.ClusterId) + assert.Empty(t, b.Config.Bundle.ComputeId) + + assert.Len(t, diags, 1) + assert.Equal(t, "compute_id is deprecated, please use cluster_id instead", diags[0].Summary) + assert.Equal(t, diag.Warning, diags[0].Severity) +} diff --git a/bundle/config/mutator/mutator.go b/bundle/config/mutator/mutator.go index 0458beff4..faf50ae6e 100644 --- a/bundle/config/mutator/mutator.go +++ b/bundle/config/mutator/mutator.go @@ -23,6 +23,7 @@ func DefaultMutators() []bundle.Mutator { VerifyCliVersion(), EnvironmentsToTargets(), + ComputeIdToClusterId(), InitializeVariables(), DefineDefaultTarget(), LoadGitDetails(), diff --git a/bundle/config/mutator/override_compute.go b/bundle/config/mutator/override_compute.go index 73fbad364..5700cdf26 100644 --- a/bundle/config/mutator/override_compute.go +++ b/bundle/config/mutator/override_compute.go @@ -39,22 +39,22 @@ func overrideJobCompute(j *resources.Job, compute string) { func (m *overrideCompute) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { if b.Config.Bundle.Mode != config.Development { - if b.Config.Bundle.ComputeID != "" { + if b.Config.Bundle.ClusterId != "" { return diag.Errorf("cannot override compute for an target that does not use 'mode: development'") } return nil } if v := env.Get(ctx, "DATABRICKS_CLUSTER_ID"); v != "" { - b.Config.Bundle.ComputeID = v + b.Config.Bundle.ClusterId = v } - if b.Config.Bundle.ComputeID == "" { + if b.Config.Bundle.ClusterId == "" { return nil } r := b.Config.Resources for i := range r.Jobs { - overrideJobCompute(r.Jobs[i], b.Config.Bundle.ComputeID) + overrideJobCompute(r.Jobs[i], b.Config.Bundle.ClusterId) } return nil diff --git a/bundle/config/mutator/override_compute_test.go b/bundle/config/mutator/override_compute_test.go index 152ee543e..369447d7e 100644 --- a/bundle/config/mutator/override_compute_test.go +++ b/bundle/config/mutator/override_compute_test.go @@ -20,7 +20,7 @@ func TestOverrideDevelopment(t *testing.T) { Config: config.Root{ Bundle: config.Bundle{ Mode: config.Development, - ComputeID: "newClusterID", + ClusterId: "newClusterID", }, Resources: config.Resources{ Jobs: map[string]*resources.Job{ @@ -144,7 +144,7 @@ func TestOverrideProduction(t *testing.T) { b := &bundle.Bundle{ Config: config.Root{ Bundle: config.Bundle{ - ComputeID: "newClusterID", + ClusterId: "newClusterID", }, Resources: config.Resources{ Jobs: map[string]*resources.Job{ diff --git a/bundle/config/mutator/process_target_mode_test.go b/bundle/config/mutator/process_target_mode_test.go index 42f1929c8..b0eb57ee1 100644 --- a/bundle/config/mutator/process_target_mode_test.go +++ b/bundle/config/mutator/process_target_mode_test.go @@ -13,6 +13,7 @@ import ( "github.com/databricks/cli/libs/tags" sdkconfig "github.com/databricks/databricks-sdk-go/config" "github.com/databricks/databricks-sdk-go/service/catalog" + "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/iam" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/databricks/databricks-sdk-go/service/ml" @@ -119,6 +120,9 @@ func mockBundle(mode config.Mode) *bundle.Bundle { Schemas: map[string]*resources.Schema{ "schema1": {CreateSchema: &catalog.CreateSchema{Name: "schema1"}}, }, + Clusters: map[string]*resources.Cluster{ + "cluster1": {ClusterSpec: &compute.ClusterSpec{ClusterName: "cluster1", SparkVersion: "13.2.x", NumWorkers: 1}}, + }, }, }, // Use AWS implementation for testing. @@ -177,6 +181,9 @@ func TestProcessTargetModeDevelopment(t *testing.T) { // Schema 1 assert.Equal(t, "dev_lennart_schema1", b.Config.Resources.Schemas["schema1"].Name) + + // Clusters + assert.Equal(t, "[dev lennart] cluster1", b.Config.Resources.Clusters["cluster1"].ClusterName) } func TestProcessTargetModeDevelopmentTagNormalizationForAws(t *testing.T) { @@ -281,6 +288,7 @@ func TestProcessTargetModeDefault(t *testing.T) { assert.Equal(t, "servingendpoint1", b.Config.Resources.ModelServingEndpoints["servingendpoint1"].Name) assert.Equal(t, "registeredmodel1", b.Config.Resources.RegisteredModels["registeredmodel1"].Name) assert.Equal(t, "qualityMonitor1", b.Config.Resources.QualityMonitors["qualityMonitor1"].TableName) + assert.Equal(t, "cluster1", b.Config.Resources.Clusters["cluster1"].ClusterName) } func TestProcessTargetModeProduction(t *testing.T) { @@ -312,6 +320,7 @@ func TestProcessTargetModeProduction(t *testing.T) { b.Config.Resources.Experiments["experiment2"].Permissions = permissions b.Config.Resources.Models["model1"].Permissions = permissions b.Config.Resources.ModelServingEndpoints["servingendpoint1"].Permissions = permissions + b.Config.Resources.Clusters["cluster1"].Permissions = permissions diags = validateProductionMode(context.Background(), b, false) require.NoError(t, diags.Error()) @@ -322,6 +331,7 @@ func TestProcessTargetModeProduction(t *testing.T) { assert.Equal(t, "servingendpoint1", b.Config.Resources.ModelServingEndpoints["servingendpoint1"].Name) assert.Equal(t, "registeredmodel1", b.Config.Resources.RegisteredModels["registeredmodel1"].Name) assert.Equal(t, "qualityMonitor1", b.Config.Resources.QualityMonitors["qualityMonitor1"].TableName) + assert.Equal(t, "cluster1", b.Config.Resources.Clusters["cluster1"].ClusterName) } func TestProcessTargetModeProductionOkForPrincipal(t *testing.T) { diff --git a/bundle/config/mutator/run_as_test.go b/bundle/config/mutator/run_as_test.go index e6cef9ba4..abeea45d0 100644 --- a/bundle/config/mutator/run_as_test.go +++ b/bundle/config/mutator/run_as_test.go @@ -32,6 +32,7 @@ func allResourceTypes(t *testing.T) []string { // the dyn library gives us the correct list of all resources supported. Please // also update this check when adding a new resource require.Equal(t, []string{ + "clusters", "experiments", "jobs", "model_serving_endpoints", @@ -133,6 +134,7 @@ func TestRunAsErrorForUnsupportedResources(t *testing.T) { // some point in the future. These resources are (implicitly) on the deny list, since // they are not on the allow list below. allowList := []string{ + "clusters", "jobs", "models", "registered_models", diff --git a/bundle/config/resources.go b/bundle/config/resources.go index 22d69ffb5..a3afb7fc3 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -19,6 +19,7 @@ type Resources struct { RegisteredModels map[string]*resources.RegisteredModel `json:"registered_models,omitempty"` QualityMonitors map[string]*resources.QualityMonitor `json:"quality_monitors,omitempty"` Schemas map[string]*resources.Schema `json:"schemas,omitempty"` + Clusters map[string]*resources.Cluster `json:"clusters,omitempty"` } type ConfigResource interface { diff --git a/bundle/config/resources/clusters.go b/bundle/config/resources/clusters.go new file mode 100644 index 000000000..632345666 --- /dev/null +++ b/bundle/config/resources/clusters.go @@ -0,0 +1,39 @@ +package resources + +import ( + "context" + + "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/marshal" + "github.com/databricks/databricks-sdk-go/service/compute" +) + +type Cluster struct { + ID string `json:"id,omitempty" bundle:"readonly"` + Permissions []Permission `json:"permissions,omitempty"` + ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"` + + *compute.ClusterSpec +} + +func (s *Cluster) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, s) +} + +func (s Cluster) MarshalJSON() ([]byte, error) { + return marshal.Marshal(s) +} + +func (s *Cluster) Exists(ctx context.Context, w *databricks.WorkspaceClient, id string) (bool, error) { + _, err := w.Clusters.GetByClusterId(ctx, id) + if err != nil { + log.Debugf(ctx, "cluster %s does not exist", id) + return false, err + } + return true, nil +} + +func (s *Cluster) TerraformResourceName() string { + return "databricks_cluster" +} diff --git a/bundle/config/root.go b/bundle/config/root.go index 884c2e1ca..92d834f0a 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -366,9 +366,9 @@ func (r *Root) MergeTargetOverrides(name string) error { } } - // Merge `compute_id`. This field must be overwritten if set, not merged. - if v := target.Get("compute_id"); v.Kind() != dyn.KindInvalid { - root, err = dyn.SetByPath(root, dyn.NewPath(dyn.Key("bundle"), dyn.Key("compute_id")), v) + // Merge `cluster_id`. This field must be overwritten if set, not merged. + if v := target.Get("cluster_id"); v.Kind() != dyn.KindInvalid { + root, err = dyn.SetByPath(root, dyn.NewPath(dyn.Key("bundle"), dyn.Key("cluster_id")), v) if err != nil { return err } diff --git a/bundle/config/target.go b/bundle/config/target.go index fc6ba7b5b..fae9c940b 100644 --- a/bundle/config/target.go +++ b/bundle/config/target.go @@ -24,8 +24,11 @@ type Target struct { // name prefix of deployed resources. Presets Presets `json:"presets,omitempty"` - // Overrides the compute used for jobs and other supported assets. - ComputeID string `json:"compute_id,omitempty"` + // DEPRECATED: Overrides the compute used for jobs and other supported assets. + ComputeId string `json:"compute_id,omitempty"` + + // Overrides the cluster used for jobs and other supported assets. + ClusterId string `json:"cluster_id,omitempty"` Bundle *Bundle `json:"bundle,omitempty"` diff --git a/bundle/deploy/terraform/convert.go b/bundle/deploy/terraform/convert.go index f13c241ce..5a548e3b5 100644 --- a/bundle/deploy/terraform/convert.go +++ b/bundle/deploy/terraform/convert.go @@ -231,6 +231,13 @@ func BundleToTerraform(config *config.Root) *schema.Root { tfroot.Resource.QualityMonitor[k] = &dst } + for k, src := range config.Resources.Clusters { + noResources = false + var dst schema.ResourceCluster + conv(src, &dst) + tfroot.Resource.Cluster[k] = &dst + } + // We explicitly set "resource" to nil to omit it from a JSON encoding. // This is required because the terraform CLI requires >= 1 resources defined // if the "resource" property is used in a .tf.json file. @@ -394,6 +401,16 @@ func TerraformToBundle(state *resourcesState, config *config.Root) error { } cur.ID = instance.Attributes.ID config.Resources.Schemas[resource.Name] = cur + case "databricks_cluster": + if config.Resources.Clusters == nil { + config.Resources.Clusters = make(map[string]*resources.Cluster) + } + cur := config.Resources.Clusters[resource.Name] + if cur == nil { + cur = &resources.Cluster{ModifiedStatus: resources.ModifiedStatusDeleted} + } + cur.ID = instance.Attributes.ID + config.Resources.Clusters[resource.Name] = cur case "databricks_permissions": case "databricks_grants": // Ignore; no need to pull these back into the configuration. @@ -443,6 +460,11 @@ func TerraformToBundle(state *resourcesState, config *config.Root) error { src.ModifiedStatus = resources.ModifiedStatusCreated } } + for _, src := range config.Resources.Clusters { + if src.ModifiedStatus == "" && src.ID == "" { + src.ModifiedStatus = resources.ModifiedStatusCreated + } + } return nil } diff --git a/bundle/deploy/terraform/convert_test.go b/bundle/deploy/terraform/convert_test.go index e4ef6114a..4c6866d9d 100644 --- a/bundle/deploy/terraform/convert_test.go +++ b/bundle/deploy/terraform/convert_test.go @@ -663,6 +663,14 @@ func TestTerraformToBundleEmptyLocalResources(t *testing.T) { {Attributes: stateInstanceAttributes{ID: "1"}}, }, }, + { + Type: "databricks_cluster", + Mode: "managed", + Name: "test_cluster", + Instances: []stateResourceInstance{ + {Attributes: stateInstanceAttributes{ID: "1"}}, + }, + }, }, } err := TerraformToBundle(&tfState, &config) @@ -692,6 +700,9 @@ func TestTerraformToBundleEmptyLocalResources(t *testing.T) { assert.Equal(t, "1", config.Resources.Schemas["test_schema"].ID) assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Schemas["test_schema"].ModifiedStatus) + assert.Equal(t, "1", config.Resources.Clusters["test_cluster"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Clusters["test_cluster"].ModifiedStatus) + AssertFullResourceCoverage(t, &config) } @@ -754,6 +765,13 @@ func TestTerraformToBundleEmptyRemoteResources(t *testing.T) { }, }, }, + Clusters: map[string]*resources.Cluster{ + "test_cluster": { + ClusterSpec: &compute.ClusterSpec{ + ClusterName: "test_cluster", + }, + }, + }, }, } var tfState = resourcesState{ @@ -786,6 +804,9 @@ func TestTerraformToBundleEmptyRemoteResources(t *testing.T) { assert.Equal(t, "", config.Resources.Schemas["test_schema"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Schemas["test_schema"].ModifiedStatus) + assert.Equal(t, "", config.Resources.Clusters["test_cluster"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Clusters["test_cluster"].ModifiedStatus) + AssertFullResourceCoverage(t, &config) } @@ -888,6 +909,18 @@ func TestTerraformToBundleModifiedResources(t *testing.T) { }, }, }, + Clusters: map[string]*resources.Cluster{ + "test_cluster": { + ClusterSpec: &compute.ClusterSpec{ + ClusterName: "test_cluster", + }, + }, + "test_cluster_new": { + ClusterSpec: &compute.ClusterSpec{ + ClusterName: "test_cluster_new", + }, + }, + }, }, } var tfState = resourcesState{ @@ -1020,6 +1053,22 @@ func TestTerraformToBundleModifiedResources(t *testing.T) { {Attributes: stateInstanceAttributes{ID: "2"}}, }, }, + { + Type: "databricks_cluster", + Mode: "managed", + Name: "test_cluster", + Instances: []stateResourceInstance{ + {Attributes: stateInstanceAttributes{ID: "1"}}, + }, + }, + { + Type: "databricks_cluster", + Mode: "managed", + Name: "test_cluster_old", + Instances: []stateResourceInstance{ + {Attributes: stateInstanceAttributes{ID: "2"}}, + }, + }, }, } err := TerraformToBundle(&tfState, &config) @@ -1081,6 +1130,13 @@ func TestTerraformToBundleModifiedResources(t *testing.T) { assert.Equal(t, "", config.Resources.Schemas["test_schema_new"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Schemas["test_schema_new"].ModifiedStatus) + assert.Equal(t, "1", config.Resources.Clusters["test_cluster"].ID) + assert.Equal(t, "", config.Resources.Clusters["test_cluster"].ModifiedStatus) + assert.Equal(t, "2", config.Resources.Clusters["test_cluster_old"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Clusters["test_cluster_old"].ModifiedStatus) + assert.Equal(t, "", config.Resources.Clusters["test_cluster_new"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Clusters["test_cluster_new"].ModifiedStatus) + AssertFullResourceCoverage(t, &config) } diff --git a/bundle/deploy/terraform/interpolate.go b/bundle/deploy/terraform/interpolate.go index faa098e1c..12894c684 100644 --- a/bundle/deploy/terraform/interpolate.go +++ b/bundle/deploy/terraform/interpolate.go @@ -58,6 +58,8 @@ func (m *interpolateMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.D path = dyn.NewPath(dyn.Key("databricks_quality_monitor")).Append(path[2:]...) case dyn.Key("schemas"): path = dyn.NewPath(dyn.Key("databricks_schema")).Append(path[2:]...) + case dyn.Key("clusters"): + path = dyn.NewPath(dyn.Key("databricks_cluster")).Append(path[2:]...) default: // Trigger "key not found" for unknown resource types. return dyn.GetByPath(root, path) diff --git a/bundle/deploy/terraform/interpolate_test.go b/bundle/deploy/terraform/interpolate_test.go index 5ceb243bc..630a904ac 100644 --- a/bundle/deploy/terraform/interpolate_test.go +++ b/bundle/deploy/terraform/interpolate_test.go @@ -31,6 +31,7 @@ func TestInterpolate(t *testing.T) { "other_model_serving": "${resources.model_serving_endpoints.other_model_serving.id}", "other_registered_model": "${resources.registered_models.other_registered_model.id}", "other_schema": "${resources.schemas.other_schema.id}", + "other_cluster": "${resources.clusters.other_cluster.id}", }, Tasks: []jobs.Task{ { @@ -67,6 +68,7 @@ func TestInterpolate(t *testing.T) { assert.Equal(t, "${databricks_model_serving.other_model_serving.id}", j.Tags["other_model_serving"]) assert.Equal(t, "${databricks_registered_model.other_registered_model.id}", j.Tags["other_registered_model"]) assert.Equal(t, "${databricks_schema.other_schema.id}", j.Tags["other_schema"]) + assert.Equal(t, "${databricks_cluster.other_cluster.id}", j.Tags["other_cluster"]) m := b.Config.Resources.Models["my_model"] assert.Equal(t, "my_model", m.Model.Name) diff --git a/bundle/deploy/terraform/tfdyn/convert_cluster.go b/bundle/deploy/terraform/tfdyn/convert_cluster.go new file mode 100644 index 000000000..f25f09ea8 --- /dev/null +++ b/bundle/deploy/terraform/tfdyn/convert_cluster.go @@ -0,0 +1,52 @@ +package tfdyn + +import ( + "context" + "fmt" + + "github.com/databricks/cli/bundle/internal/tf/schema" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go/service/compute" +) + +func convertClusterResource(ctx context.Context, vin dyn.Value) (dyn.Value, error) { + // Normalize the output value to the target schema. + vout, diags := convert.Normalize(compute.ClusterSpec{}, vin) + for _, diag := range diags { + log.Debugf(ctx, "cluster normalization diagnostic: %s", diag.Summary) + } + + return vout, nil +} + +type clusterConverter struct{} + +func (clusterConverter) Convert(ctx context.Context, key string, vin dyn.Value, out *schema.Resources) error { + vout, err := convertClusterResource(ctx, vin) + if err != nil { + return err + } + + // We always set no_wait as it allows DABs not to wait for cluster to be started. + vout, err = dyn.Set(vout, "no_wait", dyn.V(true)) + if err != nil { + return err + } + + // Add the converted resource to the output. + out.Cluster[key] = vout.AsAny() + + // Configure permissions for this resource. + if permissions := convertPermissionsResource(ctx, vin); permissions != nil { + permissions.JobId = fmt.Sprintf("${databricks_cluster.%s.id}", key) + out.Permissions["cluster_"+key] = permissions + } + + return nil +} + +func init() { + registerConverter("clusters", clusterConverter{}) +} diff --git a/bundle/deploy/terraform/tfdyn/convert_cluster_test.go b/bundle/deploy/terraform/tfdyn/convert_cluster_test.go new file mode 100644 index 000000000..e7d2542fd --- /dev/null +++ b/bundle/deploy/terraform/tfdyn/convert_cluster_test.go @@ -0,0 +1,97 @@ +package tfdyn + +import ( + "context" + "testing" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/internal/tf/schema" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/databricks-sdk-go/service/compute" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestConvertCluster(t *testing.T) { + var src = resources.Cluster{ + ClusterSpec: &compute.ClusterSpec{ + NumWorkers: 3, + SparkVersion: "13.3.x-scala2.12", + ClusterName: "cluster", + SparkConf: map[string]string{ + "spark.executor.memory": "2g", + }, + AwsAttributes: &compute.AwsAttributes{ + Availability: "ON_DEMAND", + }, + AzureAttributes: &compute.AzureAttributes{ + Availability: "SPOT", + }, + DataSecurityMode: "USER_ISOLATION", + NodeTypeId: "m5.xlarge", + Autoscale: &compute.AutoScale{ + MinWorkers: 1, + MaxWorkers: 10, + }, + }, + + Permissions: []resources.Permission{ + { + Level: "CAN_RUN", + UserName: "jack@gmail.com", + }, + { + Level: "CAN_MANAGE", + ServicePrincipalName: "sp", + }, + }, + } + + vin, err := convert.FromTyped(src, dyn.NilValue) + require.NoError(t, err) + + ctx := context.Background() + out := schema.NewResources() + err = clusterConverter{}.Convert(ctx, "my_cluster", vin, out) + require.NoError(t, err) + + cluster := out.Cluster["my_cluster"] + assert.Equal(t, map[string]any{ + "num_workers": int64(3), + "spark_version": "13.3.x-scala2.12", + "cluster_name": "cluster", + "spark_conf": map[string]any{ + "spark.executor.memory": "2g", + }, + "aws_attributes": map[string]any{ + "availability": "ON_DEMAND", + }, + "azure_attributes": map[string]any{ + "availability": "SPOT", + }, + "data_security_mode": "USER_ISOLATION", + "no_wait": true, + "node_type_id": "m5.xlarge", + "autoscale": map[string]any{ + "min_workers": int64(1), + "max_workers": int64(10), + }, + }, cluster) + + // Assert equality on the permissions + assert.Equal(t, &schema.ResourcePermissions{ + JobId: "${databricks_cluster.my_cluster.id}", + AccessControl: []schema.ResourcePermissionsAccessControl{ + { + PermissionLevel: "CAN_RUN", + UserName: "jack@gmail.com", + }, + { + PermissionLevel: "CAN_MANAGE", + ServicePrincipalName: "sp", + }, + }, + }, out.Permissions["cluster_my_cluster"]) + +} diff --git a/bundle/tests/clusters/databricks.yml b/bundle/tests/clusters/databricks.yml new file mode 100644 index 000000000..1074462a6 --- /dev/null +++ b/bundle/tests/clusters/databricks.yml @@ -0,0 +1,36 @@ +bundle: + name: clusters + +workspace: + host: https://acme.cloud.databricks.com/ + +resources: + clusters: + foo: + cluster_name: foo + num_workers: 2 + node_type_id: "i3.xlarge" + autoscale: + min_workers: 2 + max_workers: 7 + spark_version: "13.3.x-scala2.12" + spark_conf: + "spark.executor.memory": "2g" + +targets: + default: + + development: + resources: + clusters: + foo: + cluster_name: foo-override + num_workers: 3 + node_type_id: "m5.xlarge" + autoscale: + min_workers: 1 + max_workers: 3 + spark_version: "15.2.x-scala2.12" + spark_conf: + "spark.executor.memory": "4g" + "spark.executor.memory2": "4g" diff --git a/bundle/tests/clusters_test.go b/bundle/tests/clusters_test.go new file mode 100644 index 000000000..def8a2a31 --- /dev/null +++ b/bundle/tests/clusters_test.go @@ -0,0 +1,36 @@ +package config_tests + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestClusters(t *testing.T) { + b := load(t, "./clusters") + assert.Equal(t, "clusters", b.Config.Bundle.Name) + + cluster := b.Config.Resources.Clusters["foo"] + assert.Equal(t, "foo", cluster.ClusterName) + assert.Equal(t, "13.3.x-scala2.12", cluster.SparkVersion) + assert.Equal(t, "i3.xlarge", cluster.NodeTypeId) + assert.Equal(t, 2, cluster.NumWorkers) + assert.Equal(t, "2g", cluster.SparkConf["spark.executor.memory"]) + assert.Equal(t, 2, cluster.Autoscale.MinWorkers) + assert.Equal(t, 7, cluster.Autoscale.MaxWorkers) +} + +func TestClustersOverride(t *testing.T) { + b := loadTarget(t, "./clusters", "development") + assert.Equal(t, "clusters", b.Config.Bundle.Name) + + cluster := b.Config.Resources.Clusters["foo"] + assert.Equal(t, "foo-override", cluster.ClusterName) + assert.Equal(t, "15.2.x-scala2.12", cluster.SparkVersion) + assert.Equal(t, "m5.xlarge", cluster.NodeTypeId) + assert.Equal(t, 3, cluster.NumWorkers) + assert.Equal(t, "4g", cluster.SparkConf["spark.executor.memory"]) + assert.Equal(t, "4g", cluster.SparkConf["spark.executor.memory2"]) + assert.Equal(t, 1, cluster.Autoscale.MinWorkers) + assert.Equal(t, 3, cluster.Autoscale.MaxWorkers) +} diff --git a/cmd/bundle/deploy.go b/cmd/bundle/deploy.go index 492317347..f1c85cb3d 100644 --- a/cmd/bundle/deploy.go +++ b/cmd/bundle/deploy.go @@ -24,14 +24,16 @@ func newDeployCommand() *cobra.Command { var force bool var forceLock bool var failOnActiveRuns bool - var computeID string + var clusterId string var autoApprove bool var verbose bool cmd.Flags().BoolVar(&force, "force", false, "Force-override Git branch validation.") cmd.Flags().BoolVar(&forceLock, "force-lock", false, "Force acquisition of deployment lock.") cmd.Flags().BoolVar(&failOnActiveRuns, "fail-on-active-runs", false, "Fail if there are running jobs or pipelines in the deployment.") - cmd.Flags().StringVarP(&computeID, "compute-id", "c", "", "Override compute in the deployment with the given compute ID.") + cmd.Flags().StringVar(&clusterId, "compute-id", "", "Override cluster in the deployment with the given compute ID.") + cmd.Flags().StringVarP(&clusterId, "cluster-id", "c", "", "Override cluster in the deployment with the given cluster ID.") cmd.Flags().BoolVar(&autoApprove, "auto-approve", false, "Skip interactive approvals that might be required for deployment.") + cmd.Flags().MarkDeprecated("compute-id", "use --cluster-id instead") cmd.Flags().BoolVar(&verbose, "verbose", false, "Enable verbose output.") // Verbose flag currently only affects file sync output, it's used by the vscode extension cmd.Flags().MarkHidden("verbose") @@ -47,7 +49,10 @@ func newDeployCommand() *cobra.Command { b.AutoApprove = autoApprove if cmd.Flag("compute-id").Changed { - b.Config.Bundle.ComputeID = computeID + b.Config.Bundle.ClusterId = clusterId + } + if cmd.Flag("cluster-id").Changed { + b.Config.Bundle.ClusterId = clusterId } if cmd.Flag("fail-on-active-runs").Changed { b.Config.Bundle.Deployment.FailOnActiveRuns = failOnActiveRuns diff --git a/internal/bundle/bundles/clusters/databricks_template_schema.json b/internal/bundle/bundles/clusters/databricks_template_schema.json new file mode 100644 index 000000000..c1c5cf12e --- /dev/null +++ b/internal/bundle/bundles/clusters/databricks_template_schema.json @@ -0,0 +1,16 @@ +{ + "properties": { + "unique_id": { + "type": "string", + "description": "Unique ID for job name" + }, + "spark_version": { + "type": "string", + "description": "Spark version used for job cluster" + }, + "node_type_id": { + "type": "string", + "description": "Node type id for job cluster" + } + } +} diff --git a/internal/bundle/bundles/clusters/template/databricks.yml.tmpl b/internal/bundle/bundles/clusters/template/databricks.yml.tmpl new file mode 100644 index 000000000..e0d6320a3 --- /dev/null +++ b/internal/bundle/bundles/clusters/template/databricks.yml.tmpl @@ -0,0 +1,24 @@ +bundle: + name: basic + +workspace: + root_path: "~/.bundle/{{.unique_id}}" + +resources: + clusters: + test_cluster: + cluster_name: "test-cluster-{{.unique_id}}" + spark_version: "{{.spark_version}}" + node_type_id: "{{.node_type_id}}" + num_workers: 2 + spark_conf: + "spark.executor.memory": "2g" + + jobs: + foo: + name: test-job-with-cluster-{{.unique_id}} + tasks: + - task_key: my_notebook_task + existing_cluster_id: "${resources.clusters.test_cluster.cluster_id}" + spark_python_task: + python_file: ./hello_world.py diff --git a/internal/bundle/bundles/clusters/template/hello_world.py b/internal/bundle/bundles/clusters/template/hello_world.py new file mode 100644 index 000000000..f301245e2 --- /dev/null +++ b/internal/bundle/bundles/clusters/template/hello_world.py @@ -0,0 +1 @@ +print("Hello World!") diff --git a/internal/bundle/clusters_test.go b/internal/bundle/clusters_test.go new file mode 100644 index 000000000..a961f3ea8 --- /dev/null +++ b/internal/bundle/clusters_test.go @@ -0,0 +1,56 @@ +package bundle + +import ( + "fmt" + "testing" + + "github.com/databricks/cli/internal" + "github.com/databricks/cli/internal/acc" + "github.com/databricks/cli/internal/testutil" + "github.com/databricks/cli/libs/env" + "github.com/databricks/databricks-sdk-go/service/compute" + "github.com/google/uuid" + "github.com/stretchr/testify/require" +) + +func TestAccDeployBundleWithCluster(t *testing.T) { + ctx, wt := acc.WorkspaceTest(t) + + if testutil.IsAWSCloud(wt.T) { + t.Skip("Skipping test for AWS cloud because it is not permitted to create clusters") + } + + nodeTypeId := internal.GetNodeTypeId(env.Get(ctx, "CLOUD_ENV")) + uniqueId := uuid.New().String() + root, err := initTestTemplate(t, ctx, "clusters", map[string]any{ + "unique_id": uniqueId, + "node_type_id": nodeTypeId, + "spark_version": defaultSparkVersion, + }) + require.NoError(t, err) + + t.Cleanup(func() { + err = destroyBundle(t, ctx, root) + require.NoError(t, err) + + cluster, err := wt.W.Clusters.GetByClusterName(ctx, fmt.Sprintf("test-cluster-%s", uniqueId)) + if err != nil { + require.ErrorContains(t, err, "does not exist") + } else { + require.Contains(t, []compute.State{compute.StateTerminated, compute.StateTerminating}, cluster.State) + } + + }) + + err = deployBundle(t, ctx, root) + require.NoError(t, err) + + // Cluster should exists after bundle deployment + cluster, err := wt.W.Clusters.GetByClusterName(ctx, fmt.Sprintf("test-cluster-%s", uniqueId)) + require.NoError(t, err) + require.NotNil(t, cluster) + + out, err := runResource(t, ctx, root, "foo") + require.NoError(t, err) + require.Contains(t, out, "Hello World!") +} diff --git a/internal/testutil/cloud.go b/internal/testutil/cloud.go index e547069f3..ba5b75ecf 100644 --- a/internal/testutil/cloud.go +++ b/internal/testutil/cloud.go @@ -49,3 +49,7 @@ func GetCloud(t *testing.T) Cloud { } return -1 } + +func IsAWSCloud(t *testing.T) bool { + return GetCloud(t) == AWS +} From 0cc35ca05693e5989308f432f22bb0a28f8cb1dd Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Mon, 23 Sep 2024 18:12:30 +0530 Subject: [PATCH 09/34] Assert tokens are redacted in origin URL when username is not specified (#1785) TSIA --- libs/git/repository_test.go | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/libs/git/repository_test.go b/libs/git/repository_test.go index a28038eeb..93d9a03dc 100644 --- a/libs/git/repository_test.go +++ b/libs/git/repository_test.go @@ -209,7 +209,26 @@ func TestRepositoryGitConfigWhenNotARepo(t *testing.T) { } func TestRepositoryOriginUrlRemovesUserCreds(t *testing.T) { - repo := newTestRepository(t) - repo.addOriginUrl("https://username:token@github.com/databricks/foobar.git") - repo.assertOriginUrl("https://github.com/databricks/foobar.git") + tcases := []struct { + url string + expected string + }{ + { + url: "https://username:token@github.com/databricks/foobar.git", + expected: "https://github.com/databricks/foobar.git", + }, + { + // Note: The token is still considered and parsed as a username here. + // However credentials integrations by Git providers like GitHub + // allow for setting a PAT token as a username. + url: "https://token@github.com/databricks/foobar.git", + expected: "https://github.com/databricks/foobar.git", + }, + } + + for _, tc := range tcases { + repo := newTestRepository(t) + repo.addOriginUrl(tc.url) + repo.assertOriginUrl(tc.expected) + } } From 490259a14aec0a53fe6bd97f9d1ea5a384e74773 Mon Sep 17 00:00:00 2001 From: Gleb Kanterov Date: Tue, 24 Sep 2024 15:51:54 +0200 Subject: [PATCH 10/34] Refactor jobs path translation (#1782) ## Changes Extract package for other modules to transform different kinds of paths in job resources. ## Tests Unit tests --- .../config/mutator/paths/job_paths_visitor.go | 115 ++++++++++++ .../mutator/paths/job_paths_visitor_test.go | 168 ++++++++++++++++++ bundle/config/mutator/paths/visitor.go | 26 +++ bundle/config/mutator/translate_paths_jobs.go | 137 ++++---------- 4 files changed, 340 insertions(+), 106 deletions(-) create mode 100644 bundle/config/mutator/paths/job_paths_visitor.go create mode 100644 bundle/config/mutator/paths/job_paths_visitor_test.go create mode 100644 bundle/config/mutator/paths/visitor.go diff --git a/bundle/config/mutator/paths/job_paths_visitor.go b/bundle/config/mutator/paths/job_paths_visitor.go new file mode 100644 index 000000000..275a8fa53 --- /dev/null +++ b/bundle/config/mutator/paths/job_paths_visitor.go @@ -0,0 +1,115 @@ +package paths + +import ( + "github.com/databricks/cli/bundle/libraries" + "github.com/databricks/cli/libs/dyn" +) + +type jobRewritePattern struct { + pattern dyn.Pattern + kind PathKind + skipRewrite func(string) bool +} + +func noSkipRewrite(string) bool { + return false +} + +func jobTaskRewritePatterns(base dyn.Pattern) []jobRewritePattern { + return []jobRewritePattern{ + { + base.Append(dyn.Key("notebook_task"), dyn.Key("notebook_path")), + PathKindNotebook, + noSkipRewrite, + }, + { + base.Append(dyn.Key("spark_python_task"), dyn.Key("python_file")), + PathKindWorkspaceFile, + noSkipRewrite, + }, + { + base.Append(dyn.Key("dbt_task"), dyn.Key("project_directory")), + PathKindDirectory, + noSkipRewrite, + }, + { + base.Append(dyn.Key("sql_task"), dyn.Key("file"), dyn.Key("path")), + PathKindWorkspaceFile, + noSkipRewrite, + }, + { + base.Append(dyn.Key("libraries"), dyn.AnyIndex(), dyn.Key("whl")), + PathKindLibrary, + noSkipRewrite, + }, + { + base.Append(dyn.Key("libraries"), dyn.AnyIndex(), dyn.Key("jar")), + PathKindLibrary, + noSkipRewrite, + }, + { + base.Append(dyn.Key("libraries"), dyn.AnyIndex(), dyn.Key("requirements")), + PathKindWorkspaceFile, + noSkipRewrite, + }, + } +} + +func jobRewritePatterns() []jobRewritePattern { + // Base pattern to match all tasks in all jobs. + base := dyn.NewPattern( + dyn.Key("resources"), + dyn.Key("jobs"), + dyn.AnyKey(), + dyn.Key("tasks"), + dyn.AnyIndex(), + ) + + // Compile list of patterns and their respective rewrite functions. + jobEnvironmentsPatterns := []jobRewritePattern{ + { + dyn.NewPattern( + dyn.Key("resources"), + dyn.Key("jobs"), + dyn.AnyKey(), + dyn.Key("environments"), + dyn.AnyIndex(), + dyn.Key("spec"), + dyn.Key("dependencies"), + dyn.AnyIndex(), + ), + PathKindWithPrefix, + func(s string) bool { + return !libraries.IsLibraryLocal(s) + }, + }, + } + + taskPatterns := jobTaskRewritePatterns(base) + forEachPatterns := jobTaskRewritePatterns(base.Append(dyn.Key("for_each_task"), dyn.Key("task"))) + allPatterns := append(taskPatterns, jobEnvironmentsPatterns...) + allPatterns = append(allPatterns, forEachPatterns...) + return allPatterns +} + +// VisitJobPaths visits all paths in job resources and applies a function to each path. +func VisitJobPaths(value dyn.Value, fn VisitFunc) (dyn.Value, error) { + var err error + var newValue = value + + for _, rewritePattern := range jobRewritePatterns() { + newValue, err = dyn.MapByPattern(newValue, rewritePattern.pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + if rewritePattern.skipRewrite(v.MustString()) { + return v, nil + } + + return fn(p, rewritePattern.kind, v) + }) + + if err != nil { + return dyn.InvalidValue, err + } + } + + return newValue, nil +} diff --git a/bundle/config/mutator/paths/job_paths_visitor_test.go b/bundle/config/mutator/paths/job_paths_visitor_test.go new file mode 100644 index 000000000..7f0201579 --- /dev/null +++ b/bundle/config/mutator/paths/job_paths_visitor_test.go @@ -0,0 +1,168 @@ +package paths + +import ( + "testing" + + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/dyn" + assert "github.com/databricks/cli/libs/dyn/dynassert" + "github.com/databricks/databricks-sdk-go/service/compute" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/stretchr/testify/require" +) + +func TestVisitJobPaths(t *testing.T) { + task0 := jobs.Task{ + NotebookTask: &jobs.NotebookTask{ + NotebookPath: "abc", + }, + } + task1 := jobs.Task{ + SparkPythonTask: &jobs.SparkPythonTask{ + PythonFile: "abc", + }, + } + task2 := jobs.Task{ + DbtTask: &jobs.DbtTask{ + ProjectDirectory: "abc", + }, + } + task3 := jobs.Task{ + SqlTask: &jobs.SqlTask{ + File: &jobs.SqlTaskFile{ + Path: "abc", + }, + }, + } + task4 := jobs.Task{ + Libraries: []compute.Library{ + {Whl: "dist/foo.whl"}, + }, + } + task5 := jobs.Task{ + Libraries: []compute.Library{ + {Jar: "dist/foo.jar"}, + }, + } + task6 := jobs.Task{ + Libraries: []compute.Library{ + {Requirements: "requirements.txt"}, + }, + } + + job0 := &resources.Job{ + JobSettings: &jobs.JobSettings{ + Tasks: []jobs.Task{ + task0, + task1, + task2, + task3, + task4, + task5, + task6, + }, + }, + } + + root := config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job0": job0, + }, + }, + } + + actual := visitJobPaths(t, root) + expected := []dyn.Path{ + dyn.MustPathFromString("resources.jobs.job0.tasks[0].notebook_task.notebook_path"), + dyn.MustPathFromString("resources.jobs.job0.tasks[1].spark_python_task.python_file"), + dyn.MustPathFromString("resources.jobs.job0.tasks[2].dbt_task.project_directory"), + dyn.MustPathFromString("resources.jobs.job0.tasks[3].sql_task.file.path"), + dyn.MustPathFromString("resources.jobs.job0.tasks[4].libraries[0].whl"), + dyn.MustPathFromString("resources.jobs.job0.tasks[5].libraries[0].jar"), + dyn.MustPathFromString("resources.jobs.job0.tasks[6].libraries[0].requirements"), + } + + assert.ElementsMatch(t, expected, actual) +} + +func TestVisitJobPaths_environments(t *testing.T) { + environment0 := jobs.JobEnvironment{ + Spec: &compute.Environment{ + Dependencies: []string{ + "dist_0/*.whl", + "dist_1/*.whl", + }, + }, + } + job0 := &resources.Job{ + JobSettings: &jobs.JobSettings{ + Environments: []jobs.JobEnvironment{ + environment0, + }, + }, + } + + root := config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job0": job0, + }, + }, + } + + actual := visitJobPaths(t, root) + expected := []dyn.Path{ + dyn.MustPathFromString("resources.jobs.job0.environments[0].spec.dependencies[0]"), + dyn.MustPathFromString("resources.jobs.job0.environments[0].spec.dependencies[1]"), + } + + assert.ElementsMatch(t, expected, actual) +} + +func TestVisitJobPaths_foreach(t *testing.T) { + task0 := jobs.Task{ + ForEachTask: &jobs.ForEachTask{ + Task: jobs.Task{ + NotebookTask: &jobs.NotebookTask{ + NotebookPath: "abc", + }, + }, + }, + } + job0 := &resources.Job{ + JobSettings: &jobs.JobSettings{ + Tasks: []jobs.Task{ + task0, + }, + }, + } + + root := config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job0": job0, + }, + }, + } + + actual := visitJobPaths(t, root) + expected := []dyn.Path{ + dyn.MustPathFromString("resources.jobs.job0.tasks[0].for_each_task.task.notebook_task.notebook_path"), + } + + assert.ElementsMatch(t, expected, actual) +} + +func visitJobPaths(t *testing.T, root config.Root) []dyn.Path { + var actual []dyn.Path + err := root.Mutate(func(value dyn.Value) (dyn.Value, error) { + return VisitJobPaths(value, func(p dyn.Path, kind PathKind, v dyn.Value) (dyn.Value, error) { + actual = append(actual, p) + return v, nil + }) + }) + require.NoError(t, err) + return actual +} diff --git a/bundle/config/mutator/paths/visitor.go b/bundle/config/mutator/paths/visitor.go new file mode 100644 index 000000000..40d1f14ef --- /dev/null +++ b/bundle/config/mutator/paths/visitor.go @@ -0,0 +1,26 @@ +package paths + +import "github.com/databricks/cli/libs/dyn" + +type PathKind int + +const ( + // PathKindLibrary is a path to a library file + PathKindLibrary = iota + + // PathKindNotebook is a path to a notebook file + PathKindNotebook + + // PathKindWorkspaceFile is a path to a regular workspace file, + // notebooks are not allowed because they are uploaded a special + // kind of workspace object. + PathKindWorkspaceFile + + // PathKindWithPrefix is a path that starts with './' + PathKindWithPrefix + + // PathKindDirectory is a path to directory + PathKindDirectory +) + +type VisitFunc func(path dyn.Path, kind PathKind, value dyn.Value) (dyn.Value, error) diff --git a/bundle/config/mutator/translate_paths_jobs.go b/bundle/config/mutator/translate_paths_jobs.go index e34eeb2f0..c29ff0ea9 100644 --- a/bundle/config/mutator/translate_paths_jobs.go +++ b/bundle/config/mutator/translate_paths_jobs.go @@ -4,97 +4,11 @@ import ( "fmt" "slices" - "github.com/databricks/cli/bundle/libraries" + "github.com/databricks/cli/bundle/config/mutator/paths" + "github.com/databricks/cli/libs/dyn" ) -type jobRewritePattern struct { - pattern dyn.Pattern - fn rewriteFunc - skipRewrite func(string) bool -} - -func noSkipRewrite(string) bool { - return false -} - -func rewritePatterns(t *translateContext, base dyn.Pattern) []jobRewritePattern { - return []jobRewritePattern{ - { - base.Append(dyn.Key("notebook_task"), dyn.Key("notebook_path")), - t.translateNotebookPath, - noSkipRewrite, - }, - { - base.Append(dyn.Key("spark_python_task"), dyn.Key("python_file")), - t.translateFilePath, - noSkipRewrite, - }, - { - base.Append(dyn.Key("dbt_task"), dyn.Key("project_directory")), - t.translateDirectoryPath, - noSkipRewrite, - }, - { - base.Append(dyn.Key("sql_task"), dyn.Key("file"), dyn.Key("path")), - t.translateFilePath, - noSkipRewrite, - }, - { - base.Append(dyn.Key("libraries"), dyn.AnyIndex(), dyn.Key("whl")), - t.translateNoOp, - noSkipRewrite, - }, - { - base.Append(dyn.Key("libraries"), dyn.AnyIndex(), dyn.Key("jar")), - t.translateNoOp, - noSkipRewrite, - }, - { - base.Append(dyn.Key("libraries"), dyn.AnyIndex(), dyn.Key("requirements")), - t.translateFilePath, - noSkipRewrite, - }, - } -} - -func (t *translateContext) jobRewritePatterns() []jobRewritePattern { - // Base pattern to match all tasks in all jobs. - base := dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("jobs"), - dyn.AnyKey(), - dyn.Key("tasks"), - dyn.AnyIndex(), - ) - - // Compile list of patterns and their respective rewrite functions. - jobEnvironmentsPatterns := []jobRewritePattern{ - { - dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("jobs"), - dyn.AnyKey(), - dyn.Key("environments"), - dyn.AnyIndex(), - dyn.Key("spec"), - dyn.Key("dependencies"), - dyn.AnyIndex(), - ), - t.translateNoOpWithPrefix, - func(s string) bool { - return !libraries.IsLibraryLocal(s) - }, - }, - } - - taskPatterns := rewritePatterns(t, base) - forEachPatterns := rewritePatterns(t, base.Append(dyn.Key("for_each_task"), dyn.Key("task"))) - allPatterns := append(taskPatterns, jobEnvironmentsPatterns...) - allPatterns = append(allPatterns, forEachPatterns...) - return allPatterns -} - func (t *translateContext) applyJobTranslations(v dyn.Value) (dyn.Value, error) { var err error @@ -111,30 +25,41 @@ func (t *translateContext) applyJobTranslations(v dyn.Value) (dyn.Value, error) } } - for _, rewritePattern := range t.jobRewritePatterns() { - v, err = dyn.MapByPattern(v, rewritePattern.pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - key := p[2].Key() + return paths.VisitJobPaths(v, func(p dyn.Path, kind paths.PathKind, v dyn.Value) (dyn.Value, error) { + key := p[2].Key() - // Skip path translation if the job is using git source. - if slices.Contains(ignore, key) { - return v, nil - } + // Skip path translation if the job is using git source. + if slices.Contains(ignore, key) { + return v, nil + } - dir, err := v.Location().Directory() - if err != nil { - return dyn.InvalidValue, fmt.Errorf("unable to determine directory for job %s: %w", key, err) - } + dir, err := v.Location().Directory() + if err != nil { + return dyn.InvalidValue, fmt.Errorf("unable to determine directory for job %s: %w", key, err) + } - sv := v.MustString() - if rewritePattern.skipRewrite(sv) { - return v, nil - } - return t.rewriteRelativeTo(p, v, rewritePattern.fn, dir, fallback[key]) - }) + rewritePatternFn, err := t.getRewritePatternFn(kind) if err != nil { return dyn.InvalidValue, err } + + return t.rewriteRelativeTo(p, v, rewritePatternFn, dir, fallback[key]) + }) +} + +func (t *translateContext) getRewritePatternFn(kind paths.PathKind) (rewriteFunc, error) { + switch kind { + case paths.PathKindLibrary: + return t.translateNoOp, nil + case paths.PathKindNotebook: + return t.translateNotebookPath, nil + case paths.PathKindWorkspaceFile: + return t.translateFilePath, nil + case paths.PathKindDirectory: + return t.translateDirectoryPath, nil + case paths.PathKindWithPrefix: + return t.translateNoOpWithPrefix, nil } - return v, nil + return nil, fmt.Errorf("unsupported path kind: %d", kind) } From 3d9decdda9638fb5495212611307d70257b3d3e6 Mon Sep 17 00:00:00 2001 From: Gleb Kanterov Date: Wed, 25 Sep 2024 13:30:14 +0200 Subject: [PATCH 11/34] Add JobTaskClusterSpec validate mutator (#1784) ## Changes Add JobTaskClusterSpec validate mutator. It catches the case when tasks don't which cluster to use. For example, we can get this error with minor modifications to `default-python` template: ```yaml tasks: - task_key: python_file_task spark_python_task: python_file: ../src/my_project_10/main.py ``` ``` % databricks bundle validate Error: Missing required cluster or environment settings at resources.jobs.my_project_10_job.tasks[0] in resources/my_project_10_job.yml:17:11 Task "print_github_stars" requires a cluster or an environment to run. Specify one of the following fields: job_cluster_key, environment_key, existing_cluster_id, new_cluster. ``` We implicitly rely on "one of" validation, which does not exist. Many bundle fields can't co-exist, for instance, specifying: `JobTask.{existing_cluster_id,job_cluster_key}`, `Library.{whl,pypi}`, `JobTask.{notebook_task,python_wheel_task}`, etc. ## Tests Unit tests --------- Co-authored-by: Pieter Noordhuis --- .../config/validate/job_task_cluster_spec.go | 161 ++++++++++++++ .../validate/job_task_cluster_spec_test.go | 203 ++++++++++++++++++ bundle/config/validate/validate.go | 1 + 3 files changed, 365 insertions(+) create mode 100644 bundle/config/validate/job_task_cluster_spec.go create mode 100644 bundle/config/validate/job_task_cluster_spec_test.go diff --git a/bundle/config/validate/job_task_cluster_spec.go b/bundle/config/validate/job_task_cluster_spec.go new file mode 100644 index 000000000..b80befcdf --- /dev/null +++ b/bundle/config/validate/job_task_cluster_spec.go @@ -0,0 +1,161 @@ +package validate + +import ( + "context" + "fmt" + "strings" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/databricks-sdk-go/service/jobs" +) + +// JobTaskClusterSpec validates that job tasks have cluster spec defined +// if task requires a cluster +func JobTaskClusterSpec() bundle.ReadOnlyMutator { + return &jobTaskClusterSpec{} +} + +type jobTaskClusterSpec struct { +} + +func (v *jobTaskClusterSpec) Name() string { + return "validate:job_task_cluster_spec" +} + +func (v *jobTaskClusterSpec) Apply(ctx context.Context, rb bundle.ReadOnlyBundle) diag.Diagnostics { + diags := diag.Diagnostics{} + + jobsPath := dyn.NewPath(dyn.Key("resources"), dyn.Key("jobs")) + + for resourceName, job := range rb.Config().Resources.Jobs { + resourcePath := jobsPath.Append(dyn.Key(resourceName)) + + for taskIndex, task := range job.Tasks { + taskPath := resourcePath.Append(dyn.Key("tasks"), dyn.Index(taskIndex)) + + diags = diags.Extend(validateJobTask(rb, task, taskPath)) + } + } + + return diags +} + +func validateJobTask(rb bundle.ReadOnlyBundle, task jobs.Task, taskPath dyn.Path) diag.Diagnostics { + diags := diag.Diagnostics{} + + var specified []string + var unspecified []string + + if task.JobClusterKey != "" { + specified = append(specified, "job_cluster_key") + } else { + unspecified = append(unspecified, "job_cluster_key") + } + + if task.EnvironmentKey != "" { + specified = append(specified, "environment_key") + } else { + unspecified = append(unspecified, "environment_key") + } + + if task.ExistingClusterId != "" { + specified = append(specified, "existing_cluster_id") + } else { + unspecified = append(unspecified, "existing_cluster_id") + } + + if task.NewCluster != nil { + specified = append(specified, "new_cluster") + } else { + unspecified = append(unspecified, "new_cluster") + } + + if task.ForEachTask != nil { + forEachTaskPath := taskPath.Append(dyn.Key("for_each_task"), dyn.Key("task")) + + diags = diags.Extend(validateJobTask(rb, task.ForEachTask.Task, forEachTaskPath)) + } + + if isComputeTask(task) && len(specified) == 0 { + if task.NotebookTask != nil { + // notebook tasks without cluster spec will use notebook environment + } else { + // path might be not very helpful, adding user-specified task key clarifies the context + detail := fmt.Sprintf( + "Task %q requires a cluster or an environment to run.\nSpecify one of the following fields: %s.", + task.TaskKey, + strings.Join(unspecified, ", "), + ) + + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: "Missing required cluster or environment settings", + Detail: detail, + Locations: rb.Config().GetLocations(taskPath.String()), + Paths: []dyn.Path{taskPath}, + }) + } + } + + return diags +} + +// isComputeTask returns true if the task runs on a cluster or serverless GC +func isComputeTask(task jobs.Task) bool { + if task.NotebookTask != nil { + // if warehouse_id is set, it's SQL notebook that doesn't need cluster or serverless GC + if task.NotebookTask.WarehouseId != "" { + return false + } else { + // task settings don't require specifying a cluster/serverless GC, but task itself can run on one + // we handle that case separately in validateJobTask + return true + } + } + + if task.PythonWheelTask != nil { + return true + } + + if task.DbtTask != nil { + return true + } + + if task.SparkJarTask != nil { + return true + } + + if task.SparkSubmitTask != nil { + return true + } + + if task.SparkPythonTask != nil { + return true + } + + if task.SqlTask != nil { + return false + } + + if task.PipelineTask != nil { + // while pipelines use clusters, pipeline tasks don't, they only trigger pipelines + return false + } + + if task.RunJobTask != nil { + return false + } + + if task.ConditionTask != nil { + return false + } + + // for each task doesn't use clusters, underlying task(s) can though + if task.ForEachTask != nil { + return false + } + + return false +} diff --git a/bundle/config/validate/job_task_cluster_spec_test.go b/bundle/config/validate/job_task_cluster_spec_test.go new file mode 100644 index 000000000..a3a7ccf25 --- /dev/null +++ b/bundle/config/validate/job_task_cluster_spec_test.go @@ -0,0 +1,203 @@ +package validate + +import ( + "context" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go/service/compute" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/stretchr/testify/assert" +) + +func TestJobTaskClusterSpec(t *testing.T) { + expectedSummary := "Missing required cluster or environment settings" + + type testCase struct { + name string + task jobs.Task + errorPath string + errorDetail string + errorSummary string + } + + testCases := []testCase{ + { + name: "valid notebook task", + task: jobs.Task{ + // while a cluster is needed, it will use notebook environment to create one + NotebookTask: &jobs.NotebookTask{}, + }, + }, + { + name: "valid notebook task (job_cluster_key)", + task: jobs.Task{ + JobClusterKey: "cluster1", + NotebookTask: &jobs.NotebookTask{}, + }, + }, + { + name: "valid notebook task (new_cluster)", + task: jobs.Task{ + NewCluster: &compute.ClusterSpec{}, + NotebookTask: &jobs.NotebookTask{}, + }, + }, + { + name: "valid notebook task (existing_cluster_id)", + task: jobs.Task{ + ExistingClusterId: "cluster1", + NotebookTask: &jobs.NotebookTask{}, + }, + }, + { + name: "valid SQL notebook task", + task: jobs.Task{ + NotebookTask: &jobs.NotebookTask{ + WarehouseId: "warehouse1", + }, + }, + }, + { + name: "valid python wheel task", + task: jobs.Task{ + JobClusterKey: "cluster1", + PythonWheelTask: &jobs.PythonWheelTask{}, + }, + }, + { + name: "valid python wheel task (environment_key)", + task: jobs.Task{ + EnvironmentKey: "environment1", + PythonWheelTask: &jobs.PythonWheelTask{}, + }, + }, + { + name: "valid dbt task", + task: jobs.Task{ + JobClusterKey: "cluster1", + DbtTask: &jobs.DbtTask{}, + }, + }, + { + name: "valid spark jar task", + task: jobs.Task{ + JobClusterKey: "cluster1", + SparkJarTask: &jobs.SparkJarTask{}, + }, + }, + { + name: "valid spark submit", + task: jobs.Task{ + NewCluster: &compute.ClusterSpec{}, + SparkSubmitTask: &jobs.SparkSubmitTask{}, + }, + }, + { + name: "valid spark python task", + task: jobs.Task{ + JobClusterKey: "cluster1", + SparkPythonTask: &jobs.SparkPythonTask{}, + }, + }, + { + name: "valid SQL task", + task: jobs.Task{ + SqlTask: &jobs.SqlTask{}, + }, + }, + { + name: "valid pipeline task", + task: jobs.Task{ + PipelineTask: &jobs.PipelineTask{}, + }, + }, + { + name: "valid run job task", + task: jobs.Task{ + RunJobTask: &jobs.RunJobTask{}, + }, + }, + { + name: "valid condition task", + task: jobs.Task{ + ConditionTask: &jobs.ConditionTask{}, + }, + }, + { + name: "valid for each task", + task: jobs.Task{ + ForEachTask: &jobs.ForEachTask{ + Task: jobs.Task{ + JobClusterKey: "cluster1", + NotebookTask: &jobs.NotebookTask{}, + }, + }, + }, + }, + { + name: "invalid python wheel task", + task: jobs.Task{ + PythonWheelTask: &jobs.PythonWheelTask{}, + TaskKey: "my_task", + }, + errorPath: "resources.jobs.job1.tasks[0]", + errorDetail: `Task "my_task" requires a cluster or an environment to run. +Specify one of the following fields: job_cluster_key, environment_key, existing_cluster_id, new_cluster.`, + errorSummary: expectedSummary, + }, + { + name: "invalid for each task", + task: jobs.Task{ + ForEachTask: &jobs.ForEachTask{ + Task: jobs.Task{ + PythonWheelTask: &jobs.PythonWheelTask{}, + TaskKey: "my_task", + }, + }, + }, + errorPath: "resources.jobs.job1.tasks[0].for_each_task.task", + errorDetail: `Task "my_task" requires a cluster or an environment to run. +Specify one of the following fields: job_cluster_key, environment_key, existing_cluster_id, new_cluster.`, + errorSummary: expectedSummary, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + job := &resources.Job{ + JobSettings: &jobs.JobSettings{ + Tasks: []jobs.Task{tc.task}, + }, + } + + b := createBundle(map[string]*resources.Job{"job1": job}) + diags := bundle.ApplyReadOnly(context.Background(), bundle.ReadOnly(b), JobTaskClusterSpec()) + + if tc.errorPath != "" || tc.errorDetail != "" || tc.errorSummary != "" { + assert.Len(t, diags, 1) + assert.Len(t, diags[0].Paths, 1) + + diag := diags[0] + + assert.Equal(t, tc.errorPath, diag.Paths[0].String()) + assert.Equal(t, tc.errorSummary, diag.Summary) + assert.Equal(t, tc.errorDetail, diag.Detail) + } else { + assert.ElementsMatch(t, []string{}, diags) + } + }) + } +} + +func createBundle(jobs map[string]*resources.Job) *bundle.Bundle { + return &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: jobs, + }, + }, + } +} diff --git a/bundle/config/validate/validate.go b/bundle/config/validate/validate.go index b4da0bc05..79f42bd23 100644 --- a/bundle/config/validate/validate.go +++ b/bundle/config/validate/validate.go @@ -34,6 +34,7 @@ func (v *validate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics JobClusterKeyDefined(), FilesToSync(), ValidateSyncPatterns(), + JobTaskClusterSpec(), )) } From b3a3071086899dabbdf36f063d1cf892993090ff Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 25 Sep 2024 14:35:16 +0200 Subject: [PATCH 12/34] Fixed full variable override detection (#1787) ## Changes Fixes #1786 ## Tests All valid override combinations are added as test cases --- bundle/config/root.go | 37 +++++++++++++---- bundle/config/root_test.go | 85 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+), 8 deletions(-) diff --git a/bundle/config/root.go b/bundle/config/root.go index 92d834f0a..ff169e4ce 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -418,22 +418,43 @@ func isFullVariableOverrideDef(v dyn.Value) bool { return false } - // If the map has more than 2 keys, it is not a full variable override. - if mv.Len() > 2 { + // If the map has more than 3 keys, it is not a full variable override. + if mv.Len() > 3 { return false } - // If the map has 2 keys, one of them should be "default" and the other is "type" + // If the map has 3 keys, they should be "description", "type" and "default" or "lookup" + if mv.Len() == 3 { + if _, ok := mv.GetByString("type"); ok { + if _, ok := mv.GetByString("description"); ok { + if _, ok := mv.GetByString("default"); ok { + return true + } + } + } + + return false + } + + // If the map has 2 keys, one of them should be "default" or "lookup" and the other is "type" or "description" if mv.Len() == 2 { - if _, ok := mv.GetByString("type"); !ok { - return false + if _, ok := mv.GetByString("type"); ok { + if _, ok := mv.GetByString("default"); ok { + return true + } } - if _, ok := mv.GetByString("default"); !ok { - return false + if _, ok := mv.GetByString("description"); ok { + if _, ok := mv.GetByString("default"); ok { + return true + } + + if _, ok := mv.GetByString("lookup"); ok { + return true + } } - return true + return false } for _, keyword := range variableKeywords { diff --git a/bundle/config/root_test.go b/bundle/config/root_test.go index d2c7a9b1f..9e6123534 100644 --- a/bundle/config/root_test.go +++ b/bundle/config/root_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/databricks/cli/bundle/config/variable" + "github.com/databricks/cli/libs/dyn" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -169,3 +170,87 @@ func TestRootMergeTargetOverridesWithVariables(t *testing.T) { assert.Equal(t, "complex var", root.Variables["complex"].Description) } + +func TestIsFullVariableOverrideDef(t *testing.T) { + testCases := []struct { + value dyn.Value + expected bool + }{ + { + value: dyn.V(map[string]dyn.Value{ + "type": dyn.V("string"), + "default": dyn.V("foo"), + "description": dyn.V("foo var"), + }), + expected: true, + }, + { + value: dyn.V(map[string]dyn.Value{ + "type": dyn.V("string"), + "lookup": dyn.V("foo"), + "description": dyn.V("foo var"), + }), + expected: false, + }, + { + value: dyn.V(map[string]dyn.Value{ + "type": dyn.V("string"), + "default": dyn.V("foo"), + }), + expected: true, + }, + { + value: dyn.V(map[string]dyn.Value{ + "type": dyn.V("string"), + "lookup": dyn.V("foo"), + }), + expected: false, + }, + { + value: dyn.V(map[string]dyn.Value{ + "description": dyn.V("string"), + "default": dyn.V("foo"), + }), + expected: true, + }, + { + value: dyn.V(map[string]dyn.Value{ + "description": dyn.V("string"), + "lookup": dyn.V("foo"), + }), + expected: true, + }, + { + value: dyn.V(map[string]dyn.Value{ + "default": dyn.V("foo"), + }), + expected: true, + }, + { + value: dyn.V(map[string]dyn.Value{ + "lookup": dyn.V("foo"), + }), + expected: true, + }, + { + value: dyn.V(map[string]dyn.Value{ + "type": dyn.V("string"), + }), + expected: false, + }, + { + value: dyn.V(map[string]dyn.Value{ + "type": dyn.V("string"), + "default": dyn.V("foo"), + "description": dyn.V("foo var"), + "lookup": dyn.V("foo"), + }), + expected: false, + }, + } + + for i, tc := range testCases { + assert.Equal(t, tc.expected, isFullVariableOverrideDef(tc.value), "test case %d", i) + } + +} From a4ba0bbe9f332dbc497d6cc3be0e19436e2e9375 Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Wed, 25 Sep 2024 18:28:14 +0530 Subject: [PATCH 13/34] Add sub-extension to resource files in built-in templates (#1777) ## Changes We want to encourage a pattern of only specifying a single resource in a YAML file when an `..yml` (like `.job.yml`) is used. This convention could allow us to bijectively map a resource YAML file to it's corresponding resource in the Databricks workspace. This PR simply makes the built-in templates compliant to this format. ## Tests Existing tests. --- .../dbt-sql/template/{{.project_name}}/README.md.tmpl | 2 +- ...ect_name}}_job.yml.tmpl => {{.project_name}}.job.yml.tmpl} | 0 .../templates/default-python/template/__preamble.tmpl | 4 ++-- .../default-python/template/{{.project_name}}/README.md.tmpl | 2 +- ...ect_name}}_job.yml.tmpl => {{.project_name}}.job.yml.tmpl} | 2 +- ..._pipeline.yml.tmpl => {{.project_name}}.pipeline.yml.tmpl} | 0 .../template/{{.project_name}}/src/dlt_pipeline.ipynb.tmpl | 2 +- .../template/{{.project_name}}/src/notebook.ipynb.tmpl | 2 +- ...}}_sql_job.yml.tmpl => {{.project_name}}_sql.job.yml.tmpl} | 0 .../template/{{.project_name}}/src/orders_daily.sql.tmpl | 2 +- .../template/{{.project_name}}/src/orders_raw.sql.tmpl | 2 +- 11 files changed, 9 insertions(+), 9 deletions(-) rename libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{{.project_name}}_job.yml.tmpl => {{.project_name}}.job.yml.tmpl} (100%) rename libs/template/templates/default-python/template/{{.project_name}}/resources/{{{.project_name}}_job.yml.tmpl => {{.project_name}}.job.yml.tmpl} (97%) rename libs/template/templates/default-python/template/{{.project_name}}/resources/{{{.project_name}}_pipeline.yml.tmpl => {{.project_name}}.pipeline.yml.tmpl} (100%) rename libs/template/templates/default-sql/template/{{.project_name}}/resources/{{{.project_name}}_sql_job.yml.tmpl => {{.project_name}}_sql.job.yml.tmpl} (100%) diff --git a/libs/template/templates/dbt-sql/template/{{.project_name}}/README.md.tmpl b/libs/template/templates/dbt-sql/template/{{.project_name}}/README.md.tmpl index dbf8a8d85..cd4c29a76 100644 --- a/libs/template/templates/dbt-sql/template/{{.project_name}}/README.md.tmpl +++ b/libs/template/templates/dbt-sql/template/{{.project_name}}/README.md.tmpl @@ -121,7 +121,7 @@ You can find that job by opening your workpace and clicking on **Workflows**. You can also deploy to your production target directly from the command-line. The warehouse, catalog, and schema for that target are configured in databricks.yml. -When deploying to this target, note that the default job at resources/{{.project_name}}_job.yml +When deploying to this target, note that the default job at resources/{{.project_name}}.job.yml has a schedule set that runs every day. The schedule is paused when deploying in development mode (see https://docs.databricks.com/dev-tools/bundles/deployment-modes.html). diff --git a/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl b/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl similarity index 100% rename from libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl rename to libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl diff --git a/libs/template/templates/default-python/template/__preamble.tmpl b/libs/template/templates/default-python/template/__preamble.tmpl index a919a269c..69b769cde 100644 --- a/libs/template/templates/default-python/template/__preamble.tmpl +++ b/libs/template/templates/default-python/template/__preamble.tmpl @@ -18,7 +18,7 @@ This file only template directives; it is skipped for the actual output. {{if $notDLT}} {{skip "{{.project_name}}/src/dlt_pipeline.ipynb"}} - {{skip "{{.project_name}}/resources/{{.project_name}}_pipeline.yml"}} + {{skip "{{.project_name}}/resources/{{.project_name}}.pipeline.yml"}} {{end}} {{if $notNotebook}} @@ -26,7 +26,7 @@ This file only template directives; it is skipped for the actual output. {{end}} {{if (and $notDLT $notNotebook $notPython)}} - {{skip "{{.project_name}}/resources/{{.project_name}}_job.yml"}} + {{skip "{{.project_name}}/resources/{{.project_name}}.job.yml"}} {{else}} {{skip "{{.project_name}}/resources/.gitkeep"}} {{end}} diff --git a/libs/template/templates/default-python/template/{{.project_name}}/README.md.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/README.md.tmpl index 5adade0b3..53847a9c9 100644 --- a/libs/template/templates/default-python/template/{{.project_name}}/README.md.tmpl +++ b/libs/template/templates/default-python/template/{{.project_name}}/README.md.tmpl @@ -29,7 +29,7 @@ The '{{.project_name}}' project was generated by using the default-python templa ``` Note that the default job from the template has a schedule that runs every day - (defined in resources/{{.project_name}}_job.yml). The schedule + (defined in resources/{{.project_name}}.job.yml). The schedule is paused when deploying in development mode (see https://docs.databricks.com/dev-tools/bundles/deployment-modes.html). diff --git a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl similarity index 97% rename from libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl rename to libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl index d2100e908..5211e3894 100644 --- a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl +++ b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl @@ -40,7 +40,7 @@ resources: - task_key: notebook_task {{- end}} pipeline_task: - {{- /* TODO: we should find a way that doesn't use magics for the below, like ./{{project_name}}_pipeline.yml */}} + {{- /* TODO: we should find a way that doesn't use magics for the below, like ./{{project_name}}.pipeline.yml */}} pipeline_id: ${resources.pipelines.{{.project_name}}_pipeline.id} {{end -}} {{- if (eq .include_python "yes") }} diff --git a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_pipeline.yml.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.pipeline.yml.tmpl similarity index 100% rename from libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_pipeline.yml.tmpl rename to libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.pipeline.yml.tmpl diff --git a/libs/template/templates/default-python/template/{{.project_name}}/src/dlt_pipeline.ipynb.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/src/dlt_pipeline.ipynb.tmpl index b152e9a30..253ed321c 100644 --- a/libs/template/templates/default-python/template/{{.project_name}}/src/dlt_pipeline.ipynb.tmpl +++ b/libs/template/templates/default-python/template/{{.project_name}}/src/dlt_pipeline.ipynb.tmpl @@ -14,7 +14,7 @@ "source": [ "# DLT pipeline\n", "\n", - "This Delta Live Tables (DLT) definition is executed using a pipeline defined in resources/{{.project_name}}_pipeline.yml." + "This Delta Live Tables (DLT) definition is executed using a pipeline defined in resources/{{.project_name}}.pipeline.yml." ] }, { diff --git a/libs/template/templates/default-python/template/{{.project_name}}/src/notebook.ipynb.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/src/notebook.ipynb.tmpl index a228f8d18..6782a053b 100644 --- a/libs/template/templates/default-python/template/{{.project_name}}/src/notebook.ipynb.tmpl +++ b/libs/template/templates/default-python/template/{{.project_name}}/src/notebook.ipynb.tmpl @@ -14,7 +14,7 @@ "source": [ "# Default notebook\n", "\n", - "This default notebook is executed using Databricks Workflows as defined in resources/{{.project_name}}_job.yml." + "This default notebook is executed using Databricks Workflows as defined in resources/{{.project_name}}.job.yml." ] }, { diff --git a/libs/template/templates/default-sql/template/{{.project_name}}/resources/{{.project_name}}_sql_job.yml.tmpl b/libs/template/templates/default-sql/template/{{.project_name}}/resources/{{.project_name}}_sql.job.yml.tmpl similarity index 100% rename from libs/template/templates/default-sql/template/{{.project_name}}/resources/{{.project_name}}_sql_job.yml.tmpl rename to libs/template/templates/default-sql/template/{{.project_name}}/resources/{{.project_name}}_sql.job.yml.tmpl diff --git a/libs/template/templates/default-sql/template/{{.project_name}}/src/orders_daily.sql.tmpl b/libs/template/templates/default-sql/template/{{.project_name}}/src/orders_daily.sql.tmpl index e5ceb77a9..444ae4e03 100644 --- a/libs/template/templates/default-sql/template/{{.project_name}}/src/orders_daily.sql.tmpl +++ b/libs/template/templates/default-sql/template/{{.project_name}}/src/orders_daily.sql.tmpl @@ -1,4 +1,4 @@ --- This query is executed using Databricks Workflows (see resources/{{.project_name}}_sql_job.yml) +-- This query is executed using Databricks Workflows (see resources/{{.project_name}}_sql.job.yml) USE CATALOG {{"{{"}}catalog{{"}}"}}; USE IDENTIFIER({{"{{"}}schema{{"}}"}}); diff --git a/libs/template/templates/default-sql/template/{{.project_name}}/src/orders_raw.sql.tmpl b/libs/template/templates/default-sql/template/{{.project_name}}/src/orders_raw.sql.tmpl index c73606ef1..80f6773cb 100644 --- a/libs/template/templates/default-sql/template/{{.project_name}}/src/orders_raw.sql.tmpl +++ b/libs/template/templates/default-sql/template/{{.project_name}}/src/orders_raw.sql.tmpl @@ -1,4 +1,4 @@ --- This query is executed using Databricks Workflows (see resources/{{.project_name}}_sql_job.yml) +-- This query is executed using Databricks Workflows (see resources/{{.project_name}}_sql.job.yml) -- -- The streaming table below ingests all JSON files in /databricks-datasets/retail-org/sales_orders/ -- See also https://docs.databricks.com/sql/language-manual/sql-ref-syntax-ddl-create-streaming-table.html From 7f1121d8d85900db0fc333ee901dfe6eb8488b3b Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Wed, 25 Sep 2024 17:45:28 +0200 Subject: [PATCH 14/34] Pin Go toolchain to 1.22.7 (#1790) ## Changes Relates to https://github.com/databricks/cli/pull/1758. More information about toolchains: * https://go.dev/blog/toolchain * https://go.dev/doc/toolchain We need to specify the toolchain as we need to bump Go to 1.22.0 for the `mod` upgrade and want to use the latest toolchain on the 1.22 series. ## Tests The previous release was made with Go 1.22.7 so we should continue to use it. --- .github/workflows/push.yml | 6 +++--- .github/workflows/release-snapshot.yml | 2 +- .github/workflows/release.yml | 2 +- go.mod | 4 +++- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 02bf73784..ee60da9da 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -33,7 +33,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v5 with: - go-version: 1.22.x + go-version: 1.22.7 - name: Setup Python uses: actions/setup-python@v5 @@ -68,7 +68,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v5 with: - go-version: 1.22.x + go-version: 1.22.7 # No need to download cached dependencies when running gofmt. cache: false @@ -100,7 +100,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v5 with: - go-version: 1.22.x + go-version: 1.22.7 # Github repo: https://github.com/ajv-validator/ajv-cli - name: Install ajv-cli diff --git a/.github/workflows/release-snapshot.yml b/.github/workflows/release-snapshot.yml index defd1c535..6a601a5f9 100644 --- a/.github/workflows/release-snapshot.yml +++ b/.github/workflows/release-snapshot.yml @@ -21,7 +21,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v5 with: - go-version: 1.22.x + go-version: 1.22.7 # The default cache key for this action considers only the `go.sum` file. # We include .goreleaser.yaml here to differentiate from the cache used by the push action diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 531fb39bf..f9742a19d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -22,7 +22,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v5 with: - go-version: 1.22.x + go-version: 1.22.7 # The default cache key for this action considers only the `go.sum` file. # We include .goreleaser.yaml here to differentiate from the cache used by the push action diff --git a/go.mod b/go.mod index ba41ef3ac..e1c7519fd 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/databricks/cli -go 1.22 +go 1.22.0 + +toolchain go1.22.7 require ( github.com/Masterminds/semver/v3 v3.3.0 // MIT From 495040e4cd2d8fbbbcc09aff6cf3b88cb4daee78 Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Wed, 25 Sep 2024 21:43:48 +0530 Subject: [PATCH 15/34] Modify SetLocation test utility to take full locations as argument (#1788) I plan to use this in https://github.com/databricks/cli/pull/1780, to set the line and column numbers as well for the locations. gopatch file used: ``` @@ var x expression var y expression var z expression @@ -bundletest.SetLocation(x, y, z) +bundletest.SetLocation(x, y, []dyn.Location{{File: z}}) ``` --- bundle/artifacts/expand_globs_test.go | 7 ++-- .../expand_pipeline_glob_paths_test.go | 5 +-- .../config/mutator/rewrite_sync_paths_test.go | 25 ++++++++------- bundle/config/mutator/sync_infer_root_test.go | 3 +- bundle/config/mutator/translate_paths_test.go | 32 +++++++++---------- bundle/deploy/metadata/compute_test.go | 7 ++-- bundle/internal/bundletest/location.go | 6 ++-- .../libraries/expand_glob_references_test.go | 7 ++-- 8 files changed, 48 insertions(+), 44 deletions(-) diff --git a/bundle/artifacts/expand_globs_test.go b/bundle/artifacts/expand_globs_test.go index c9c478448..1665a4806 100644 --- a/bundle/artifacts/expand_globs_test.go +++ b/bundle/artifacts/expand_globs_test.go @@ -10,6 +10,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/internal/bundletest" "github.com/databricks/cli/internal/testutil" + "github.com/databricks/cli/libs/dyn" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -36,7 +37,7 @@ func TestExpandGlobs_Nominal(t *testing.T) { }, } - bundletest.SetLocation(b, "artifacts", filepath.Join(tmpDir, "databricks.yml")) + bundletest.SetLocation(b, "artifacts", []dyn.Location{{File: filepath.Join(tmpDir, "databricks.yml")}}) ctx := context.Background() diags := bundle.Apply(ctx, b, bundle.Seq( @@ -77,7 +78,7 @@ func TestExpandGlobs_InvalidPattern(t *testing.T) { }, } - bundletest.SetLocation(b, "artifacts", filepath.Join(tmpDir, "databricks.yml")) + bundletest.SetLocation(b, "artifacts", []dyn.Location{{File: filepath.Join(tmpDir, "databricks.yml")}}) ctx := context.Background() diags := bundle.Apply(ctx, b, bundle.Seq( @@ -125,7 +126,7 @@ func TestExpandGlobs_NoMatches(t *testing.T) { }, } - bundletest.SetLocation(b, "artifacts", filepath.Join(tmpDir, "databricks.yml")) + bundletest.SetLocation(b, "artifacts", []dyn.Location{{File: filepath.Join(tmpDir, "databricks.yml")}}) ctx := context.Background() diags := bundle.Apply(ctx, b, bundle.Seq( diff --git a/bundle/config/mutator/expand_pipeline_glob_paths_test.go b/bundle/config/mutator/expand_pipeline_glob_paths_test.go index d1671c256..07dd20215 100644 --- a/bundle/config/mutator/expand_pipeline_glob_paths_test.go +++ b/bundle/config/mutator/expand_pipeline_glob_paths_test.go @@ -10,6 +10,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/internal/bundletest" + "github.com/databricks/cli/libs/dyn" "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/pipelines" "github.com/stretchr/testify/require" @@ -105,8 +106,8 @@ func TestExpandGlobPathsInPipelines(t *testing.T) { }, } - bundletest.SetLocation(b, ".", filepath.Join(dir, "resource.yml")) - bundletest.SetLocation(b, "resources.pipelines.pipeline.libraries[3]", filepath.Join(dir, "relative", "resource.yml")) + bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "resource.yml")}}) + bundletest.SetLocation(b, "resources.pipelines.pipeline.libraries[3]", []dyn.Location{{File: filepath.Join(dir, "relative", "resource.yml")}}) m := ExpandPipelineGlobPaths() diags := bundle.Apply(context.Background(), b, m) diff --git a/bundle/config/mutator/rewrite_sync_paths_test.go b/bundle/config/mutator/rewrite_sync_paths_test.go index fa7f124b7..a66f2763a 100644 --- a/bundle/config/mutator/rewrite_sync_paths_test.go +++ b/bundle/config/mutator/rewrite_sync_paths_test.go @@ -9,6 +9,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/internal/bundletest" + "github.com/databricks/cli/libs/dyn" "github.com/stretchr/testify/assert" ) @@ -33,12 +34,12 @@ func TestRewriteSyncPathsRelative(t *testing.T) { }, } - bundletest.SetLocation(b, "sync.paths[0]", "./databricks.yml") - bundletest.SetLocation(b, "sync.paths[1]", "./databricks.yml") - bundletest.SetLocation(b, "sync.include[0]", "./file.yml") - bundletest.SetLocation(b, "sync.include[1]", "./a/file.yml") - bundletest.SetLocation(b, "sync.exclude[0]", "./a/b/file.yml") - bundletest.SetLocation(b, "sync.exclude[1]", "./a/b/c/file.yml") + bundletest.SetLocation(b, "sync.paths[0]", []dyn.Location{{File: "./databricks.yml"}}) + bundletest.SetLocation(b, "sync.paths[1]", []dyn.Location{{File: "./databricks.yml"}}) + bundletest.SetLocation(b, "sync.include[0]", []dyn.Location{{File: "./file.yml"}}) + bundletest.SetLocation(b, "sync.include[1]", []dyn.Location{{File: "./a/file.yml"}}) + bundletest.SetLocation(b, "sync.exclude[0]", []dyn.Location{{File: "./a/b/file.yml"}}) + bundletest.SetLocation(b, "sync.exclude[1]", []dyn.Location{{File: "./a/b/c/file.yml"}}) diags := bundle.Apply(context.Background(), b, mutator.RewriteSyncPaths()) assert.NoError(t, diags.Error()) @@ -72,12 +73,12 @@ func TestRewriteSyncPathsAbsolute(t *testing.T) { }, } - bundletest.SetLocation(b, "sync.paths[0]", "/tmp/dir/databricks.yml") - bundletest.SetLocation(b, "sync.paths[1]", "/tmp/dir/databricks.yml") - bundletest.SetLocation(b, "sync.include[0]", "/tmp/dir/file.yml") - bundletest.SetLocation(b, "sync.include[1]", "/tmp/dir/a/file.yml") - bundletest.SetLocation(b, "sync.exclude[0]", "/tmp/dir/a/b/file.yml") - bundletest.SetLocation(b, "sync.exclude[1]", "/tmp/dir/a/b/c/file.yml") + bundletest.SetLocation(b, "sync.paths[0]", []dyn.Location{{File: "/tmp/dir/databricks.yml"}}) + bundletest.SetLocation(b, "sync.paths[1]", []dyn.Location{{File: "/tmp/dir/databricks.yml"}}) + bundletest.SetLocation(b, "sync.include[0]", []dyn.Location{{File: "/tmp/dir/file.yml"}}) + bundletest.SetLocation(b, "sync.include[1]", []dyn.Location{{File: "/tmp/dir/a/file.yml"}}) + bundletest.SetLocation(b, "sync.exclude[0]", []dyn.Location{{File: "/tmp/dir/a/b/file.yml"}}) + bundletest.SetLocation(b, "sync.exclude[1]", []dyn.Location{{File: "/tmp/dir/a/b/c/file.yml"}}) diags := bundle.Apply(context.Background(), b, mutator.RewriteSyncPaths()) assert.NoError(t, diags.Error()) diff --git a/bundle/config/mutator/sync_infer_root_test.go b/bundle/config/mutator/sync_infer_root_test.go index 383e56769..85e40adc6 100644 --- a/bundle/config/mutator/sync_infer_root_test.go +++ b/bundle/config/mutator/sync_infer_root_test.go @@ -9,6 +9,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/internal/bundletest" + "github.com/databricks/cli/libs/dyn" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -184,7 +185,7 @@ func TestSyncInferRoot_Error(t *testing.T) { }, } - bundletest.SetLocation(b, "sync.paths", "databricks.yml") + bundletest.SetLocation(b, "sync.paths", []dyn.Location{{File: "databricks.yml"}}) ctx := context.Background() diags := bundle.Apply(ctx, b, mutator.SyncInferRoot()) diff --git a/bundle/config/mutator/translate_paths_test.go b/bundle/config/mutator/translate_paths_test.go index 50fcd3b07..c03cee73e 100644 --- a/bundle/config/mutator/translate_paths_test.go +++ b/bundle/config/mutator/translate_paths_test.go @@ -82,7 +82,7 @@ func TestTranslatePathsSkippedWithGitSource(t *testing.T) { }, } - bundletest.SetLocation(b, ".", filepath.Join(dir, "resource.yml")) + bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "resource.yml")}}) diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths()) require.NoError(t, diags.Error()) @@ -210,7 +210,7 @@ func TestTranslatePaths(t *testing.T) { }, } - bundletest.SetLocation(b, ".", filepath.Join(dir, "resource.yml")) + bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "resource.yml")}}) diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths()) require.NoError(t, diags.Error()) @@ -346,8 +346,8 @@ func TestTranslatePathsInSubdirectories(t *testing.T) { }, } - bundletest.SetLocation(b, "resources.jobs", filepath.Join(dir, "job/resource.yml")) - bundletest.SetLocation(b, "resources.pipelines", filepath.Join(dir, "pipeline/resource.yml")) + bundletest.SetLocation(b, "resources.jobs", []dyn.Location{{File: filepath.Join(dir, "job/resource.yml")}}) + bundletest.SetLocation(b, "resources.pipelines", []dyn.Location{{File: filepath.Join(dir, "pipeline/resource.yml")}}) diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths()) require.NoError(t, diags.Error()) @@ -408,7 +408,7 @@ func TestTranslatePathsOutsideSyncRoot(t *testing.T) { }, } - bundletest.SetLocation(b, ".", filepath.Join(dir, "../resource.yml")) + bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "../resource.yml")}}) diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths()) assert.ErrorContains(t, diags.Error(), "is not contained in sync root path") @@ -439,7 +439,7 @@ func TestJobNotebookDoesNotExistError(t *testing.T) { }, } - bundletest.SetLocation(b, ".", filepath.Join(dir, "fake.yml")) + bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "fake.yml")}}) diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths()) assert.EqualError(t, diags.Error(), "notebook ./doesnt_exist.py not found") @@ -470,7 +470,7 @@ func TestJobFileDoesNotExistError(t *testing.T) { }, } - bundletest.SetLocation(b, ".", filepath.Join(dir, "fake.yml")) + bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "fake.yml")}}) diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths()) assert.EqualError(t, diags.Error(), "file ./doesnt_exist.py not found") @@ -501,7 +501,7 @@ func TestPipelineNotebookDoesNotExistError(t *testing.T) { }, } - bundletest.SetLocation(b, ".", filepath.Join(dir, "fake.yml")) + bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "fake.yml")}}) diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths()) assert.EqualError(t, diags.Error(), "notebook ./doesnt_exist.py not found") @@ -532,7 +532,7 @@ func TestPipelineFileDoesNotExistError(t *testing.T) { }, } - bundletest.SetLocation(b, ".", filepath.Join(dir, "fake.yml")) + bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "fake.yml")}}) diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths()) assert.EqualError(t, diags.Error(), "file ./doesnt_exist.py not found") @@ -567,7 +567,7 @@ func TestJobSparkPythonTaskWithNotebookSourceError(t *testing.T) { }, } - bundletest.SetLocation(b, ".", filepath.Join(dir, "resource.yml")) + bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "resource.yml")}}) diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths()) assert.ErrorContains(t, diags.Error(), `expected a file for "resources.jobs.job.tasks[0].spark_python_task.python_file" but got a notebook`) @@ -602,7 +602,7 @@ func TestJobNotebookTaskWithFileSourceError(t *testing.T) { }, } - bundletest.SetLocation(b, ".", filepath.Join(dir, "resource.yml")) + bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "resource.yml")}}) diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths()) assert.ErrorContains(t, diags.Error(), `expected a notebook for "resources.jobs.job.tasks[0].notebook_task.notebook_path" but got a file`) @@ -637,7 +637,7 @@ func TestPipelineNotebookLibraryWithFileSourceError(t *testing.T) { }, } - bundletest.SetLocation(b, ".", filepath.Join(dir, "resource.yml")) + bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "resource.yml")}}) diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths()) assert.ErrorContains(t, diags.Error(), `expected a notebook for "resources.pipelines.pipeline.libraries[0].notebook.path" but got a file`) @@ -672,7 +672,7 @@ func TestPipelineFileLibraryWithNotebookSourceError(t *testing.T) { }, } - bundletest.SetLocation(b, ".", filepath.Join(dir, "resource.yml")) + bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "resource.yml")}}) diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths()) assert.ErrorContains(t, diags.Error(), `expected a file for "resources.pipelines.pipeline.libraries[0].file.path" but got a notebook`) @@ -710,7 +710,7 @@ func TestTranslatePathJobEnvironments(t *testing.T) { }, } - bundletest.SetLocation(b, "resources.jobs", filepath.Join(dir, "job/resource.yml")) + bundletest.SetLocation(b, "resources.jobs", []dyn.Location{{File: filepath.Join(dir, "job/resource.yml")}}) diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths()) require.NoError(t, diags.Error()) @@ -753,8 +753,8 @@ func TestTranslatePathWithComplexVariables(t *testing.T) { }, } - bundletest.SetLocation(b, "variables", filepath.Join(dir, "variables/variables.yml")) - bundletest.SetLocation(b, "resources.jobs", filepath.Join(dir, "job/resource.yml")) + bundletest.SetLocation(b, "variables", []dyn.Location{{File: filepath.Join(dir, "variables/variables.yml")}}) + bundletest.SetLocation(b, "resources.jobs", []dyn.Location{{File: filepath.Join(dir, "job/resource.yml")}}) ctx := context.Background() // Assign the variables to the dynamic configuration. diff --git a/bundle/deploy/metadata/compute_test.go b/bundle/deploy/metadata/compute_test.go index 6d43f845b..2c2c72376 100644 --- a/bundle/deploy/metadata/compute_test.go +++ b/bundle/deploy/metadata/compute_test.go @@ -9,6 +9,7 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/internal/bundletest" "github.com/databricks/cli/bundle/metadata" + "github.com/databricks/cli/libs/dyn" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -55,9 +56,9 @@ func TestComputeMetadataMutator(t *testing.T) { }, } - bundletest.SetLocation(b, "resources.jobs.my-job-1", "a/b/c") - bundletest.SetLocation(b, "resources.jobs.my-job-2", "d/e/f") - bundletest.SetLocation(b, "resources.pipelines.my-pipeline", "abc") + bundletest.SetLocation(b, "resources.jobs.my-job-1", []dyn.Location{{File: "a/b/c"}}) + bundletest.SetLocation(b, "resources.jobs.my-job-2", []dyn.Location{{File: "d/e/f"}}) + bundletest.SetLocation(b, "resources.pipelines.my-pipeline", []dyn.Location{{File: "abc"}}) expectedMetadata := metadata.Metadata{ Version: metadata.Version, diff --git a/bundle/internal/bundletest/location.go b/bundle/internal/bundletest/location.go index 380d6e17d..2ffd621bf 100644 --- a/bundle/internal/bundletest/location.go +++ b/bundle/internal/bundletest/location.go @@ -8,15 +8,13 @@ import ( // SetLocation sets the location of all values in the bundle to the given path. // This is useful for testing where we need to associate configuration // with the path it is loaded from. -func SetLocation(b *bundle.Bundle, prefix string, filePath string) { +func SetLocation(b *bundle.Bundle, prefix string, locations []dyn.Location) { start := dyn.MustPathFromString(prefix) b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) { return dyn.Walk(root, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { // If the path has the given prefix, set the location. if p.HasPrefix(start) { - return v.WithLocations([]dyn.Location{{ - File: filePath, - }}), nil + return v.WithLocations(locations), nil } // The path is not nested under the given prefix. diff --git a/bundle/libraries/expand_glob_references_test.go b/bundle/libraries/expand_glob_references_test.go index e7f2e1693..2dfbddb74 100644 --- a/bundle/libraries/expand_glob_references_test.go +++ b/bundle/libraries/expand_glob_references_test.go @@ -10,6 +10,7 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/internal/bundletest" "github.com/databricks/cli/internal/testutil" + "github.com/databricks/cli/libs/dyn" "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/require" @@ -61,7 +62,7 @@ func TestGlobReferencesExpandedForTaskLibraries(t *testing.T) { }, } - bundletest.SetLocation(b, ".", filepath.Join(dir, "resource.yml")) + bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "resource.yml")}}) diags := bundle.Apply(context.Background(), b, ExpandGlobReferences()) require.Empty(t, diags) @@ -146,7 +147,7 @@ func TestGlobReferencesExpandedForForeachTaskLibraries(t *testing.T) { }, } - bundletest.SetLocation(b, ".", filepath.Join(dir, "resource.yml")) + bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "resource.yml")}}) diags := bundle.Apply(context.Background(), b, ExpandGlobReferences()) require.Empty(t, diags) @@ -221,7 +222,7 @@ func TestGlobReferencesExpandedForEnvironmentsDeps(t *testing.T) { }, } - bundletest.SetLocation(b, ".", filepath.Join(dir, "resource.yml")) + bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(dir, "resource.yml")}}) diags := bundle.Apply(context.Background(), b, ExpandGlobReferences()) require.Empty(t, diags) From 875b112f801c8b04694e077cc07ed88a335db31b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 26 Sep 2024 06:07:01 +0000 Subject: [PATCH 16/34] Bump golang.org/x/mod from 0.20.0 to 0.21.0 (#1758) Bumps [golang.org/x/mod](https://github.com/golang/mod) from 0.20.0 to 0.21.0.
Commits
  • 46a3137 zip: set GIT_DIR in test when using bare repositories
  • 3afcd4e go.mod: set go version to 1.22.0
  • b1d336c go.mod: update required go version to go1.22
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golang.org/x/mod&package-manager=go_modules&previous-version=0.20.0&new-version=0.21.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Andrew Nester Co-authored-by: Pieter Noordhuis --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e1c7519fd..b6478a915 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/spf13/pflag v1.0.5 // BSD-3-Clause github.com/stretchr/testify v1.9.0 // MIT golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 - golang.org/x/mod v0.20.0 + golang.org/x/mod v0.21.0 golang.org/x/oauth2 v0.23.0 golang.org/x/sync v0.8.0 golang.org/x/term v0.24.0 diff --git a/go.sum b/go.sum index 3d4a2cdce..80fa43fdd 100644 --- a/go.sum +++ b/go.sum @@ -180,8 +180,8 @@ golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0= -golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= +golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= From 94d8c3ba1e18abb82ca4dde85210cc1d2134f303 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 26 Sep 2024 06:29:34 +0000 Subject: [PATCH 17/34] Bump github.com/hashicorp/hc-install from 0.7.0 to 0.9.0 (#1772) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/hashicorp/hc-install](https://github.com/hashicorp/hc-install) from 0.7.0 to 0.9.0.
Release notes

Sourced from github.com/hashicorp/hc-install's releases.

v0.9.0

What's Changed

Full Changelog: https://github.com/hashicorp/hc-install/compare/v0.8.1...v0.9.0

v0.8.1

What's Changed

New Contributors

Full Changelog: https://github.com/hashicorp/hc-install/compare/v0.8.0...v0.8.1

v0.8.0

ENHANCEMENTS:

BUG FIXES:

INTERNAL:

... (truncated)

Commits
  • 157a802 Merge pull request #250 from hashicorp/release-0.9.0
  • 4c734fc Prepare for v0.9.0 release
  • d78b328 Merge pull request #249 from hashicorp/d-contributing-md-update
  • 34f38b0 docs: Update release instructions
  • 6a5aa83 build(deps): bump golang.org/x/mod from 0.20.0 to 0.21.0 (#242)
  • 1784fcc Merge pull request #248 from hashicorp/revert-version-contents
  • ea2c69b Finish Release of 0.8.1 by updating VERSION
  • 4f3e00e Releasing 0.8.1
  • c6d1ced Merge pull request #246 from hashicorp/update-contributing
  • eea12f1 Update CONTRIBUTING.md to add clean up step
  • Additional commits viewable in compare view

Most Recent Ignore Conditions Applied to This Pull Request | Dependency Name | Ignore Conditions | | --- | --- | | github.com/hashicorp/hc-install | [>= 0.8.a, < 0.9] |
[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/hashicorp/hc-install&package-manager=go_modules&previous-version=0.7.0&new-version=0.9.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 3 ++- go.sum | 8 ++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b6478a915..0cf3ef8a7 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/ghodss/yaml v1.0.0 // MIT + NOTICE github.com/google/uuid v1.6.0 // BSD-3-Clause github.com/hashicorp/go-version v1.7.0 // MPL 2.0 - github.com/hashicorp/hc-install v0.7.0 // MPL 2.0 + github.com/hashicorp/hc-install v0.9.0 // MPL 2.0 github.com/hashicorp/terraform-exec v0.21.0 // MPL 2.0 github.com/hashicorp/terraform-json v0.22.1 // MPL 2.0 github.com/manifoldco/promptui v0.9.0 // BSD-3-Clause @@ -51,6 +51,7 @@ require ( github.com/google/s2a-go v0.1.7 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-retryablehttp v0.7.7 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect diff --git a/go.sum b/go.sum index 80fa43fdd..d88667751 100644 --- a/go.sum +++ b/go.sum @@ -99,10 +99,14 @@ github.com/googleapis/gax-go/v2 v2.12.4 h1:9gWcmF85Wvq4ryPFvGFaOgPIs1AQX0d0bcbGw github.com/googleapis/gax-go/v2 v2.12.4/go.mod h1:KYEYLorsnIGDi/rPC8b5TdlB9kbKoFubselGIoBMCwI= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= +github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= +github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/hc-install v0.7.0 h1:Uu9edVqjKQxxuD28mR5TikkKDd/p55S8vzPC1659aBk= -github.com/hashicorp/hc-install v0.7.0/go.mod h1:ELmmzZlGnEcqoUMKUuykHaPCIR1sYLYX+KSggWSKZuA= +github.com/hashicorp/hc-install v0.9.0 h1:2dIk8LcvANwtv3QZLckxcjyF5w8KVtiMxu6G6eLhghE= +github.com/hashicorp/hc-install v0.9.0/go.mod h1:+6vOP+mf3tuGgMApVYtmsnDoKWMDcFXeTxCACYZ8SFg= github.com/hashicorp/terraform-exec v0.21.0 h1:uNkLAe95ey5Uux6KJdua6+cv8asgILFVWkd/RG0D2XQ= github.com/hashicorp/terraform-exec v0.21.0/go.mod h1:1PPeMYou+KDUSSeRE9szMZ/oHf4fYUmB923Wzbq1ICg= github.com/hashicorp/terraform-json v0.22.1 h1:xft84GZR0QzjPVWs4lRUwvTcPnegqlyS7orfb5Ltvec= From 66f2ba64a8a479d45efb9b23eab096a5ffda1367 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 26 Sep 2024 14:55:07 +0200 Subject: [PATCH 18/34] Simplified isFullVariableOverrideDef implementation (#1791) ## Changes Simplified isFullVariableOverrideDef implementation Follow up on https://github.com/databricks/cli/pull/1787 --- bundle/config/root.go | 54 ++++++++++++++++--------------------------- 1 file changed, 20 insertions(+), 34 deletions(-) diff --git a/bundle/config/root.go b/bundle/config/root.go index ff169e4ce..4b1467456 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -406,7 +406,14 @@ func (r *Root) MergeTargetOverrides(name string) error { return r.updateWithDynamicValue(root) } -var variableKeywords = []string{"default", "lookup"} +var allowedVariableDefinitions = []([]string){ + {"default", "type", "description"}, + {"default", "type"}, + {"default", "description"}, + {"lookup", "description"}, + {"default"}, + {"lookup"}, +} // isFullVariableOverrideDef checks if the given value is a full syntax varaible override. // A full syntax variable override is a map with either 1 of 2 keys. @@ -423,42 +430,21 @@ func isFullVariableOverrideDef(v dyn.Value) bool { return false } - // If the map has 3 keys, they should be "description", "type" and "default" or "lookup" - if mv.Len() == 3 { - if _, ok := mv.GetByString("type"); ok { - if _, ok := mv.GetByString("description"); ok { - if _, ok := mv.GetByString("default"); ok { - return true - } + for _, keys := range allowedVariableDefinitions { + if len(keys) != mv.Len() { + continue + } + + // Check if the keys are the same. + match := true + for _, key := range keys { + if _, ok := mv.GetByString(key); !ok { + match = false + break } } - return false - } - - // If the map has 2 keys, one of them should be "default" or "lookup" and the other is "type" or "description" - if mv.Len() == 2 { - if _, ok := mv.GetByString("type"); ok { - if _, ok := mv.GetByString("default"); ok { - return true - } - } - - if _, ok := mv.GetByString("description"); ok { - if _, ok := mv.GetByString("default"); ok { - return true - } - - if _, ok := mv.GetByString("lookup"); ok { - return true - } - } - - return false - } - - for _, keyword := range variableKeywords { - if _, ok := mv.GetByString(keyword); ok { + if match { return true } } From 4e8e02738081b74017bcb9d7b440e75ffa08c0d7 Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Thu, 26 Sep 2024 18:52:22 +0530 Subject: [PATCH 19/34] Sort tasks by `task_key` before generating the Terraform configuration (#1776) ## Changes Sort the tasks in the resultant `bundle.tf.json`. This is important because configuration from one task can leak into another if the tasks are not sorted. For more details see: 1. https://github.com/databricks/terraform-provider-databricks/issues/3951 2. https://github.com/databricks/terraform-provider-databricks/issues/4011 ## Tests Unit test and manually. For manual testing I used the following configuration: ``` resources: jobs: foo: tasks: - task_key: task-Z notebook_task: notebook_path: nb.py source: GIT existing_cluster_id: 0715-133738-ju0ma84z - task_key: task-1 notebook_task: notebook_path: ${workspace.file_path}/local.py source: WORKSPACE existing_cluster_id: 0715-133738-ju0ma84z depends_on: - task_key: task-Z git_source: git_provider: gitHub git_url: https://github.com/shreyas-goenka/job-source-tmp.git git_branch: main ``` Steps (1): 1. Deploy this bundle. 2. Comment out "source: GIT" 3. Deploy again Before: Deploying this bundle twice would fail. This is because the "source: GIT" would carry over to the next deployment. After: There was no error on the subsequent deployment. Steps (2): 1. Deploy once 2. Deploy again Before: Works correctly but leads to a update API call every time. After: No diff is detected by terraform. --- bundle/deploy/terraform/convert.go | 5 +++ bundle/deploy/terraform/tfdyn/convert_job.go | 33 ++++++++++++++++++- .../terraform/tfdyn/convert_job_test.go | 30 ++++++++++++++--- 3 files changed, 63 insertions(+), 5 deletions(-) diff --git a/bundle/deploy/terraform/convert.go b/bundle/deploy/terraform/convert.go index 5a548e3b5..b8993c031 100644 --- a/bundle/deploy/terraform/convert.go +++ b/bundle/deploy/terraform/convert.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "sort" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" @@ -82,6 +83,10 @@ func BundleToTerraform(config *config.Root) *schema.Root { conv(src, &dst) if src.JobSettings != nil { + sort.Slice(src.JobSettings.Tasks, func(i, j int) bool { + return src.JobSettings.Tasks[i].TaskKey < src.JobSettings.Tasks[j].TaskKey + }) + for _, v := range src.Tasks { var t schema.ResourceJobTask conv(v, &t) diff --git a/bundle/deploy/terraform/tfdyn/convert_job.go b/bundle/deploy/terraform/tfdyn/convert_job.go index d1e7e73e2..8948e3baf 100644 --- a/bundle/deploy/terraform/tfdyn/convert_job.go +++ b/bundle/deploy/terraform/tfdyn/convert_job.go @@ -3,6 +3,7 @@ package tfdyn import ( "context" "fmt" + "sort" "github.com/databricks/cli/bundle/internal/tf/schema" "github.com/databricks/cli/libs/dyn" @@ -19,8 +20,38 @@ func convertJobResource(ctx context.Context, vin dyn.Value) (dyn.Value, error) { log.Debugf(ctx, "job normalization diagnostic: %s", diag.Summary) } + // Sort the tasks of each job in the bundle by task key. Sorting + // the task keys ensures that the diff computed by terraform is correct and avoids + // recreates. For more details see the NOTE at + // https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/job#example-usage + // and https://github.com/databricks/terraform-provider-databricks/issues/4011 + // and https://github.com/databricks/cli/pull/1776 + vout := vin + var err error + tasks, ok := vin.Get("tasks").AsSequence() + if ok { + sort.Slice(tasks, func(i, j int) bool { + // We sort the tasks by their task key. Tasks without task keys are ordered + // before tasks with task keys. We do not error for those tasks + // since presence of a task_key is validated for in the Jobs backend. + tk1, ok := tasks[i].Get("task_key").AsString() + if !ok { + return true + } + tk2, ok := tasks[j].Get("task_key").AsString() + if !ok { + return false + } + return tk1 < tk2 + }) + vout, err = dyn.Set(vin, "tasks", dyn.V(tasks)) + if err != nil { + return dyn.InvalidValue, err + } + } + // Modify top-level keys. - vout, err := renameKeys(vin, map[string]string{ + vout, err = renameKeys(vout, map[string]string{ "tasks": "task", "job_clusters": "job_cluster", "parameters": "parameter", diff --git a/bundle/deploy/terraform/tfdyn/convert_job_test.go b/bundle/deploy/terraform/tfdyn/convert_job_test.go index b9e1f967f..695b9ba24 100644 --- a/bundle/deploy/terraform/tfdyn/convert_job_test.go +++ b/bundle/deploy/terraform/tfdyn/convert_job_test.go @@ -42,8 +42,8 @@ func TestConvertJob(t *testing.T) { }, Tasks: []jobs.Task{ { - TaskKey: "task_key", - JobClusterKey: "job_cluster_key", + TaskKey: "task_key_b", + JobClusterKey: "job_cluster_key_b", Libraries: []compute.Library{ { Pypi: &compute.PythonPyPiLibrary{ @@ -55,6 +55,17 @@ func TestConvertJob(t *testing.T) { }, }, }, + { + TaskKey: "task_key_a", + JobClusterKey: "job_cluster_key_a", + }, + { + TaskKey: "task_key_c", + JobClusterKey: "job_cluster_key_c", + }, + { + Description: "missing task key 😱", + }, }, }, Permissions: []resources.Permission{ @@ -100,8 +111,15 @@ func TestConvertJob(t *testing.T) { }, "task": []any{ map[string]any{ - "task_key": "task_key", - "job_cluster_key": "job_cluster_key", + "description": "missing task key 😱", + }, + map[string]any{ + "task_key": "task_key_a", + "job_cluster_key": "job_cluster_key_a", + }, + map[string]any{ + "task_key": "task_key_b", + "job_cluster_key": "job_cluster_key_b", "library": []any{ map[string]any{ "pypi": map[string]any{ @@ -113,6 +131,10 @@ func TestConvertJob(t *testing.T) { }, }, }, + map[string]any{ + "task_key": "task_key_c", + "job_cluster_key": "job_cluster_key_c", + }, }, }, out.Job["my_job"]) From a1dca56abfb16879d55e113d98735e10a97f558e Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Fri, 27 Sep 2024 11:30:39 +0200 Subject: [PATCH 20/34] Trim trailing whitespace (#1794) ## Changes Trailing whitespace is trimmed per the VS Code settings for this repository. ## Tests n/a --- bundle/config/mutator/python/python_mutator.go | 4 ++-- bundle/config/mutator/python/python_mutator_test.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bundle/config/mutator/python/python_mutator.go b/bundle/config/mutator/python/python_mutator.go index fbf3b7e0b..3d4a502f7 100644 --- a/bundle/config/mutator/python/python_mutator.go +++ b/bundle/config/mutator/python/python_mutator.go @@ -228,12 +228,12 @@ func (m *pythonMutator) runPythonMutator(ctx context.Context, cacheDir string, r return output, pythonDiagnostics } -const installExplanation = `If using Python wheels, ensure that 'databricks-pydabs' is included in the dependencies, +const installExplanation = `If using Python wheels, ensure that 'databricks-pydabs' is included in the dependencies, and that the wheel is installed in the Python environment: $ .venv/bin/pip install -e . -If using a virtual environment, ensure it is specified as the venv_path property in databricks.yml, +If using a virtual environment, ensure it is specified as the venv_path property in databricks.yml, or activate the environment before running CLI commands: experimental: diff --git a/bundle/config/mutator/python/python_mutator_test.go b/bundle/config/mutator/python/python_mutator_test.go index bf12b2499..7a419d799 100644 --- a/bundle/config/mutator/python/python_mutator_test.go +++ b/bundle/config/mutator/python/python_mutator_test.go @@ -570,12 +570,12 @@ func TestExplainProcessErr(t *testing.T) { Explanation: 'databricks-pydabs' library is not installed in the Python environment. -If using Python wheels, ensure that 'databricks-pydabs' is included in the dependencies, +If using Python wheels, ensure that 'databricks-pydabs' is included in the dependencies, and that the wheel is installed in the Python environment: $ .venv/bin/pip install -e . -If using a virtual environment, ensure it is specified as the venv_path property in databricks.yml, +If using a virtual environment, ensure it is specified as the venv_path property in databricks.yml, or activate the environment before running CLI commands: experimental: From 56cd96cb939207df3403546e998579a4cc768cf6 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Fri, 27 Sep 2024 11:32:54 +0200 Subject: [PATCH 21/34] Move trampoline code into trampoline package (#1793) ## Changes Doing this to make room for PyDABs under `bundle/python`. ## Tests n/a --- bundle/phases/deploy.go | 4 ++-- bundle/phases/initialize.go | 4 ++-- .../conditional_transform_test.go | 2 +- .../warning.go => trampoline/python_dbr_warning.go} | 2 +- .../python_dbr_warning_test.go} | 2 +- .../transform.go => trampoline/python_wheel.go} | 10 +++++----- .../python_wheel_test.go} | 2 +- bundle/{config/mutator => trampoline}/trampoline.go | 3 ++- .../{config/mutator => trampoline}/trampoline_test.go | 2 +- 9 files changed, 16 insertions(+), 15 deletions(-) rename bundle/{python => trampoline}/conditional_transform_test.go (99%) rename bundle/{python/warning.go => trampoline/python_dbr_warning.go} (99%) rename bundle/{python/warning_test.go => trampoline/python_dbr_warning_test.go} (99%) rename bundle/{python/transform.go => trampoline/python_wheel.go} (94%) rename bundle/{python/transform_test.go => trampoline/python_wheel_test.go} (99%) rename bundle/{config/mutator => trampoline}/trampoline.go (99%) rename bundle/{config/mutator => trampoline}/trampoline_test.go (99%) diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 097c561eb..cb0ecf75d 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -15,8 +15,8 @@ import ( "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/libraries" "github.com/databricks/cli/bundle/permissions" - "github.com/databricks/cli/bundle/python" "github.com/databricks/cli/bundle/scripts" + "github.com/databricks/cli/bundle/trampoline" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/sync" terraformlib "github.com/databricks/cli/libs/terraform" @@ -157,7 +157,7 @@ func Deploy(outputHandler sync.OutputHandler) bundle.Mutator { artifacts.CleanUp(), libraries.ExpandGlobReferences(), libraries.Upload(), - python.TransformWheelTask(), + trampoline.TransformWheelTask(), files.Upload(outputHandler), deploy.StateUpdate(), deploy.StatePush(), diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index 8039a4f13..93ce61b25 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -9,8 +9,8 @@ import ( "github.com/databricks/cli/bundle/deploy/metadata" "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/permissions" - "github.com/databricks/cli/bundle/python" "github.com/databricks/cli/bundle/scripts" + "github.com/databricks/cli/bundle/trampoline" ) // The initialize phase fills in defaults and connects to the workspace. @@ -66,7 +66,7 @@ func Initialize() bundle.Mutator { mutator.ConfigureWSFS(), mutator.TranslatePaths(), - python.WrapperWarning(), + trampoline.WrapperWarning(), permissions.ApplyBundlePermissions(), permissions.FilterCurrentUser(), metadata.AnnotateJobs(), diff --git a/bundle/python/conditional_transform_test.go b/bundle/trampoline/conditional_transform_test.go similarity index 99% rename from bundle/python/conditional_transform_test.go rename to bundle/trampoline/conditional_transform_test.go index 1d397f7a7..26e67154e 100644 --- a/bundle/python/conditional_transform_test.go +++ b/bundle/trampoline/conditional_transform_test.go @@ -1,4 +1,4 @@ -package python +package trampoline import ( "context" diff --git a/bundle/python/warning.go b/bundle/trampoline/python_dbr_warning.go similarity index 99% rename from bundle/python/warning.go rename to bundle/trampoline/python_dbr_warning.go index 0e9d8bef0..f62e9eab4 100644 --- a/bundle/python/warning.go +++ b/bundle/trampoline/python_dbr_warning.go @@ -1,4 +1,4 @@ -package python +package trampoline import ( "context" diff --git a/bundle/python/warning_test.go b/bundle/trampoline/python_dbr_warning_test.go similarity index 99% rename from bundle/python/warning_test.go rename to bundle/trampoline/python_dbr_warning_test.go index a5ab75632..d293c9477 100644 --- a/bundle/python/warning_test.go +++ b/bundle/trampoline/python_dbr_warning_test.go @@ -1,4 +1,4 @@ -package python +package trampoline import ( "context" diff --git a/bundle/python/transform.go b/bundle/trampoline/python_wheel.go similarity index 94% rename from bundle/python/transform.go rename to bundle/trampoline/python_wheel.go index 9d3b1ab6a..8e309a625 100644 --- a/bundle/python/transform.go +++ b/bundle/trampoline/python_wheel.go @@ -1,4 +1,4 @@ -package python +package trampoline import ( "context" @@ -69,7 +69,7 @@ func TransformWheelTask() bundle.Mutator { res := b.Config.Experimental != nil && b.Config.Experimental.PythonWheelWrapper return res, nil }, - mutator.NewTrampoline( + NewTrampoline( "python_wheel", &pythonTrampoline{}, NOTEBOOK_TEMPLATE, @@ -94,9 +94,9 @@ func (t *pythonTrampoline) CleanUp(task *jobs.Task) error { return nil } -func (t *pythonTrampoline) GetTasks(b *bundle.Bundle) []mutator.TaskWithJobKey { +func (t *pythonTrampoline) GetTasks(b *bundle.Bundle) []TaskWithJobKey { r := b.Config.Resources - result := make([]mutator.TaskWithJobKey, 0) + result := make([]TaskWithJobKey, 0) for k := range b.Config.Resources.Jobs { tasks := r.Jobs[k].JobSettings.Tasks for i := range tasks { @@ -110,7 +110,7 @@ func (t *pythonTrampoline) GetTasks(b *bundle.Bundle) []mutator.TaskWithJobKey { continue } - result = append(result, mutator.TaskWithJobKey{ + result = append(result, TaskWithJobKey{ JobKey: k, Task: task, }) diff --git a/bundle/python/transform_test.go b/bundle/trampoline/python_wheel_test.go similarity index 99% rename from bundle/python/transform_test.go rename to bundle/trampoline/python_wheel_test.go index c7bddca14..40c3b38f3 100644 --- a/bundle/python/transform_test.go +++ b/bundle/trampoline/python_wheel_test.go @@ -1,4 +1,4 @@ -package python +package trampoline import ( "context" diff --git a/bundle/config/mutator/trampoline.go b/bundle/trampoline/trampoline.go similarity index 99% rename from bundle/config/mutator/trampoline.go rename to bundle/trampoline/trampoline.go index dcca50149..1dc1c4463 100644 --- a/bundle/config/mutator/trampoline.go +++ b/bundle/trampoline/trampoline.go @@ -1,4 +1,4 @@ -package mutator +package trampoline import ( "context" @@ -23,6 +23,7 @@ type TrampolineFunctions interface { GetTasks(b *bundle.Bundle) []TaskWithJobKey CleanUp(task *jobs.Task) error } + type trampoline struct { name string functions TrampolineFunctions diff --git a/bundle/config/mutator/trampoline_test.go b/bundle/trampoline/trampoline_test.go similarity index 99% rename from bundle/config/mutator/trampoline_test.go rename to bundle/trampoline/trampoline_test.go index 08d3c8220..08a290f93 100644 --- a/bundle/config/mutator/trampoline_test.go +++ b/bundle/trampoline/trampoline_test.go @@ -1,4 +1,4 @@ -package mutator +package trampoline import ( "context" From 1d1aa0a4166903b20b0f2e6321134a1aff0c4204 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Fri, 27 Sep 2024 12:03:05 +0200 Subject: [PATCH 22/34] Rename `RootPath` -> `BundleRootPath` (#1792) ## Changes After introducing the `SyncRootPath` field on the bundle (#1694), the previous `RootPath` became ambiguous. Does it mean the bundle root path or the sync root path? This PR renames to field to `BundleRootPath` to remove the ambiguity. ## Tests n/a --------- Co-authored-by: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> --- bundle/artifacts/expand_globs_test.go | 6 ++-- bundle/artifacts/prepare.go | 2 +- bundle/artifacts/whl/autodetect.go | 6 ++-- bundle/bundle.go | 30 +++++++++++-------- bundle/bundle_read_only.go | 2 +- bundle/bundle_test.go | 4 +-- bundle/config/loader/entry_point.go | 2 +- bundle/config/loader/entry_point_test.go | 2 +- bundle/config/loader/process_include_test.go | 4 +-- bundle/config/loader/process_root_includes.go | 6 ++-- .../loader/process_root_includes_test.go | 24 +++++++-------- .../expand_pipeline_glob_paths_test.go | 2 +- bundle/config/mutator/load_git_details.go | 2 +- .../config/mutator/python/python_mutator.go | 2 +- bundle/config/mutator/rewrite_sync_paths.go | 6 ++-- .../config/mutator/rewrite_sync_paths_test.go | 8 ++--- .../config/mutator/sync_default_path_test.go | 8 ++--- bundle/config/mutator/sync_infer_root.go | 2 +- bundle/config/mutator/sync_infer_root_test.go | 12 ++++---- bundle/deploy/metadata/compute.go | 2 +- bundle/deploy/state_pull_test.go | 8 ++--- bundle/deploy/state_push_test.go | 2 +- bundle/deploy/state_update_test.go | 2 +- bundle/deploy/terraform/init_test.go | 22 +++++++------- bundle/deploy/terraform/load_test.go | 2 +- bundle/deploy/terraform/state_pull_test.go | 2 +- bundle/deploy/terraform/state_push_test.go | 2 +- bundle/deploy/terraform/util_test.go | 4 +-- bundle/render/render_text_output.go | 2 +- bundle/scripts/scripts.go | 2 +- bundle/scripts/scripts_test.go | 2 +- .../trampoline/conditional_transform_test.go | 8 ++--- bundle/trampoline/python_wheel_test.go | 2 +- bundle/trampoline/trampoline_test.go | 4 +-- cmd/bundle/generate/generate_test.go | 4 +-- cmd/sync/sync_test.go | 8 ++--- internal/bundle/artifacts_test.go | 12 ++++---- 37 files changed, 112 insertions(+), 108 deletions(-) diff --git a/bundle/artifacts/expand_globs_test.go b/bundle/artifacts/expand_globs_test.go index 1665a4806..dc7c77de7 100644 --- a/bundle/artifacts/expand_globs_test.go +++ b/bundle/artifacts/expand_globs_test.go @@ -24,7 +24,7 @@ func TestExpandGlobs_Nominal(t *testing.T) { testutil.Touch(t, tmpDir, "bc.txt") b := &bundle.Bundle{ - RootPath: tmpDir, + BundleRootPath: tmpDir, Config: config.Root{ Artifacts: config.Artifacts{ "test": { @@ -63,7 +63,7 @@ func TestExpandGlobs_InvalidPattern(t *testing.T) { tmpDir := t.TempDir() b := &bundle.Bundle{ - RootPath: tmpDir, + BundleRootPath: tmpDir, Config: config.Root{ Artifacts: config.Artifacts{ "test": { @@ -111,7 +111,7 @@ func TestExpandGlobs_NoMatches(t *testing.T) { testutil.Touch(t, tmpDir, "b2.txt") b := &bundle.Bundle{ - RootPath: tmpDir, + BundleRootPath: tmpDir, Config: config.Root{ Artifacts: config.Artifacts{ "test": { diff --git a/bundle/artifacts/prepare.go b/bundle/artifacts/prepare.go index fb61ed9e2..91e0bd091 100644 --- a/bundle/artifacts/prepare.go +++ b/bundle/artifacts/prepare.go @@ -47,7 +47,7 @@ func (m *prepare) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics // If artifact path is not provided, use bundle root dir if artifact.Path == "" { - artifact.Path = b.RootPath + artifact.Path = b.BundleRootPath } if !filepath.IsAbs(artifact.Path) { diff --git a/bundle/artifacts/whl/autodetect.go b/bundle/artifacts/whl/autodetect.go index 1601767f6..88dc742c1 100644 --- a/bundle/artifacts/whl/autodetect.go +++ b/bundle/artifacts/whl/autodetect.go @@ -35,21 +35,21 @@ func (m *detectPkg) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostic log.Infof(ctx, "Detecting Python wheel project...") // checking if there is setup.py in the bundle root - setupPy := filepath.Join(b.RootPath, "setup.py") + setupPy := filepath.Join(b.BundleRootPath, "setup.py") _, err := os.Stat(setupPy) if err != nil { log.Infof(ctx, "No Python wheel project found at bundle root folder") return nil } - log.Infof(ctx, fmt.Sprintf("Found Python wheel project at %s", b.RootPath)) + log.Infof(ctx, fmt.Sprintf("Found Python wheel project at %s", b.BundleRootPath)) module := extractModuleName(setupPy) if b.Config.Artifacts == nil { b.Config.Artifacts = make(map[string]*config.Artifact) } - pkgPath, err := filepath.Abs(b.RootPath) + pkgPath, err := filepath.Abs(b.BundleRootPath) if err != nil { return diag.FromErr(err) } diff --git a/bundle/bundle.go b/bundle/bundle.go index 8b5ff976d..856255685 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -31,22 +31,26 @@ import ( const internalFolder = ".internal" type Bundle struct { - // RootPath contains the directory path to the root of the bundle. + // BundleRootPath is the local path to the root directory of the bundle. // It is set when we instantiate a new bundle instance. - RootPath string + BundleRootPath string - // BundleRoot is a virtual filesystem path to the root of the bundle. + // BundleRoot is a virtual filesystem path to [BundleRootPath]. // Exclusively use this field for filesystem operations. BundleRoot vfs.Path - // SyncRoot is a virtual filesystem path to the root directory of the files that are synchronized to the workspace. - // It can be an ancestor to [BundleRoot], but not a descendant; that is, [SyncRoot] must contain [BundleRoot]. - SyncRoot vfs.Path - // SyncRootPath is the local path to the root directory of files that are synchronized to the workspace. - // It is equal to `SyncRoot.Native()` and included as dedicated field for convenient access. + // By default, it is the same as [BundleRootPath]. + // If it is different, it must be an ancestor to [BundleRootPath]. + // That is, [SyncRootPath] must contain [BundleRootPath]. SyncRootPath string + // SyncRoot is a virtual filesystem path to [SyncRootPath]. + // Exclusively use this field for filesystem operations. + SyncRoot vfs.Path + + // Config contains the bundle configuration. + // It is loaded from the bundle configuration files and mutators may update it. Config config.Root // Metadata about the bundle deployment. This is the interface Databricks services @@ -84,14 +88,14 @@ type Bundle struct { func Load(ctx context.Context, path string) (*Bundle, error) { b := &Bundle{ - RootPath: filepath.Clean(path), - BundleRoot: vfs.MustNew(path), + BundleRootPath: filepath.Clean(path), + BundleRoot: vfs.MustNew(path), } configFile, err := config.FileNames.FindInPath(path) if err != nil { return nil, err } - log.Debugf(ctx, "Found bundle root at %s (file %s)", b.RootPath, configFile) + log.Debugf(ctx, "Found bundle root at %s (file %s)", b.BundleRootPath, configFile) return b, nil } @@ -160,7 +164,7 @@ func (b *Bundle) CacheDir(ctx context.Context, paths ...string) (string, error) if !exists || cacheDirName == "" { cacheDirName = filepath.Join( // Anchor at bundle root directory. - b.RootPath, + b.BundleRootPath, // Static cache directory. ".databricks", "bundle", @@ -212,7 +216,7 @@ func (b *Bundle) GetSyncIncludePatterns(ctx context.Context) ([]string, error) { if err != nil { return nil, err } - internalDirRel, err := filepath.Rel(b.RootPath, internalDir) + internalDirRel, err := filepath.Rel(b.BundleRootPath, internalDir) if err != nil { return nil, err } diff --git a/bundle/bundle_read_only.go b/bundle/bundle_read_only.go index 74b9d94de..ceab95c0b 100644 --- a/bundle/bundle_read_only.go +++ b/bundle/bundle_read_only.go @@ -21,7 +21,7 @@ func (r ReadOnlyBundle) Config() config.Root { } func (r ReadOnlyBundle) RootPath() string { - return r.b.RootPath + return r.b.BundleRootPath } func (r ReadOnlyBundle) BundleRoot() vfs.Path { diff --git a/bundle/bundle_test.go b/bundle/bundle_test.go index a29aa024b..1c3102357 100644 --- a/bundle/bundle_test.go +++ b/bundle/bundle_test.go @@ -79,7 +79,7 @@ func TestBundleMustLoadSuccess(t *testing.T) { t.Setenv(env.RootVariable, "./tests/basic") b, err := MustLoad(context.Background()) require.NoError(t, err) - assert.Equal(t, "tests/basic", filepath.ToSlash(b.RootPath)) + assert.Equal(t, "tests/basic", filepath.ToSlash(b.BundleRootPath)) } func TestBundleMustLoadFailureWithEnv(t *testing.T) { @@ -98,7 +98,7 @@ func TestBundleTryLoadSuccess(t *testing.T) { t.Setenv(env.RootVariable, "./tests/basic") b, err := TryLoad(context.Background()) require.NoError(t, err) - assert.Equal(t, "tests/basic", filepath.ToSlash(b.RootPath)) + assert.Equal(t, "tests/basic", filepath.ToSlash(b.BundleRootPath)) } func TestBundleTryLoadFailureWithEnv(t *testing.T) { diff --git a/bundle/config/loader/entry_point.go b/bundle/config/loader/entry_point.go index 2c73a5825..d476cb221 100644 --- a/bundle/config/loader/entry_point.go +++ b/bundle/config/loader/entry_point.go @@ -20,7 +20,7 @@ func (m *entryPoint) Name() string { } func (m *entryPoint) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { - path, err := config.FileNames.FindInPath(b.RootPath) + path, err := config.FileNames.FindInPath(b.BundleRootPath) if err != nil { return diag.FromErr(err) } diff --git a/bundle/config/loader/entry_point_test.go b/bundle/config/loader/entry_point_test.go index 80271f0b7..406b9b67c 100644 --- a/bundle/config/loader/entry_point_test.go +++ b/bundle/config/loader/entry_point_test.go @@ -18,7 +18,7 @@ func TestEntryPointNoRootPath(t *testing.T) { func TestEntryPoint(t *testing.T) { b := &bundle.Bundle{ - RootPath: "testdata", + BundleRootPath: "testdata", } diags := bundle.Apply(context.Background(), b, loader.EntryPoint()) require.NoError(t, diags.Error()) diff --git a/bundle/config/loader/process_include_test.go b/bundle/config/loader/process_include_test.go index da4da9ff6..2ccd84b31 100644 --- a/bundle/config/loader/process_include_test.go +++ b/bundle/config/loader/process_include_test.go @@ -14,7 +14,7 @@ import ( func TestProcessInclude(t *testing.T) { b := &bundle.Bundle{ - RootPath: "testdata", + BundleRootPath: "testdata", Config: config.Root{ Workspace: config.Workspace{ Host: "foo", @@ -22,7 +22,7 @@ func TestProcessInclude(t *testing.T) { }, } - m := loader.ProcessInclude(filepath.Join(b.RootPath, "host.yml"), "host.yml") + m := loader.ProcessInclude(filepath.Join(b.BundleRootPath, "host.yml"), "host.yml") assert.Equal(t, "ProcessInclude(host.yml)", m.Name()) // Assert the host value prior to applying the mutator diff --git a/bundle/config/loader/process_root_includes.go b/bundle/config/loader/process_root_includes.go index 25f284fd3..c14fb7ce1 100644 --- a/bundle/config/loader/process_root_includes.go +++ b/bundle/config/loader/process_root_includes.go @@ -47,7 +47,7 @@ func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) diag. } // Anchor includes to the bundle root path. - matches, err := filepath.Glob(filepath.Join(b.RootPath, entry)) + matches, err := filepath.Glob(filepath.Join(b.BundleRootPath, entry)) if err != nil { return diag.FromErr(err) } @@ -61,7 +61,7 @@ func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) diag. // Filter matches to ones we haven't seen yet. var includes []string for _, match := range matches { - rel, err := filepath.Rel(b.RootPath, match) + rel, err := filepath.Rel(b.BundleRootPath, match) if err != nil { return diag.FromErr(err) } @@ -76,7 +76,7 @@ func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) diag. slices.Sort(includes) files = append(files, includes...) for _, include := range includes { - out = append(out, ProcessInclude(filepath.Join(b.RootPath, include), include)) + out = append(out, ProcessInclude(filepath.Join(b.BundleRootPath, include), include)) } } diff --git a/bundle/config/loader/process_root_includes_test.go b/bundle/config/loader/process_root_includes_test.go index 737dbbefd..27ff9b05f 100644 --- a/bundle/config/loader/process_root_includes_test.go +++ b/bundle/config/loader/process_root_includes_test.go @@ -15,7 +15,7 @@ import ( func TestProcessRootIncludesEmpty(t *testing.T) { b := &bundle.Bundle{ - RootPath: ".", + BundleRootPath: ".", } diags := bundle.Apply(context.Background(), b, loader.ProcessRootIncludes()) require.NoError(t, diags.Error()) @@ -30,7 +30,7 @@ func TestProcessRootIncludesAbs(t *testing.T) { } b := &bundle.Bundle{ - RootPath: ".", + BundleRootPath: ".", Config: config.Root{ Include: []string{ "/tmp/*.yml", @@ -44,7 +44,7 @@ func TestProcessRootIncludesAbs(t *testing.T) { func TestProcessRootIncludesSingleGlob(t *testing.T) { b := &bundle.Bundle{ - RootPath: t.TempDir(), + BundleRootPath: t.TempDir(), Config: config.Root{ Include: []string{ "*.yml", @@ -52,9 +52,9 @@ func TestProcessRootIncludesSingleGlob(t *testing.T) { }, } - testutil.Touch(t, b.RootPath, "databricks.yml") - testutil.Touch(t, b.RootPath, "a.yml") - testutil.Touch(t, b.RootPath, "b.yml") + testutil.Touch(t, b.BundleRootPath, "databricks.yml") + testutil.Touch(t, b.BundleRootPath, "a.yml") + testutil.Touch(t, b.BundleRootPath, "b.yml") diags := bundle.Apply(context.Background(), b, loader.ProcessRootIncludes()) require.NoError(t, diags.Error()) @@ -63,7 +63,7 @@ func TestProcessRootIncludesSingleGlob(t *testing.T) { func TestProcessRootIncludesMultiGlob(t *testing.T) { b := &bundle.Bundle{ - RootPath: t.TempDir(), + BundleRootPath: t.TempDir(), Config: config.Root{ Include: []string{ "a*.yml", @@ -72,8 +72,8 @@ func TestProcessRootIncludesMultiGlob(t *testing.T) { }, } - testutil.Touch(t, b.RootPath, "a1.yml") - testutil.Touch(t, b.RootPath, "b1.yml") + testutil.Touch(t, b.BundleRootPath, "a1.yml") + testutil.Touch(t, b.BundleRootPath, "b1.yml") diags := bundle.Apply(context.Background(), b, loader.ProcessRootIncludes()) require.NoError(t, diags.Error()) @@ -82,7 +82,7 @@ func TestProcessRootIncludesMultiGlob(t *testing.T) { func TestProcessRootIncludesRemoveDups(t *testing.T) { b := &bundle.Bundle{ - RootPath: t.TempDir(), + BundleRootPath: t.TempDir(), Config: config.Root{ Include: []string{ "*.yml", @@ -91,7 +91,7 @@ func TestProcessRootIncludesRemoveDups(t *testing.T) { }, } - testutil.Touch(t, b.RootPath, "a.yml") + testutil.Touch(t, b.BundleRootPath, "a.yml") diags := bundle.Apply(context.Background(), b, loader.ProcessRootIncludes()) require.NoError(t, diags.Error()) @@ -100,7 +100,7 @@ func TestProcessRootIncludesRemoveDups(t *testing.T) { func TestProcessRootIncludesNotExists(t *testing.T) { b := &bundle.Bundle{ - RootPath: t.TempDir(), + BundleRootPath: t.TempDir(), Config: config.Root{ Include: []string{ "notexist.yml", diff --git a/bundle/config/mutator/expand_pipeline_glob_paths_test.go b/bundle/config/mutator/expand_pipeline_glob_paths_test.go index 07dd20215..9f70b74ae 100644 --- a/bundle/config/mutator/expand_pipeline_glob_paths_test.go +++ b/bundle/config/mutator/expand_pipeline_glob_paths_test.go @@ -42,7 +42,7 @@ func TestExpandGlobPathsInPipelines(t *testing.T) { touchEmptyFile(t, filepath.Join(dir, "skip/test7.py")) b := &bundle.Bundle{ - RootPath: dir, + BundleRootPath: dir, Config: config.Root{ Resources: config.Resources{ Pipelines: map[string]*resources.Pipeline{ diff --git a/bundle/config/mutator/load_git_details.go b/bundle/config/mutator/load_git_details.go index 9b1c963c9..00e7f54d1 100644 --- a/bundle/config/mutator/load_git_details.go +++ b/bundle/config/mutator/load_git_details.go @@ -56,7 +56,7 @@ func (m *loadGitDetails) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagn } // Compute relative path of the bundle root from the Git repo root. - absBundlePath, err := filepath.Abs(b.RootPath) + absBundlePath, err := filepath.Abs(b.BundleRootPath) if err != nil { return diag.FromErr(err) } diff --git a/bundle/config/mutator/python/python_mutator.go b/bundle/config/mutator/python/python_mutator.go index 3d4a502f7..da6c4d210 100644 --- a/bundle/config/mutator/python/python_mutator.go +++ b/bundle/config/mutator/python/python_mutator.go @@ -108,7 +108,7 @@ func (m *pythonMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno return dyn.InvalidValue, fmt.Errorf("failed to create cache dir: %w", err) } - rightRoot, diags := m.runPythonMutator(ctx, cacheDir, b.RootPath, pythonPath, leftRoot) + rightRoot, diags := m.runPythonMutator(ctx, cacheDir, b.BundleRootPath, pythonPath, leftRoot) mutateDiags = diags if diags.HasError() { return dyn.InvalidValue, mutateDiagsHasError diff --git a/bundle/config/mutator/rewrite_sync_paths.go b/bundle/config/mutator/rewrite_sync_paths.go index 888714abe..f9a023696 100644 --- a/bundle/config/mutator/rewrite_sync_paths.go +++ b/bundle/config/mutator/rewrite_sync_paths.go @@ -45,15 +45,15 @@ func (m *rewriteSyncPaths) makeRelativeTo(root string) dyn.MapFunc { func (m *rewriteSyncPaths) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { return dyn.Map(v, "sync", func(_ dyn.Path, v dyn.Value) (nv dyn.Value, err error) { - v, err = dyn.Map(v, "paths", dyn.Foreach(m.makeRelativeTo(b.RootPath))) + v, err = dyn.Map(v, "paths", dyn.Foreach(m.makeRelativeTo(b.BundleRootPath))) if err != nil { return dyn.InvalidValue, err } - v, err = dyn.Map(v, "include", dyn.Foreach(m.makeRelativeTo(b.RootPath))) + v, err = dyn.Map(v, "include", dyn.Foreach(m.makeRelativeTo(b.BundleRootPath))) if err != nil { return dyn.InvalidValue, err } - v, err = dyn.Map(v, "exclude", dyn.Foreach(m.makeRelativeTo(b.RootPath))) + v, err = dyn.Map(v, "exclude", dyn.Foreach(m.makeRelativeTo(b.BundleRootPath))) if err != nil { return dyn.InvalidValue, err } diff --git a/bundle/config/mutator/rewrite_sync_paths_test.go b/bundle/config/mutator/rewrite_sync_paths_test.go index a66f2763a..0e4dfc4ce 100644 --- a/bundle/config/mutator/rewrite_sync_paths_test.go +++ b/bundle/config/mutator/rewrite_sync_paths_test.go @@ -15,7 +15,7 @@ import ( func TestRewriteSyncPathsRelative(t *testing.T) { b := &bundle.Bundle{ - RootPath: ".", + BundleRootPath: ".", Config: config.Root{ Sync: config.Sync{ Paths: []string{ @@ -54,7 +54,7 @@ func TestRewriteSyncPathsRelative(t *testing.T) { func TestRewriteSyncPathsAbsolute(t *testing.T) { b := &bundle.Bundle{ - RootPath: "/tmp/dir", + BundleRootPath: "/tmp/dir", Config: config.Root{ Sync: config.Sync{ Paths: []string{ @@ -94,7 +94,7 @@ func TestRewriteSyncPathsAbsolute(t *testing.T) { func TestRewriteSyncPathsErrorPaths(t *testing.T) { t.Run("no sync block", func(t *testing.T) { b := &bundle.Bundle{ - RootPath: ".", + BundleRootPath: ".", } diags := bundle.Apply(context.Background(), b, mutator.RewriteSyncPaths()) @@ -103,7 +103,7 @@ func TestRewriteSyncPathsErrorPaths(t *testing.T) { t.Run("empty include/exclude blocks", func(t *testing.T) { b := &bundle.Bundle{ - RootPath: ".", + BundleRootPath: ".", Config: config.Root{ Sync: config.Sync{ Include: []string{}, diff --git a/bundle/config/mutator/sync_default_path_test.go b/bundle/config/mutator/sync_default_path_test.go index a37e913d2..c82a687b7 100644 --- a/bundle/config/mutator/sync_default_path_test.go +++ b/bundle/config/mutator/sync_default_path_test.go @@ -15,8 +15,8 @@ import ( func TestSyncDefaultPath_DefaultIfUnset(t *testing.T) { b := &bundle.Bundle{ - RootPath: "/tmp/some/dir", - Config: config.Root{}, + BundleRootPath: "/tmp/some/dir", + Config: config.Root{}, } ctx := context.Background() @@ -51,8 +51,8 @@ func TestSyncDefaultPath_SkipIfSet(t *testing.T) { for _, tcase := range tcases { t.Run(tcase.name, func(t *testing.T) { b := &bundle.Bundle{ - RootPath: "/tmp/some/dir", - Config: config.Root{}, + BundleRootPath: "/tmp/some/dir", + Config: config.Root{}, } diags := bundle.ApplyFunc(context.Background(), b, func(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { diff --git a/bundle/config/mutator/sync_infer_root.go b/bundle/config/mutator/sync_infer_root.go index 012acf800..512adcdbf 100644 --- a/bundle/config/mutator/sync_infer_root.go +++ b/bundle/config/mutator/sync_infer_root.go @@ -57,7 +57,7 @@ func (m *syncInferRoot) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno var diags diag.Diagnostics // Use the bundle root path as the starting point for inferring the sync root path. - bundleRootPath := filepath.Clean(b.RootPath) + bundleRootPath := filepath.Clean(b.BundleRootPath) // Infer the sync root path by looking at each one of the sync paths. // Every sync path must be a descendant of the final sync root path. diff --git a/bundle/config/mutator/sync_infer_root_test.go b/bundle/config/mutator/sync_infer_root_test.go index 85e40adc6..f507cbc7f 100644 --- a/bundle/config/mutator/sync_infer_root_test.go +++ b/bundle/config/mutator/sync_infer_root_test.go @@ -16,7 +16,7 @@ import ( func TestSyncInferRoot_NominalAbsolute(t *testing.T) { b := &bundle.Bundle{ - RootPath: "/tmp/some/dir", + BundleRootPath: "/tmp/some/dir", Config: config.Root{ Sync: config.Sync{ Paths: []string{ @@ -47,7 +47,7 @@ func TestSyncInferRoot_NominalAbsolute(t *testing.T) { func TestSyncInferRoot_NominalRelative(t *testing.T) { b := &bundle.Bundle{ - RootPath: "./some/dir", + BundleRootPath: "./some/dir", Config: config.Root{ Sync: config.Sync{ Paths: []string{ @@ -78,7 +78,7 @@ func TestSyncInferRoot_NominalRelative(t *testing.T) { func TestSyncInferRoot_ParentDirectory(t *testing.T) { b := &bundle.Bundle{ - RootPath: "/tmp/some/dir", + BundleRootPath: "/tmp/some/dir", Config: config.Root{ Sync: config.Sync{ Paths: []string{ @@ -109,7 +109,7 @@ func TestSyncInferRoot_ParentDirectory(t *testing.T) { func TestSyncInferRoot_ManyParentDirectories(t *testing.T) { b := &bundle.Bundle{ - RootPath: "/tmp/some/dir/that/is/very/deeply/nested", + BundleRootPath: "/tmp/some/dir/that/is/very/deeply/nested", Config: config.Root{ Sync: config.Sync{ Paths: []string{ @@ -146,7 +146,7 @@ func TestSyncInferRoot_ManyParentDirectories(t *testing.T) { func TestSyncInferRoot_MultiplePaths(t *testing.T) { b := &bundle.Bundle{ - RootPath: "/tmp/some/bundle/root", + BundleRootPath: "/tmp/some/bundle/root", Config: config.Root{ Sync: config.Sync{ Paths: []string{ @@ -173,7 +173,7 @@ func TestSyncInferRoot_MultiplePaths(t *testing.T) { func TestSyncInferRoot_Error(t *testing.T) { b := &bundle.Bundle{ - RootPath: "/tmp/some/dir", + BundleRootPath: "/tmp/some/dir", Config: config.Root{ Sync: config.Sync{ Paths: []string{ diff --git a/bundle/deploy/metadata/compute.go b/bundle/deploy/metadata/compute.go index 6ab997e27..bc8767de4 100644 --- a/bundle/deploy/metadata/compute.go +++ b/bundle/deploy/metadata/compute.go @@ -40,7 +40,7 @@ func (m *compute) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { // Compute config file path the job is defined in, relative to the bundle // root l := b.Config.GetLocation("resources.jobs." + name) - relativePath, err := filepath.Rel(b.RootPath, l.File) + relativePath, err := filepath.Rel(b.BundleRootPath, l.File) if err != nil { return diag.Errorf("failed to compute relative path for job %s: %v", name, err) } diff --git a/bundle/deploy/state_pull_test.go b/bundle/deploy/state_pull_test.go index f75193065..42701eb26 100644 --- a/bundle/deploy/state_pull_test.go +++ b/bundle/deploy/state_pull_test.go @@ -62,8 +62,8 @@ func testStatePull(t *testing.T, opts statePullOpts) { tmpDir := t.TempDir() b := &bundle.Bundle{ - RootPath: tmpDir, - BundleRoot: vfs.MustNew(tmpDir), + BundleRootPath: tmpDir, + BundleRoot: vfs.MustNew(tmpDir), SyncRootPath: tmpDir, SyncRoot: vfs.MustNew(tmpDir), @@ -259,7 +259,7 @@ func TestStatePullNoState(t *testing.T) { }} b := &bundle.Bundle{ - RootPath: t.TempDir(), + BundleRootPath: t.TempDir(), Config: config.Root{ Bundle: config.Bundle{ Target: "default", @@ -447,7 +447,7 @@ func TestStatePullNewerDeploymentStateVersion(t *testing.T) { }} b := &bundle.Bundle{ - RootPath: t.TempDir(), + BundleRootPath: t.TempDir(), Config: config.Root{ Bundle: config.Bundle{ Target: "default", diff --git a/bundle/deploy/state_push_test.go b/bundle/deploy/state_push_test.go index 39e4d13a5..038b75341 100644 --- a/bundle/deploy/state_push_test.go +++ b/bundle/deploy/state_push_test.go @@ -45,7 +45,7 @@ func TestStatePush(t *testing.T) { }} b := &bundle.Bundle{ - RootPath: t.TempDir(), + BundleRootPath: t.TempDir(), Config: config.Root{ Bundle: config.Bundle{ Target: "default", diff --git a/bundle/deploy/state_update_test.go b/bundle/deploy/state_update_test.go index 72096d142..1f5010b52 100644 --- a/bundle/deploy/state_update_test.go +++ b/bundle/deploy/state_update_test.go @@ -27,7 +27,7 @@ func setupBundleForStateUpdate(t *testing.T) *bundle.Bundle { require.NoError(t, err) return &bundle.Bundle{ - RootPath: tmpDir, + BundleRootPath: tmpDir, Config: config.Root{ Bundle: config.Bundle{ Target: "default", diff --git a/bundle/deploy/terraform/init_test.go b/bundle/deploy/terraform/init_test.go index 450e7eb6a..e3621c6c3 100644 --- a/bundle/deploy/terraform/init_test.go +++ b/bundle/deploy/terraform/init_test.go @@ -33,7 +33,7 @@ func TestInitEnvironmentVariables(t *testing.T) { } b := &bundle.Bundle{ - RootPath: t.TempDir(), + BundleRootPath: t.TempDir(), Config: config.Root{ Bundle: config.Bundle{ Target: "whatever", @@ -60,7 +60,7 @@ func TestSetTempDirEnvVarsForUnixWithTmpDirSet(t *testing.T) { } b := &bundle.Bundle{ - RootPath: t.TempDir(), + BundleRootPath: t.TempDir(), Config: config.Root{ Bundle: config.Bundle{ Target: "whatever", @@ -88,7 +88,7 @@ func TestSetTempDirEnvVarsForUnixWithTmpDirNotSet(t *testing.T) { } b := &bundle.Bundle{ - RootPath: t.TempDir(), + BundleRootPath: t.TempDir(), Config: config.Root{ Bundle: config.Bundle{ Target: "whatever", @@ -114,7 +114,7 @@ func TestSetTempDirEnvVarsForWindowWithAllTmpDirEnvVarsSet(t *testing.T) { } b := &bundle.Bundle{ - RootPath: t.TempDir(), + BundleRootPath: t.TempDir(), Config: config.Root{ Bundle: config.Bundle{ Target: "whatever", @@ -144,7 +144,7 @@ func TestSetTempDirEnvVarsForWindowWithUserProfileAndTempSet(t *testing.T) { } b := &bundle.Bundle{ - RootPath: t.TempDir(), + BundleRootPath: t.TempDir(), Config: config.Root{ Bundle: config.Bundle{ Target: "whatever", @@ -174,7 +174,7 @@ func TestSetTempDirEnvVarsForWindowsWithoutAnyTempDirEnvVarsSet(t *testing.T) { } b := &bundle.Bundle{ - RootPath: t.TempDir(), + BundleRootPath: t.TempDir(), Config: config.Root{ Bundle: config.Bundle{ Target: "whatever", @@ -202,7 +202,7 @@ func TestSetTempDirEnvVarsForWindowsWithoutAnyTempDirEnvVarsSet(t *testing.T) { func TestSetProxyEnvVars(t *testing.T) { b := &bundle.Bundle{ - RootPath: t.TempDir(), + BundleRootPath: t.TempDir(), Config: config.Root{ Bundle: config.Bundle{ Target: "whatever", @@ -250,7 +250,7 @@ func TestSetProxyEnvVars(t *testing.T) { func TestSetUserAgentExtraEnvVar(t *testing.T) { b := &bundle.Bundle{ - RootPath: t.TempDir(), + BundleRootPath: t.TempDir(), Config: config.Root{ Experimental: &config.Experimental{ PyDABs: config.PyDABs{ @@ -333,7 +333,7 @@ func TestFindExecPathFromEnvironmentWithWrongVersion(t *testing.T) { ctx := context.Background() m := &initialize{} b := &bundle.Bundle{ - RootPath: t.TempDir(), + BundleRootPath: t.TempDir(), Config: config.Root{ Bundle: config.Bundle{ Target: "whatever", @@ -357,7 +357,7 @@ func TestFindExecPathFromEnvironmentWithCorrectVersionAndNoBinary(t *testing.T) ctx := context.Background() m := &initialize{} b := &bundle.Bundle{ - RootPath: t.TempDir(), + BundleRootPath: t.TempDir(), Config: config.Root{ Bundle: config.Bundle{ Target: "whatever", @@ -380,7 +380,7 @@ func TestFindExecPathFromEnvironmentWithCorrectVersionAndBinary(t *testing.T) { ctx := context.Background() m := &initialize{} b := &bundle.Bundle{ - RootPath: t.TempDir(), + BundleRootPath: t.TempDir(), Config: config.Root{ Bundle: config.Bundle{ Target: "whatever", diff --git a/bundle/deploy/terraform/load_test.go b/bundle/deploy/terraform/load_test.go index c62217187..b7243ca19 100644 --- a/bundle/deploy/terraform/load_test.go +++ b/bundle/deploy/terraform/load_test.go @@ -17,7 +17,7 @@ func TestLoadWithNoState(t *testing.T) { } b := &bundle.Bundle{ - RootPath: t.TempDir(), + BundleRootPath: t.TempDir(), Config: config.Root{ Bundle: config.Bundle{ Target: "whatever", diff --git a/bundle/deploy/terraform/state_pull_test.go b/bundle/deploy/terraform/state_pull_test.go index 39937a3cc..c4798c578 100644 --- a/bundle/deploy/terraform/state_pull_test.go +++ b/bundle/deploy/terraform/state_pull_test.go @@ -32,7 +32,7 @@ func mockStateFilerForPull(t *testing.T, contents map[string]any, merr error) fi func statePullTestBundle(t *testing.T) *bundle.Bundle { return &bundle.Bundle{ - RootPath: t.TempDir(), + BundleRootPath: t.TempDir(), Config: config.Root{ Bundle: config.Bundle{ Target: "default", diff --git a/bundle/deploy/terraform/state_push_test.go b/bundle/deploy/terraform/state_push_test.go index ac74f345d..e022dee1b 100644 --- a/bundle/deploy/terraform/state_push_test.go +++ b/bundle/deploy/terraform/state_push_test.go @@ -29,7 +29,7 @@ func mockStateFilerForPush(t *testing.T, fn func(body io.Reader)) filer.Filer { func statePushTestBundle(t *testing.T) *bundle.Bundle { return &bundle.Bundle{ - RootPath: t.TempDir(), + BundleRootPath: t.TempDir(), Config: config.Root{ Bundle: config.Bundle{ Target: "default", diff --git a/bundle/deploy/terraform/util_test.go b/bundle/deploy/terraform/util_test.go index 251a7c256..74b329259 100644 --- a/bundle/deploy/terraform/util_test.go +++ b/bundle/deploy/terraform/util_test.go @@ -13,7 +13,7 @@ import ( func TestParseResourcesStateWithNoFile(t *testing.T) { b := &bundle.Bundle{ - RootPath: t.TempDir(), + BundleRootPath: t.TempDir(), Config: config.Root{ Bundle: config.Bundle{ Target: "whatever", @@ -31,7 +31,7 @@ func TestParseResourcesStateWithNoFile(t *testing.T) { func TestParseResourcesStateWithExistingStateFile(t *testing.T) { ctx := context.Background() b := &bundle.Bundle{ - RootPath: t.TempDir(), + BundleRootPath: t.TempDir(), Config: config.Root{ Bundle: config.Bundle{ Target: "whatever", diff --git a/bundle/render/render_text_output.go b/bundle/render/render_text_output.go index ea0b9a944..e1fad98a3 100644 --- a/bundle/render/render_text_output.go +++ b/bundle/render/render_text_output.go @@ -148,7 +148,7 @@ func renderDiagnostics(out io.Writer, b *bundle.Bundle, diags diag.Diagnostics) // Make location relative to bundle root if d.Locations[i].File != "" { - out, err := filepath.Rel(b.RootPath, d.Locations[i].File) + out, err := filepath.Rel(b.BundleRootPath, d.Locations[i].File) // if we can't relativize the path, just use path as-is if err == nil { d.Locations[i].File = out diff --git a/bundle/scripts/scripts.go b/bundle/scripts/scripts.go index 629b3a8ab..f9e1541e8 100644 --- a/bundle/scripts/scripts.go +++ b/bundle/scripts/scripts.go @@ -30,7 +30,7 @@ func (m *script) Name() string { } func (m *script) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - executor, err := exec.NewCommandExecutor(b.RootPath) + executor, err := exec.NewCommandExecutor(b.BundleRootPath) if err != nil { return diag.FromErr(err) } diff --git a/bundle/scripts/scripts_test.go b/bundle/scripts/scripts_test.go index 1bc216b61..0c92bc2c3 100644 --- a/bundle/scripts/scripts_test.go +++ b/bundle/scripts/scripts_test.go @@ -23,7 +23,7 @@ func TestExecutesHook(t *testing.T) { }, } - executor, err := exec.NewCommandExecutor(b.RootPath) + executor, err := exec.NewCommandExecutor(b.BundleRootPath) require.NoError(t, err) _, out, err := executeHook(context.Background(), executor, b, config.ScriptPreBuild) require.NoError(t, err) diff --git a/bundle/trampoline/conditional_transform_test.go b/bundle/trampoline/conditional_transform_test.go index 26e67154e..57aa9aac3 100644 --- a/bundle/trampoline/conditional_transform_test.go +++ b/bundle/trampoline/conditional_transform_test.go @@ -17,8 +17,8 @@ func TestNoTransformByDefault(t *testing.T) { tmpDir := t.TempDir() b := &bundle.Bundle{ - RootPath: filepath.Join(tmpDir, "parent", "my_bundle"), - SyncRootPath: filepath.Join(tmpDir, "parent"), + BundleRootPath: filepath.Join(tmpDir, "parent", "my_bundle"), + SyncRootPath: filepath.Join(tmpDir, "parent"), Config: config.Root{ Bundle: config.Bundle{ Target: "development", @@ -66,8 +66,8 @@ func TestTransformWithExperimentalSettingSetToTrue(t *testing.T) { tmpDir := t.TempDir() b := &bundle.Bundle{ - RootPath: filepath.Join(tmpDir, "parent", "my_bundle"), - SyncRootPath: filepath.Join(tmpDir, "parent"), + BundleRootPath: filepath.Join(tmpDir, "parent", "my_bundle"), + SyncRootPath: filepath.Join(tmpDir, "parent"), Config: config.Root{ Bundle: config.Bundle{ Target: "development", diff --git a/bundle/trampoline/python_wheel_test.go b/bundle/trampoline/python_wheel_test.go index 40c3b38f3..517be35e4 100644 --- a/bundle/trampoline/python_wheel_test.go +++ b/bundle/trampoline/python_wheel_test.go @@ -115,7 +115,7 @@ func TestTransformFiltersWheelTasksOnly(t *testing.T) { func TestNoPanicWithNoPythonWheelTasks(t *testing.T) { tmpDir := t.TempDir() b := &bundle.Bundle{ - RootPath: tmpDir, + BundleRootPath: tmpDir, Config: config.Root{ Bundle: config.Bundle{ Target: "development", diff --git a/bundle/trampoline/trampoline_test.go b/bundle/trampoline/trampoline_test.go index 08a290f93..4682d8fa0 100644 --- a/bundle/trampoline/trampoline_test.go +++ b/bundle/trampoline/trampoline_test.go @@ -56,8 +56,8 @@ func TestGenerateTrampoline(t *testing.T) { } b := &bundle.Bundle{ - RootPath: filepath.Join(tmpDir, "parent", "my_bundle"), - SyncRootPath: filepath.Join(tmpDir, "parent"), + BundleRootPath: filepath.Join(tmpDir, "parent", "my_bundle"), + SyncRootPath: filepath.Join(tmpDir, "parent"), Config: config.Root{ Workspace: config.Workspace{ FilePath: "/Workspace/files", diff --git a/cmd/bundle/generate/generate_test.go b/cmd/bundle/generate/generate_test.go index 7de6805fb..943f721c9 100644 --- a/cmd/bundle/generate/generate_test.go +++ b/cmd/bundle/generate/generate_test.go @@ -24,7 +24,7 @@ func TestGeneratePipelineCommand(t *testing.T) { root := t.TempDir() b := &bundle.Bundle{ - RootPath: root, + BundleRootPath: root, } m := mocks.NewMockWorkspaceClient(t) @@ -122,7 +122,7 @@ func TestGenerateJobCommand(t *testing.T) { root := t.TempDir() b := &bundle.Bundle{ - RootPath: root, + BundleRootPath: root, } m := mocks.NewMockWorkspaceClient(t) diff --git a/cmd/sync/sync_test.go b/cmd/sync/sync_test.go index bd03eec91..8f65aedba 100644 --- a/cmd/sync/sync_test.go +++ b/cmd/sync/sync_test.go @@ -17,10 +17,10 @@ import ( func TestSyncOptionsFromBundle(t *testing.T) { tempDir := t.TempDir() b := &bundle.Bundle{ - RootPath: tempDir, - BundleRoot: vfs.MustNew(tempDir), - SyncRootPath: tempDir, - SyncRoot: vfs.MustNew(tempDir), + BundleRootPath: tempDir, + BundleRoot: vfs.MustNew(tempDir), + SyncRootPath: tempDir, + SyncRoot: vfs.MustNew(tempDir), Config: config.Root{ Bundle: config.Bundle{ Target: "default", diff --git a/internal/bundle/artifacts_test.go b/internal/bundle/artifacts_test.go index fa052e223..775327e18 100644 --- a/internal/bundle/artifacts_test.go +++ b/internal/bundle/artifacts_test.go @@ -36,8 +36,8 @@ func TestAccUploadArtifactFileToCorrectRemotePath(t *testing.T) { wsDir := internal.TemporaryWorkspaceDir(t, w) b := &bundle.Bundle{ - RootPath: dir, - SyncRootPath: dir, + BundleRootPath: dir, + SyncRootPath: dir, Config: config.Root{ Bundle: config.Bundle{ Target: "whatever", @@ -101,8 +101,8 @@ func TestAccUploadArtifactFileToCorrectRemotePathWithEnvironments(t *testing.T) wsDir := internal.TemporaryWorkspaceDir(t, w) b := &bundle.Bundle{ - RootPath: dir, - SyncRootPath: dir, + BundleRootPath: dir, + SyncRootPath: dir, Config: config.Root{ Bundle: config.Bundle{ Target: "whatever", @@ -171,8 +171,8 @@ func TestAccUploadArtifactFileToCorrectRemotePathForVolumes(t *testing.T) { touchEmptyFile(t, whlPath) b := &bundle.Bundle{ - RootPath: dir, - SyncRootPath: dir, + BundleRootPath: dir, + SyncRootPath: dir, Config: config.Root{ Bundle: config.Bundle{ Target: "whatever", From da3b4f7c72ab6c48168ca968637846f61e7899c8 Mon Sep 17 00:00:00 2001 From: "Lennart Kats (databricks)" Date: Sun, 29 Sep 2024 16:08:10 +0200 Subject: [PATCH 23/34] Fix panic in `apply_presets.go` (#1796) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Changes This fixes the user-reported panic in `apply_presets.go`. I'm still unsure how to reproduce this, since the CLI just reports `ob broken_job is not defined` when I try to use `bundle deploy` with an empty job. That said — we may as well be defensive here and I see we have lots of checks for empty job/cluster/etc. settings scattered throughout our code base so at least we're somewhat consistent. --- bundle/config/mutator/apply_presets.go | 79 ++++++++++---- bundle/config/mutator/apply_presets_test.go | 113 ++++++++++++++++++++ 2 files changed, 171 insertions(+), 21 deletions(-) diff --git a/bundle/config/mutator/apply_presets.go b/bundle/config/mutator/apply_presets.go index 27af82e54..1fd49206f 100644 --- a/bundle/config/mutator/apply_presets.go +++ b/bundle/config/mutator/apply_presets.go @@ -35,8 +35,10 @@ func (m *applyPresets) Name() string { } func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + var diags diag.Diagnostics + if d := validatePauseStatus(b); d != nil { - return d + diags = diags.Extend(d) } r := b.Config.Resources @@ -45,7 +47,11 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos tags := toTagArray(t.Tags) // Jobs presets: Prefix, Tags, JobsMaxConcurrentRuns, TriggerPauseStatus - for _, j := range r.Jobs { + for key, j := range r.Jobs { + if j.JobSettings == nil { + diags = diags.Extend(diag.Errorf("job %s is not defined", key)) + continue + } j.Name = prefix + j.Name if j.Tags == nil { j.Tags = make(map[string]string) @@ -77,20 +83,27 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos } // Pipelines presets: Prefix, PipelinesDevelopment - for i := range r.Pipelines { - r.Pipelines[i].Name = prefix + r.Pipelines[i].Name + for key, p := range r.Pipelines { + if p.PipelineSpec == nil { + diags = diags.Extend(diag.Errorf("pipeline %s is not defined", key)) + continue + } + p.Name = prefix + p.Name if config.IsExplicitlyEnabled(t.PipelinesDevelopment) { - r.Pipelines[i].Development = true + p.Development = true } if t.TriggerPauseStatus == config.Paused { - r.Pipelines[i].Continuous = false + p.Continuous = false } - // As of 2024-06, pipelines don't yet support tags } // Models presets: Prefix, Tags - for _, m := range r.Models { + for key, m := range r.Models { + if m.Model == nil { + diags = diags.Extend(diag.Errorf("model %s is not defined", key)) + continue + } m.Name = prefix + m.Name for _, t := range tags { exists := slices.ContainsFunc(m.Tags, func(modelTag ml.ModelTag) bool { @@ -104,7 +117,11 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos } // Experiments presets: Prefix, Tags - for _, e := range r.Experiments { + for key, e := range r.Experiments { + if e.Experiment == nil { + diags = diags.Extend(diag.Errorf("experiment %s is not defined", key)) + continue + } filepath := e.Name dir := path.Dir(filepath) base := path.Base(filepath) @@ -128,40 +145,60 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos } // Model serving endpoint presets: Prefix - for i := range r.ModelServingEndpoints { - r.ModelServingEndpoints[i].Name = normalizePrefix(prefix) + r.ModelServingEndpoints[i].Name + for key, e := range r.ModelServingEndpoints { + if e.CreateServingEndpoint == nil { + diags = diags.Extend(diag.Errorf("model serving endpoint %s is not defined", key)) + continue + } + e.Name = normalizePrefix(prefix) + e.Name // As of 2024-06, model serving endpoints don't yet support tags } // Registered models presets: Prefix - for i := range r.RegisteredModels { - r.RegisteredModels[i].Name = normalizePrefix(prefix) + r.RegisteredModels[i].Name + for key, m := range r.RegisteredModels { + if m.CreateRegisteredModelRequest == nil { + diags = diags.Extend(diag.Errorf("registered model %s is not defined", key)) + continue + } + m.Name = normalizePrefix(prefix) + m.Name // As of 2024-06, registered models don't yet support tags } - // Quality monitors presets: Prefix + // Quality monitors presets: Schedule if t.TriggerPauseStatus == config.Paused { - for i := range r.QualityMonitors { + for key, q := range r.QualityMonitors { + if q.CreateMonitor == nil { + diags = diags.Extend(diag.Errorf("quality monitor %s is not defined", key)) + continue + } // Remove all schedules from monitors, since they don't support pausing/unpausing. // Quality monitors might support the "pause" property in the future, so at the // CLI level we do respect that property if it is set to "unpaused." - if r.QualityMonitors[i].Schedule != nil && r.QualityMonitors[i].Schedule.PauseStatus != catalog.MonitorCronSchedulePauseStatusUnpaused { - r.QualityMonitors[i].Schedule = nil + if q.Schedule != nil && q.Schedule.PauseStatus != catalog.MonitorCronSchedulePauseStatusUnpaused { + q.Schedule = nil } } } // Schemas: Prefix - for i := range r.Schemas { - r.Schemas[i].Name = normalizePrefix(prefix) + r.Schemas[i].Name + for key, s := range r.Schemas { + if s.CreateSchema == nil { + diags = diags.Extend(diag.Errorf("schema %s is not defined", key)) + continue + } + s.Name = normalizePrefix(prefix) + s.Name // HTTP API for schemas doesn't yet support tags. It's only supported in // the Databricks UI and via the SQL API. } // Clusters: Prefix, Tags - for _, c := range r.Clusters { + for key, c := range r.Clusters { + if c.ClusterSpec == nil { + diags = diags.Extend(diag.Errorf("cluster %s is not defined", key)) + continue + } c.ClusterName = prefix + c.ClusterName if c.CustomTags == nil { c.CustomTags = make(map[string]string) @@ -175,7 +212,7 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos } } - return nil + return diags } func validatePauseStatus(b *bundle.Bundle) diag.Diagnostics { diff --git a/bundle/config/mutator/apply_presets_test.go b/bundle/config/mutator/apply_presets_test.go index ab2478aee..24295da48 100644 --- a/bundle/config/mutator/apply_presets_test.go +++ b/bundle/config/mutator/apply_presets_test.go @@ -251,3 +251,116 @@ func TestApplyPresetsJobsMaxConcurrentRuns(t *testing.T) { }) } } + +func TestApplyPresetsPrefixWithoutJobSettings(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job1": {}, // no jobsettings inside + }, + }, + Presets: config.Presets{ + NamePrefix: "prefix-", + }, + }, + } + + ctx := context.Background() + diags := bundle.Apply(ctx, b, mutator.ApplyPresets()) + + require.ErrorContains(t, diags.Error(), "job job1 is not defined") +} + +func TestApplyPresetsResourceNotDefined(t *testing.T) { + tests := []struct { + resources config.Resources + error string + }{ + { + resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job1": {}, // no jobsettings inside + }, + }, + error: "job job1 is not defined", + }, + { + resources: config.Resources{ + Pipelines: map[string]*resources.Pipeline{ + "pipeline1": {}, // no pipelinespec inside + }, + }, + error: "pipeline pipeline1 is not defined", + }, + { + resources: config.Resources{ + Models: map[string]*resources.MlflowModel{ + "model1": {}, // no model inside + }, + }, + error: "model model1 is not defined", + }, + { + resources: config.Resources{ + Experiments: map[string]*resources.MlflowExperiment{ + "experiment1": {}, // no experiment inside + }, + }, + error: "experiment experiment1 is not defined", + }, + { + resources: config.Resources{ + ModelServingEndpoints: map[string]*resources.ModelServingEndpoint{ + "endpoint1": {}, // no CreateServingEndpoint inside + }, + RegisteredModels: map[string]*resources.RegisteredModel{ + "model1": {}, // no CreateRegisteredModelRequest inside + }, + }, + error: "model serving endpoint endpoint1 is not defined", + }, + { + resources: config.Resources{ + QualityMonitors: map[string]*resources.QualityMonitor{ + "monitor1": {}, // no CreateMonitor inside + }, + }, + error: "quality monitor monitor1 is not defined", + }, + { + resources: config.Resources{ + Schemas: map[string]*resources.Schema{ + "schema1": {}, // no CreateSchema inside + }, + }, + error: "schema schema1 is not defined", + }, + { + resources: config.Resources{ + Clusters: map[string]*resources.Cluster{ + "cluster1": {}, // no ClusterSpec inside + }, + }, + error: "cluster cluster1 is not defined", + }, + } + + for _, tt := range tests { + t.Run(tt.error, func(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: tt.resources, + Presets: config.Presets{ + TriggerPauseStatus: config.Paused, + }, + }, + } + + ctx := context.Background() + diags := bundle.Apply(ctx, b, mutator.ApplyPresets()) + + require.ErrorContains(t, diags.Error(), tt.error) + }) + } +} From 84fc1ed13162fafd54b6cebc3df991202acc431e Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 1 Oct 2024 12:44:47 +0200 Subject: [PATCH 24/34] Upgrade to Go SDK 0.47.0 (#1799) ## Changes Upgrade to Go SDK 0.47.0 ## Tests --- .codegen/_openapi_sha | 2 +- .gitattributes | 3 + bundle/schema/jsonschema.json | 152 +++++++++++- .../disable-legacy-features.go | 215 +++++++++++++++++ cmd/account/settings/settings.go | 2 + cmd/workspace/apps/apps.go | 83 ++++--- cmd/workspace/clusters/clusters.go | 10 +- cmd/workspace/cmd.go | 2 + .../disable-legacy-access.go | 217 ++++++++++++++++++ cmd/workspace/pipelines/pipelines.go | 1 + .../serving-endpoints/serving-endpoints.go | 80 ++++++- cmd/workspace/settings/settings.go | 2 + cmd/workspace/tables/tables.go | 3 + .../temporary-table-credentials.go | 122 ++++++++++ go.mod | 2 +- go.sum | 4 +- 16 files changed, 851 insertions(+), 49 deletions(-) create mode 100755 cmd/account/disable-legacy-features/disable-legacy-features.go create mode 100755 cmd/workspace/disable-legacy-access/disable-legacy-access.go create mode 100755 cmd/workspace/temporary-table-credentials/temporary-table-credentials.go diff --git a/.codegen/_openapi_sha b/.codegen/_openapi_sha index 4ceeab3d3..ffd6f58dd 100644 --- a/.codegen/_openapi_sha +++ b/.codegen/_openapi_sha @@ -1 +1 @@ -d05898328669a3f8ab0c2ecee37db2673d3ea3f7 \ No newline at end of file +6f6b1371e640f2dfeba72d365ac566368656f6b6 \ No newline at end of file diff --git a/.gitattributes b/.gitattributes index f35c4f81d..2470eb33d 100755 --- a/.gitattributes +++ b/.gitattributes @@ -6,6 +6,7 @@ cmd/account/cmd.go linguist-generated=true cmd/account/credentials/credentials.go linguist-generated=true cmd/account/csp-enablement-account/csp-enablement-account.go linguist-generated=true cmd/account/custom-app-integration/custom-app-integration.go linguist-generated=true +cmd/account/disable-legacy-features/disable-legacy-features.go linguist-generated=true cmd/account/encryption-keys/encryption-keys.go linguist-generated=true cmd/account/esm-enablement-account/esm-enablement-account.go linguist-generated=true cmd/account/groups/groups.go linguist-generated=true @@ -52,6 +53,7 @@ cmd/workspace/dashboard-widgets/dashboard-widgets.go linguist-generated=true cmd/workspace/dashboards/dashboards.go linguist-generated=true cmd/workspace/data-sources/data-sources.go linguist-generated=true cmd/workspace/default-namespace/default-namespace.go linguist-generated=true +cmd/workspace/disable-legacy-access/disable-legacy-access.go linguist-generated=true cmd/workspace/enhanced-security-monitoring/enhanced-security-monitoring.go linguist-generated=true cmd/workspace/experiments/experiments.go linguist-generated=true cmd/workspace/external-locations/external-locations.go linguist-generated=true @@ -108,6 +110,7 @@ cmd/workspace/storage-credentials/storage-credentials.go linguist-generated=true cmd/workspace/system-schemas/system-schemas.go linguist-generated=true cmd/workspace/table-constraints/table-constraints.go linguist-generated=true cmd/workspace/tables/tables.go linguist-generated=true +cmd/workspace/temporary-table-credentials/temporary-table-credentials.go linguist-generated=true cmd/workspace/token-management/token-management.go linguist-generated=true cmd/workspace/tokens/tokens.go linguist-generated=true cmd/workspace/users/users.go linguist-generated=true diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 2db1a5ab4..afdf9fb9e 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -59,6 +59,127 @@ "cli": { "bundle": { "config": { + "resources.Cluster": { + "anyOf": [ + { + "type": "object", + "properties": { + "apply_policy_default_values": { + "description": "When set to true, fixed and default values from the policy will be used for fields that are omitted. When set to false, only fixed values from the policy will be applied.", + "$ref": "#/$defs/bool" + }, + "autoscale": { + "description": "Parameters needed in order to automatically scale clusters up and down based on load.\nNote: autoscaling works best with DB runtime versions 3.0 or later.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.AutoScale" + }, + "autotermination_minutes": { + "description": "Automatically terminates the cluster after it is inactive for this time in minutes. If not set,\nthis cluster will not be automatically terminated. If specified, the threshold must be between\n10 and 10000 minutes.\nUsers can also set this value to 0 to explicitly disable automatic termination.", + "$ref": "#/$defs/int" + }, + "aws_attributes": { + "description": "Attributes related to clusters running on Amazon Web Services.\nIf not specified at cluster creation, a set of default values will be used.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.AwsAttributes" + }, + "azure_attributes": { + "description": "Attributes related to clusters running on Microsoft Azure.\nIf not specified at cluster creation, a set of default values will be used.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.AzureAttributes" + }, + "cluster_log_conf": { + "description": "The configuration for delivering spark logs to a long-term storage destination.\nTwo kinds of destinations (dbfs and s3) are supported. Only one destination can be specified\nfor one cluster. If the conf is given, the logs will be delivered to the destination every\n`5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while\nthe destination of executor logs is `$destination/$clusterId/executor`.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.ClusterLogConf" + }, + "cluster_name": { + "description": "Cluster name requested by the user. This doesn't have to be unique.\nIf not specified at creation, the cluster name will be an empty string.\n", + "$ref": "#/$defs/string" + }, + "custom_tags": { + "description": "Additional tags for cluster resources. Databricks will tag all cluster resources (e.g., AWS\ninstances and EBS volumes) with these tags in addition to `default_tags`. Notes:\n\n- Currently, Databricks allows at most 45 custom tags\n\n- Clusters can only reuse cloud resources if the resources' tags are a subset of the cluster tags", + "$ref": "#/$defs/map/string" + }, + "data_security_mode": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.DataSecurityMode" + }, + "docker_image": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.DockerImage" + }, + "driver_instance_pool_id": { + "description": "The optional ID of the instance pool for the driver of the cluster belongs.\nThe pool cluster uses the instance pool with id (instance_pool_id) if the driver pool is not\nassigned.", + "$ref": "#/$defs/string" + }, + "driver_node_type_id": { + "description": "The node type of the Spark driver. Note that this field is optional;\nif unset, the driver node type will be set as the same value\nas `node_type_id` defined above.\n", + "$ref": "#/$defs/string" + }, + "enable_elastic_disk": { + "description": "Autoscaling Local Storage: when enabled, this cluster will dynamically acquire additional disk\nspace when its Spark workers are running low on disk space. This feature requires specific AWS\npermissions to function correctly - refer to the User Guide for more details.", + "$ref": "#/$defs/bool" + }, + "enable_local_disk_encryption": { + "description": "Whether to enable LUKS on cluster VMs' local disks", + "$ref": "#/$defs/bool" + }, + "gcp_attributes": { + "description": "Attributes related to clusters running on Google Cloud Platform.\nIf not specified at cluster creation, a set of default values will be used.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.GcpAttributes" + }, + "init_scripts": { + "description": "The configuration for storing init scripts. Any number of destinations can be specified. The scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, init script logs are sent to `\u003cdestination\u003e/\u003ccluster-ID\u003e/init_scripts`.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/compute.InitScriptInfo" + }, + "instance_pool_id": { + "description": "The optional ID of the instance pool to which the cluster belongs.", + "$ref": "#/$defs/string" + }, + "node_type_id": { + "description": "This field encodes, through a single value, the resources available to each of\nthe Spark nodes in this cluster. For example, the Spark nodes can be provisioned\nand optimized for memory or compute intensive workloads. A list of available node\ntypes can be retrieved by using the :method:clusters/listNodeTypes API call.\n", + "$ref": "#/$defs/string" + }, + "num_workers": { + "description": "Number of worker nodes that this cluster should have. A cluster has one Spark Driver\nand `num_workers` Executors for a total of `num_workers` + 1 Spark nodes.\n\nNote: When reading the properties of a cluster, this field reflects the desired number\nof workers rather than the actual current number of workers. For instance, if a cluster\nis resized from 5 to 10 workers, this field will immediately be updated to reflect\nthe target size of 10 workers, whereas the workers listed in `spark_info` will gradually\nincrease from 5 to 10 as the new nodes are provisioned.", + "$ref": "#/$defs/int" + }, + "permissions": { + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Permission" + }, + "policy_id": { + "description": "The ID of the cluster policy used to create the cluster if applicable.", + "$ref": "#/$defs/string" + }, + "runtime_engine": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.RuntimeEngine" + }, + "single_user_name": { + "description": "Single user name if data_security_mode is `SINGLE_USER`", + "$ref": "#/$defs/string" + }, + "spark_conf": { + "description": "An object containing a set of optional, user-specified Spark configuration key-value pairs.\nUsers can also pass in a string of extra JVM options to the driver and the executors via\n`spark.driver.extraJavaOptions` and `spark.executor.extraJavaOptions` respectively.\n", + "$ref": "#/$defs/map/string" + }, + "spark_env_vars": { + "description": "An object containing a set of optional, user-specified environment variable key-value pairs.\nPlease note that key-value pair of the form (X,Y) will be exported as is (i.e.,\n`export X='Y'`) while launching the driver and workers.\n\nIn order to specify an additional set of `SPARK_DAEMON_JAVA_OPTS`, we recommend appending\nthem to `$SPARK_DAEMON_JAVA_OPTS` as shown in the example below. This ensures that all\ndefault databricks managed environmental variables are included as well.\n\nExample Spark environment variables:\n`{\"SPARK_WORKER_MEMORY\": \"28000m\", \"SPARK_LOCAL_DIRS\": \"/local_disk0\"}` or\n`{\"SPARK_DAEMON_JAVA_OPTS\": \"$SPARK_DAEMON_JAVA_OPTS -Dspark.shuffle.service.enabled=true\"}`", + "$ref": "#/$defs/map/string" + }, + "spark_version": { + "description": "The Spark version of the cluster, e.g. `3.3.x-scala2.11`.\nA list of available Spark versions can be retrieved by using\nthe :method:clusters/sparkVersions API call.\n", + "$ref": "#/$defs/string" + }, + "ssh_public_keys": { + "description": "SSH public key contents that will be added to each Spark node in this cluster. The\ncorresponding private keys can be used to login with the user name `ubuntu` on port `2200`.\nUp to 10 keys can be specified.", + "$ref": "#/$defs/slice/string" + }, + "workload_type": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.WorkloadType" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.Grant": { "anyOf": [ { @@ -109,7 +230,7 @@ "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobEmailNotifications" }, "environments": { - "description": "A list of task execution environment specifications that can be referenced by tasks of this job.", + "description": "A list of task execution environment specifications that can be referenced by serverless tasks of this job.\nAn environment is required to be present for serverless tasks.\nFor serverless notebook tasks, the environment is accessible in the notebook environment panel.\nFor other serverless tasks, the task environment is required to be specified using environment_key in the task settings.", "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.JobEnvironment" }, "format": { @@ -293,7 +414,7 @@ "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Permission" }, "rate_limits": { - "description": "Rate limits to be applied to the serving endpoint. NOTE: only external and foundation model endpoints are supported as of now.", + "description": "Rate limits to be applied to the serving endpoint. NOTE: this field is deprecated, please use AI Gateway to manage rate limits.", "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/serving.RateLimit" }, "route_optimized": { @@ -747,6 +868,9 @@ { "type": "object", "properties": { + "cluster_id": { + "$ref": "#/$defs/string" + }, "compute_id": { "$ref": "#/$defs/string" }, @@ -923,6 +1047,9 @@ { "type": "object", "properties": { + "clusters": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Cluster" + }, "experiments": { "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.MlflowExperiment" }, @@ -990,6 +1117,9 @@ "bundle": { "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Bundle" }, + "cluster_id": { + "$ref": "#/$defs/string" + }, "compute_id": { "$ref": "#/$defs/string" }, @@ -2028,7 +2158,7 @@ }, "compute.RuntimeEngine": { "type": "string", - "description": "Decides which runtime engine to be use, e.g. Standard vs. Photon. If unspecified, the runtime\nengine is inferred from spark_version.", + "description": "Determines the cluster's runtime engine, either standard or Photon.\n\nThis field is not compatible with legacy `spark_version` values that contain `-photon-`.\nRemove `-photon-` from the `spark_version` and set `runtime_engine` to `PHOTON`.\n\nIf left unspecified, the runtime engine defaults to standard unless the spark_version\ncontains -photon-, in which case Photon will be used.\n", "enum": [ "NULL", "STANDARD", @@ -2610,7 +2740,7 @@ "anyOf": [ { "type": "object", - "description": "Write-only setting, available only in Create/Update/Reset and Submit calls. Specifies the user or service principal that the job runs as. If not specified, the job runs as the user who created the job.\n\nOnly `user_name` or `service_principal_name` can be specified. If both are specified, an error is thrown.", + "description": "Write-only setting. Specifies the user, service principal or group that the job/pipeline runs as. If not specified, the job/pipeline runs as the user who created the job/pipeline.\n\nExactly one of `user_name`, `service_principal_name`, `group_name` should be specified. If not, an error is thrown.", "properties": { "service_principal_name": { "description": "Application ID of an active service principal. Setting this field requires the `servicePrincipal/user` role.", @@ -4904,6 +5034,20 @@ "cli": { "bundle": { "config": { + "resources.Cluster": { + "anyOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Cluster" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.Job": { "anyOf": [ { diff --git a/cmd/account/disable-legacy-features/disable-legacy-features.go b/cmd/account/disable-legacy-features/disable-legacy-features.go new file mode 100755 index 000000000..6d25b943d --- /dev/null +++ b/cmd/account/disable-legacy-features/disable-legacy-features.go @@ -0,0 +1,215 @@ +// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT. + +package disable_legacy_features + +import ( + "fmt" + + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/flags" + "github.com/databricks/databricks-sdk-go/service/settings" + "github.com/spf13/cobra" +) + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var cmdOverrides []func(*cobra.Command) + +func New() *cobra.Command { + cmd := &cobra.Command{ + Use: "disable-legacy-features", + Short: `Disable legacy features for new Databricks workspaces.`, + Long: `Disable legacy features for new Databricks workspaces. + + For newly created workspaces: 1. Disables the use of DBFS root and mounts. 2. + Hive Metastore will not be provisioned. 3. Disables the use of ‘No-isolation + clusters’. 4. Disables Databricks Runtime versions prior to 13.3LTS.`, + + // This service is being previewed; hide from help output. + Hidden: true, + } + + // Add methods + cmd.AddCommand(newDelete()) + cmd.AddCommand(newGet()) + cmd.AddCommand(newUpdate()) + + // Apply optional overrides to this command. + for _, fn := range cmdOverrides { + fn(cmd) + } + + return cmd +} + +// start delete command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var deleteOverrides []func( + *cobra.Command, + *settings.DeleteDisableLegacyFeaturesRequest, +) + +func newDelete() *cobra.Command { + cmd := &cobra.Command{} + + var deleteReq settings.DeleteDisableLegacyFeaturesRequest + + // TODO: short flags + + cmd.Flags().StringVar(&deleteReq.Etag, "etag", deleteReq.Etag, `etag used for versioning.`) + + cmd.Use = "delete" + cmd.Short = `Delete the disable legacy features setting.` + cmd.Long = `Delete the disable legacy features setting. + + Deletes the disable legacy features setting.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + + cmd.PreRunE = root.MustAccountClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + a := root.AccountClient(ctx) + + response, err := a.Settings.DisableLegacyFeatures().Delete(ctx, deleteReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range deleteOverrides { + fn(cmd, &deleteReq) + } + + return cmd +} + +// start get command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var getOverrides []func( + *cobra.Command, + *settings.GetDisableLegacyFeaturesRequest, +) + +func newGet() *cobra.Command { + cmd := &cobra.Command{} + + var getReq settings.GetDisableLegacyFeaturesRequest + + // TODO: short flags + + cmd.Flags().StringVar(&getReq.Etag, "etag", getReq.Etag, `etag used for versioning.`) + + cmd.Use = "get" + cmd.Short = `Get the disable legacy features setting.` + cmd.Long = `Get the disable legacy features setting. + + Gets the value of the disable legacy features setting.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + + cmd.PreRunE = root.MustAccountClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + a := root.AccountClient(ctx) + + response, err := a.Settings.DisableLegacyFeatures().Get(ctx, getReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range getOverrides { + fn(cmd, &getReq) + } + + return cmd +} + +// start update command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var updateOverrides []func( + *cobra.Command, + *settings.UpdateDisableLegacyFeaturesRequest, +) + +func newUpdate() *cobra.Command { + cmd := &cobra.Command{} + + var updateReq settings.UpdateDisableLegacyFeaturesRequest + var updateJson flags.JsonFlag + + // TODO: short flags + cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + cmd.Use = "update" + cmd.Short = `Update the disable legacy features setting.` + cmd.Long = `Update the disable legacy features setting. + + Updates the value of the disable legacy features setting.` + + cmd.Annotations = make(map[string]string) + + cmd.PreRunE = root.MustAccountClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + a := root.AccountClient(ctx) + + if cmd.Flags().Changed("json") { + err = updateJson.Unmarshal(&updateReq) + if err != nil { + return err + } + } else { + return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") + } + + response, err := a.Settings.DisableLegacyFeatures().Update(ctx, updateReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range updateOverrides { + fn(cmd, &updateReq) + } + + return cmd +} + +// end service DisableLegacyFeatures diff --git a/cmd/account/settings/settings.go b/cmd/account/settings/settings.go index a750e81e0..9a9cd44bf 100755 --- a/cmd/account/settings/settings.go +++ b/cmd/account/settings/settings.go @@ -6,6 +6,7 @@ import ( "github.com/spf13/cobra" csp_enablement_account "github.com/databricks/cli/cmd/account/csp-enablement-account" + disable_legacy_features "github.com/databricks/cli/cmd/account/disable-legacy-features" esm_enablement_account "github.com/databricks/cli/cmd/account/esm-enablement-account" personal_compute "github.com/databricks/cli/cmd/account/personal-compute" ) @@ -27,6 +28,7 @@ func New() *cobra.Command { // Add subservices cmd.AddCommand(csp_enablement_account.New()) + cmd.AddCommand(disable_legacy_features.New()) cmd.AddCommand(esm_enablement_account.New()) cmd.AddCommand(personal_compute.New()) diff --git a/cmd/workspace/apps/apps.go b/cmd/workspace/apps/apps.go index bc3fbe920..baec6d03c 100755 --- a/cmd/workspace/apps/apps.go +++ b/cmd/workspace/apps/apps.go @@ -75,8 +75,8 @@ func newCreate() *cobra.Command { var createSkipWait bool var createTimeout time.Duration - cmd.Flags().BoolVar(&createSkipWait, "no-wait", createSkipWait, `do not wait to reach IDLE state`) - cmd.Flags().DurationVar(&createTimeout, "timeout", 20*time.Minute, `maximum amount of time to reach IDLE state`) + cmd.Flags().BoolVar(&createSkipWait, "no-wait", createSkipWait, `do not wait to reach ACTIVE state`) + cmd.Flags().DurationVar(&createTimeout, "timeout", 20*time.Minute, `maximum amount of time to reach ACTIVE state`) // TODO: short flags cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`) @@ -130,13 +130,13 @@ func newCreate() *cobra.Command { } spinner := cmdio.Spinner(ctx) info, err := wait.OnProgress(func(i *apps.App) { - if i.Status == nil { + if i.ComputeStatus == nil { return } - status := i.Status.State + status := i.ComputeStatus.State statusMessage := fmt.Sprintf("current status: %s", status) - if i.Status != nil { - statusMessage = i.Status.Message + if i.ComputeStatus != nil { + statusMessage = i.ComputeStatus.Message } spinner <- statusMessage }).GetWithTimeout(createTimeout) @@ -198,11 +198,11 @@ func newDelete() *cobra.Command { deleteReq.Name = args[0] - err = w.Apps.Delete(ctx, deleteReq) + response, err := w.Apps.Delete(ctx, deleteReq) if err != nil { return err } - return nil + return cmdio.Render(ctx, response) } // Disable completions since they are not applicable. @@ -240,35 +240,23 @@ func newDeploy() *cobra.Command { // TODO: short flags cmd.Flags().Var(&deployJson, "json", `either inline JSON string or @path/to/file.json with request body`) + cmd.Flags().StringVar(&deployReq.DeploymentId, "deployment-id", deployReq.DeploymentId, `The unique id of the deployment.`) cmd.Flags().Var(&deployReq.Mode, "mode", `The mode of which the deployment will manage the source code. Supported values: [AUTO_SYNC, SNAPSHOT]`) + cmd.Flags().StringVar(&deployReq.SourceCodePath, "source-code-path", deployReq.SourceCodePath, `The workspace file system path of the source code used to create the app deployment.`) - cmd.Use = "deploy APP_NAME SOURCE_CODE_PATH" + cmd.Use = "deploy APP_NAME" cmd.Short = `Create an app deployment.` cmd.Long = `Create an app deployment. Creates an app deployment for the app with the supplied name. Arguments: - APP_NAME: The name of the app. - SOURCE_CODE_PATH: The workspace file system path of the source code used to create the app - deployment. This is different from - deployment_artifacts.source_code_path, which is the path used by the - deployed app. The former refers to the original source code location of - the app in the workspace during deployment creation, whereas the latter - provides a system generated stable snapshotted source code path used by - the deployment.` + APP_NAME: The name of the app.` cmd.Annotations = make(map[string]string) cmd.Args = func(cmd *cobra.Command, args []string) error { - if cmd.Flags().Changed("json") { - err := root.ExactArgs(1)(cmd, args) - if err != nil { - return fmt.Errorf("when --json flag is specified, provide only APP_NAME as positional arguments. Provide 'source_code_path' in your JSON input") - } - return nil - } - check := root.ExactArgs(2) + check := root.ExactArgs(1) return check(cmd, args) } @@ -284,9 +272,6 @@ func newDeploy() *cobra.Command { } } deployReq.AppName = args[0] - if !cmd.Flags().Changed("json") { - deployReq.SourceCodePath = args[1] - } wait, err := w.Apps.Deploy(ctx, deployReq) if err != nil { @@ -759,8 +744,8 @@ func newStart() *cobra.Command { var startSkipWait bool var startTimeout time.Duration - cmd.Flags().BoolVar(&startSkipWait, "no-wait", startSkipWait, `do not wait to reach SUCCEEDED state`) - cmd.Flags().DurationVar(&startTimeout, "timeout", 20*time.Minute, `maximum amount of time to reach SUCCEEDED state`) + cmd.Flags().BoolVar(&startSkipWait, "no-wait", startSkipWait, `do not wait to reach ACTIVE state`) + cmd.Flags().DurationVar(&startTimeout, "timeout", 20*time.Minute, `maximum amount of time to reach ACTIVE state`) // TODO: short flags cmd.Use = "start NAME" @@ -794,14 +779,14 @@ func newStart() *cobra.Command { return cmdio.Render(ctx, wait.Response) } spinner := cmdio.Spinner(ctx) - info, err := wait.OnProgress(func(i *apps.AppDeployment) { - if i.Status == nil { + info, err := wait.OnProgress(func(i *apps.App) { + if i.ComputeStatus == nil { return } - status := i.Status.State + status := i.ComputeStatus.State statusMessage := fmt.Sprintf("current status: %s", status) - if i.Status != nil { - statusMessage = i.Status.Message + if i.ComputeStatus != nil { + statusMessage = i.ComputeStatus.Message } spinner <- statusMessage }).GetWithTimeout(startTimeout) @@ -838,6 +823,11 @@ func newStop() *cobra.Command { var stopReq apps.StopAppRequest + var stopSkipWait bool + var stopTimeout time.Duration + + cmd.Flags().BoolVar(&stopSkipWait, "no-wait", stopSkipWait, `do not wait to reach STOPPED state`) + cmd.Flags().DurationVar(&stopTimeout, "timeout", 20*time.Minute, `maximum amount of time to reach STOPPED state`) // TODO: short flags cmd.Use = "stop NAME" @@ -863,11 +853,30 @@ func newStop() *cobra.Command { stopReq.Name = args[0] - err = w.Apps.Stop(ctx, stopReq) + wait, err := w.Apps.Stop(ctx, stopReq) if err != nil { return err } - return nil + if stopSkipWait { + return cmdio.Render(ctx, wait.Response) + } + spinner := cmdio.Spinner(ctx) + info, err := wait.OnProgress(func(i *apps.App) { + if i.ComputeStatus == nil { + return + } + status := i.ComputeStatus.State + statusMessage := fmt.Sprintf("current status: %s", status) + if i.ComputeStatus != nil { + statusMessage = i.ComputeStatus.Message + } + spinner <- statusMessage + }).GetWithTimeout(stopTimeout) + close(spinner) + if err != nil { + return err + } + return cmdio.Render(ctx, info) } // Disable completions since they are not applicable. diff --git a/cmd/workspace/clusters/clusters.go b/cmd/workspace/clusters/clusters.go index a64a6ab7c..b36102d98 100755 --- a/cmd/workspace/clusters/clusters.go +++ b/cmd/workspace/clusters/clusters.go @@ -217,7 +217,7 @@ func newCreate() *cobra.Command { cmd.Flags().StringVar(&createReq.NodeTypeId, "node-type-id", createReq.NodeTypeId, `This field encodes, through a single value, the resources available to each of the Spark nodes in this cluster.`) cmd.Flags().IntVar(&createReq.NumWorkers, "num-workers", createReq.NumWorkers, `Number of worker nodes that this cluster should have.`) cmd.Flags().StringVar(&createReq.PolicyId, "policy-id", createReq.PolicyId, `The ID of the cluster policy used to create the cluster if applicable.`) - cmd.Flags().Var(&createReq.RuntimeEngine, "runtime-engine", `Decides which runtime engine to be use, e.g. Supported values: [NULL, PHOTON, STANDARD]`) + cmd.Flags().Var(&createReq.RuntimeEngine, "runtime-engine", `Determines the cluster's runtime engine, either standard or Photon. Supported values: [NULL, PHOTON, STANDARD]`) cmd.Flags().StringVar(&createReq.SingleUserName, "single-user-name", createReq.SingleUserName, `Single user name if data_security_mode is SINGLE_USER.`) // TODO: map via StringToStringVar: spark_conf // TODO: map via StringToStringVar: spark_env_vars @@ -236,6 +236,12 @@ func newCreate() *cobra.Command { If Databricks acquires at least 85% of the requested on-demand nodes, cluster creation will succeed. Otherwise the cluster will terminate with an informative error message. + + Rather than authoring the cluster's JSON definition from scratch, Databricks + recommends filling out the [create compute UI] and then copying the generated + JSON definition from the UI. + + [create compute UI]: https://docs.databricks.com/compute/configure.html Arguments: SPARK_VERSION: The Spark version of the cluster, e.g. 3.3.x-scala2.11. A list of @@ -463,7 +469,7 @@ func newEdit() *cobra.Command { cmd.Flags().StringVar(&editReq.NodeTypeId, "node-type-id", editReq.NodeTypeId, `This field encodes, through a single value, the resources available to each of the Spark nodes in this cluster.`) cmd.Flags().IntVar(&editReq.NumWorkers, "num-workers", editReq.NumWorkers, `Number of worker nodes that this cluster should have.`) cmd.Flags().StringVar(&editReq.PolicyId, "policy-id", editReq.PolicyId, `The ID of the cluster policy used to create the cluster if applicable.`) - cmd.Flags().Var(&editReq.RuntimeEngine, "runtime-engine", `Decides which runtime engine to be use, e.g. Supported values: [NULL, PHOTON, STANDARD]`) + cmd.Flags().Var(&editReq.RuntimeEngine, "runtime-engine", `Determines the cluster's runtime engine, either standard or Photon. Supported values: [NULL, PHOTON, STANDARD]`) cmd.Flags().StringVar(&editReq.SingleUserName, "single-user-name", editReq.SingleUserName, `Single user name if data_security_mode is SINGLE_USER.`) // TODO: map via StringToStringVar: spark_conf // TODO: map via StringToStringVar: spark_env_vars diff --git a/cmd/workspace/cmd.go b/cmd/workspace/cmd.go index 11be8077a..3fe5b2686 100755 --- a/cmd/workspace/cmd.go +++ b/cmd/workspace/cmd.go @@ -76,6 +76,7 @@ import ( system_schemas "github.com/databricks/cli/cmd/workspace/system-schemas" table_constraints "github.com/databricks/cli/cmd/workspace/table-constraints" tables "github.com/databricks/cli/cmd/workspace/tables" + temporary_table_credentials "github.com/databricks/cli/cmd/workspace/temporary-table-credentials" token_management "github.com/databricks/cli/cmd/workspace/token-management" tokens "github.com/databricks/cli/cmd/workspace/tokens" users "github.com/databricks/cli/cmd/workspace/users" @@ -165,6 +166,7 @@ func All() []*cobra.Command { out = append(out, system_schemas.New()) out = append(out, table_constraints.New()) out = append(out, tables.New()) + out = append(out, temporary_table_credentials.New()) out = append(out, token_management.New()) out = append(out, tokens.New()) out = append(out, users.New()) diff --git a/cmd/workspace/disable-legacy-access/disable-legacy-access.go b/cmd/workspace/disable-legacy-access/disable-legacy-access.go new file mode 100755 index 000000000..fea2b3c40 --- /dev/null +++ b/cmd/workspace/disable-legacy-access/disable-legacy-access.go @@ -0,0 +1,217 @@ +// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT. + +package disable_legacy_access + +import ( + "fmt" + + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/flags" + "github.com/databricks/databricks-sdk-go/service/settings" + "github.com/spf13/cobra" +) + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var cmdOverrides []func(*cobra.Command) + +func New() *cobra.Command { + cmd := &cobra.Command{ + Use: "disable-legacy-access", + Short: `'Disabling legacy access' has the following impacts: 1.`, + Long: `'Disabling legacy access' has the following impacts: + + 1. Disables direct access to the Hive Metastore. However, you can still access + Hive Metastore through HMS Federation. 2. Disables Fallback Mode (docs link) + on any External Location access from the workspace. 3. Alters DBFS path access + to use External Location permissions in place of legacy credentials. 4. + Enforces Unity Catalog access on all path based access.`, + + // This service is being previewed; hide from help output. + Hidden: true, + } + + // Add methods + cmd.AddCommand(newDelete()) + cmd.AddCommand(newGet()) + cmd.AddCommand(newUpdate()) + + // Apply optional overrides to this command. + for _, fn := range cmdOverrides { + fn(cmd) + } + + return cmd +} + +// start delete command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var deleteOverrides []func( + *cobra.Command, + *settings.DeleteDisableLegacyAccessRequest, +) + +func newDelete() *cobra.Command { + cmd := &cobra.Command{} + + var deleteReq settings.DeleteDisableLegacyAccessRequest + + // TODO: short flags + + cmd.Flags().StringVar(&deleteReq.Etag, "etag", deleteReq.Etag, `etag used for versioning.`) + + cmd.Use = "delete" + cmd.Short = `Delete Legacy Access Disablement Status.` + cmd.Long = `Delete Legacy Access Disablement Status. + + Deletes legacy access disablement status.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + response, err := w.Settings.DisableLegacyAccess().Delete(ctx, deleteReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range deleteOverrides { + fn(cmd, &deleteReq) + } + + return cmd +} + +// start get command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var getOverrides []func( + *cobra.Command, + *settings.GetDisableLegacyAccessRequest, +) + +func newGet() *cobra.Command { + cmd := &cobra.Command{} + + var getReq settings.GetDisableLegacyAccessRequest + + // TODO: short flags + + cmd.Flags().StringVar(&getReq.Etag, "etag", getReq.Etag, `etag used for versioning.`) + + cmd.Use = "get" + cmd.Short = `Retrieve Legacy Access Disablement Status.` + cmd.Long = `Retrieve Legacy Access Disablement Status. + + Retrieves legacy access disablement Status.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + response, err := w.Settings.DisableLegacyAccess().Get(ctx, getReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range getOverrides { + fn(cmd, &getReq) + } + + return cmd +} + +// start update command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var updateOverrides []func( + *cobra.Command, + *settings.UpdateDisableLegacyAccessRequest, +) + +func newUpdate() *cobra.Command { + cmd := &cobra.Command{} + + var updateReq settings.UpdateDisableLegacyAccessRequest + var updateJson flags.JsonFlag + + // TODO: short flags + cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + cmd.Use = "update" + cmd.Short = `Update Legacy Access Disablement Status.` + cmd.Long = `Update Legacy Access Disablement Status. + + Updates legacy access disablement status.` + + cmd.Annotations = make(map[string]string) + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + err = updateJson.Unmarshal(&updateReq) + if err != nil { + return err + } + } else { + return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") + } + + response, err := w.Settings.DisableLegacyAccess().Update(ctx, updateReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range updateOverrides { + fn(cmd, &updateReq) + } + + return cmd +} + +// end service DisableLegacyAccess diff --git a/cmd/workspace/pipelines/pipelines.go b/cmd/workspace/pipelines/pipelines.go index f1cc4e3f7..5b4d9645e 100755 --- a/cmd/workspace/pipelines/pipelines.go +++ b/cmd/workspace/pipelines/pipelines.go @@ -935,6 +935,7 @@ func newUpdate() *cobra.Command { cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`) cmd.Flags().BoolVar(&updateReq.AllowDuplicateNames, "allow-duplicate-names", updateReq.AllowDuplicateNames, `If false, deployment will fail if name has changed and conflicts the name of another pipeline.`) + cmd.Flags().StringVar(&updateReq.BudgetPolicyId, "budget-policy-id", updateReq.BudgetPolicyId, `Budget policy of this pipeline.`) cmd.Flags().StringVar(&updateReq.Catalog, "catalog", updateReq.Catalog, `A catalog in Unity Catalog to publish data from this pipeline to.`) cmd.Flags().StringVar(&updateReq.Channel, "channel", updateReq.Channel, `DLT Release Channel that specifies which version to use.`) // TODO: array: clusters diff --git a/cmd/workspace/serving-endpoints/serving-endpoints.go b/cmd/workspace/serving-endpoints/serving-endpoints.go index b92f824d3..0837652db 100755 --- a/cmd/workspace/serving-endpoints/serving-endpoints.go +++ b/cmd/workspace/serving-endpoints/serving-endpoints.go @@ -53,6 +53,7 @@ func New() *cobra.Command { cmd.AddCommand(newLogs()) cmd.AddCommand(newPatch()) cmd.AddCommand(newPut()) + cmd.AddCommand(newPutAiGateway()) cmd.AddCommand(newQuery()) cmd.AddCommand(newSetPermissions()) cmd.AddCommand(newUpdateConfig()) @@ -151,6 +152,7 @@ func newCreate() *cobra.Command { // TODO: short flags cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`) + // TODO: complex arg: ai_gateway // TODO: array: rate_limits cmd.Flags().BoolVar(&createReq.RouteOptimized, "route-optimized", createReq.RouteOptimized, `Enable route optimization for the serving endpoint.`) // TODO: array: tags @@ -754,8 +756,9 @@ func newPut() *cobra.Command { cmd.Short = `Update rate limits of a serving endpoint.` cmd.Long = `Update rate limits of a serving endpoint. - Used to update the rate limits of a serving endpoint. NOTE: only external and - foundation model endpoints are supported as of now. + Used to update the rate limits of a serving endpoint. NOTE: Only foundation + model endpoints are currently supported. For external models, use AI Gateway + to manage rate limits. Arguments: NAME: The name of the serving endpoint whose rate limits are being updated. This @@ -800,6 +803,79 @@ func newPut() *cobra.Command { return cmd } +// start put-ai-gateway command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var putAiGatewayOverrides []func( + *cobra.Command, + *serving.PutAiGatewayRequest, +) + +func newPutAiGateway() *cobra.Command { + cmd := &cobra.Command{} + + var putAiGatewayReq serving.PutAiGatewayRequest + var putAiGatewayJson flags.JsonFlag + + // TODO: short flags + cmd.Flags().Var(&putAiGatewayJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + // TODO: complex arg: guardrails + // TODO: complex arg: inference_table_config + // TODO: array: rate_limits + // TODO: complex arg: usage_tracking_config + + cmd.Use = "put-ai-gateway NAME" + cmd.Short = `Update AI Gateway of a serving endpoint.` + cmd.Long = `Update AI Gateway of a serving endpoint. + + Used to update the AI Gateway of a serving endpoint. NOTE: Only external model + endpoints are currently supported. + + Arguments: + NAME: The name of the serving endpoint whose AI Gateway is being updated. This + field is required.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + err = putAiGatewayJson.Unmarshal(&putAiGatewayReq) + if err != nil { + return err + } + } + putAiGatewayReq.Name = args[0] + + response, err := w.ServingEndpoints.PutAiGateway(ctx, putAiGatewayReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range putAiGatewayOverrides { + fn(cmd, &putAiGatewayReq) + } + + return cmd +} + // start query command // Slice with functions to override default command behavior. diff --git a/cmd/workspace/settings/settings.go b/cmd/workspace/settings/settings.go index 214986c76..aaeecf41b 100755 --- a/cmd/workspace/settings/settings.go +++ b/cmd/workspace/settings/settings.go @@ -8,6 +8,7 @@ import ( automatic_cluster_update "github.com/databricks/cli/cmd/workspace/automatic-cluster-update" compliance_security_profile "github.com/databricks/cli/cmd/workspace/compliance-security-profile" default_namespace "github.com/databricks/cli/cmd/workspace/default-namespace" + disable_legacy_access "github.com/databricks/cli/cmd/workspace/disable-legacy-access" enhanced_security_monitoring "github.com/databricks/cli/cmd/workspace/enhanced-security-monitoring" restrict_workspace_admins "github.com/databricks/cli/cmd/workspace/restrict-workspace-admins" ) @@ -31,6 +32,7 @@ func New() *cobra.Command { cmd.AddCommand(automatic_cluster_update.New()) cmd.AddCommand(compliance_security_profile.New()) cmd.AddCommand(default_namespace.New()) + cmd.AddCommand(disable_legacy_access.New()) cmd.AddCommand(enhanced_security_monitoring.New()) cmd.AddCommand(restrict_workspace_admins.New()) diff --git a/cmd/workspace/tables/tables.go b/cmd/workspace/tables/tables.go index 4564b4fe6..ec297f294 100755 --- a/cmd/workspace/tables/tables.go +++ b/cmd/workspace/tables/tables.go @@ -220,6 +220,7 @@ func newGet() *cobra.Command { cmd.Flags().BoolVar(&getReq.IncludeBrowse, "include-browse", getReq.IncludeBrowse, `Whether to include tables in the response for which the principal can only access selective metadata for.`) cmd.Flags().BoolVar(&getReq.IncludeDeltaMetadata, "include-delta-metadata", getReq.IncludeDeltaMetadata, `Whether delta metadata should be included in the response.`) + cmd.Flags().BoolVar(&getReq.IncludeManifestCapabilities, "include-manifest-capabilities", getReq.IncludeManifestCapabilities, `Whether to include a manifest containing capabilities the table has.`) cmd.Use = "get FULL_NAME" cmd.Short = `Get a table.` @@ -299,6 +300,7 @@ func newList() *cobra.Command { cmd.Flags().BoolVar(&listReq.IncludeBrowse, "include-browse", listReq.IncludeBrowse, `Whether to include tables in the response for which the principal can only access selective metadata for.`) cmd.Flags().BoolVar(&listReq.IncludeDeltaMetadata, "include-delta-metadata", listReq.IncludeDeltaMetadata, `Whether delta metadata should be included in the response.`) + cmd.Flags().BoolVar(&listReq.IncludeManifestCapabilities, "include-manifest-capabilities", listReq.IncludeManifestCapabilities, `Whether to include a manifest containing capabilities the table has.`) cmd.Flags().IntVar(&listReq.MaxResults, "max-results", listReq.MaxResults, `Maximum number of tables to return.`) cmd.Flags().BoolVar(&listReq.OmitColumns, "omit-columns", listReq.OmitColumns, `Whether to omit the columns of the table from the response or not.`) cmd.Flags().BoolVar(&listReq.OmitProperties, "omit-properties", listReq.OmitProperties, `Whether to omit the properties of the table from the response or not.`) @@ -366,6 +368,7 @@ func newListSummaries() *cobra.Command { // TODO: short flags + cmd.Flags().BoolVar(&listSummariesReq.IncludeManifestCapabilities, "include-manifest-capabilities", listSummariesReq.IncludeManifestCapabilities, `Whether to include a manifest containing capabilities the table has.`) cmd.Flags().IntVar(&listSummariesReq.MaxResults, "max-results", listSummariesReq.MaxResults, `Maximum number of summaries for tables to return.`) cmd.Flags().StringVar(&listSummariesReq.PageToken, "page-token", listSummariesReq.PageToken, `Opaque pagination token to go to next page based on previous query.`) cmd.Flags().StringVar(&listSummariesReq.SchemaNamePattern, "schema-name-pattern", listSummariesReq.SchemaNamePattern, `A sql LIKE pattern (% and _) for schema names.`) diff --git a/cmd/workspace/temporary-table-credentials/temporary-table-credentials.go b/cmd/workspace/temporary-table-credentials/temporary-table-credentials.go new file mode 100755 index 000000000..8718f7ba1 --- /dev/null +++ b/cmd/workspace/temporary-table-credentials/temporary-table-credentials.go @@ -0,0 +1,122 @@ +// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT. + +package temporary_table_credentials + +import ( + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/flags" + "github.com/databricks/databricks-sdk-go/service/catalog" + "github.com/spf13/cobra" +) + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var cmdOverrides []func(*cobra.Command) + +func New() *cobra.Command { + cmd := &cobra.Command{ + Use: "temporary-table-credentials", + Short: `Temporary Table Credentials refer to short-lived, downscoped credentials used to access cloud storage locationswhere table data is stored in Databricks.`, + Long: `Temporary Table Credentials refer to short-lived, downscoped credentials used + to access cloud storage locationswhere table data is stored in Databricks. + These credentials are employed to provide secure and time-limitedaccess to + data in cloud environments such as AWS, Azure, and Google Cloud. Each cloud + provider has its own typeof credentials: AWS uses temporary session tokens via + AWS Security Token Service (STS), Azure utilizesShared Access Signatures (SAS) + for its data storage services, and Google Cloud supports temporary + credentialsthrough OAuth 2.0.Temporary table credentials ensure that data + access is limited in scope and duration, reducing the risk ofunauthorized + access or misuse. To use the temporary table credentials API, a metastore + admin needs to enable the external_access_enabled flag (off by default) at the + metastore level, and user needs to be granted the EXTERNAL USE SCHEMA + permission at the schema level by catalog admin. Note that EXTERNAL USE SCHEMA + is a schema level permission that can only be granted by catalog admin + explicitly and is not included in schema ownership or ALL PRIVILEGES on the + schema for security reason.`, + GroupID: "catalog", + Annotations: map[string]string{ + "package": "catalog", + }, + } + + // Add methods + cmd.AddCommand(newGenerateTemporaryTableCredentials()) + + // Apply optional overrides to this command. + for _, fn := range cmdOverrides { + fn(cmd) + } + + return cmd +} + +// start generate-temporary-table-credentials command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var generateTemporaryTableCredentialsOverrides []func( + *cobra.Command, + *catalog.GenerateTemporaryTableCredentialRequest, +) + +func newGenerateTemporaryTableCredentials() *cobra.Command { + cmd := &cobra.Command{} + + var generateTemporaryTableCredentialsReq catalog.GenerateTemporaryTableCredentialRequest + var generateTemporaryTableCredentialsJson flags.JsonFlag + + // TODO: short flags + cmd.Flags().Var(&generateTemporaryTableCredentialsJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + cmd.Flags().Var(&generateTemporaryTableCredentialsReq.Operation, "operation", `The operation performed against the table data, either READ or READ_WRITE. Supported values: [READ, READ_WRITE]`) + cmd.Flags().StringVar(&generateTemporaryTableCredentialsReq.TableId, "table-id", generateTemporaryTableCredentialsReq.TableId, `UUID of the table to read or write.`) + + cmd.Use = "generate-temporary-table-credentials" + cmd.Short = `Generate a temporary table credential.` + cmd.Long = `Generate a temporary table credential. + + Get a short-lived credential for directly accessing the table data on cloud + storage. The metastore must have external_access_enabled flag set to true + (default false). The caller must have EXTERNAL_USE_SCHEMA privilege on the + parent schema and this privilege can only be granted by catalog owners.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + err = generateTemporaryTableCredentialsJson.Unmarshal(&generateTemporaryTableCredentialsReq) + if err != nil { + return err + } + } + + response, err := w.TemporaryTableCredentials.GenerateTemporaryTableCredentials(ctx, generateTemporaryTableCredentialsReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range generateTemporaryTableCredentialsOverrides { + fn(cmd, &generateTemporaryTableCredentialsReq) + } + + return cmd +} + +// end service TemporaryTableCredentials diff --git a/go.mod b/go.mod index 0cf3ef8a7..9141274c2 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ toolchain go1.22.7 require ( github.com/Masterminds/semver/v3 v3.3.0 // MIT github.com/briandowns/spinner v1.23.1 // Apache 2.0 - github.com/databricks/databricks-sdk-go v0.46.0 // Apache 2.0 + github.com/databricks/databricks-sdk-go v0.47.0 // Apache 2.0 github.com/fatih/color v1.17.0 // MIT github.com/ghodss/yaml v1.0.0 // MIT + NOTICE github.com/google/uuid v1.6.0 // BSD-3-Clause diff --git a/go.sum b/go.sum index d88667751..177707a50 100644 --- a/go.sum +++ b/go.sum @@ -32,8 +32,8 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= -github.com/databricks/databricks-sdk-go v0.46.0 h1:D0TxmtSVAOsdnfzH4OGtAmcq+8TyA7Z6fA6JEYhupeY= -github.com/databricks/databricks-sdk-go v0.46.0/go.mod h1:ds+zbv5mlQG7nFEU5ojLtgN/u0/9YzZmKQES/CfedzU= +github.com/databricks/databricks-sdk-go v0.47.0 h1:eE7dN9axviL8+s10jnQAayOYDaR+Mfu7E9COGjO4lrQ= +github.com/databricks/databricks-sdk-go v0.47.0/go.mod h1:ds+zbv5mlQG7nFEU5ojLtgN/u0/9YzZmKQES/CfedzU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= From c28d64f2dc3885becbddc3b44b1bdc5893ba6b7c Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 1 Oct 2024 15:20:17 +0200 Subject: [PATCH 25/34] [Release] Release v0.229.0 (#1801) Bundles: * Added support for creating all-purpose clusters ([#1698](https://github.com/databricks/cli/pull/1698)). * Reduce time until the prompt is shown for bundle run ([#1727](https://github.com/databricks/cli/pull/1727)). * Use Unity Catalog for pipelines in the default-python template ([#1766](https://github.com/databricks/cli/pull/1766)). * Add verbose flag to the "bundle deploy" command ([#1774](https://github.com/databricks/cli/pull/1774)). * Fixed full variable override detection ([#1787](https://github.com/databricks/cli/pull/1787)). * Add sub-extension to resource files in built-in templates ([#1777](https://github.com/databricks/cli/pull/1777)). * Fix panic in `apply_presets.go` ([#1796](https://github.com/databricks/cli/pull/1796)). Internal: * Assert tokens are redacted in origin URL when username is not specified ([#1785](https://github.com/databricks/cli/pull/1785)). * Refactor jobs path translation ([#1782](https://github.com/databricks/cli/pull/1782)). * Add JobTaskClusterSpec validate mutator ([#1784](https://github.com/databricks/cli/pull/1784)). * Pin Go toolchain to 1.22.7 ([#1790](https://github.com/databricks/cli/pull/1790)). * Modify SetLocation test utility to take full locations as argument ([#1788](https://github.com/databricks/cli/pull/1788)). * Simplified isFullVariableOverrideDef implementation ([#1791](https://github.com/databricks/cli/pull/1791)). * Sort tasks by `task_key` before generating the Terraform configuration ([#1776](https://github.com/databricks/cli/pull/1776)). * Trim trailing whitespace ([#1794](https://github.com/databricks/cli/pull/1794)). * Move trampoline code into trampoline package ([#1793](https://github.com/databricks/cli/pull/1793)). * Rename `RootPath` -> `BundleRootPath` ([#1792](https://github.com/databricks/cli/pull/1792)). API Changes: * Changed `databricks apps delete` command to return . * Changed `databricks apps deploy` command with new required argument order. * Changed `databricks apps start` command to return . * Changed `databricks apps stop` command to return . * Added `databricks temporary-table-credentials` command group. * Added `databricks serving-endpoints put-ai-gateway` command. * Added `databricks disable-legacy-access` command group. * Added `databricks account disable-legacy-features` command group. OpenAPI commit 6f6b1371e640f2dfeba72d365ac566368656f6b6 (2024-09-19) Dependency updates: * Upgrade to Go SDK 0.47.0 ([#1799](https://github.com/databricks/cli/pull/1799)). * Upgrade to TF provider 1.52 ([#1781](https://github.com/databricks/cli/pull/1781)). * Bump golang.org/x/mod from 0.20.0 to 0.21.0 ([#1758](https://github.com/databricks/cli/pull/1758)). * Bump github.com/hashicorp/hc-install from 0.7.0 to 0.9.0 ([#1772](https://github.com/databricks/cli/pull/1772)). --- CHANGELOG.md | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 32a7e5cfa..4f5f68ac6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,45 @@ # Version changelog +## [Release] Release v0.229.0 + +Bundles: + * Added support for creating all-purpose clusters ([#1698](https://github.com/databricks/cli/pull/1698)). + * Reduce time until the prompt is shown for bundle run ([#1727](https://github.com/databricks/cli/pull/1727)). + * Use Unity Catalog for pipelines in the default-python template ([#1766](https://github.com/databricks/cli/pull/1766)). + * Add verbose flag to the "bundle deploy" command ([#1774](https://github.com/databricks/cli/pull/1774)). + * Fixed full variable override detection ([#1787](https://github.com/databricks/cli/pull/1787)). + * Add sub-extension to resource files in built-in templates ([#1777](https://github.com/databricks/cli/pull/1777)). + * Fix panic in `apply_presets.go` ([#1796](https://github.com/databricks/cli/pull/1796)). + +Internal: + * Assert tokens are redacted in origin URL when username is not specified ([#1785](https://github.com/databricks/cli/pull/1785)). + * Refactor jobs path translation ([#1782](https://github.com/databricks/cli/pull/1782)). + * Add JobTaskClusterSpec validate mutator ([#1784](https://github.com/databricks/cli/pull/1784)). + * Pin Go toolchain to 1.22.7 ([#1790](https://github.com/databricks/cli/pull/1790)). + * Modify SetLocation test utility to take full locations as argument ([#1788](https://github.com/databricks/cli/pull/1788)). + * Simplified isFullVariableOverrideDef implementation ([#1791](https://github.com/databricks/cli/pull/1791)). + * Sort tasks by `task_key` before generating the Terraform configuration ([#1776](https://github.com/databricks/cli/pull/1776)). + * Trim trailing whitespace ([#1794](https://github.com/databricks/cli/pull/1794)). + * Move trampoline code into trampoline package ([#1793](https://github.com/databricks/cli/pull/1793)). + * Rename `RootPath` -> `BundleRootPath` ([#1792](https://github.com/databricks/cli/pull/1792)). + +API Changes: + * Changed `databricks apps delete` command to return . + * Changed `databricks apps deploy` command with new required argument order. + * Changed `databricks apps start` command to return . + * Changed `databricks apps stop` command to return . + * Added `databricks temporary-table-credentials` command group. + * Added `databricks serving-endpoints put-ai-gateway` command. + * Added `databricks disable-legacy-access` command group. + * Added `databricks account disable-legacy-features` command group. + +OpenAPI commit 6f6b1371e640f2dfeba72d365ac566368656f6b6 (2024-09-19) +Dependency updates: + * Upgrade to Go SDK 0.47.0 ([#1799](https://github.com/databricks/cli/pull/1799)). + * Upgrade to TF provider 1.52 ([#1781](https://github.com/databricks/cli/pull/1781)). + * Bump golang.org/x/mod from 0.20.0 to 0.21.0 ([#1758](https://github.com/databricks/cli/pull/1758)). + * Bump github.com/hashicorp/hc-install from 0.7.0 to 0.9.0 ([#1772](https://github.com/databricks/cli/pull/1772)). + ## [Release] Release v0.228.1 Bundles: From 044a00c7f9fc2f2d35794062b63a54d01081ffa4 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 2 Oct 2024 15:53:24 +0200 Subject: [PATCH 26/34] Add an error if state files grow bigger than the export limit (#1795) ## Changes Currently API limits on exporting files from workspaces are set at 10 MBs while uploading to is 500 MBs. We want to prevent users running into deadlock when they won't be able to pull state file anymore so we prevent from uploading large state files (over 10 MBs) to Databricks workspace. --- bundle/deploy/state_push.go | 13 +++++++++++ bundle/deploy/terraform/state_push.go | 11 +++++++++ bundle/deploy/terraform/state_push_test.go | 27 ++++++++++++++++++++++ 3 files changed, 51 insertions(+) diff --git a/bundle/deploy/state_push.go b/bundle/deploy/state_push.go index 176a907c8..6912414c1 100644 --- a/bundle/deploy/state_push.go +++ b/bundle/deploy/state_push.go @@ -10,6 +10,8 @@ import ( "github.com/databricks/cli/libs/log" ) +const MaxStateFileSize = 10 * 1024 * 1024 // 10MB + type statePush struct { filerFactory FilerFactory } @@ -35,6 +37,17 @@ func (s *statePush) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostic } defer local.Close() + if !b.Config.Bundle.Force { + state, err := local.Stat() + if err != nil { + return diag.FromErr(err) + } + + if state.Size() > MaxStateFileSize { + return diag.Errorf("Deployment state file size exceeds the maximum allowed size of %d bytes. Please reduce the number of resources in your bundle, split your bundle into multiple or re-run the command with --force flag.", MaxStateFileSize) + } + } + log.Infof(ctx, "Writing local deployment state file to remote state directory") err = f.Write(ctx, DeploymentStateFileName, local, filer.CreateParentDirectories, filer.OverwriteIfExists) if err != nil { diff --git a/bundle/deploy/terraform/state_push.go b/bundle/deploy/terraform/state_push.go index 6cdde1371..84d8e7670 100644 --- a/bundle/deploy/terraform/state_push.go +++ b/bundle/deploy/terraform/state_push.go @@ -47,6 +47,17 @@ func (l *statePush) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostic } defer local.Close() + if !b.Config.Bundle.Force { + state, err := local.Stat() + if err != nil { + return diag.FromErr(err) + } + + if state.Size() > deploy.MaxStateFileSize { + return diag.Errorf("Terraform state file size exceeds the maximum allowed size of %d bytes. Please reduce the number of resources in your bundle, split your bundle into multiple or re-run the command with --force flag", deploy.MaxStateFileSize) + } + } + // Upload state file from local cache directory to filer. cmdio.LogString(ctx, "Updating deployment state...") log.Infof(ctx, "Writing local state file to remote state directory") diff --git a/bundle/deploy/terraform/state_push_test.go b/bundle/deploy/terraform/state_push_test.go index e022dee1b..4cc52b7a7 100644 --- a/bundle/deploy/terraform/state_push_test.go +++ b/bundle/deploy/terraform/state_push_test.go @@ -3,6 +3,7 @@ package terraform import ( "context" "encoding/json" + "fmt" "io" "testing" @@ -59,3 +60,29 @@ func TestStatePush(t *testing.T) { diags := bundle.Apply(ctx, b, m) assert.NoError(t, diags.Error()) } + +func TestStatePushLargeState(t *testing.T) { + mock := mockfiler.NewMockFiler(t) + m := &statePush{ + identityFiler(mock), + } + + ctx := context.Background() + b := statePushTestBundle(t) + + largeState := map[string]any{} + for i := 0; i < 1000000; i++ { + largeState[fmt.Sprintf("field_%d", i)] = i + } + + // Write a stale local state file. + writeLocalState(t, ctx, b, largeState) + diags := bundle.Apply(ctx, b, m) + assert.ErrorContains(t, diags.Error(), "Terraform state file size exceeds the maximum allowed size of 10485760 bytes. Please reduce the number of resources in your bundle, split your bundle into multiple or re-run the command with --force flag") + + // Force the write. + b = statePushTestBundle(t) + b.Config.Bundle.Force = true + diags = bundle.Apply(ctx, b, m) + assert.NoError(t, diags.Error()) +} From 80d55f454066a9c1666cb53387f9e735494fe72a Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Wed, 2 Oct 2024 15:55:40 +0200 Subject: [PATCH 27/34] Add resource path field to bundle workspace configuration (#1800) ## Changes Default workspace path for resources with a presence in the workspace tree. Note: this path is **not** created automatically (yet). We need this only for dashboards (so far), so can take care of creation if one or more dashboards are part of a deployment. This saves an API call for deployments where this is not necessary. ## Tests Expanded existing tests. --- bundle/config/mutator/default_workspace_paths.go | 4 ++++ bundle/config/mutator/default_workspace_paths_test.go | 3 +++ bundle/config/mutator/process_target_mode.go | 9 ++++++--- bundle/config/workspace.go | 5 +++++ 4 files changed, 18 insertions(+), 3 deletions(-) diff --git a/bundle/config/mutator/default_workspace_paths.go b/bundle/config/mutator/default_workspace_paths.go index 71e562b51..02a1ddb3b 100644 --- a/bundle/config/mutator/default_workspace_paths.go +++ b/bundle/config/mutator/default_workspace_paths.go @@ -29,6 +29,10 @@ func (m *defineDefaultWorkspacePaths) Apply(ctx context.Context, b *bundle.Bundl b.Config.Workspace.FilePath = path.Join(root, "files") } + if b.Config.Workspace.ResourcePath == "" { + b.Config.Workspace.ResourcePath = path.Join(root, "resources") + } + if b.Config.Workspace.ArtifactPath == "" { b.Config.Workspace.ArtifactPath = path.Join(root, "artifacts") } diff --git a/bundle/config/mutator/default_workspace_paths_test.go b/bundle/config/mutator/default_workspace_paths_test.go index 0ba20ea2b..6779c3732 100644 --- a/bundle/config/mutator/default_workspace_paths_test.go +++ b/bundle/config/mutator/default_workspace_paths_test.go @@ -22,6 +22,7 @@ func TestDefineDefaultWorkspacePaths(t *testing.T) { diags := bundle.Apply(context.Background(), b, mutator.DefineDefaultWorkspacePaths()) require.NoError(t, diags.Error()) assert.Equal(t, "/files", b.Config.Workspace.FilePath) + assert.Equal(t, "/resources", b.Config.Workspace.ResourcePath) assert.Equal(t, "/artifacts", b.Config.Workspace.ArtifactPath) assert.Equal(t, "/state", b.Config.Workspace.StatePath) } @@ -32,6 +33,7 @@ func TestDefineDefaultWorkspacePathsAlreadySet(t *testing.T) { Workspace: config.Workspace{ RootPath: "/", FilePath: "/foo/bar", + ResourcePath: "/foo/bar", ArtifactPath: "/foo/bar", StatePath: "/foo/bar", }, @@ -40,6 +42,7 @@ func TestDefineDefaultWorkspacePathsAlreadySet(t *testing.T) { diags := bundle.Apply(context.Background(), b, mutator.DefineDefaultWorkspacePaths()) require.NoError(t, diags.Error()) assert.Equal(t, "/foo/bar", b.Config.Workspace.FilePath) + assert.Equal(t, "/foo/bar", b.Config.Workspace.ResourcePath) assert.Equal(t, "/foo/bar", b.Config.Workspace.ArtifactPath) assert.Equal(t, "/foo/bar", b.Config.Workspace.StatePath) } diff --git a/bundle/config/mutator/process_target_mode.go b/bundle/config/mutator/process_target_mode.go index 70382f054..9944f6ffd 100644 --- a/bundle/config/mutator/process_target_mode.go +++ b/bundle/config/mutator/process_target_mode.go @@ -118,15 +118,18 @@ func findNonUserPath(b *bundle.Bundle) string { if b.Config.Workspace.RootPath != "" && !containsName(b.Config.Workspace.RootPath) { return "root_path" } - if b.Config.Workspace.StatePath != "" && !containsName(b.Config.Workspace.StatePath) { - return "state_path" - } if b.Config.Workspace.FilePath != "" && !containsName(b.Config.Workspace.FilePath) { return "file_path" } + if b.Config.Workspace.ResourcePath != "" && !containsName(b.Config.Workspace.ResourcePath) { + return "resource_path" + } if b.Config.Workspace.ArtifactPath != "" && !containsName(b.Config.Workspace.ArtifactPath) { return "artifact_path" } + if b.Config.Workspace.StatePath != "" && !containsName(b.Config.Workspace.StatePath) { + return "state_path" + } return "" } diff --git a/bundle/config/workspace.go b/bundle/config/workspace.go index efc5caa66..878d07838 100644 --- a/bundle/config/workspace.go +++ b/bundle/config/workspace.go @@ -54,6 +54,11 @@ type Workspace struct { // This defaults to "${workspace.root}/files". FilePath string `json:"file_path,omitempty"` + // Remote workspace path for resources with a presence in the workspace. + // These are kept outside [FilePath] to avoid potential naming collisions. + // This defaults to "${workspace.root}/resources". + ResourcePath string `json:"resource_path,omitempty"` + // Remote workspace path for build artifacts. // This defaults to "${workspace.root}/artifacts". ArtifactPath string `json:"artifact_path,omitempty"` From a8cff48c0b4d747cb769d09724fecdf3c5a3f25b Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 2 Oct 2024 17:34:00 +0200 Subject: [PATCH 28/34] Always prepend bundle remote paths with /Workspace (#1724) ## Changes Due to platform changes, all libraries, notebooks and etc. paths used in Databricks must be started with either /Workspace or /Volumes prefix. This PR makes sure that all bundle paths are correctly prefixed. Note: this change is a breaking change if user previously configured and used `/Workspace/Workspace` folder in their workspace file system or having `/Workspace/${workspace.root_path}...` pattern configured anywhere in their bundle config Fixes: #1751 AI: - [x] Scan DABs config and error out on `/Workspace/${workspace.root_path}...` pattern usage ## Tests Added unit tests --------- Co-authored-by: Pieter Noordhuis --- .../config/mutator/expand_workspace_root.go | 2 +- .../mutator/expand_workspace_root_test.go | 2 +- .../mutator/prepend_workspace_prefix.go | 67 +++++++++++++++ .../mutator/prepend_workspace_prefix_test.go | 79 +++++++++++++++++ .../mutator/rewrite_workspace_prefix.go | 72 ++++++++++++++++ .../mutator/rewrite_workspace_prefix_test.go | 85 +++++++++++++++++++ bundle/config/workspace.go | 2 +- bundle/phases/initialize.go | 7 ++ bundle/tests/pipeline_glob_paths_test.go | 2 +- .../tests/relative_path_translation_test.go | 8 +- internal/bundle/deploy_test.go | 2 +- internal/bundle/helpers.go | 2 +- .../{{.project_name}}/databricks.yml.tmpl | 4 +- .../{{.project_name}}/databricks.yml.tmpl | 4 +- .../{{.project_name}}.pipeline.yml.tmpl | 2 +- .../{{.project_name}}/databricks.yml.tmpl | 4 +- 16 files changed, 327 insertions(+), 17 deletions(-) create mode 100644 bundle/config/mutator/prepend_workspace_prefix.go create mode 100644 bundle/config/mutator/prepend_workspace_prefix_test.go create mode 100644 bundle/config/mutator/rewrite_workspace_prefix.go create mode 100644 bundle/config/mutator/rewrite_workspace_prefix_test.go diff --git a/bundle/config/mutator/expand_workspace_root.go b/bundle/config/mutator/expand_workspace_root.go index 8954abd46..3f0547de1 100644 --- a/bundle/config/mutator/expand_workspace_root.go +++ b/bundle/config/mutator/expand_workspace_root.go @@ -33,7 +33,7 @@ func (m *expandWorkspaceRoot) Apply(ctx context.Context, b *bundle.Bundle) diag. } if strings.HasPrefix(root, "~/") { - home := fmt.Sprintf("/Users/%s", currentUser.UserName) + home := fmt.Sprintf("/Workspace/Users/%s", currentUser.UserName) b.Config.Workspace.RootPath = path.Join(home, root[2:]) } diff --git a/bundle/config/mutator/expand_workspace_root_test.go b/bundle/config/mutator/expand_workspace_root_test.go index e6260dbd8..40bf35ca7 100644 --- a/bundle/config/mutator/expand_workspace_root_test.go +++ b/bundle/config/mutator/expand_workspace_root_test.go @@ -27,7 +27,7 @@ func TestExpandWorkspaceRoot(t *testing.T) { } diags := bundle.Apply(context.Background(), b, mutator.ExpandWorkspaceRoot()) require.NoError(t, diags.Error()) - assert.Equal(t, "/Users/jane@doe.com/foo", b.Config.Workspace.RootPath) + assert.Equal(t, "/Workspace/Users/jane@doe.com/foo", b.Config.Workspace.RootPath) } func TestExpandWorkspaceRootDoesNothing(t *testing.T) { diff --git a/bundle/config/mutator/prepend_workspace_prefix.go b/bundle/config/mutator/prepend_workspace_prefix.go new file mode 100644 index 000000000..dd467344b --- /dev/null +++ b/bundle/config/mutator/prepend_workspace_prefix.go @@ -0,0 +1,67 @@ +package mutator + +import ( + "context" + "fmt" + "strings" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +type prependWorkspacePrefix struct{} + +// PrependWorkspacePrefix prepends the workspace root path to all paths in the bundle. +func PrependWorkspacePrefix() bundle.Mutator { + return &prependWorkspacePrefix{} +} + +func (m *prependWorkspacePrefix) Name() string { + return "PrependWorkspacePrefix" +} + +var skipPrefixes = []string{ + "/Workspace/", + "/Volumes/", +} + +func (m *prependWorkspacePrefix) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + patterns := []dyn.Pattern{ + dyn.NewPattern(dyn.Key("workspace"), dyn.Key("root_path")), + dyn.NewPattern(dyn.Key("workspace"), dyn.Key("file_path")), + dyn.NewPattern(dyn.Key("workspace"), dyn.Key("artifact_path")), + dyn.NewPattern(dyn.Key("workspace"), dyn.Key("state_path")), + } + + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + var err error + for _, pattern := range patterns { + v, err = dyn.MapByPattern(v, pattern, func(p dyn.Path, pv dyn.Value) (dyn.Value, error) { + path, ok := pv.AsString() + if !ok { + return dyn.InvalidValue, fmt.Errorf("expected string, got %s", v.Kind()) + } + + for _, prefix := range skipPrefixes { + if strings.HasPrefix(path, prefix) { + return pv, nil + } + } + + return dyn.NewValue(fmt.Sprintf("/Workspace%s", path), v.Locations()), nil + }) + + if err != nil { + return dyn.InvalidValue, err + } + } + return v, nil + }) + + if err != nil { + return diag.FromErr(err) + } + + return nil +} diff --git a/bundle/config/mutator/prepend_workspace_prefix_test.go b/bundle/config/mutator/prepend_workspace_prefix_test.go new file mode 100644 index 000000000..287c694d3 --- /dev/null +++ b/bundle/config/mutator/prepend_workspace_prefix_test.go @@ -0,0 +1,79 @@ +package mutator + +import ( + "context" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/databricks-sdk-go/service/iam" + "github.com/stretchr/testify/require" +) + +func TestPrependWorkspacePrefix(t *testing.T) { + testCases := []struct { + path string + expected string + }{ + { + path: "/Users/test", + expected: "/Workspace/Users/test", + }, + { + path: "/Shared/test", + expected: "/Workspace/Shared/test", + }, + { + path: "/Workspace/Users/test", + expected: "/Workspace/Users/test", + }, + { + path: "/Volumes/Users/test", + expected: "/Volumes/Users/test", + }, + } + + for _, tc := range testCases { + b := &bundle.Bundle{ + Config: config.Root{ + Workspace: config.Workspace{ + RootPath: tc.path, + ArtifactPath: tc.path, + FilePath: tc.path, + StatePath: tc.path, + }, + }, + } + + diags := bundle.Apply(context.Background(), b, PrependWorkspacePrefix()) + require.Empty(t, diags) + require.Equal(t, tc.expected, b.Config.Workspace.RootPath) + require.Equal(t, tc.expected, b.Config.Workspace.ArtifactPath) + require.Equal(t, tc.expected, b.Config.Workspace.FilePath) + require.Equal(t, tc.expected, b.Config.Workspace.StatePath) + } +} + +func TestPrependWorkspaceForDefaultConfig(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Bundle: config.Bundle{ + Name: "test", + Target: "dev", + }, + Workspace: config.Workspace{ + CurrentUser: &config.User{ + User: &iam.User{ + UserName: "jane@doe.com", + }, + }, + }, + }, + } + diags := bundle.Apply(context.Background(), b, bundle.Seq(DefineDefaultWorkspaceRoot(), ExpandWorkspaceRoot(), DefineDefaultWorkspacePaths(), PrependWorkspacePrefix())) + require.Empty(t, diags) + require.Equal(t, "/Workspace/Users/jane@doe.com/.bundle/test/dev", b.Config.Workspace.RootPath) + require.Equal(t, "/Workspace/Users/jane@doe.com/.bundle/test/dev/artifacts", b.Config.Workspace.ArtifactPath) + require.Equal(t, "/Workspace/Users/jane@doe.com/.bundle/test/dev/files", b.Config.Workspace.FilePath) + require.Equal(t, "/Workspace/Users/jane@doe.com/.bundle/test/dev/state", b.Config.Workspace.StatePath) +} diff --git a/bundle/config/mutator/rewrite_workspace_prefix.go b/bundle/config/mutator/rewrite_workspace_prefix.go new file mode 100644 index 000000000..8a39ee8a1 --- /dev/null +++ b/bundle/config/mutator/rewrite_workspace_prefix.go @@ -0,0 +1,72 @@ +package mutator + +import ( + "context" + "fmt" + "strings" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +type rewriteWorkspacePrefix struct{} + +// RewriteWorkspacePrefix finds any strings in bundle configration that have +// workspace prefix plus workspace path variable used and removes workspace prefix from it. +func RewriteWorkspacePrefix() bundle.Mutator { + return &rewriteWorkspacePrefix{} +} + +func (m *rewriteWorkspacePrefix) Name() string { + return "RewriteWorkspacePrefix" +} + +func (m *rewriteWorkspacePrefix) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + diags := diag.Diagnostics{} + paths := map[string]string{ + "/Workspace/${workspace.root_path}": "${workspace.root_path}", + "/Workspace${workspace.root_path}": "${workspace.root_path}", + "/Workspace/${workspace.file_path}": "${workspace.file_path}", + "/Workspace${workspace.file_path}": "${workspace.file_path}", + "/Workspace/${workspace.artifact_path}": "${workspace.artifact_path}", + "/Workspace${workspace.artifact_path}": "${workspace.artifact_path}", + "/Workspace/${workspace.state_path}": "${workspace.state_path}", + "/Workspace${workspace.state_path}": "${workspace.state_path}", + } + + err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) { + // Walk through the bundle configuration, check all the string leafs and + // see if any of the prefixes are used in the remote path. + return dyn.Walk(root, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + vv, ok := v.AsString() + if !ok { + return v, nil + } + + for path, replacePath := range paths { + if strings.Contains(vv, path) { + newPath := strings.Replace(vv, path, replacePath, 1) + diags = append(diags, diag.Diagnostic{ + Severity: diag.Warning, + Summary: fmt.Sprintf("substring %q found in %q. Please update this to %q.", path, vv, newPath), + Detail: "For more information, please refer to: https://docs.databricks.com/en/release-notes/dev-tools/bundles.html#workspace-paths", + Locations: v.Locations(), + Paths: []dyn.Path{p}, + }) + + // Remove the workspace prefix from the string. + return dyn.NewValue(newPath, v.Locations()), nil + } + } + + return v, nil + }) + }) + + if err != nil { + return diag.FromErr(err) + } + + return diags +} diff --git a/bundle/config/mutator/rewrite_workspace_prefix_test.go b/bundle/config/mutator/rewrite_workspace_prefix_test.go new file mode 100644 index 000000000..d75ec89db --- /dev/null +++ b/bundle/config/mutator/rewrite_workspace_prefix_test.go @@ -0,0 +1,85 @@ +package mutator + +import ( + "context" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/databricks-sdk-go/service/compute" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/stretchr/testify/require" +) + +func TestNoWorkspacePrefixUsed(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Workspace: config.Workspace{ + RootPath: "/Workspace/Users/test", + ArtifactPath: "/Workspace/Users/test/artifacts", + FilePath: "/Workspace/Users/test/files", + StatePath: "/Workspace/Users/test/state", + }, + + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "test_job": { + JobSettings: &jobs.JobSettings{ + Tasks: []jobs.Task{ + { + SparkPythonTask: &jobs.SparkPythonTask{ + PythonFile: "/Workspace/${workspace.root_path}/file1.py", + }, + }, + { + NotebookTask: &jobs.NotebookTask{ + NotebookPath: "/Workspace${workspace.file_path}/notebook1", + }, + Libraries: []compute.Library{ + { + Jar: "/Workspace/${workspace.artifact_path}/jar1.jar", + }, + }, + }, + { + NotebookTask: &jobs.NotebookTask{ + NotebookPath: "${workspace.file_path}/notebook2", + }, + Libraries: []compute.Library{ + { + Jar: "${workspace.artifact_path}/jar2.jar", + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + diags := bundle.Apply(context.Background(), b, RewriteWorkspacePrefix()) + require.Len(t, diags, 3) + + expectedErrors := map[string]bool{ + `substring "/Workspace/${workspace.root_path}" found in "/Workspace/${workspace.root_path}/file1.py". Please update this to "${workspace.root_path}/file1.py".`: true, + `substring "/Workspace${workspace.file_path}" found in "/Workspace${workspace.file_path}/notebook1". Please update this to "${workspace.file_path}/notebook1".`: true, + `substring "/Workspace/${workspace.artifact_path}" found in "/Workspace/${workspace.artifact_path}/jar1.jar". Please update this to "${workspace.artifact_path}/jar1.jar".`: true, + } + + for _, d := range diags { + require.Equal(t, d.Severity, diag.Warning) + require.Contains(t, expectedErrors, d.Summary) + delete(expectedErrors, d.Summary) + } + + require.Equal(t, "${workspace.root_path}/file1.py", b.Config.Resources.Jobs["test_job"].JobSettings.Tasks[0].SparkPythonTask.PythonFile) + require.Equal(t, "${workspace.file_path}/notebook1", b.Config.Resources.Jobs["test_job"].JobSettings.Tasks[1].NotebookTask.NotebookPath) + require.Equal(t, "${workspace.artifact_path}/jar1.jar", b.Config.Resources.Jobs["test_job"].JobSettings.Tasks[1].Libraries[0].Jar) + require.Equal(t, "${workspace.file_path}/notebook2", b.Config.Resources.Jobs["test_job"].JobSettings.Tasks[2].NotebookTask.NotebookPath) + require.Equal(t, "${workspace.artifact_path}/jar2.jar", b.Config.Resources.Jobs["test_job"].JobSettings.Tasks[2].Libraries[0].Jar) + +} diff --git a/bundle/config/workspace.go b/bundle/config/workspace.go index 878d07838..e64808ab3 100644 --- a/bundle/config/workspace.go +++ b/bundle/config/workspace.go @@ -47,7 +47,7 @@ type Workspace struct { // Remote workspace base path for deployment state, for artifacts, as synchronization target. // This defaults to "~/.bundle/${bundle.name}/${bundle.target}" where "~" expands to - // the current user's home directory in the workspace (e.g. `/Users/jane@doe.com`). + // the current user's home directory in the workspace (e.g. `/Workspace/Users/jane@doe.com`). RootPath string `json:"root_path,omitempty"` // Remote workspace path to synchronize local files to. diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index 93ce61b25..a41819c76 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -39,9 +39,16 @@ func Initialize() bundle.Mutator { mutator.MergePipelineClusters(), mutator.InitializeWorkspaceClient(), mutator.PopulateCurrentUser(), + mutator.DefineDefaultWorkspaceRoot(), mutator.ExpandWorkspaceRoot(), mutator.DefineDefaultWorkspacePaths(), + mutator.PrependWorkspacePrefix(), + + // This mutator needs to be run before variable interpolation because it + // searches for strings with variable references in them. + mutator.RewriteWorkspacePrefix(), + mutator.SetVariables(), // Intentionally placed before ResolveVariableReferencesInLookup, ResolveResourceReferences, // ResolveVariableReferencesInComplexVariables and ResolveVariableReferences. diff --git a/bundle/tests/pipeline_glob_paths_test.go b/bundle/tests/pipeline_glob_paths_test.go index c1c62cfb6..d17f0404f 100644 --- a/bundle/tests/pipeline_glob_paths_test.go +++ b/bundle/tests/pipeline_glob_paths_test.go @@ -11,7 +11,7 @@ func TestExpandPipelineGlobPaths(t *testing.T) { require.NoError(t, diags.Error()) require.Equal( t, - "/Users/user@domain.com/.bundle/pipeline_glob_paths/default/files/dlt/nyc_taxi_loader", + "/Workspace/Users/user@domain.com/.bundle/pipeline_glob_paths/default/files/dlt/nyc_taxi_loader", b.Config.Resources.Pipelines["nyc_taxi_pipeline"].Libraries[0].Notebook.Path, ) } diff --git a/bundle/tests/relative_path_translation_test.go b/bundle/tests/relative_path_translation_test.go index 199871d23..0f553ac3d 100644 --- a/bundle/tests/relative_path_translation_test.go +++ b/bundle/tests/relative_path_translation_test.go @@ -12,9 +12,9 @@ func TestRelativePathTranslationDefault(t *testing.T) { require.NoError(t, diags.Error()) t0 := b.Config.Resources.Jobs["job"].Tasks[0] - assert.Equal(t, "/remote/src/file1.py", t0.SparkPythonTask.PythonFile) + assert.Equal(t, "/Workspace/remote/src/file1.py", t0.SparkPythonTask.PythonFile) t1 := b.Config.Resources.Jobs["job"].Tasks[1] - assert.Equal(t, "/remote/src/file1.py", t1.SparkPythonTask.PythonFile) + assert.Equal(t, "/Workspace/remote/src/file1.py", t1.SparkPythonTask.PythonFile) } func TestRelativePathTranslationOverride(t *testing.T) { @@ -22,7 +22,7 @@ func TestRelativePathTranslationOverride(t *testing.T) { require.NoError(t, diags.Error()) t0 := b.Config.Resources.Jobs["job"].Tasks[0] - assert.Equal(t, "/remote/src/file2.py", t0.SparkPythonTask.PythonFile) + assert.Equal(t, "/Workspace/remote/src/file2.py", t0.SparkPythonTask.PythonFile) t1 := b.Config.Resources.Jobs["job"].Tasks[1] - assert.Equal(t, "/remote/src/file2.py", t1.SparkPythonTask.PythonFile) + assert.Equal(t, "/Workspace/remote/src/file2.py", t1.SparkPythonTask.PythonFile) } diff --git a/internal/bundle/deploy_test.go b/internal/bundle/deploy_test.go index 736c880db..885435855 100644 --- a/internal/bundle/deploy_test.go +++ b/internal/bundle/deploy_test.go @@ -236,7 +236,7 @@ func TestAccDeployBasicBundleLogs(t *testing.T) { stdout, stderr := blackBoxRun(t, root, "bundle", "deploy") assert.Equal(t, strings.Join([]string{ - fmt.Sprintf("Uploading bundle files to /Users/%s/.bundle/%s/files...", currentUser.UserName, uniqueId), + fmt.Sprintf("Uploading bundle files to /Workspace/Users/%s/.bundle/%s/files...", currentUser.UserName, uniqueId), "Deploying resources...", "Updating deployment state...", "Deployment complete!\n", diff --git a/internal/bundle/helpers.go b/internal/bundle/helpers.go index 3547c1755..b8c81a8d2 100644 --- a/internal/bundle/helpers.go +++ b/internal/bundle/helpers.go @@ -114,7 +114,7 @@ func getBundleRemoteRootPath(w *databricks.WorkspaceClient, t *testing.T, unique // Compute root path for the bundle deployment me, err := w.CurrentUser.Me(context.Background()) require.NoError(t, err) - root := fmt.Sprintf("/Users/%s/.bundle/%s", me.UserName, uniqueId) + root := fmt.Sprintf("/Workspace/Users/%s/.bundle/%s", me.UserName, uniqueId) return root } diff --git a/libs/template/templates/dbt-sql/template/{{.project_name}}/databricks.yml.tmpl b/libs/template/templates/dbt-sql/template/{{.project_name}}/databricks.yml.tmpl index f96ce4fe6..5594749a9 100644 --- a/libs/template/templates/dbt-sql/template/{{.project_name}}/databricks.yml.tmpl +++ b/libs/template/templates/dbt-sql/template/{{.project_name}}/databricks.yml.tmpl @@ -24,8 +24,8 @@ targets: mode: production workspace: host: {{workspace_host}} - # We explicitly specify /Users/{{user_name}} to make sure we only have a single copy. - root_path: /Users/{{user_name}}/.bundle/${bundle.name}/${bundle.target} + # We explicitly specify /Workspace/Users/{{user_name}} to make sure we only have a single copy. + root_path: /Workspace/Users/{{user_name}}/.bundle/${bundle.name}/${bundle.target} permissions: - {{if is_service_principal}}service_principal{{else}}user{{end}}_name: {{user_name}} level: CAN_MANAGE diff --git a/libs/template/templates/default-python/template/{{.project_name}}/databricks.yml.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/databricks.yml.tmpl index 8544dc834..c42b822a8 100644 --- a/libs/template/templates/default-python/template/{{.project_name}}/databricks.yml.tmpl +++ b/libs/template/templates/default-python/template/{{.project_name}}/databricks.yml.tmpl @@ -21,8 +21,8 @@ targets: mode: production workspace: host: {{workspace_host}} - # We explicitly specify /Users/{{user_name}} to make sure we only have a single copy. - root_path: /Users/{{user_name}}/.bundle/${bundle.name}/${bundle.target} + # We explicitly specify /Workspace/Users/{{user_name}} to make sure we only have a single copy. + root_path: /Workspace/Users/{{user_name}}/.bundle/${bundle.name}/${bundle.target} permissions: - {{if is_service_principal}}service_principal{{else}}user{{end}}_name: {{user_name}} level: CAN_MANAGE diff --git a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.pipeline.yml.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.pipeline.yml.tmpl index bf4690461..50e5ad97c 100644 --- a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.pipeline.yml.tmpl +++ b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.pipeline.yml.tmpl @@ -15,4 +15,4 @@ resources: path: ../src/dlt_pipeline.ipynb configuration: - bundle.sourcePath: /Workspace/${workspace.file_path}/src + bundle.sourcePath: ${workspace.file_path}/src diff --git a/libs/template/templates/default-sql/template/{{.project_name}}/databricks.yml.tmpl b/libs/template/templates/default-sql/template/{{.project_name}}/databricks.yml.tmpl index 55c1aae4a..51d03e99a 100644 --- a/libs/template/templates/default-sql/template/{{.project_name}}/databricks.yml.tmpl +++ b/libs/template/templates/default-sql/template/{{.project_name}}/databricks.yml.tmpl @@ -41,8 +41,8 @@ targets: mode: production workspace: host: {{workspace_host}} - # We explicitly specify /Users/{{user_name}} to make sure we only have a single copy. - root_path: /Users/{{user_name}}/.bundle/${bundle.name}/${bundle.target} + # We explicitly specify /Workspace/Users/{{user_name}} to make sure we only have a single copy. + root_path: /Workspace/Users/{{user_name}}/.bundle/${bundle.name}/${bundle.target} variables: warehouse_id: {{index ((regexp "[^/]+$").FindStringSubmatch .http_path) 0}} catalog: {{.default_catalog}} From bca9c2eda4cf66d1baaaf78a11abf67d5e587503 Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Mon, 7 Oct 2024 14:46:20 +0530 Subject: [PATCH 29/34] Add validation for files with a `.(resource-name).yml` extension (#1780) ## Changes We want to encourage a pattern of specifying only a single resource in a YAML file when the `.(resource-type).yml` extension is used (for example, `.job.yml`). This convention could allow us to bijectively map a resource YAML file to its corresponding resource in the Databricks workspace. This PR: 1. Emits a recommendation diagnostic when we detect this convention is being violated. We can promote this to a warning when we want to encourage this pattern more strongly. 2. Visualises the recommendation diagnostics in the `bundle validate` command. **NOTE:** While this PR also shows the recommendation for `.yaml` files, we do not encourage users to use this extension. We only support it here since it's part of the YAML standard and some existing users might already be using `.yaml`. ## Tests Unit tests and manually. Here's what an example output looks like: ``` Recommendation: define a single job in a file with the .job.yml extension. at resources.jobs.bar resources.jobs.foo in foo.job.yml:13:7 foo.job.yml:5:7 The following resources are defined or configured in this file: - bar (job) - foo (job) ``` --------- Co-authored-by: Lennart Kats (databricks) --- bundle/config/loader/entry_point_test.go | 2 +- bundle/config/loader/process_include.go | 130 ++++++++++++ bundle/config/loader/process_include_test.go | 185 +++++++++++++++++- .../testdata/{ => basic}/databricks.yml | 0 .../loader/testdata/{ => basic}/host.yml | 0 .../format_match/job_and_pipeline.yml | 11 ++ .../format_match/multiple_resources.yml | 43 ++++ .../testdata/format_match/one_job.job.yml | 11 ++ .../format_match/one_pipeline.pipeline.yaml | 4 + .../loader/testdata/format_match/two_job.yml | 7 + .../job_and_pipeline.experiment.yml | 11 ++ .../format_not_match/job_and_pipeline.job.yml | 11 ++ ...tiple_resources.model_serving_endpoint.yml | 43 ++++ .../second_job_in_target.job.yml | 11 ++ .../format_not_match/single_job.pipeline.yaml | 11 ++ .../format_not_match/two_jobs.job.yml | 7 + .../two_jobs_in_target.job.yml | 8 + bundle/config/resources.go | 19 ++ bundle/config/resources_test.go | 16 ++ bundle/render/render_text_output.go | 34 +++- bundle/render/render_text_output_test.go | 164 +++++++++++++++- libs/diag/severity.go | 1 + 22 files changed, 721 insertions(+), 8 deletions(-) rename bundle/config/loader/testdata/{ => basic}/databricks.yml (100%) rename bundle/config/loader/testdata/{ => basic}/host.yml (100%) create mode 100644 bundle/config/loader/testdata/format_match/job_and_pipeline.yml create mode 100644 bundle/config/loader/testdata/format_match/multiple_resources.yml create mode 100644 bundle/config/loader/testdata/format_match/one_job.job.yml create mode 100644 bundle/config/loader/testdata/format_match/one_pipeline.pipeline.yaml create mode 100644 bundle/config/loader/testdata/format_match/two_job.yml create mode 100644 bundle/config/loader/testdata/format_not_match/job_and_pipeline.experiment.yml create mode 100644 bundle/config/loader/testdata/format_not_match/job_and_pipeline.job.yml create mode 100644 bundle/config/loader/testdata/format_not_match/multiple_resources.model_serving_endpoint.yml create mode 100644 bundle/config/loader/testdata/format_not_match/second_job_in_target.job.yml create mode 100644 bundle/config/loader/testdata/format_not_match/single_job.pipeline.yaml create mode 100644 bundle/config/loader/testdata/format_not_match/two_jobs.job.yml create mode 100644 bundle/config/loader/testdata/format_not_match/two_jobs_in_target.job.yml diff --git a/bundle/config/loader/entry_point_test.go b/bundle/config/loader/entry_point_test.go index 406b9b67c..0723c056c 100644 --- a/bundle/config/loader/entry_point_test.go +++ b/bundle/config/loader/entry_point_test.go @@ -18,7 +18,7 @@ func TestEntryPointNoRootPath(t *testing.T) { func TestEntryPoint(t *testing.T) { b := &bundle.Bundle{ - BundleRootPath: "testdata", + BundleRootPath: "testdata/basic", } diags := bundle.Apply(context.Background(), b, loader.EntryPoint()) require.NoError(t, diags.Error()) diff --git a/bundle/config/loader/process_include.go b/bundle/config/loader/process_include.go index 7cf9a17d7..f82f5db1e 100644 --- a/bundle/config/loader/process_include.go +++ b/bundle/config/loader/process_include.go @@ -3,12 +3,135 @@ package loader import ( "context" "fmt" + "slices" + "sort" + "strings" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" ) +func validateFileFormat(configRoot dyn.Value, filePath string) diag.Diagnostics { + for _, resourceDescription := range config.SupportedResources() { + singularName := resourceDescription.SingularName + + for _, yamlExt := range []string{"yml", "yaml"} { + ext := fmt.Sprintf(".%s.%s", singularName, yamlExt) + if strings.HasSuffix(filePath, ext) { + return validateSingleResourceDefined(configRoot, ext, singularName) + } + } + } + + return nil +} + +func validateSingleResourceDefined(configRoot dyn.Value, ext, typ string) diag.Diagnostics { + type resource struct { + path dyn.Path + value dyn.Value + typ string + key string + } + + resources := []resource{} + supportedResources := config.SupportedResources() + + // Gather all resources defined in the resources block. + _, err := dyn.MapByPattern( + configRoot, + dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), + func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + // The key for the resource, e.g. "my_job" for jobs.my_job. + k := p[2].Key() + // The type of the resource, e.g. "job" for jobs.my_job. + typ := supportedResources[p[1].Key()].SingularName + + resources = append(resources, resource{path: p, value: v, typ: typ, key: k}) + return v, nil + }) + if err != nil { + return diag.FromErr(err) + } + + // Gather all resources defined in a target block. + _, err = dyn.MapByPattern( + configRoot, + dyn.NewPattern(dyn.Key("targets"), dyn.AnyKey(), dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), + func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + // The key for the resource, e.g. "my_job" for jobs.my_job. + k := p[4].Key() + // The type of the resource, e.g. "job" for jobs.my_job. + typ := supportedResources[p[3].Key()].SingularName + + resources = append(resources, resource{path: p, value: v, typ: typ, key: k}) + return v, nil + }) + if err != nil { + return diag.FromErr(err) + } + + typeMatch := true + seenKeys := map[string]struct{}{} + for _, rr := range resources { + // case: The resource is not of the correct type. + if rr.typ != typ { + typeMatch = false + break + } + + seenKeys[rr.key] = struct{}{} + } + + // Format matches. There's at most one resource defined in the file. + // The resource is also of the correct type. + if typeMatch && len(seenKeys) <= 1 { + return nil + } + + detail := strings.Builder{} + detail.WriteString("The following resources are defined or configured in this file:\n") + lines := []string{} + for _, r := range resources { + lines = append(lines, fmt.Sprintf(" - %s (%s)\n", r.key, r.typ)) + } + // Sort the lines to print to make the output deterministic. + sort.Strings(lines) + // Compact the lines before writing them to the message to remove any duplicate lines. + // This is needed because we do not dedup earlier when gathering the resources + // and it's valid to define the same resource in both the resources and targets block. + lines = slices.Compact(lines) + for _, l := range lines { + detail.WriteString(l) + } + + locations := []dyn.Location{} + paths := []dyn.Path{} + for _, rr := range resources { + locations = append(locations, rr.value.Locations()...) + paths = append(paths, rr.path) + } + // Sort the locations and paths to make the output deterministic. + sort.Slice(locations, func(i, j int) bool { + return locations[i].String() < locations[j].String() + }) + sort.Slice(paths, func(i, j int) bool { + return paths[i].String() < paths[j].String() + }) + + return diag.Diagnostics{ + { + Severity: diag.Recommendation, + Summary: fmt.Sprintf("define a single %s in a file with the %s extension.", strings.ReplaceAll(typ, "_", " "), ext), + Detail: detail.String(), + Locations: locations, + Paths: paths, + }, + } +} + type processInclude struct { fullPath string relPath string @@ -31,6 +154,13 @@ func (m *processInclude) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnos if diags.HasError() { return diags } + + // Add any diagnostics associated with the file format. + diags = append(diags, validateFileFormat(this.Value(), m.relPath)...) + if diags.HasError() { + return diags + } + err := b.Config.Merge(this) if err != nil { diags = diags.Extend(diag.FromErr(err)) diff --git a/bundle/config/loader/process_include_test.go b/bundle/config/loader/process_include_test.go index 2ccd84b31..66c695e17 100644 --- a/bundle/config/loader/process_include_test.go +++ b/bundle/config/loader/process_include_test.go @@ -8,13 +8,15 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/loader" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestProcessInclude(t *testing.T) { b := &bundle.Bundle{ - BundleRootPath: "testdata", + BundleRootPath: "testdata/basic", Config: config.Root{ Workspace: config.Workspace{ Host: "foo", @@ -33,3 +35,184 @@ func TestProcessInclude(t *testing.T) { require.NoError(t, diags.Error()) assert.Equal(t, "bar", b.Config.Workspace.Host) } + +func TestProcessIncludeFormatMatch(t *testing.T) { + for _, fileName := range []string{ + "one_job.job.yml", + "one_pipeline.pipeline.yaml", + "two_job.yml", + "job_and_pipeline.yml", + "multiple_resources.yml", + } { + t.Run(fileName, func(t *testing.T) { + b := &bundle.Bundle{ + BundleRootPath: "testdata/format_match", + Config: config.Root{ + Bundle: config.Bundle{ + Name: "format_test", + }, + }, + } + + m := loader.ProcessInclude(filepath.Join(b.BundleRootPath, fileName), fileName) + diags := bundle.Apply(context.Background(), b, m) + assert.Empty(t, diags) + }) + } +} + +func TestProcessIncludeFormatNotMatch(t *testing.T) { + for fileName, expectedDiags := range map[string]diag.Diagnostics{ + "single_job.pipeline.yaml": { + { + Severity: diag.Recommendation, + Summary: "define a single pipeline in a file with the .pipeline.yaml extension.", + Detail: "The following resources are defined or configured in this file:\n - job1 (job)\n", + Locations: []dyn.Location{ + {File: filepath.FromSlash("testdata/format_not_match/single_job.pipeline.yaml"), Line: 11, Column: 11}, + {File: filepath.FromSlash("testdata/format_not_match/single_job.pipeline.yaml"), Line: 4, Column: 7}, + }, + Paths: []dyn.Path{ + dyn.MustPathFromString("resources.jobs.job1"), + dyn.MustPathFromString("targets.target1.resources.jobs.job1"), + }, + }, + }, + "job_and_pipeline.job.yml": { + { + Severity: diag.Recommendation, + Summary: "define a single job in a file with the .job.yml extension.", + Detail: "The following resources are defined or configured in this file:\n - job1 (job)\n - pipeline1 (pipeline)\n", + Locations: []dyn.Location{ + {File: filepath.FromSlash("testdata/format_not_match/job_and_pipeline.job.yml"), Line: 11, Column: 11}, + {File: filepath.FromSlash("testdata/format_not_match/job_and_pipeline.job.yml"), Line: 4, Column: 7}, + }, + Paths: []dyn.Path{ + dyn.MustPathFromString("resources.pipelines.pipeline1"), + dyn.MustPathFromString("targets.target1.resources.jobs.job1"), + }, + }, + }, + "job_and_pipeline.experiment.yml": { + { + Severity: diag.Recommendation, + Summary: "define a single experiment in a file with the .experiment.yml extension.", + Detail: "The following resources are defined or configured in this file:\n - job1 (job)\n - pipeline1 (pipeline)\n", + Locations: []dyn.Location{ + {File: filepath.FromSlash("testdata/format_not_match/job_and_pipeline.experiment.yml"), Line: 11, Column: 11}, + {File: filepath.FromSlash("testdata/format_not_match/job_and_pipeline.experiment.yml"), Line: 4, Column: 7}, + }, + Paths: []dyn.Path{ + dyn.MustPathFromString("resources.pipelines.pipeline1"), + dyn.MustPathFromString("targets.target1.resources.jobs.job1"), + }, + }, + }, + "two_jobs.job.yml": { + { + Severity: diag.Recommendation, + Summary: "define a single job in a file with the .job.yml extension.", + Detail: "The following resources are defined or configured in this file:\n - job1 (job)\n - job2 (job)\n", + Locations: []dyn.Location{ + {File: filepath.FromSlash("testdata/format_not_match/two_jobs.job.yml"), Line: 4, Column: 7}, + {File: filepath.FromSlash("testdata/format_not_match/two_jobs.job.yml"), Line: 7, Column: 7}, + }, + Paths: []dyn.Path{ + dyn.MustPathFromString("resources.jobs.job1"), + dyn.MustPathFromString("resources.jobs.job2"), + }, + }, + }, + "second_job_in_target.job.yml": { + { + Severity: diag.Recommendation, + Summary: "define a single job in a file with the .job.yml extension.", + Detail: "The following resources are defined or configured in this file:\n - job1 (job)\n - job2 (job)\n", + Locations: []dyn.Location{ + {File: filepath.FromSlash("testdata/format_not_match/second_job_in_target.job.yml"), Line: 11, Column: 11}, + {File: filepath.FromSlash("testdata/format_not_match/second_job_in_target.job.yml"), Line: 4, Column: 7}, + }, + Paths: []dyn.Path{ + dyn.MustPathFromString("resources.jobs.job1"), + dyn.MustPathFromString("targets.target1.resources.jobs.job2"), + }, + }, + }, + "two_jobs_in_target.job.yml": { + { + Severity: diag.Recommendation, + Summary: "define a single job in a file with the .job.yml extension.", + Detail: "The following resources are defined or configured in this file:\n - job1 (job)\n - job2 (job)\n", + Locations: []dyn.Location{ + {File: filepath.FromSlash("testdata/format_not_match/two_jobs_in_target.job.yml"), Line: 6, Column: 11}, + {File: filepath.FromSlash("testdata/format_not_match/two_jobs_in_target.job.yml"), Line: 8, Column: 11}, + }, + Paths: []dyn.Path{ + dyn.MustPathFromString("targets.target1.resources.jobs.job1"), + dyn.MustPathFromString("targets.target1.resources.jobs.job2"), + }, + }, + }, + "multiple_resources.model_serving_endpoint.yml": { + { + Severity: diag.Recommendation, + Summary: "define a single model serving endpoint in a file with the .model_serving_endpoint.yml extension.", + Detail: `The following resources are defined or configured in this file: + - experiment1 (experiment) + - job1 (job) + - job2 (job) + - job3 (job) + - model1 (model) + - model_serving_endpoint1 (model_serving_endpoint) + - pipeline1 (pipeline) + - pipeline2 (pipeline) + - quality_monitor1 (quality_monitor) + - registered_model1 (registered_model) + - schema1 (schema) +`, + Locations: []dyn.Location{ + {File: filepath.FromSlash("testdata/format_not_match/multiple_resources.model_serving_endpoint.yml"), Line: 12, Column: 7}, + {File: filepath.FromSlash("testdata/format_not_match/multiple_resources.model_serving_endpoint.yml"), Line: 14, Column: 7}, + {File: filepath.FromSlash("testdata/format_not_match/multiple_resources.model_serving_endpoint.yml"), Line: 18, Column: 7}, + {File: filepath.FromSlash("testdata/format_not_match/multiple_resources.model_serving_endpoint.yml"), Line: 22, Column: 7}, + {File: filepath.FromSlash("testdata/format_not_match/multiple_resources.model_serving_endpoint.yml"), Line: 24, Column: 7}, + {File: filepath.FromSlash("testdata/format_not_match/multiple_resources.model_serving_endpoint.yml"), Line: 28, Column: 7}, + {File: filepath.FromSlash("testdata/format_not_match/multiple_resources.model_serving_endpoint.yml"), Line: 35, Column: 11}, + {File: filepath.FromSlash("testdata/format_not_match/multiple_resources.model_serving_endpoint.yml"), Line: 39, Column: 11}, + {File: filepath.FromSlash("testdata/format_not_match/multiple_resources.model_serving_endpoint.yml"), Line: 43, Column: 11}, + {File: filepath.FromSlash("testdata/format_not_match/multiple_resources.model_serving_endpoint.yml"), Line: 4, Column: 7}, + {File: filepath.FromSlash("testdata/format_not_match/multiple_resources.model_serving_endpoint.yml"), Line: 8, Column: 7}, + }, + Paths: []dyn.Path{ + dyn.MustPathFromString("resources.experiments.experiment1"), + dyn.MustPathFromString("resources.jobs.job1"), + dyn.MustPathFromString("resources.jobs.job2"), + dyn.MustPathFromString("resources.model_serving_endpoints.model_serving_endpoint1"), + dyn.MustPathFromString("resources.models.model1"), + dyn.MustPathFromString("resources.pipelines.pipeline1"), + dyn.MustPathFromString("resources.pipelines.pipeline2"), + dyn.MustPathFromString("resources.schemas.schema1"), + dyn.MustPathFromString("targets.target1.resources.jobs.job3"), + dyn.MustPathFromString("targets.target1.resources.quality_monitors.quality_monitor1"), + dyn.MustPathFromString("targets.target1.resources.registered_models.registered_model1"), + }, + }, + }, + } { + t.Run(fileName, func(t *testing.T) { + b := &bundle.Bundle{ + BundleRootPath: "testdata/format_not_match", + Config: config.Root{ + Bundle: config.Bundle{ + Name: "format_test", + }, + }, + } + + m := loader.ProcessInclude(filepath.Join(b.BundleRootPath, fileName), fileName) + diags := bundle.Apply(context.Background(), b, m) + require.Len(t, diags, 1) + assert.Equal(t, expectedDiags, diags) + }) + } +} diff --git a/bundle/config/loader/testdata/databricks.yml b/bundle/config/loader/testdata/basic/databricks.yml similarity index 100% rename from bundle/config/loader/testdata/databricks.yml rename to bundle/config/loader/testdata/basic/databricks.yml diff --git a/bundle/config/loader/testdata/host.yml b/bundle/config/loader/testdata/basic/host.yml similarity index 100% rename from bundle/config/loader/testdata/host.yml rename to bundle/config/loader/testdata/basic/host.yml diff --git a/bundle/config/loader/testdata/format_match/job_and_pipeline.yml b/bundle/config/loader/testdata/format_match/job_and_pipeline.yml new file mode 100644 index 000000000..0867fcae4 --- /dev/null +++ b/bundle/config/loader/testdata/format_match/job_and_pipeline.yml @@ -0,0 +1,11 @@ +resources: + pipelines: + pipeline1: + name: pipeline1 + +targets: + target1: + resources: + jobs: + job1: + name: job1 diff --git a/bundle/config/loader/testdata/format_match/multiple_resources.yml b/bundle/config/loader/testdata/format_match/multiple_resources.yml new file mode 100644 index 000000000..dc8e837c6 --- /dev/null +++ b/bundle/config/loader/testdata/format_match/multiple_resources.yml @@ -0,0 +1,43 @@ +resources: + experiments: + experiment1: + name: experiment1 + + model_serving_endpoints: + model_serving_endpoint1: + name: model_serving_endpoint1 + + jobs: + job1: + name: job1 + job2: + name: job2 + + models: + model1: + name: model1 + + pipelines: + pipeline1: + name: pipeline1 + pipeline2: + name: pipeline2 + + schemas: + schema1: + name: schema1 + +targets: + target1: + resources: + quality_monitors: + quality_monitor1: + baseline_table_name: quality_monitor1 + + jobs: + job3: + name: job3 + + registered_models: + registered_model1: + name: registered_model1 diff --git a/bundle/config/loader/testdata/format_match/one_job.job.yml b/bundle/config/loader/testdata/format_match/one_job.job.yml new file mode 100644 index 000000000..91af87cdc --- /dev/null +++ b/bundle/config/loader/testdata/format_match/one_job.job.yml @@ -0,0 +1,11 @@ +resources: + jobs: + job1: + name: job1 + +targets: + target1: + resources: + jobs: + job1: + description: job1 diff --git a/bundle/config/loader/testdata/format_match/one_pipeline.pipeline.yaml b/bundle/config/loader/testdata/format_match/one_pipeline.pipeline.yaml new file mode 100644 index 000000000..85cb0d7fc --- /dev/null +++ b/bundle/config/loader/testdata/format_match/one_pipeline.pipeline.yaml @@ -0,0 +1,4 @@ +resources: + pipelines: + pipeline1: + name: pipeline1 diff --git a/bundle/config/loader/testdata/format_match/two_job.yml b/bundle/config/loader/testdata/format_match/two_job.yml new file mode 100644 index 000000000..81ff90a75 --- /dev/null +++ b/bundle/config/loader/testdata/format_match/two_job.yml @@ -0,0 +1,7 @@ +resources: + jobs: + job1: + name: job1 + + job2: + name: job2 diff --git a/bundle/config/loader/testdata/format_not_match/job_and_pipeline.experiment.yml b/bundle/config/loader/testdata/format_not_match/job_and_pipeline.experiment.yml new file mode 100644 index 000000000..0867fcae4 --- /dev/null +++ b/bundle/config/loader/testdata/format_not_match/job_and_pipeline.experiment.yml @@ -0,0 +1,11 @@ +resources: + pipelines: + pipeline1: + name: pipeline1 + +targets: + target1: + resources: + jobs: + job1: + name: job1 diff --git a/bundle/config/loader/testdata/format_not_match/job_and_pipeline.job.yml b/bundle/config/loader/testdata/format_not_match/job_and_pipeline.job.yml new file mode 100644 index 000000000..0867fcae4 --- /dev/null +++ b/bundle/config/loader/testdata/format_not_match/job_and_pipeline.job.yml @@ -0,0 +1,11 @@ +resources: + pipelines: + pipeline1: + name: pipeline1 + +targets: + target1: + resources: + jobs: + job1: + name: job1 diff --git a/bundle/config/loader/testdata/format_not_match/multiple_resources.model_serving_endpoint.yml b/bundle/config/loader/testdata/format_not_match/multiple_resources.model_serving_endpoint.yml new file mode 100644 index 000000000..dc8e837c6 --- /dev/null +++ b/bundle/config/loader/testdata/format_not_match/multiple_resources.model_serving_endpoint.yml @@ -0,0 +1,43 @@ +resources: + experiments: + experiment1: + name: experiment1 + + model_serving_endpoints: + model_serving_endpoint1: + name: model_serving_endpoint1 + + jobs: + job1: + name: job1 + job2: + name: job2 + + models: + model1: + name: model1 + + pipelines: + pipeline1: + name: pipeline1 + pipeline2: + name: pipeline2 + + schemas: + schema1: + name: schema1 + +targets: + target1: + resources: + quality_monitors: + quality_monitor1: + baseline_table_name: quality_monitor1 + + jobs: + job3: + name: job3 + + registered_models: + registered_model1: + name: registered_model1 diff --git a/bundle/config/loader/testdata/format_not_match/second_job_in_target.job.yml b/bundle/config/loader/testdata/format_not_match/second_job_in_target.job.yml new file mode 100644 index 000000000..628b9879f --- /dev/null +++ b/bundle/config/loader/testdata/format_not_match/second_job_in_target.job.yml @@ -0,0 +1,11 @@ +resources: + jobs: + job1: + name: job1 + +targets: + target1: + resources: + jobs: + job2: + name: job2 diff --git a/bundle/config/loader/testdata/format_not_match/single_job.pipeline.yaml b/bundle/config/loader/testdata/format_not_match/single_job.pipeline.yaml new file mode 100644 index 000000000..91af87cdc --- /dev/null +++ b/bundle/config/loader/testdata/format_not_match/single_job.pipeline.yaml @@ -0,0 +1,11 @@ +resources: + jobs: + job1: + name: job1 + +targets: + target1: + resources: + jobs: + job1: + description: job1 diff --git a/bundle/config/loader/testdata/format_not_match/two_jobs.job.yml b/bundle/config/loader/testdata/format_not_match/two_jobs.job.yml new file mode 100644 index 000000000..81ff90a75 --- /dev/null +++ b/bundle/config/loader/testdata/format_not_match/two_jobs.job.yml @@ -0,0 +1,7 @@ +resources: + jobs: + job1: + name: job1 + + job2: + name: job2 diff --git a/bundle/config/loader/testdata/format_not_match/two_jobs_in_target.job.yml b/bundle/config/loader/testdata/format_not_match/two_jobs_in_target.job.yml new file mode 100644 index 000000000..3b489c1f7 --- /dev/null +++ b/bundle/config/loader/testdata/format_not_match/two_jobs_in_target.job.yml @@ -0,0 +1,8 @@ +targets: + target1: + resources: + jobs: + job1: + description: job1 + job2: + description: job2 diff --git a/bundle/config/resources.go b/bundle/config/resources.go index a3afb7fc3..dc51a7caf 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -59,3 +59,22 @@ func (r *Resources) FindResourceByConfigKey(key string) (ConfigResource, error) return found[0], nil } + +type ResourceDescription struct { + SingularName string +} + +// The keys of the map corresponds to the resource key in the bundle configuration. +func SupportedResources() map[string]ResourceDescription { + return map[string]ResourceDescription{ + "jobs": {SingularName: "job"}, + "pipelines": {SingularName: "pipeline"}, + "models": {SingularName: "model"}, + "experiments": {SingularName: "experiment"}, + "model_serving_endpoints": {SingularName: "model_serving_endpoint"}, + "registered_models": {SingularName: "registered_model"}, + "quality_monitors": {SingularName: "quality_monitor"}, + "schemas": {SingularName: "schema"}, + "clusters": {SingularName: "cluster"}, + } +} diff --git a/bundle/config/resources_test.go b/bundle/config/resources_test.go index 6860d73da..c1b76118c 100644 --- a/bundle/config/resources_test.go +++ b/bundle/config/resources_test.go @@ -3,6 +3,7 @@ package config import ( "encoding/json" "reflect" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -61,3 +62,18 @@ func TestCustomMarshallerIsImplemented(t *testing.T) { }, "Resource %s does not have a custom unmarshaller", field.Name) } } + +func TestSupportedResources(t *testing.T) { + expected := map[string]ResourceDescription{} + typ := reflect.TypeOf(Resources{}) + for i := 0; i < typ.NumField(); i++ { + field := typ.Field(i) + jsonTags := strings.Split(field.Tag.Get("json"), ",") + singularName := strings.TrimSuffix(jsonTags[0], "s") + expected[jsonTags[0]] = ResourceDescription{SingularName: singularName} + } + + // Please add your resource to the SupportedResources() function in resources.go + // if you are adding a new resource. + assert.Equal(t, expected, SupportedResources()) +} diff --git a/bundle/render/render_text_output.go b/bundle/render/render_text_output.go index e1fad98a3..56387c386 100644 --- a/bundle/render/render_text_output.go +++ b/bundle/render/render_text_output.go @@ -56,6 +56,20 @@ const warningTemplate = `{{ "Warning" | yellow }}: {{ .Summary }} ` +const recommendationTemplate = `{{ "Recommendation" | blue }}: {{ .Summary }} +{{- range $index, $element := .Paths }} + {{ if eq $index 0 }}at {{else}} {{ end}}{{ $element.String | green }} +{{- end }} +{{- range $index, $element := .Locations }} + {{ if eq $index 0 }}in {{else}} {{ end}}{{ $element.String | cyan }} +{{- end }} +{{- if .Detail }} + +{{ .Detail }} +{{- end }} + +` + const summaryTemplate = `{{- if .Name -}} Name: {{ .Name | bold }} {{- if .Target }} @@ -94,9 +108,20 @@ func buildTrailer(diags diag.Diagnostics) string { if warnings := len(diags.Filter(diag.Warning)); warnings > 0 { parts = append(parts, color.YellowString(pluralize(warnings, "warning", "warnings"))) } - if len(parts) > 0 { - return fmt.Sprintf("Found %s", strings.Join(parts, " and ")) - } else { + if recommendations := len(diags.Filter(diag.Recommendation)); recommendations > 0 { + parts = append(parts, color.BlueString(pluralize(recommendations, "recommendation", "recommendations"))) + } + switch { + case len(parts) >= 3: + first := strings.Join(parts[:len(parts)-1], ", ") + last := parts[len(parts)-1] + return fmt.Sprintf("Found %s, and %s", first, last) + case len(parts) == 2: + return fmt.Sprintf("Found %s and %s", parts[0], parts[1]) + case len(parts) == 1: + return fmt.Sprintf("Found %s", parts[0]) + default: + // No diagnostics to print. return color.GreenString("Validation OK!") } } @@ -130,6 +155,7 @@ func renderSummaryTemplate(out io.Writer, b *bundle.Bundle, diags diag.Diagnosti func renderDiagnostics(out io.Writer, b *bundle.Bundle, diags diag.Diagnostics) error { errorT := template.Must(template.New("error").Funcs(renderFuncMap).Parse(errorTemplate)) warningT := template.Must(template.New("warning").Funcs(renderFuncMap).Parse(warningTemplate)) + recommendationT := template.Must(template.New("recommendation").Funcs(renderFuncMap).Parse(recommendationTemplate)) // Print errors and warnings. for _, d := range diags { @@ -139,6 +165,8 @@ func renderDiagnostics(out io.Writer, b *bundle.Bundle, diags diag.Diagnostics) t = errorT case diag.Warning: t = warningT + case diag.Recommendation: + t = recommendationT } for i := range d.Locations { diff --git a/bundle/render/render_text_output_test.go b/bundle/render/render_text_output_test.go index 976f86e79..1a41fa01c 100644 --- a/bundle/render/render_text_output_test.go +++ b/bundle/render/render_text_output_test.go @@ -45,6 +45,19 @@ func TestRenderTextOutput(t *testing.T) { "\n" + "Found 1 error\n", }, + { + name: "nil bundle and 1 recommendation", + diags: diag.Diagnostics{ + { + Severity: diag.Recommendation, + Summary: "recommendation", + }, + }, + opts: RenderOptions{RenderSummaryTable: true}, + expected: "Recommendation: recommendation\n" + + "\n" + + "Found 1 recommendation\n", + }, { name: "bundle during 'load' and 1 error", bundle: loadingBundle, @@ -84,7 +97,7 @@ func TestRenderTextOutput(t *testing.T) { "Found 2 warnings\n", }, { - name: "bundle during 'load' and 2 errors, 1 warning with details", + name: "bundle during 'load' and 2 errors, 1 warning and 1 recommendation with details", bundle: loadingBundle, diags: diag.Diagnostics{ diag.Diagnostic{ @@ -105,6 +118,12 @@ func TestRenderTextOutput(t *testing.T) { Detail: "detail (3)", Locations: []dyn.Location{{File: "foo.py", Line: 3, Column: 1}}, }, + diag.Diagnostic{ + Severity: diag.Recommendation, + Summary: "recommendation (4)", + Detail: "detail (4)", + Locations: []dyn.Location{{File: "foo.py", Line: 4, Column: 1}}, + }, }, opts: RenderOptions{RenderSummaryTable: true}, expected: "Error: error (1)\n" + @@ -122,10 +141,114 @@ func TestRenderTextOutput(t *testing.T) { "\n" + "detail (3)\n" + "\n" + + "Recommendation: recommendation (4)\n" + + " in foo.py:4:1\n" + + "\n" + + "detail (4)\n" + + "\n" + "Name: test-bundle\n" + "Target: test-target\n" + "\n" + - "Found 2 errors and 1 warning\n", + "Found 2 errors, 1 warning, and 1 recommendation\n", + }, + { + name: "bundle during 'load' and 1 error and 1 warning", + bundle: loadingBundle, + diags: diag.Diagnostics{ + diag.Diagnostic{ + Severity: diag.Error, + Summary: "error (1)", + Detail: "detail (1)", + Locations: []dyn.Location{{File: "foo.py", Line: 1, Column: 1}}, + }, + diag.Diagnostic{ + Severity: diag.Warning, + Summary: "warning (2)", + Detail: "detail (2)", + Locations: []dyn.Location{{File: "foo.py", Line: 2, Column: 1}}, + }, + }, + opts: RenderOptions{RenderSummaryTable: true}, + expected: "Error: error (1)\n" + + " in foo.py:1:1\n" + + "\n" + + "detail (1)\n" + + "\n" + + "Warning: warning (2)\n" + + " in foo.py:2:1\n" + + "\n" + + "detail (2)\n" + + "\n" + + "Name: test-bundle\n" + + "Target: test-target\n" + + "\n" + + "Found 1 error and 1 warning\n", + }, + { + name: "bundle during 'load' and 1 errors, 2 warning and 2 recommendations with details", + bundle: loadingBundle, + diags: diag.Diagnostics{ + diag.Diagnostic{ + Severity: diag.Error, + Summary: "error (1)", + Detail: "detail (1)", + Locations: []dyn.Location{{File: "foo.py", Line: 1, Column: 1}}, + }, + diag.Diagnostic{ + Severity: diag.Warning, + Summary: "warning (2)", + Detail: "detail (2)", + Locations: []dyn.Location{{File: "foo.py", Line: 2, Column: 1}}, + }, + diag.Diagnostic{ + Severity: diag.Warning, + Summary: "warning (3)", + Detail: "detail (3)", + Locations: []dyn.Location{{File: "foo.py", Line: 3, Column: 1}}, + }, + diag.Diagnostic{ + Severity: diag.Recommendation, + Summary: "recommendation (4)", + Detail: "detail (4)", + Locations: []dyn.Location{{File: "foo.py", Line: 4, Column: 1}}, + }, + diag.Diagnostic{ + Severity: diag.Recommendation, + Summary: "recommendation (5)", + Detail: "detail (5)", + Locations: []dyn.Location{{File: "foo.py", Line: 5, Column: 1}}, + }, + }, + opts: RenderOptions{RenderSummaryTable: true}, + expected: "Error: error (1)\n" + + " in foo.py:1:1\n" + + "\n" + + "detail (1)\n" + + "\n" + + "Warning: warning (2)\n" + + " in foo.py:2:1\n" + + "\n" + + "detail (2)\n" + + "\n" + + "Warning: warning (3)\n" + + " in foo.py:3:1\n" + + "\n" + + "detail (3)\n" + + "\n" + + "Recommendation: recommendation (4)\n" + + " in foo.py:4:1\n" + + "\n" + + "detail (4)\n" + + "\n" + + "Recommendation: recommendation (5)\n" + + " in foo.py:5:1\n" + + "\n" + + "detail (5)\n" + + "\n" + + "Name: test-bundle\n" + + "Target: test-target\n" + + "\n" + + "Found 1 error, 2 warnings, and 2 recommendations\n", }, { name: "bundle during 'init'", @@ -158,7 +281,7 @@ func TestRenderTextOutput(t *testing.T) { "Validation OK!\n", }, { - name: "nil bundle without summary with 1 error and 1 warning", + name: "nil bundle without summary with 1 error, 1 warning and 1 recommendation", bundle: nil, diags: diag.Diagnostics{ diag.Diagnostic{ @@ -173,6 +296,12 @@ func TestRenderTextOutput(t *testing.T) { Detail: "detail (2)", Locations: []dyn.Location{{File: "foo.py", Line: 3, Column: 1}}, }, + diag.Diagnostic{ + Severity: diag.Recommendation, + Summary: "recommendation (3)", + Detail: "detail (3)", + Locations: []dyn.Location{{File: "foo.py", Line: 5, Column: 1}}, + }, }, opts: RenderOptions{RenderSummaryTable: false}, expected: "Error: error (1)\n" + @@ -184,6 +313,11 @@ func TestRenderTextOutput(t *testing.T) { " in foo.py:3:1\n" + "\n" + "detail (2)\n" + + "\n" + + "Recommendation: recommendation (3)\n" + + " in foo.py:5:1\n" + + "\n" + + "detail (3)\n" + "\n", }, } @@ -304,6 +438,30 @@ func TestRenderDiagnostics(t *testing.T) { "\n" + "'name' is required\n\n", }, + { + name: "recommendation with multiple paths and locations", + diags: diag.Diagnostics{ + { + Severity: diag.Recommendation, + Summary: "summary", + Detail: "detail", + Paths: []dyn.Path{ + dyn.MustPathFromString("resources.jobs.xxx"), + dyn.MustPathFromString("resources.jobs.yyy"), + }, + Locations: []dyn.Location{ + {File: "foo.yaml", Line: 1, Column: 2}, + {File: "bar.yaml", Line: 3, Column: 4}, + }, + }, + }, + expected: "Recommendation: summary\n" + + " at resources.jobs.xxx\n" + + " resources.jobs.yyy\n" + + " in foo.yaml:1:2\n" + + " bar.yaml:3:4\n\n" + + "detail\n\n", + }, } for _, tc := range testCases { diff --git a/libs/diag/severity.go b/libs/diag/severity.go index d25c12806..0e88085f5 100644 --- a/libs/diag/severity.go +++ b/libs/diag/severity.go @@ -6,4 +6,5 @@ const ( Error Severity = iota Warning Info + Recommendation ) From 589e5ff1a44b3632d69fa5d2b992530a54ab7247 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Oct 2024 14:58:25 +0200 Subject: [PATCH 30/34] Bump golang.org/x/term from 0.24.0 to 0.25.0 (#1811) Bumps [golang.org/x/term](https://github.com/golang/term) from 0.24.0 to 0.25.0.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golang.org/x/term&package-manager=go_modules&previous-version=0.24.0&new-version=0.25.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 9141274c2..ae56b513a 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( golang.org/x/mod v0.21.0 golang.org/x/oauth2 v0.23.0 golang.org/x/sync v0.8.0 - golang.org/x/term v0.24.0 + golang.org/x/term v0.25.0 golang.org/x/text v0.18.0 gopkg.in/ini.v1 v1.67.0 // Apache 2.0 gopkg.in/yaml.v3 v3.0.1 @@ -64,7 +64,7 @@ require ( go.opentelemetry.io/otel/trace v1.24.0 // indirect golang.org/x/crypto v0.24.0 // indirect golang.org/x/net v0.26.0 // indirect - golang.org/x/sys v0.25.0 // indirect + golang.org/x/sys v0.26.0 // indirect golang.org/x/time v0.5.0 // indirect google.golang.org/api v0.182.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e // indirect diff --git a/go.sum b/go.sum index 177707a50..e797ac549 100644 --- a/go.sum +++ b/go.sum @@ -212,10 +212,10 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= -golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM= -golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24= +golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= From 02bfd397eff71380b3b9e466f6a821eee6794fb5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Oct 2024 13:14:02 +0000 Subject: [PATCH 31/34] Bump golang.org/x/text from 0.18.0 to 0.19.0 (#1812) Bumps [golang.org/x/text](https://github.com/golang/text) from 0.18.0 to 0.19.0.
Commits
  • 3043346 x/text: Correct examples in number/doc
  • 38a95c2 all: fix some comments
  • 20097e4 all: fix printf(var) mistakes detected by latest printf checker
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golang.org/x/text&package-manager=go_modules&previous-version=0.18.0&new-version=0.19.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index ae56b513a..c029e5da0 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( golang.org/x/oauth2 v0.23.0 golang.org/x/sync v0.8.0 golang.org/x/term v0.25.0 - golang.org/x/text v0.18.0 + golang.org/x/text v0.19.0 gopkg.in/ini.v1 v1.67.0 // Apache 2.0 gopkg.in/yaml.v3 v3.0.1 ) diff --git a/go.sum b/go.sum index e797ac549..9f05b1451 100644 --- a/go.sum +++ b/go.sum @@ -218,8 +218,8 @@ golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24= golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= -golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= +golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From 7a2141fc5ba1af695ba55aeac524bf80644d7606 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Oct 2024 15:21:05 +0200 Subject: [PATCH 32/34] Bump github.com/databricks/databricks-sdk-go from 0.47.0 to 0.48.0 (#1810) Bumps [github.com/databricks/databricks-sdk-go](https://github.com/databricks/databricks-sdk-go) from 0.47.0 to 0.48.0.
Release notes

Sourced from github.com/databricks/databricks-sdk-go's releases.

v0.48.0

Internal Changes

  • Update SDK to latest OpenAPI spec (#1057).

Note: This release contains breaking changes, please see the API changes below for more details.

API Changes:

OpenAPI SHA: 0c86ea6dbd9a730c24ff0d4e509603e476955ac5, Date: 2024-10-02

Changelog

Sourced from github.com/databricks/databricks-sdk-go's changelog.

[Release] Release v0.48.0

Internal Changes

  • Update SDK to latest OpenAPI spec (#1057).

Note: This release contains breaking changes, please see the API changes below for more details.

API Changes:

OpenAPI SHA: 0c86ea6dbd9a730c24ff0d4e509603e476955ac5, Date: 2024-10-02

Commits

Most Recent Ignore Conditions Applied to This Pull Request | Dependency Name | Ignore Conditions | | --- | --- | | github.com/databricks/databricks-sdk-go | [>= 0.28.a, < 0.29] |
[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/databricks/databricks-sdk-go&package-manager=go_modules&previous-version=0.47.0&new-version=0.48.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Andrew Nester --- .codegen/_openapi_sha | 2 +- bundle/schema/jsonschema.json | 235 +++++++++++++++++- cmd/workspace/apps/apps.go | 2 + .../git-credentials/git-credentials.go | 62 ++--- cmd/workspace/pipelines/pipelines.go | 1 + cmd/workspace/repos/overrides.go | 4 +- cmd/workspace/repos/repos.go | 35 +-- go.mod | 2 +- go.sum | 4 +- internal/helpers.go | 2 +- internal/locker_test.go | 2 +- internal/repos_test.go | 2 +- internal/sync_test.go | 2 +- 13 files changed, 296 insertions(+), 59 deletions(-) diff --git a/.codegen/_openapi_sha b/.codegen/_openapi_sha index ffd6f58dd..303c78553 100644 --- a/.codegen/_openapi_sha +++ b/.codegen/_openapi_sha @@ -1 +1 @@ -6f6b1371e640f2dfeba72d365ac566368656f6b6 \ No newline at end of file +0c86ea6dbd9a730c24ff0d4e509603e476955ac5 \ No newline at end of file diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index afdf9fb9e..ae209c01c 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -402,6 +402,10 @@ { "type": "object", "properties": { + "ai_gateway": { + "description": "The AI Gateway configuration for the serving endpoint. NOTE: only external model endpoints are supported as of now.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayConfig" + }, "config": { "description": "The core config of the serving endpoint.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.EndpointCoreConfigInput" @@ -472,6 +476,10 @@ { "type": "object", "properties": { + "budget_policy_id": { + "description": "Budget policy of this pipeline.", + "$ref": "#/$defs/string" + }, "catalog": { "description": "A catalog in Unity Catalog to publish data from this pipeline to. If `target` is specified, tables in this pipeline are published to a `target` schema inside `catalog` (for example, `catalog`.`target`.`table`). If `target` is not specified, no data is published to Unity Catalog.", "$ref": "#/$defs/string" @@ -539,6 +547,10 @@ "description": "Whether Photon is enabled for this pipeline.", "$ref": "#/$defs/bool" }, + "schema": { + "description": "The default schema (database) where tables are read from or published to. The presence of this field implies that the pipeline is in direct publishing mode.", + "$ref": "#/$defs/string" + }, "serverless": { "description": "Whether serverless compute is enabled for this pipeline.", "$ref": "#/$defs/bool" @@ -1206,6 +1218,9 @@ "profile": { "$ref": "#/$defs/string" }, + "resource_path": { + "$ref": "#/$defs/string" + }, "root_path": { "$ref": "#/$defs/string" }, @@ -2632,7 +2647,7 @@ "type": "object", "properties": { "no_alert_for_skipped_runs": { - "description": "If true, do not send email to recipients specified in `on_failure` if the run is skipped.", + "description": "If true, do not send email to recipients specified in `on_failure` if the run is skipped.\nThis field is `deprecated`. Please use the `notification_settings.no_alert_for_skipped_runs` field.", "$ref": "#/$defs/bool" }, "on_duration_warning_threshold_exceeded": { @@ -3073,6 +3088,7 @@ "$ref": "#/$defs/map/string" }, "pipeline_params": { + "description": "Controls whether the pipeline should perform a full refresh", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.PipelineParams" }, "python_named_params": { @@ -3547,7 +3563,7 @@ "type": "object", "properties": { "no_alert_for_skipped_runs": { - "description": "If true, do not send email to recipients specified in `on_failure` if the run is skipped.", + "description": "If true, do not send email to recipients specified in `on_failure` if the run is skipped.\nThis field is `deprecated`. Please use the `notification_settings.no_alert_for_skipped_runs` field.", "$ref": "#/$defs/bool" }, "on_duration_warning_threshold_exceeded": { @@ -4365,6 +4381,207 @@ } ] }, + "serving.AiGatewayConfig": { + "anyOf": [ + { + "type": "object", + "properties": { + "guardrails": { + "description": "Configuration for AI Guardrails to prevent unwanted data and unsafe data in requests and responses.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrails" + }, + "inference_table_config": { + "description": "Configuration for payload logging using inference tables. Use these tables to monitor and audit data being sent to and received from model APIs and to improve model quality.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayInferenceTableConfig" + }, + "rate_limits": { + "description": "Configuration for rate limits which can be set to limit endpoint traffic.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayRateLimit" + }, + "usage_tracking_config": { + "description": "Configuration to enable usage tracking using system tables. These tables allow you to monitor operational usage on endpoints and their associated costs.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayUsageTrackingConfig" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.AiGatewayGuardrailParameters": { + "anyOf": [ + { + "type": "object", + "properties": { + "invalid_keywords": { + "description": "List of invalid keywords. AI guardrail uses keyword or string matching to decide if the keyword exists in the request or response content.", + "$ref": "#/$defs/slice/string" + }, + "pii": { + "description": "Configuration for guardrail PII filter.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrailPiiBehavior" + }, + "safety": { + "description": "Indicates whether the safety filter is enabled.", + "$ref": "#/$defs/bool" + }, + "valid_topics": { + "description": "The list of allowed topics. Given a chat request, this guardrail flags the request if its topic is not in the allowed topics.", + "$ref": "#/$defs/slice/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.AiGatewayGuardrailPiiBehavior": { + "anyOf": [ + { + "type": "object", + "properties": { + "behavior": { + "description": "Behavior for PII filter. Currently only 'BLOCK' is supported. If 'BLOCK' is set for the input guardrail and the request contains PII, the request is not sent to the model server and 400 status code is returned; if 'BLOCK' is set for the output guardrail and the model response contains PII, the PII info in the response is redacted and 400 status code is returned.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrailPiiBehaviorBehavior", + "enum": [ + "NONE", + "BLOCK" + ] + } + }, + "additionalProperties": false, + "required": [ + "behavior" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.AiGatewayGuardrailPiiBehaviorBehavior": { + "type": "string" + }, + "serving.AiGatewayGuardrails": { + "anyOf": [ + { + "type": "object", + "properties": { + "input": { + "description": "Configuration for input guardrail filters.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrailParameters" + }, + "output": { + "description": "Configuration for output guardrail filters.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrailParameters" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.AiGatewayInferenceTableConfig": { + "anyOf": [ + { + "type": "object", + "properties": { + "catalog_name": { + "description": "The name of the catalog in Unity Catalog. Required when enabling inference tables. NOTE: On update, you have to disable inference table first in order to change the catalog name.", + "$ref": "#/$defs/string" + }, + "enabled": { + "description": "Indicates whether the inference table is enabled.", + "$ref": "#/$defs/bool" + }, + "schema_name": { + "description": "The name of the schema in Unity Catalog. Required when enabling inference tables. NOTE: On update, you have to disable inference table first in order to change the schema name.", + "$ref": "#/$defs/string" + }, + "table_name_prefix": { + "description": "The prefix of the table in Unity Catalog. NOTE: On update, you have to disable inference table first in order to change the prefix name.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.AiGatewayRateLimit": { + "anyOf": [ + { + "type": "object", + "properties": { + "calls": { + "description": "Used to specify how many calls are allowed for a key within the renewal_period.", + "$ref": "#/$defs/int" + }, + "key": { + "description": "Key field for a rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayRateLimitKey", + "enum": [ + "user", + "endpoint" + ] + }, + "renewal_period": { + "description": "Renewal period field for a rate limit. Currently, only 'minute' is supported.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayRateLimitRenewalPeriod", + "enum": [ + "minute" + ] + } + }, + "additionalProperties": false, + "required": [ + "calls", + "renewal_period" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.AiGatewayRateLimitKey": { + "type": "string" + }, + "serving.AiGatewayRateLimitRenewalPeriod": { + "type": "string" + }, + "serving.AiGatewayUsageTrackingConfig": { + "anyOf": [ + { + "type": "object", + "properties": { + "enabled": { + "description": "Whether to enable usage tracking.", + "$ref": "#/$defs/bool" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "serving.AmazonBedrockConfig": { "anyOf": [ { @@ -5569,6 +5786,20 @@ } ] }, + "serving.AiGatewayRateLimit": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayRateLimit" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "serving.EndpointTag": { "anyOf": [ { diff --git a/cmd/workspace/apps/apps.go b/cmd/workspace/apps/apps.go index baec6d03c..780f55945 100755 --- a/cmd/workspace/apps/apps.go +++ b/cmd/workspace/apps/apps.go @@ -81,6 +81,7 @@ func newCreate() *cobra.Command { cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`) cmd.Flags().StringVar(&createReq.Description, "description", createReq.Description, `The description of the app.`) + // TODO: array: resources cmd.Use = "create NAME" cmd.Short = `Create an app.` @@ -910,6 +911,7 @@ func newUpdate() *cobra.Command { cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`) cmd.Flags().StringVar(&updateReq.Description, "description", updateReq.Description, `The description of the app.`) + // TODO: array: resources cmd.Use = "update NAME" cmd.Short = `Update an app.` diff --git a/cmd/workspace/git-credentials/git-credentials.go b/cmd/workspace/git-credentials/git-credentials.go index 2e8cc2cd4..b5082d311 100755 --- a/cmd/workspace/git-credentials/git-credentials.go +++ b/cmd/workspace/git-credentials/git-credentials.go @@ -53,13 +53,13 @@ func New() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var createOverrides []func( *cobra.Command, - *workspace.CreateCredentials, + *workspace.CreateCredentialsRequest, ) func newCreate() *cobra.Command { cmd := &cobra.Command{} - var createReq workspace.CreateCredentials + var createReq workspace.CreateCredentialsRequest var createJson flags.JsonFlag // TODO: short flags @@ -79,8 +79,9 @@ func newCreate() *cobra.Command { Arguments: GIT_PROVIDER: Git provider. This field is case-insensitive. The available Git providers - are gitHub, bitbucketCloud, gitLab, azureDevOpsServices, gitHubEnterprise, - bitbucketServer, gitLabEnterpriseEdition and awsCodeCommit.` + are gitHub, bitbucketCloud, gitLab, azureDevOpsServices, + gitHubEnterprise, bitbucketServer, gitLabEnterpriseEdition and + awsCodeCommit.` cmd.Annotations = make(map[string]string) @@ -136,13 +137,13 @@ func newCreate() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var deleteOverrides []func( *cobra.Command, - *workspace.DeleteGitCredentialRequest, + *workspace.DeleteCredentialsRequest, ) func newDelete() *cobra.Command { cmd := &cobra.Command{} - var deleteReq workspace.DeleteGitCredentialRequest + var deleteReq workspace.DeleteCredentialsRequest // TODO: short flags @@ -209,13 +210,13 @@ func newDelete() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var getOverrides []func( *cobra.Command, - *workspace.GetGitCredentialRequest, + *workspace.GetCredentialsRequest, ) func newGet() *cobra.Command { cmd := &cobra.Command{} - var getReq workspace.GetGitCredentialRequest + var getReq workspace.GetCredentialsRequest // TODO: short flags @@ -322,33 +323,48 @@ func newList() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var updateOverrides []func( *cobra.Command, - *workspace.UpdateCredentials, + *workspace.UpdateCredentialsRequest, ) func newUpdate() *cobra.Command { cmd := &cobra.Command{} - var updateReq workspace.UpdateCredentials + var updateReq workspace.UpdateCredentialsRequest var updateJson flags.JsonFlag // TODO: short flags cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`) - cmd.Flags().StringVar(&updateReq.GitProvider, "git-provider", updateReq.GitProvider, `Git provider.`) cmd.Flags().StringVar(&updateReq.GitUsername, "git-username", updateReq.GitUsername, `The username or email provided with your Git provider account, depending on which provider you are using.`) cmd.Flags().StringVar(&updateReq.PersonalAccessToken, "personal-access-token", updateReq.PersonalAccessToken, `The personal access token used to authenticate to the corresponding Git provider.`) - cmd.Use = "update CREDENTIAL_ID" + cmd.Use = "update CREDENTIAL_ID GIT_PROVIDER" cmd.Short = `Update a credential.` cmd.Long = `Update a credential. Updates the specified Git credential. Arguments: - CREDENTIAL_ID: The ID for the corresponding credential to access.` + CREDENTIAL_ID: The ID for the corresponding credential to access. + GIT_PROVIDER: Git provider. This field is case-insensitive. The available Git providers + are gitHub, bitbucketCloud, gitLab, azureDevOpsServices, + gitHubEnterprise, bitbucketServer, gitLabEnterpriseEdition and + awsCodeCommit.` cmd.Annotations = make(map[string]string) + cmd.Args = func(cmd *cobra.Command, args []string) error { + if cmd.Flags().Changed("json") { + err := root.ExactArgs(1)(cmd, args) + if err != nil { + return fmt.Errorf("when --json flag is specified, provide only CREDENTIAL_ID as positional arguments. Provide 'git_provider' in your JSON input") + } + return nil + } + check := root.ExactArgs(2) + return check(cmd, args) + } + cmd.PreRunE = root.MustWorkspaceClient cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() @@ -360,27 +376,13 @@ func newUpdate() *cobra.Command { return err } } - if len(args) == 0 { - promptSpinner := cmdio.Spinner(ctx) - promptSpinner <- "No CREDENTIAL_ID argument specified. Loading names for Git Credentials drop-down." - names, err := w.GitCredentials.CredentialInfoGitProviderToCredentialIdMap(ctx) - close(promptSpinner) - if err != nil { - return fmt.Errorf("failed to load names for Git Credentials drop-down. Please manually specify required arguments. Original error: %w", err) - } - id, err := cmdio.Select(ctx, names, "The ID for the corresponding credential to access") - if err != nil { - return err - } - args = append(args, id) - } - if len(args) != 1 { - return fmt.Errorf("expected to have the id for the corresponding credential to access") - } _, err = fmt.Sscan(args[0], &updateReq.CredentialId) if err != nil { return fmt.Errorf("invalid CREDENTIAL_ID: %s", args[0]) } + if !cmd.Flags().Changed("json") { + updateReq.GitProvider = args[1] + } err = w.GitCredentials.Update(ctx, updateReq) if err != nil { diff --git a/cmd/workspace/pipelines/pipelines.go b/cmd/workspace/pipelines/pipelines.go index 5b4d9645e..ac361e313 100755 --- a/cmd/workspace/pipelines/pipelines.go +++ b/cmd/workspace/pipelines/pipelines.go @@ -954,6 +954,7 @@ func newUpdate() *cobra.Command { // TODO: array: notifications cmd.Flags().BoolVar(&updateReq.Photon, "photon", updateReq.Photon, `Whether Photon is enabled for this pipeline.`) cmd.Flags().StringVar(&updateReq.PipelineId, "pipeline-id", updateReq.PipelineId, `Unique identifier for this pipeline.`) + cmd.Flags().StringVar(&updateReq.Schema, "schema", updateReq.Schema, `The default schema (database) where tables are read from or published to.`) cmd.Flags().BoolVar(&updateReq.Serverless, "serverless", updateReq.Serverless, `Whether serverless compute is enabled for this pipeline.`) cmd.Flags().StringVar(&updateReq.Storage, "storage", updateReq.Storage, `DBFS root directory for storing checkpoints and tables.`) cmd.Flags().StringVar(&updateReq.Target, "target", updateReq.Target, `Target schema (database) to add tables in this pipeline to.`) diff --git a/cmd/workspace/repos/overrides.go b/cmd/workspace/repos/overrides.go index 96d645efb..9546d1c1e 100644 --- a/cmd/workspace/repos/overrides.go +++ b/cmd/workspace/repos/overrides.go @@ -19,7 +19,7 @@ func listOverride(listCmd *cobra.Command, listReq *workspace.ListReposRequest) { {{end}}`) } -func createOverride(createCmd *cobra.Command, createReq *workspace.CreateRepo) { +func createOverride(createCmd *cobra.Command, createReq *workspace.CreateRepoRequest) { createCmd.Use = "create URL [PROVIDER]" createCmd.Args = func(cmd *cobra.Command, args []string) error { // If the provider argument is not specified, we try to detect it from the URL. @@ -95,7 +95,7 @@ func getOverride(getCmd *cobra.Command, getReq *workspace.GetRepoRequest) { } } -func updateOverride(updateCmd *cobra.Command, updateReq *workspace.UpdateRepo) { +func updateOverride(updateCmd *cobra.Command, updateReq *workspace.UpdateRepoRequest) { updateCmd.Use = "update REPO_ID_OR_PATH" updateJson := updateCmd.Flag("json").Value.(*flags.JsonFlag) diff --git a/cmd/workspace/repos/repos.go b/cmd/workspace/repos/repos.go index fb3d51b06..f11dd3ace 100755 --- a/cmd/workspace/repos/repos.go +++ b/cmd/workspace/repos/repos.go @@ -61,13 +61,13 @@ func New() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var createOverrides []func( *cobra.Command, - *workspace.CreateRepo, + *workspace.CreateRepoRequest, ) func newCreate() *cobra.Command { cmd := &cobra.Command{} - var createReq workspace.CreateRepo + var createReq workspace.CreateRepoRequest var createJson flags.JsonFlag // TODO: short flags @@ -87,8 +87,9 @@ func newCreate() *cobra.Command { Arguments: URL: URL of the Git repository to be linked. PROVIDER: Git provider. This field is case-insensitive. The available Git providers - are gitHub, bitbucketCloud, gitLab, azureDevOpsServices, gitHubEnterprise, - bitbucketServer, gitLabEnterpriseEdition and awsCodeCommit.` + are gitHub, bitbucketCloud, gitLab, azureDevOpsServices, + gitHubEnterprise, bitbucketServer, gitLabEnterpriseEdition and + awsCodeCommit.` cmd.Annotations = make(map[string]string) @@ -164,7 +165,7 @@ func newDelete() *cobra.Command { Deletes the specified repo. Arguments: - REPO_ID: The ID for the corresponding repo to access.` + REPO_ID: ID of the Git folder (repo) object in the workspace.` cmd.Annotations = make(map[string]string) @@ -181,14 +182,14 @@ func newDelete() *cobra.Command { if err != nil { return fmt.Errorf("failed to load names for Repos drop-down. Please manually specify required arguments. Original error: %w", err) } - id, err := cmdio.Select(ctx, names, "The ID for the corresponding repo to access") + id, err := cmdio.Select(ctx, names, "ID of the Git folder (repo) object in the workspace") if err != nil { return err } args = append(args, id) } if len(args) != 1 { - return fmt.Errorf("expected to have the id for the corresponding repo to access") + return fmt.Errorf("expected to have id of the git folder (repo) object in the workspace") } _, err = fmt.Sscan(args[0], &deleteReq.RepoId) if err != nil { @@ -237,7 +238,7 @@ func newGet() *cobra.Command { Returns the repo with the given repo ID. Arguments: - REPO_ID: The ID for the corresponding repo to access.` + REPO_ID: ID of the Git folder (repo) object in the workspace.` cmd.Annotations = make(map[string]string) @@ -254,14 +255,14 @@ func newGet() *cobra.Command { if err != nil { return fmt.Errorf("failed to load names for Repos drop-down. Please manually specify required arguments. Original error: %w", err) } - id, err := cmdio.Select(ctx, names, "The ID for the corresponding repo to access") + id, err := cmdio.Select(ctx, names, "ID of the Git folder (repo) object in the workspace") if err != nil { return err } args = append(args, id) } if len(args) != 1 { - return fmt.Errorf("expected to have the id for the corresponding repo to access") + return fmt.Errorf("expected to have id of the git folder (repo) object in the workspace") } _, err = fmt.Sscan(args[0], &getReq.RepoId) if err != nil { @@ -451,8 +452,8 @@ func newList() *cobra.Command { cmd.Short = `Get repos.` cmd.Long = `Get repos. - Returns repos that the calling user has Manage permissions on. Results are - paginated with each page containing twenty repos.` + Returns repos that the calling user has Manage permissions on. Use + next_page_token to iterate through additional pages.` cmd.Annotations = make(map[string]string) @@ -569,13 +570,13 @@ func newSetPermissions() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var updateOverrides []func( *cobra.Command, - *workspace.UpdateRepo, + *workspace.UpdateRepoRequest, ) func newUpdate() *cobra.Command { cmd := &cobra.Command{} - var updateReq workspace.UpdateRepo + var updateReq workspace.UpdateRepoRequest var updateJson flags.JsonFlag // TODO: short flags @@ -593,7 +594,7 @@ func newUpdate() *cobra.Command { latest commit on the same branch. Arguments: - REPO_ID: The ID for the corresponding repo to access.` + REPO_ID: ID of the Git folder (repo) object in the workspace.` cmd.Annotations = make(map[string]string) @@ -616,14 +617,14 @@ func newUpdate() *cobra.Command { if err != nil { return fmt.Errorf("failed to load names for Repos drop-down. Please manually specify required arguments. Original error: %w", err) } - id, err := cmdio.Select(ctx, names, "The ID for the corresponding repo to access") + id, err := cmdio.Select(ctx, names, "ID of the Git folder (repo) object in the workspace") if err != nil { return err } args = append(args, id) } if len(args) != 1 { - return fmt.Errorf("expected to have the id for the corresponding repo to access") + return fmt.Errorf("expected to have id of the git folder (repo) object in the workspace") } _, err = fmt.Sscan(args[0], &updateReq.RepoId) if err != nil { diff --git a/go.mod b/go.mod index c029e5da0..697205f33 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ toolchain go1.22.7 require ( github.com/Masterminds/semver/v3 v3.3.0 // MIT github.com/briandowns/spinner v1.23.1 // Apache 2.0 - github.com/databricks/databricks-sdk-go v0.47.0 // Apache 2.0 + github.com/databricks/databricks-sdk-go v0.48.0 // Apache 2.0 github.com/fatih/color v1.17.0 // MIT github.com/ghodss/yaml v1.0.0 // MIT + NOTICE github.com/google/uuid v1.6.0 // BSD-3-Clause diff --git a/go.sum b/go.sum index 9f05b1451..03698b20a 100644 --- a/go.sum +++ b/go.sum @@ -32,8 +32,8 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= -github.com/databricks/databricks-sdk-go v0.47.0 h1:eE7dN9axviL8+s10jnQAayOYDaR+Mfu7E9COGjO4lrQ= -github.com/databricks/databricks-sdk-go v0.47.0/go.mod h1:ds+zbv5mlQG7nFEU5ojLtgN/u0/9YzZmKQES/CfedzU= +github.com/databricks/databricks-sdk-go v0.48.0 h1:46KtsnRo+FGhC3izUXbpL0PXBNomvsdignYDhJZlm9s= +github.com/databricks/databricks-sdk-go v0.48.0/go.mod h1:ds+zbv5mlQG7nFEU5ojLtgN/u0/9YzZmKQES/CfedzU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/internal/helpers.go b/internal/helpers.go index 419fa419c..9387706bb 100644 --- a/internal/helpers.go +++ b/internal/helpers.go @@ -519,7 +519,7 @@ func TemporaryRepo(t *testing.T, w *databricks.WorkspaceClient) string { repoPath := fmt.Sprintf("/Repos/%s/%s", me.UserName, RandomName("integration-test-repo-")) t.Logf("Creating repo:%s", repoPath) - repoInfo, err := w.Repos.Create(ctx, workspace.CreateRepo{ + repoInfo, err := w.Repos.Create(ctx, workspace.CreateRepoRequest{ Url: "https://github.com/databricks/cli", Provider: "github", Path: repoPath, diff --git a/internal/locker_test.go b/internal/locker_test.go index 21e08f732..3ae783d1b 100644 --- a/internal/locker_test.go +++ b/internal/locker_test.go @@ -29,7 +29,7 @@ func createRemoteTestProject(t *testing.T, projectNamePrefix string, wsc *databr assert.NoError(t, err) remoteProjectRoot := fmt.Sprintf("/Repos/%s/%s", me.UserName, RandomName(projectNamePrefix)) - repoInfo, err := wsc.Repos.Create(ctx, workspace.CreateRepo{ + repoInfo, err := wsc.Repos.Create(ctx, workspace.CreateRepoRequest{ Path: remoteProjectRoot, Url: EmptyRepoUrl, Provider: "gitHub", diff --git a/internal/repos_test.go b/internal/repos_test.go index de0d926ad..1ad0e8775 100644 --- a/internal/repos_test.go +++ b/internal/repos_test.go @@ -34,7 +34,7 @@ func synthesizeTemporaryRepoPath(t *testing.T, w *databricks.WorkspaceClient, ct func createTemporaryRepo(t *testing.T, w *databricks.WorkspaceClient, ctx context.Context) (int64, string) { repoPath := synthesizeTemporaryRepoPath(t, w, ctx) - repoInfo, err := w.Repos.Create(ctx, workspace.CreateRepo{ + repoInfo, err := w.Repos.Create(ctx, workspace.CreateRepoRequest{ Path: repoPath, Url: repoUrl, Provider: "gitHub", diff --git a/internal/sync_test.go b/internal/sync_test.go index 4021e6490..6f8b1827b 100644 --- a/internal/sync_test.go +++ b/internal/sync_test.go @@ -38,7 +38,7 @@ func setupRepo(t *testing.T, wsc *databricks.WorkspaceClient, ctx context.Contex require.NoError(t, err) repoPath := fmt.Sprintf("/Repos/%s/%s", me.UserName, RandomName("empty-repo-sync-integration-")) - repoInfo, err := wsc.Repos.Create(ctx, workspace.CreateRepo{ + repoInfo, err := wsc.Repos.Create(ctx, workspace.CreateRepoRequest{ Path: repoPath, Url: repoUrl, Provider: "gitHub", From 88318d384acd29ed58143e0bfbbb0c382fa785c8 Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Mon, 7 Oct 2024 20:43:42 +0530 Subject: [PATCH 33/34] Remove deprecated or readonly fields from the bundle schema (#1809) ## Changes We do not need the user to specify these fields in their bundle configuration, so we remove them from the JSON schema. ## Tests Manually and end to end tests. The JSON schema has also been regenerated after these changes. --- bundle/internal/schema/main.go | 28 +++++++++++++++++++ .../fail/deprecated_job_field_format.yml | 4 +++ .../fail/hidden_job_field_deployment.yml | 6 ++++ .../fail/hidden_job_field_edit_mode.yml | 6 ++++ .../fail/readonly_job_field_git_snapshot.yml | 8 ++++++ .../fail/readonly_job_field_job_source.yml | 9 ++++++ bundle/internal/schema/testdata/pass/job.yml | 5 ++-- bundle/schema/embed_test.go | 4 +-- bundle/schema/jsonschema.json | 19 ------------- 9 files changed, 66 insertions(+), 23 deletions(-) create mode 100644 bundle/internal/schema/testdata/fail/deprecated_job_field_format.yml create mode 100644 bundle/internal/schema/testdata/fail/hidden_job_field_deployment.yml create mode 100644 bundle/internal/schema/testdata/fail/hidden_job_field_edit_mode.yml create mode 100644 bundle/internal/schema/testdata/fail/readonly_job_field_git_snapshot.yml create mode 100644 bundle/internal/schema/testdata/fail/readonly_job_field_job_source.yml diff --git a/bundle/internal/schema/main.go b/bundle/internal/schema/main.go index 4a2371472..ddeffe2fd 100644 --- a/bundle/internal/schema/main.go +++ b/bundle/internal/schema/main.go @@ -8,8 +8,10 @@ import ( "reflect" "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/config/variable" "github.com/databricks/cli/libs/jsonschema" + "github.com/databricks/databricks-sdk-go/service/jobs" ) func interpolationPattern(s string) string { @@ -66,6 +68,31 @@ func addInterpolationPatterns(typ reflect.Type, s jsonschema.Schema) jsonschema. } } +func removeJobsFields(typ reflect.Type, s jsonschema.Schema) jsonschema.Schema { + switch typ { + case reflect.TypeOf(resources.Job{}): + // This field has been deprecated in jobs API v2.1 and is always set to + // "MULTI_TASK" in the backend. We should not expose it to the user. + delete(s.Properties, "format") + + // These fields are only meant to be set by the DABs client (ie the CLI) + // and thus should not be exposed to the user. These are used to annotate + // jobs that were created by DABs. + delete(s.Properties, "deployment") + delete(s.Properties, "edit_mode") + + case reflect.TypeOf(jobs.GitSource{}): + // These fields are readonly and are not meant to be set by the user. + delete(s.Properties, "job_source") + delete(s.Properties, "git_snapshot") + + default: + // Do nothing + } + + return s +} + func main() { if len(os.Args) != 2 { fmt.Println("Usage: go run main.go ") @@ -90,6 +117,7 @@ func main() { s, err := jsonschema.FromType(reflect.TypeOf(config.Root{}), []func(reflect.Type, jsonschema.Schema) jsonschema.Schema{ p.addDescriptions, p.addEnums, + removeJobsFields, addInterpolationPatterns, }) if err != nil { diff --git a/bundle/internal/schema/testdata/fail/deprecated_job_field_format.yml b/bundle/internal/schema/testdata/fail/deprecated_job_field_format.yml new file mode 100644 index 000000000..62e490b0c --- /dev/null +++ b/bundle/internal/schema/testdata/fail/deprecated_job_field_format.yml @@ -0,0 +1,4 @@ +resources: + jobs: + foo: + format: SINGLE_TASK diff --git a/bundle/internal/schema/testdata/fail/hidden_job_field_deployment.yml b/bundle/internal/schema/testdata/fail/hidden_job_field_deployment.yml new file mode 100644 index 000000000..705ce9516 --- /dev/null +++ b/bundle/internal/schema/testdata/fail/hidden_job_field_deployment.yml @@ -0,0 +1,6 @@ +resources: + jobs: + foo: + deployment: + kind: BUNDLE + metadata_file_path: /a/b/c diff --git a/bundle/internal/schema/testdata/fail/hidden_job_field_edit_mode.yml b/bundle/internal/schema/testdata/fail/hidden_job_field_edit_mode.yml new file mode 100644 index 000000000..9cbe95f03 --- /dev/null +++ b/bundle/internal/schema/testdata/fail/hidden_job_field_edit_mode.yml @@ -0,0 +1,6 @@ +targets: + foo: + resources: + jobs: + bar: + edit_mode: whatever diff --git a/bundle/internal/schema/testdata/fail/readonly_job_field_git_snapshot.yml b/bundle/internal/schema/testdata/fail/readonly_job_field_git_snapshot.yml new file mode 100644 index 000000000..c57a560aa --- /dev/null +++ b/bundle/internal/schema/testdata/fail/readonly_job_field_git_snapshot.yml @@ -0,0 +1,8 @@ +resources: + jobs: + foo: + git_source: + git_provider: GITHUB + git_url: www.whatever.com + git_snapshot: + used_commit: abcdef diff --git a/bundle/internal/schema/testdata/fail/readonly_job_field_job_source.yml b/bundle/internal/schema/testdata/fail/readonly_job_field_job_source.yml new file mode 100644 index 000000000..9973e3bd4 --- /dev/null +++ b/bundle/internal/schema/testdata/fail/readonly_job_field_job_source.yml @@ -0,0 +1,9 @@ +resources: + jobs: + foo: + git_source: + git_provider: GITHUB + git_url: www.whatever.com + job_source: + import_from_git_branch: master + job_config_path: def diff --git a/bundle/internal/schema/testdata/pass/job.yml b/bundle/internal/schema/testdata/pass/job.yml index d9b0e832f..e13a52c03 100644 --- a/bundle/internal/schema/testdata/pass/job.yml +++ b/bundle/internal/schema/testdata/pass/job.yml @@ -32,7 +32,6 @@ resources: name: myjob continuous: pause_status: PAUSED - edit_mode: EDITABLE max_concurrent_runs: 10 description: "my job description" email_notifications: @@ -43,10 +42,12 @@ resources: dependencies: - python=3.7 client: "myclient" - format: MULTI_TASK tags: foo: bar bar: baz + git_source: + git_provider: gitHub + git_url: www.github.com/a/b tasks: - task_key: mytask notebook_task: diff --git a/bundle/schema/embed_test.go b/bundle/schema/embed_test.go index ee0b5a615..dcb381b83 100644 --- a/bundle/schema/embed_test.go +++ b/bundle/schema/embed_test.go @@ -39,7 +39,7 @@ func TestJsonSchema(t *testing.T) { // Assert job fields have their descriptions loaded. resourceJob := walk(s.Definitions, "github.com", "databricks", "cli", "bundle", "config", "resources.Job") - fields := []string{"name", "continuous", "deployment", "tasks", "trigger"} + fields := []string{"name", "continuous", "tasks", "trigger"} for _, field := range fields { assert.NotEmpty(t, resourceJob.AnyOf[0].Properties[field].Description) } @@ -53,7 +53,7 @@ func TestJsonSchema(t *testing.T) { // Assert descriptions are loaded for pipelines pipeline := walk(s.Definitions, "github.com", "databricks", "cli", "bundle", "config", "resources.Pipeline") - fields = []string{"name", "catalog", "clusters", "channel", "continuous", "deployment", "development"} + fields = []string{"name", "catalog", "clusters", "channel", "continuous", "development"} for _, field := range fields { assert.NotEmpty(t, pipeline.AnyOf[0].Properties[field].Description) } diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index ae209c01c..06b9cc15a 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -213,18 +213,10 @@ "description": "An optional continuous property for this job. The continuous property will ensure that there is always one run executing. Only one of `schedule` and `continuous` can be used.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.Continuous" }, - "deployment": { - "description": "Deployment information for jobs managed by external sources.", - "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobDeployment" - }, "description": { "description": "An optional description for the job. The maximum length is 27700 characters in UTF-8 encoding.", "$ref": "#/$defs/string" }, - "edit_mode": { - "description": "Edit mode of the job.\n\n* `UI_LOCKED`: The job is in a locked UI state and cannot be modified.\n* `EDITABLE`: The job is in an editable state and can be modified.", - "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobEditMode" - }, "email_notifications": { "description": "An optional set of email addresses that is notified when runs of this job begin or complete as well as when this job is deleted.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobEmailNotifications" @@ -233,10 +225,6 @@ "description": "A list of task execution environment specifications that can be referenced by serverless tasks of this job.\nAn environment is required to be present for serverless tasks.\nFor serverless notebook tasks, the environment is accessible in the notebook environment panel.\nFor other serverless tasks, the task environment is required to be specified using environment_key in the task settings.", "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.JobEnvironment" }, - "format": { - "description": "Used to tell what is the format of the job. This field is ignored in Create/Update/Reset calls. When using the Jobs API 2.1 this value is always set to `\"MULTI_TASK\"`.", - "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.Format" - }, "git_source": { "description": "An optional specification for a remote Git repository containing the source code used by tasks. Version-controlled source code is supported by notebook, dbt, Python script, and SQL File tasks.\n\nIf `git_source` is set, these tasks retrieve the file from the remote repository by default. However, this behavior can be overridden by setting `source` to `WORKSPACE` on the task.\n\nNote: dbt and SQL File tasks support only version-controlled sources. If dbt or SQL File tasks are used, `git_source` must be defined on the job.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.GitSource" @@ -2547,9 +2535,6 @@ "description": "Unique identifier of the service used to host the Git repository. The value is case insensitive.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.GitProvider" }, - "git_snapshot": { - "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.GitSnapshot" - }, "git_tag": { "description": "Name of the tag to be checked out and used by this job. This field cannot be specified in conjunction with git_branch or git_commit.", "$ref": "#/$defs/string" @@ -2557,10 +2542,6 @@ "git_url": { "description": "URL of the repository to be cloned by this job.", "$ref": "#/$defs/string" - }, - "job_source": { - "description": "The source of the job specification in the remote repository when the job is source controlled.", - "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobSource" } }, "additionalProperties": false, From ae568743c5ccbfb8ae9d3298c7aeee96d0dcf1b4 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Tue, 8 Oct 2024 13:41:32 +0200 Subject: [PATCH 34/34] Upgrade TF provider to 1.53.0 (#1815) ## Changes See https://github.com/databricks/terraform-provider-databricks/releases/tag/v1.53.0 ## Tests Integration tests pass. --- bundle/internal/tf/codegen/schema/version.go | 2 +- .../schema/data_source_current_metastore.go | 1 + .../tf/schema/data_source_metastore.go | 1 + .../tf/schema/data_source_mlflow_models.go | 8 +++ bundle/internal/tf/schema/data_sources.go | 2 + bundle/internal/tf/schema/resource_budget.go | 49 +++++++++++++++++ .../tf/schema/resource_model_serving.go | 52 +++++++++++++++++++ .../tf/schema/resource_permissions.go | 2 +- .../internal/tf/schema/resource_pipeline.go | 2 + .../internal/tf/schema/resource_sql_table.go | 2 + bundle/internal/tf/schema/resources.go | 2 + bundle/internal/tf/schema/root.go | 2 +- 12 files changed, 122 insertions(+), 3 deletions(-) create mode 100644 bundle/internal/tf/schema/data_source_mlflow_models.go create mode 100644 bundle/internal/tf/schema/resource_budget.go diff --git a/bundle/internal/tf/codegen/schema/version.go b/bundle/internal/tf/codegen/schema/version.go index b71ea7d1c..49e48a6e3 100644 --- a/bundle/internal/tf/codegen/schema/version.go +++ b/bundle/internal/tf/codegen/schema/version.go @@ -1,3 +1,3 @@ package schema -const ProviderVersion = "1.52.0" +const ProviderVersion = "1.53.0" diff --git a/bundle/internal/tf/schema/data_source_current_metastore.go b/bundle/internal/tf/schema/data_source_current_metastore.go index 11e647fd3..4f8c135a5 100644 --- a/bundle/internal/tf/schema/data_source_current_metastore.go +++ b/bundle/internal/tf/schema/data_source_current_metastore.go @@ -10,6 +10,7 @@ type DataSourceCurrentMetastoreMetastoreInfo struct { DeltaSharingOrganizationName string `json:"delta_sharing_organization_name,omitempty"` DeltaSharingRecipientTokenLifetimeInSeconds int `json:"delta_sharing_recipient_token_lifetime_in_seconds,omitempty"` DeltaSharingScope string `json:"delta_sharing_scope,omitempty"` + ExternalAccessEnabled bool `json:"external_access_enabled,omitempty"` GlobalMetastoreId string `json:"global_metastore_id,omitempty"` MetastoreId string `json:"metastore_id,omitempty"` Name string `json:"name,omitempty"` diff --git a/bundle/internal/tf/schema/data_source_metastore.go b/bundle/internal/tf/schema/data_source_metastore.go index ce2064794..4244febc6 100644 --- a/bundle/internal/tf/schema/data_source_metastore.go +++ b/bundle/internal/tf/schema/data_source_metastore.go @@ -10,6 +10,7 @@ type DataSourceMetastoreMetastoreInfo struct { DeltaSharingOrganizationName string `json:"delta_sharing_organization_name,omitempty"` DeltaSharingRecipientTokenLifetimeInSeconds int `json:"delta_sharing_recipient_token_lifetime_in_seconds,omitempty"` DeltaSharingScope string `json:"delta_sharing_scope,omitempty"` + ExternalAccessEnabled bool `json:"external_access_enabled,omitempty"` GlobalMetastoreId string `json:"global_metastore_id,omitempty"` MetastoreId string `json:"metastore_id,omitempty"` Name string `json:"name,omitempty"` diff --git a/bundle/internal/tf/schema/data_source_mlflow_models.go b/bundle/internal/tf/schema/data_source_mlflow_models.go new file mode 100644 index 000000000..360924e5c --- /dev/null +++ b/bundle/internal/tf/schema/data_source_mlflow_models.go @@ -0,0 +1,8 @@ +// Generated from Databricks Terraform provider schema. DO NOT EDIT. + +package schema + +type DataSourceMlflowModels struct { + Id string `json:"id,omitempty"` + Names []string `json:"names,omitempty"` +} diff --git a/bundle/internal/tf/schema/data_sources.go b/bundle/internal/tf/schema/data_sources.go index 4ac78613f..10829b994 100644 --- a/bundle/internal/tf/schema/data_sources.go +++ b/bundle/internal/tf/schema/data_sources.go @@ -30,6 +30,7 @@ type DataSources struct { Metastores map[string]any `json:"databricks_metastores,omitempty"` MlflowExperiment map[string]any `json:"databricks_mlflow_experiment,omitempty"` MlflowModel map[string]any `json:"databricks_mlflow_model,omitempty"` + MlflowModels map[string]any `json:"databricks_mlflow_models,omitempty"` MwsCredentials map[string]any `json:"databricks_mws_credentials,omitempty"` MwsWorkspaces map[string]any `json:"databricks_mws_workspaces,omitempty"` NodeType map[string]any `json:"databricks_node_type,omitempty"` @@ -85,6 +86,7 @@ func NewDataSources() *DataSources { Metastores: make(map[string]any), MlflowExperiment: make(map[string]any), MlflowModel: make(map[string]any), + MlflowModels: make(map[string]any), MwsCredentials: make(map[string]any), MwsWorkspaces: make(map[string]any), NodeType: make(map[string]any), diff --git a/bundle/internal/tf/schema/resource_budget.go b/bundle/internal/tf/schema/resource_budget.go new file mode 100644 index 000000000..5566eb939 --- /dev/null +++ b/bundle/internal/tf/schema/resource_budget.go @@ -0,0 +1,49 @@ +// Generated from Databricks Terraform provider schema. DO NOT EDIT. + +package schema + +type ResourceBudgetAlertConfigurationsActionConfigurations struct { + ActionConfigurationId string `json:"action_configuration_id,omitempty"` + ActionType string `json:"action_type,omitempty"` + Target string `json:"target,omitempty"` +} + +type ResourceBudgetAlertConfigurations struct { + AlertConfigurationId string `json:"alert_configuration_id,omitempty"` + QuantityThreshold string `json:"quantity_threshold,omitempty"` + QuantityType string `json:"quantity_type,omitempty"` + TimePeriod string `json:"time_period,omitempty"` + TriggerType string `json:"trigger_type,omitempty"` + ActionConfigurations []ResourceBudgetAlertConfigurationsActionConfigurations `json:"action_configurations,omitempty"` +} + +type ResourceBudgetFilterTagsValue struct { + Operator string `json:"operator,omitempty"` + Values []string `json:"values,omitempty"` +} + +type ResourceBudgetFilterTags struct { + Key string `json:"key,omitempty"` + Value *ResourceBudgetFilterTagsValue `json:"value,omitempty"` +} + +type ResourceBudgetFilterWorkspaceId struct { + Operator string `json:"operator,omitempty"` + Values []int `json:"values,omitempty"` +} + +type ResourceBudgetFilter struct { + Tags []ResourceBudgetFilterTags `json:"tags,omitempty"` + WorkspaceId *ResourceBudgetFilterWorkspaceId `json:"workspace_id,omitempty"` +} + +type ResourceBudget struct { + AccountId string `json:"account_id,omitempty"` + BudgetConfigurationId string `json:"budget_configuration_id,omitempty"` + CreateTime int `json:"create_time,omitempty"` + DisplayName string `json:"display_name,omitempty"` + Id string `json:"id,omitempty"` + UpdateTime int `json:"update_time,omitempty"` + AlertConfigurations []ResourceBudgetAlertConfigurations `json:"alert_configurations,omitempty"` + Filter *ResourceBudgetFilter `json:"filter,omitempty"` +} diff --git a/bundle/internal/tf/schema/resource_model_serving.go b/bundle/internal/tf/schema/resource_model_serving.go index 29d55cd5f..71cf8925d 100644 --- a/bundle/internal/tf/schema/resource_model_serving.go +++ b/bundle/internal/tf/schema/resource_model_serving.go @@ -2,6 +2,57 @@ package schema +type ResourceModelServingAiGatewayGuardrailsInputPii struct { + Behavior string `json:"behavior"` +} + +type ResourceModelServingAiGatewayGuardrailsInput struct { + InvalidKeywords []string `json:"invalid_keywords,omitempty"` + Safety bool `json:"safety,omitempty"` + ValidTopics []string `json:"valid_topics,omitempty"` + Pii *ResourceModelServingAiGatewayGuardrailsInputPii `json:"pii,omitempty"` +} + +type ResourceModelServingAiGatewayGuardrailsOutputPii struct { + Behavior string `json:"behavior"` +} + +type ResourceModelServingAiGatewayGuardrailsOutput struct { + InvalidKeywords []string `json:"invalid_keywords,omitempty"` + Safety bool `json:"safety,omitempty"` + ValidTopics []string `json:"valid_topics,omitempty"` + Pii *ResourceModelServingAiGatewayGuardrailsOutputPii `json:"pii,omitempty"` +} + +type ResourceModelServingAiGatewayGuardrails struct { + Input *ResourceModelServingAiGatewayGuardrailsInput `json:"input,omitempty"` + Output *ResourceModelServingAiGatewayGuardrailsOutput `json:"output,omitempty"` +} + +type ResourceModelServingAiGatewayInferenceTableConfig struct { + CatalogName string `json:"catalog_name,omitempty"` + Enabled bool `json:"enabled,omitempty"` + SchemaName string `json:"schema_name,omitempty"` + TableNamePrefix string `json:"table_name_prefix,omitempty"` +} + +type ResourceModelServingAiGatewayRateLimits struct { + Calls int `json:"calls"` + Key string `json:"key,omitempty"` + RenewalPeriod string `json:"renewal_period"` +} + +type ResourceModelServingAiGatewayUsageTrackingConfig struct { + Enabled bool `json:"enabled,omitempty"` +} + +type ResourceModelServingAiGateway struct { + Guardrails *ResourceModelServingAiGatewayGuardrails `json:"guardrails,omitempty"` + InferenceTableConfig *ResourceModelServingAiGatewayInferenceTableConfig `json:"inference_table_config,omitempty"` + RateLimits []ResourceModelServingAiGatewayRateLimits `json:"rate_limits,omitempty"` + UsageTrackingConfig *ResourceModelServingAiGatewayUsageTrackingConfig `json:"usage_tracking_config,omitempty"` +} + type ResourceModelServingConfigAutoCaptureConfig struct { CatalogName string `json:"catalog_name,omitempty"` Enabled bool `json:"enabled,omitempty"` @@ -139,6 +190,7 @@ type ResourceModelServing struct { Name string `json:"name"` RouteOptimized bool `json:"route_optimized,omitempty"` ServingEndpointId string `json:"serving_endpoint_id,omitempty"` + AiGateway *ResourceModelServingAiGateway `json:"ai_gateway,omitempty"` Config *ResourceModelServingConfig `json:"config,omitempty"` RateLimits []ResourceModelServingRateLimits `json:"rate_limits,omitempty"` Tags []ResourceModelServingTags `json:"tags,omitempty"` diff --git a/bundle/internal/tf/schema/resource_permissions.go b/bundle/internal/tf/schema/resource_permissions.go index ee94a1a8f..0c3b90ed3 100644 --- a/bundle/internal/tf/schema/resource_permissions.go +++ b/bundle/internal/tf/schema/resource_permissions.go @@ -4,7 +4,7 @@ package schema type ResourcePermissionsAccessControl struct { GroupName string `json:"group_name,omitempty"` - PermissionLevel string `json:"permission_level"` + PermissionLevel string `json:"permission_level,omitempty"` ServicePrincipalName string `json:"service_principal_name,omitempty"` UserName string `json:"user_name,omitempty"` } diff --git a/bundle/internal/tf/schema/resource_pipeline.go b/bundle/internal/tf/schema/resource_pipeline.go index 154686463..1bed91fcb 100644 --- a/bundle/internal/tf/schema/resource_pipeline.go +++ b/bundle/internal/tf/schema/resource_pipeline.go @@ -238,6 +238,7 @@ type ResourcePipelineTrigger struct { type ResourcePipeline struct { AllowDuplicateNames bool `json:"allow_duplicate_names,omitempty"` + BudgetPolicyId string `json:"budget_policy_id,omitempty"` Catalog string `json:"catalog,omitempty"` Cause string `json:"cause,omitempty"` Channel string `json:"channel,omitempty"` @@ -254,6 +255,7 @@ type ResourcePipeline struct { Name string `json:"name,omitempty"` Photon bool `json:"photon,omitempty"` RunAsUserName string `json:"run_as_user_name,omitempty"` + Schema string `json:"schema,omitempty"` Serverless bool `json:"serverless,omitempty"` State string `json:"state,omitempty"` Storage string `json:"storage,omitempty"` diff --git a/bundle/internal/tf/schema/resource_sql_table.go b/bundle/internal/tf/schema/resource_sql_table.go index 4f305c52e..bcf2a8e84 100644 --- a/bundle/internal/tf/schema/resource_sql_table.go +++ b/bundle/internal/tf/schema/resource_sql_table.go @@ -4,9 +4,11 @@ package schema type ResourceSqlTableColumn struct { Comment string `json:"comment,omitempty"` + Identity string `json:"identity,omitempty"` Name string `json:"name"` Nullable bool `json:"nullable,omitempty"` Type string `json:"type,omitempty"` + TypeJson string `json:"type_json,omitempty"` } type ResourceSqlTable struct { diff --git a/bundle/internal/tf/schema/resources.go b/bundle/internal/tf/schema/resources.go index 737b77a2a..53f558df6 100644 --- a/bundle/internal/tf/schema/resources.go +++ b/bundle/internal/tf/schema/resources.go @@ -10,6 +10,7 @@ type Resources struct { AzureAdlsGen1Mount map[string]any `json:"databricks_azure_adls_gen1_mount,omitempty"` AzureAdlsGen2Mount map[string]any `json:"databricks_azure_adls_gen2_mount,omitempty"` AzureBlobMount map[string]any `json:"databricks_azure_blob_mount,omitempty"` + Budget map[string]any `json:"databricks_budget,omitempty"` Catalog map[string]any `json:"databricks_catalog,omitempty"` CatalogWorkspaceBinding map[string]any `json:"databricks_catalog_workspace_binding,omitempty"` Cluster map[string]any `json:"databricks_cluster,omitempty"` @@ -112,6 +113,7 @@ func NewResources() *Resources { AzureAdlsGen1Mount: make(map[string]any), AzureAdlsGen2Mount: make(map[string]any), AzureBlobMount: make(map[string]any), + Budget: make(map[string]any), Catalog: make(map[string]any), CatalogWorkspaceBinding: make(map[string]any), Cluster: make(map[string]any), diff --git a/bundle/internal/tf/schema/root.go b/bundle/internal/tf/schema/root.go index 5fc34d6b4..7a0cc01f9 100644 --- a/bundle/internal/tf/schema/root.go +++ b/bundle/internal/tf/schema/root.go @@ -21,7 +21,7 @@ type Root struct { const ProviderHost = "registry.terraform.io" const ProviderSource = "databricks/databricks" -const ProviderVersion = "1.52.0" +const ProviderVersion = "1.53.0" func NewRoot() *Root { return &Root{