Compare commits

...

5 Commits

Author SHA1 Message Date
Ilya Kuznetsov 99dacc144f
Merge e9b72895dc into 7f3fb10c4a 2024-11-15 20:44:39 +05:30
Andrew Nester 7f3fb10c4a
Do not prepend paths starting with ~ or variable reference (#1905)
## Changes
Fixes #1904 

## Tests
Added regression test
2024-11-15 15:03:59 +00:00
Ilya Kuznetsov e9b72895dc
test: Process target mode 2024-11-15 12:32:44 +01:00
Ilya Kuznetsov 8cd95eb0ea
test: Apply presets unit test 2024-11-15 12:09:53 +01:00
Pieter Noordhuis 1db384018c
Make `TableName` field part of quality monitor schema (#1903)
## Changes

This field was special-cased in #1307 because it's not part of the JSON
payload in the SDK struct.

This approach, while pragmatic, meant it didn't show up in the JSON
schema. While debugging an issue with quality monitors in #1900, I
couldn't figure out why I was getting schema errors on this field, or
how it was passed through to the TF representation. This commit removes
the special case and makes it behave like everything else.

## Tests

* Unit tests pass.
* Confirmed that the updated schema failed validation before this
change.
2024-11-14 17:39:38 +00:00
10 changed files with 153 additions and 27 deletions

View File

@ -8,6 +8,7 @@ import (
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/config/mutator"
"github.com/databricks/cli/bundle/config/resources"
"github.com/databricks/cli/libs/dbr"
"github.com/databricks/cli/libs/vfs"
"github.com/databricks/databricks-sdk-go/service/catalog"
"github.com/databricks/databricks-sdk-go/service/jobs"
@ -371,3 +372,86 @@ func TestApplyPresetsResourceNotDefined(t *testing.T) {
})
}
}
func TestApplyPresetsInPlaceDeployment(t *testing.T) {
testContext := context.Background()
enabled := true
disabled := false
remotePath := "/Users/files"
workspacePath := "/Workspace/user.name@company.com"
tests := []struct {
bundlePath string
ctx context.Context
name string
initialValue *bool
expectedValue *bool
expectedFilePath string
}{
{
name: "preset enabled, bundle in Workspace, databricks runtime",
bundlePath: workspacePath,
ctx: dbr.MockRuntime(testContext, true),
initialValue: &enabled,
expectedValue: &enabled,
expectedFilePath: workspacePath,
},
{
name: "preset enabled, bundle not in Workspace, databricks runtime",
bundlePath: "/Users/user.name@company.com",
ctx: dbr.MockRuntime(testContext, true),
initialValue: &enabled,
expectedValue: &disabled,
expectedFilePath: remotePath,
},
{
name: "preset enabled, bundle in Workspace, not databricks runtime",
bundlePath: workspacePath,
ctx: dbr.MockRuntime(testContext, false),
initialValue: &enabled,
expectedValue: &disabled,
expectedFilePath: remotePath,
},
{
name: "preset disabled, bundle in Workspace, databricks runtime",
bundlePath: workspacePath,
ctx: dbr.MockRuntime(testContext, true),
initialValue: &disabled,
expectedValue: &disabled,
expectedFilePath: remotePath,
},
{
name: "preset nil, bundle in Workspace, databricks runtime",
bundlePath: workspacePath,
ctx: dbr.MockRuntime(testContext, true),
initialValue: nil,
expectedValue: nil,
expectedFilePath: remotePath,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
b := &bundle.Bundle{
Config: config.Root{
Presets: config.Presets{
InPlaceDeployment: tt.initialValue,
},
Workspace: config.Workspace{
FilePath: remotePath,
},
},
SyncRoot: vfs.MustNew(tt.bundlePath),
SyncRootPath: tt.bundlePath,
}
diags := bundle.Apply(tt.ctx, b, mutator.ApplyPresets())
if diags.HasError() {
t.Fatalf("unexpected error: %v", diags)
}
require.Equal(t, tt.expectedFilePath, b.Config.Workspace.FilePath)
require.Equal(t, tt.expectedValue, b.Config.Presets.InPlaceDeployment)
})
}
}

View File

@ -65,9 +65,8 @@ func TestInitializeURLs(t *testing.T) {
},
QualityMonitors: map[string]*resources.QualityMonitor{
"qualityMonitor1": {
CreateMonitor: &catalog.CreateMonitor{
TableName: "catalog.schema.qualityMonitor1",
},
TableName: "catalog.schema.qualityMonitor1",
CreateMonitor: &catalog.CreateMonitor{},
},
},
Schemas: map[string]*resources.Schema{

View File

@ -44,6 +44,11 @@ func (m *prependWorkspacePrefix) Apply(ctx context.Context, b *bundle.Bundle) di
return dyn.InvalidValue, fmt.Errorf("expected string, got %s", v.Kind())
}
// Skip prefixing if the path does not start with /, it might be variable reference or smth else.
if !strings.HasPrefix(path, "/") {
return pv, nil
}
for _, prefix := range skipPrefixes {
if strings.HasPrefix(path, prefix) {
return pv, nil

View File

@ -31,6 +31,14 @@ func TestPrependWorkspacePrefix(t *testing.T) {
path: "/Volumes/Users/test",
expected: "/Volumes/Users/test",
},
{
path: "~/test",
expected: "~/test",
},
{
path: "${workspace.file_path}/test",
expected: "${workspace.file_path}/test",
},
}
for _, tc := range testCases {

View File

@ -9,6 +9,7 @@ import (
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/config/resources"
"github.com/databricks/cli/libs/dbr"
"github.com/databricks/cli/libs/diag"
"github.com/databricks/cli/libs/tags"
"github.com/databricks/cli/libs/vfs"
@ -103,16 +104,23 @@ func mockBundle(mode config.Mode) *bundle.Bundle {
"registeredmodel1": {CreateRegisteredModelRequest: &catalog.CreateRegisteredModelRequest{Name: "registeredmodel1"}},
},
QualityMonitors: map[string]*resources.QualityMonitor{
"qualityMonitor1": {CreateMonitor: &catalog.CreateMonitor{TableName: "qualityMonitor1"}},
"qualityMonitor2": {
"qualityMonitor1": {
TableName: "qualityMonitor1",
CreateMonitor: &catalog.CreateMonitor{
TableName: "qualityMonitor2",
Schedule: &catalog.MonitorCronSchedule{},
OutputSchemaName: "catalog.schema",
},
},
"qualityMonitor2": {
TableName: "qualityMonitor2",
CreateMonitor: &catalog.CreateMonitor{
OutputSchemaName: "catalog.schema",
Schedule: &catalog.MonitorCronSchedule{},
},
},
"qualityMonitor3": {
TableName: "qualityMonitor3",
CreateMonitor: &catalog.CreateMonitor{
TableName: "qualityMonitor3",
OutputSchemaName: "catalog.schema",
Schedule: &catalog.MonitorCronSchedule{
PauseStatus: catalog.MonitorCronSchedulePauseStatusUnpaused,
},
@ -517,3 +525,31 @@ func TestPipelinesDevelopmentDisabled(t *testing.T) {
assert.False(t, b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development)
}
func TestInPlaceDeploymentEnabled(t *testing.T) {
b, diags := processInPlaceBundle(true)
require.NoError(t, diags.Error())
assert.True(t, *b.Config.Presets.InPlaceDeployment)
assert.Equal(t, b.Config.Workspace.FilePath, b.SyncRootPath)
}
func TestInPlaceDeploymentDisabled(t *testing.T) {
b, diags := processInPlaceBundle(false)
require.NoError(t, diags.Error())
assert.False(t, *b.Config.Presets.InPlaceDeployment)
assert.NotEqual(t, b.Config.Workspace.FilePath, b.SyncRootPath)
}
func processInPlaceBundle(presetEnabled bool) (*bundle.Bundle, diag.Diagnostics) {
b := mockBundle(config.Development)
workspacePath := "/Workspace/lennart@company.com/"
b.SyncRoot = vfs.MustNew(workspacePath)
b.SyncRootPath = workspacePath
b.Config.Presets.InPlaceDeployment = &presetEnabled
ctx := dbr.MockRuntime(context.Background(), true)
m := bundle.Seq(ProcessTargetMode(), ApplyPresets())
diags := bundle.Apply(ctx, b, m)
return b, diags
}

View File

@ -13,17 +13,15 @@ import (
)
type QualityMonitor struct {
// Represents the Input Arguments for Terraform and will get
// converted to a HCL representation for CRUD
*catalog.CreateMonitor
// This represents the id which is the full name of the monitor
// (catalog_name.schema_name.table_name) that can be used
// as a reference in other resources. This value is returned by terraform.
ID string `json:"id,omitempty" bundle:"readonly"`
ID string `json:"id,omitempty" bundle:"readonly"`
ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"`
URL string `json:"url,omitempty" bundle:"internal"`
// The table name is a required field but not included as a JSON field in [catalog.CreateMonitor].
TableName string `json:"table_name"`
// This struct defines the creation payload for a monitor.
*catalog.CreateMonitor
}
func (s *QualityMonitor) UnmarshalJSON(b []byte) error {

View File

@ -15,8 +15,8 @@ import (
func TestConvertQualityMonitor(t *testing.T) {
var src = resources.QualityMonitor{
TableName: "test_table_name",
CreateMonitor: &catalog.CreateMonitor{
TableName: "test_table_name",
AssetsDir: "assets_dir",
OutputSchemaName: "output_schema_name",
InferenceLog: &catalog.MonitorInferenceLog{

View File

@ -4,6 +4,7 @@ bundle:
resources:
quality_monitors:
myqualitymonitor:
table_name: catalog.schema.quality_monitor
inference_log:
granularities:
- a

View File

@ -684,6 +684,9 @@
"description": "Configuration for monitoring snapshot tables.",
"$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.MonitorSnapshot"
},
"table_name": {
"$ref": "#/$defs/string"
},
"time_series": {
"description": "Configuration for monitoring time series tables.",
"$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.MonitorTimeSeries"
@ -695,6 +698,7 @@
},
"additionalProperties": false,
"required": [
"table_name",
"assets_dir",
"output_schema_name"
]

View File

@ -6,7 +6,6 @@ import (
"sync"
"github.com/databricks/cli/libs/dyn"
"github.com/databricks/cli/libs/textutil"
)
// structInfo holds the type information we need to efficiently
@ -85,14 +84,6 @@ func buildStructInfo(typ reflect.Type) structInfo {
}
name, _, _ := strings.Cut(sf.Tag.Get("json"), ",")
if typ.Name() == "QualityMonitor" && name == "-" {
urlName, _, _ := strings.Cut(sf.Tag.Get("url"), ",")
if urlName == "" || urlName == "-" {
name = textutil.CamelToSnakeCase(sf.Name)
} else {
name = urlName
}
}
if name == "" || name == "-" {
continue
}