2023-08-23 16:47:07 +00:00
package config_tests
import (
"context"
2024-03-27 16:13:53 +00:00
"fmt"
"path/filepath"
2023-08-23 16:47:07 +00:00
"testing"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/config/mutator"
2024-03-25 14:18:47 +00:00
"github.com/databricks/cli/libs/diag"
2024-03-27 16:13:53 +00:00
"github.com/databricks/databricks-sdk-go/service/catalog"
2023-08-23 16:47:07 +00:00
"github.com/databricks/databricks-sdk-go/service/iam"
2024-03-27 16:13:53 +00:00
"github.com/databricks/databricks-sdk-go/service/ml"
2023-08-23 16:47:07 +00:00
"github.com/stretchr/testify/assert"
)
2024-03-27 16:13:53 +00:00
func TestRunAsForAllowed ( t * testing . T ) {
b := load ( t , "./run_as/allowed" )
Use dynamic configuration model in bundles (#1098)
## Changes
This is a fundamental change to how we load and process bundle
configuration. We now depend on the configuration being represented as a
`dyn.Value`. This representation is functionally equivalent to Go's
`any` (it is variadic) and allows us to capture metadata associated with
a value, such as where it was defined (e.g. file, line, and column). It
also allows us to represent Go's zero values properly (e.g. empty
string, integer equal to 0, or boolean false).
Using this representation allows us to let the configuration model
deviate from the typed structure we have been relying on so far
(`config.Root`). We need to deviate from these types when using
variables for fields that are not a string themselves. For example,
using `${var.num_workers}` for an integer `workers` field was impossible
until now (though not implemented in this change).
The loader for a `dyn.Value` includes functionality to capture any and
all type mismatches between the user-defined configuration and the
expected types. These mismatches can be surfaced as validation errors in
future PRs.
Given that many mutators expect the typed struct to be the source of
truth, this change converts between the dynamic representation and the
typed representation on mutator entry and exit. Existing mutators can
continue to modify the typed representation and these modifications are
reflected in the dynamic representation (see `MarkMutatorEntry` and
`MarkMutatorExit` in `bundle/config/root.go`).
Required changes included in this change:
* The existing interpolation package is removed in favor of
`libs/dyn/dynvar`.
* Functionality to merge job clusters, job tasks, and pipeline clusters
are now all broken out into their own mutators.
To be implemented later:
* Allow variable references for non-string types.
* Surface diagnostics about the configuration provided by the user in
the validation output.
* Some mutators use a resource's configuration file path to resolve
related relative paths. These depend on `bundle/config/paths.Path` being
set and populated through `ConfigureConfigFilePath`. Instead, they
should interact with the dynamically typed configuration directly. Doing
this also unlocks being able to differentiate different base paths used
within a job (e.g. a task override with a relative path defined in a
directory other than the base job).
## Tests
* Existing unit tests pass (some have been modified to accommodate)
* Integration tests pass
2024-02-16 19:41:58 +00:00
2023-08-23 16:47:07 +00:00
ctx := context . Background ( )
2024-03-25 14:18:47 +00:00
bundle . ApplyFunc ( ctx , b , func ( ctx context . Context , b * bundle . Bundle ) diag . Diagnostics {
Use dynamic configuration model in bundles (#1098)
## Changes
This is a fundamental change to how we load and process bundle
configuration. We now depend on the configuration being represented as a
`dyn.Value`. This representation is functionally equivalent to Go's
`any` (it is variadic) and allows us to capture metadata associated with
a value, such as where it was defined (e.g. file, line, and column). It
also allows us to represent Go's zero values properly (e.g. empty
string, integer equal to 0, or boolean false).
Using this representation allows us to let the configuration model
deviate from the typed structure we have been relying on so far
(`config.Root`). We need to deviate from these types when using
variables for fields that are not a string themselves. For example,
using `${var.num_workers}` for an integer `workers` field was impossible
until now (though not implemented in this change).
The loader for a `dyn.Value` includes functionality to capture any and
all type mismatches between the user-defined configuration and the
expected types. These mismatches can be surfaced as validation errors in
future PRs.
Given that many mutators expect the typed struct to be the source of
truth, this change converts between the dynamic representation and the
typed representation on mutator entry and exit. Existing mutators can
continue to modify the typed representation and these modifications are
reflected in the dynamic representation (see `MarkMutatorEntry` and
`MarkMutatorExit` in `bundle/config/root.go`).
Required changes included in this change:
* The existing interpolation package is removed in favor of
`libs/dyn/dynvar`.
* Functionality to merge job clusters, job tasks, and pipeline clusters
are now all broken out into their own mutators.
To be implemented later:
* Allow variable references for non-string types.
* Surface diagnostics about the configuration provided by the user in
the validation output.
* Some mutators use a resource's configuration file path to resolve
related relative paths. These depend on `bundle/config/paths.Path` being
set and populated through `ConfigureConfigFilePath`. Instead, they
should interact with the dynamically typed configuration directly. Doing
this also unlocks being able to differentiate different base paths used
within a job (e.g. a task override with a relative path defined in a
directory other than the base job).
## Tests
* Existing unit tests pass (some have been modified to accommodate)
* Integration tests pass
2024-02-16 19:41:58 +00:00
b . Config . Workspace . CurrentUser = & config . User {
User : & iam . User {
UserName : "jane@doe.com" ,
} ,
}
return nil
} )
2024-03-25 14:18:47 +00:00
diags := bundle . Apply ( ctx , b , mutator . SetRunAs ( ) )
assert . NoError ( t , diags . Error ( ) )
2023-08-23 16:47:07 +00:00
assert . Len ( t , b . Config . Resources . Jobs , 3 )
jobs := b . Config . Resources . Jobs
2024-03-27 16:13:53 +00:00
// job_one and job_two should have the same run_as identity as the bundle.
2023-08-23 16:47:07 +00:00
assert . NotNil ( t , jobs [ "job_one" ] . RunAs )
assert . Equal ( t , "my_service_principal" , jobs [ "job_one" ] . RunAs . ServicePrincipalName )
assert . Equal ( t , "" , jobs [ "job_one" ] . RunAs . UserName )
assert . NotNil ( t , jobs [ "job_two" ] . RunAs )
assert . Equal ( t , "my_service_principal" , jobs [ "job_two" ] . RunAs . ServicePrincipalName )
assert . Equal ( t , "" , jobs [ "job_two" ] . RunAs . UserName )
2024-03-27 16:13:53 +00:00
// job_three should retain the job level run_as identity.
2023-08-23 16:47:07 +00:00
assert . NotNil ( t , jobs [ "job_three" ] . RunAs )
assert . Equal ( t , "my_service_principal_for_job" , jobs [ "job_three" ] . RunAs . ServicePrincipalName )
assert . Equal ( t , "" , jobs [ "job_three" ] . RunAs . UserName )
2024-03-27 16:13:53 +00:00
// Assert other resources are not affected.
assert . Equal ( t , ml . Model { Name : "skynet" } , * b . Config . Resources . Models [ "model_one" ] . Model )
assert . Equal ( t , catalog . CreateRegisteredModelRequest { Name : "skynet (in UC)" } , * b . Config . Resources . RegisteredModels [ "model_two" ] . CreateRegisteredModelRequest )
assert . Equal ( t , ml . Experiment { Name : "experiment_one" } , * b . Config . Resources . Experiments [ "experiment_one" ] . Experiment )
2023-08-23 16:47:07 +00:00
}
2024-03-27 16:13:53 +00:00
func TestRunAsForAllowedWithTargetOverride ( t * testing . T ) {
b := loadTarget ( t , "./run_as/allowed" , "development" )
Use dynamic configuration model in bundles (#1098)
## Changes
This is a fundamental change to how we load and process bundle
configuration. We now depend on the configuration being represented as a
`dyn.Value`. This representation is functionally equivalent to Go's
`any` (it is variadic) and allows us to capture metadata associated with
a value, such as where it was defined (e.g. file, line, and column). It
also allows us to represent Go's zero values properly (e.g. empty
string, integer equal to 0, or boolean false).
Using this representation allows us to let the configuration model
deviate from the typed structure we have been relying on so far
(`config.Root`). We need to deviate from these types when using
variables for fields that are not a string themselves. For example,
using `${var.num_workers}` for an integer `workers` field was impossible
until now (though not implemented in this change).
The loader for a `dyn.Value` includes functionality to capture any and
all type mismatches between the user-defined configuration and the
expected types. These mismatches can be surfaced as validation errors in
future PRs.
Given that many mutators expect the typed struct to be the source of
truth, this change converts between the dynamic representation and the
typed representation on mutator entry and exit. Existing mutators can
continue to modify the typed representation and these modifications are
reflected in the dynamic representation (see `MarkMutatorEntry` and
`MarkMutatorExit` in `bundle/config/root.go`).
Required changes included in this change:
* The existing interpolation package is removed in favor of
`libs/dyn/dynvar`.
* Functionality to merge job clusters, job tasks, and pipeline clusters
are now all broken out into their own mutators.
To be implemented later:
* Allow variable references for non-string types.
* Surface diagnostics about the configuration provided by the user in
the validation output.
* Some mutators use a resource's configuration file path to resolve
related relative paths. These depend on `bundle/config/paths.Path` being
set and populated through `ConfigureConfigFilePath`. Instead, they
should interact with the dynamically typed configuration directly. Doing
this also unlocks being able to differentiate different base paths used
within a job (e.g. a task override with a relative path defined in a
directory other than the base job).
## Tests
* Existing unit tests pass (some have been modified to accommodate)
* Integration tests pass
2024-02-16 19:41:58 +00:00
2023-08-23 16:47:07 +00:00
ctx := context . Background ( )
2024-03-25 14:18:47 +00:00
bundle . ApplyFunc ( ctx , b , func ( ctx context . Context , b * bundle . Bundle ) diag . Diagnostics {
Use dynamic configuration model in bundles (#1098)
## Changes
This is a fundamental change to how we load and process bundle
configuration. We now depend on the configuration being represented as a
`dyn.Value`. This representation is functionally equivalent to Go's
`any` (it is variadic) and allows us to capture metadata associated with
a value, such as where it was defined (e.g. file, line, and column). It
also allows us to represent Go's zero values properly (e.g. empty
string, integer equal to 0, or boolean false).
Using this representation allows us to let the configuration model
deviate from the typed structure we have been relying on so far
(`config.Root`). We need to deviate from these types when using
variables for fields that are not a string themselves. For example,
using `${var.num_workers}` for an integer `workers` field was impossible
until now (though not implemented in this change).
The loader for a `dyn.Value` includes functionality to capture any and
all type mismatches between the user-defined configuration and the
expected types. These mismatches can be surfaced as validation errors in
future PRs.
Given that many mutators expect the typed struct to be the source of
truth, this change converts between the dynamic representation and the
typed representation on mutator entry and exit. Existing mutators can
continue to modify the typed representation and these modifications are
reflected in the dynamic representation (see `MarkMutatorEntry` and
`MarkMutatorExit` in `bundle/config/root.go`).
Required changes included in this change:
* The existing interpolation package is removed in favor of
`libs/dyn/dynvar`.
* Functionality to merge job clusters, job tasks, and pipeline clusters
are now all broken out into their own mutators.
To be implemented later:
* Allow variable references for non-string types.
* Surface diagnostics about the configuration provided by the user in
the validation output.
* Some mutators use a resource's configuration file path to resolve
related relative paths. These depend on `bundle/config/paths.Path` being
set and populated through `ConfigureConfigFilePath`. Instead, they
should interact with the dynamically typed configuration directly. Doing
this also unlocks being able to differentiate different base paths used
within a job (e.g. a task override with a relative path defined in a
directory other than the base job).
## Tests
* Existing unit tests pass (some have been modified to accommodate)
* Integration tests pass
2024-02-16 19:41:58 +00:00
b . Config . Workspace . CurrentUser = & config . User {
User : & iam . User {
UserName : "jane@doe.com" ,
} ,
}
return nil
} )
2024-03-25 14:18:47 +00:00
diags := bundle . Apply ( ctx , b , mutator . SetRunAs ( ) )
assert . NoError ( t , diags . Error ( ) )
2023-08-23 16:47:07 +00:00
assert . Len ( t , b . Config . Resources . Jobs , 3 )
jobs := b . Config . Resources . Jobs
2024-03-27 16:13:53 +00:00
// job_one and job_two should have the same run_as identity as the bundle's
// development target.
2023-08-23 16:47:07 +00:00
assert . NotNil ( t , jobs [ "job_one" ] . RunAs )
assert . Equal ( t , "" , jobs [ "job_one" ] . RunAs . ServicePrincipalName )
assert . Equal ( t , "my_user_name" , jobs [ "job_one" ] . RunAs . UserName )
assert . NotNil ( t , jobs [ "job_two" ] . RunAs )
assert . Equal ( t , "" , jobs [ "job_two" ] . RunAs . ServicePrincipalName )
assert . Equal ( t , "my_user_name" , jobs [ "job_two" ] . RunAs . UserName )
2024-03-27 16:13:53 +00:00
// job_three should retain the job level run_as identity.
2023-08-23 16:47:07 +00:00
assert . NotNil ( t , jobs [ "job_three" ] . RunAs )
assert . Equal ( t , "my_service_principal_for_job" , jobs [ "job_three" ] . RunAs . ServicePrincipalName )
assert . Equal ( t , "" , jobs [ "job_three" ] . RunAs . UserName )
2024-03-27 16:13:53 +00:00
// Assert other resources are not affected.
assert . Equal ( t , ml . Model { Name : "skynet" } , * b . Config . Resources . Models [ "model_one" ] . Model )
assert . Equal ( t , catalog . CreateRegisteredModelRequest { Name : "skynet (in UC)" } , * b . Config . Resources . RegisteredModels [ "model_two" ] . CreateRegisteredModelRequest )
assert . Equal ( t , ml . Experiment { Name : "experiment_one" } , * b . Config . Resources . Experiments [ "experiment_one" ] . Experiment )
}
func TestRunAsErrorForPipelines ( t * testing . T ) {
b := load ( t , "./run_as/not_allowed/pipelines" )
ctx := context . Background ( )
bundle . ApplyFunc ( ctx , b , func ( ctx context . Context , b * bundle . Bundle ) diag . Diagnostics {
b . Config . Workspace . CurrentUser = & config . User {
User : & iam . User {
UserName : "jane@doe.com" ,
} ,
}
return nil
} )
diags := bundle . Apply ( ctx , b , mutator . SetRunAs ( ) )
err := diags . Error ( )
configPath := filepath . FromSlash ( "run_as/not_allowed/pipelines/databricks.yml" )
assert . EqualError ( t , err , fmt . Sprintf ( "pipelines are not supported when the current deployment user is different from the bundle's run_as identity. Please deploy as the run_as identity. Location of the unsupported resource: %s:14:5. Current identity: jane@doe.com. Run as identity: my_service_principal" , configPath ) )
}
func TestRunAsNoErrorForPipelines ( t * testing . T ) {
b := load ( t , "./run_as/not_allowed/pipelines" )
// We should not error because the pipeline is being deployed with the same
// identity as the bundle run_as identity.
ctx := context . Background ( )
bundle . ApplyFunc ( ctx , b , func ( ctx context . Context , b * bundle . Bundle ) diag . Diagnostics {
b . Config . Workspace . CurrentUser = & config . User {
User : & iam . User {
UserName : "my_service_principal" ,
} ,
}
return nil
} )
diags := bundle . Apply ( ctx , b , mutator . SetRunAs ( ) )
assert . NoError ( t , diags . Error ( ) )
}
func TestRunAsErrorForModelServing ( t * testing . T ) {
b := load ( t , "./run_as/not_allowed/model_serving" )
ctx := context . Background ( )
bundle . ApplyFunc ( ctx , b , func ( ctx context . Context , b * bundle . Bundle ) diag . Diagnostics {
b . Config . Workspace . CurrentUser = & config . User {
User : & iam . User {
UserName : "jane@doe.com" ,
} ,
}
return nil
} )
diags := bundle . Apply ( ctx , b , mutator . SetRunAs ( ) )
err := diags . Error ( )
configPath := filepath . FromSlash ( "run_as/not_allowed/model_serving/databricks.yml" )
assert . EqualError ( t , err , fmt . Sprintf ( "model_serving_endpoints are not supported when the current deployment user is different from the bundle's run_as identity. Please deploy as the run_as identity. Location of the unsupported resource: %s:14:5. Current identity: jane@doe.com. Run as identity: my_service_principal" , configPath ) )
}
func TestRunAsNoErrorForModelServingEndpoints ( t * testing . T ) {
b := load ( t , "./run_as/not_allowed/model_serving" )
// We should not error because the model serving endpoint is being deployed
// with the same identity as the bundle run_as identity.
ctx := context . Background ( )
bundle . ApplyFunc ( ctx , b , func ( ctx context . Context , b * bundle . Bundle ) diag . Diagnostics {
b . Config . Workspace . CurrentUser = & config . User {
User : & iam . User {
UserName : "my_service_principal" ,
} ,
}
return nil
} )
diags := bundle . Apply ( ctx , b , mutator . SetRunAs ( ) )
assert . NoError ( t , diags . Error ( ) )
}
func TestRunAsErrorWhenBothUserAndSpSpecified ( t * testing . T ) {
b := load ( t , "./run_as/not_allowed/both_sp_and_user" )
ctx := context . Background ( )
bundle . ApplyFunc ( ctx , b , func ( ctx context . Context , b * bundle . Bundle ) diag . Diagnostics {
b . Config . Workspace . CurrentUser = & config . User {
User : & iam . User {
UserName : "my_service_principal" ,
} ,
}
return nil
} )
diags := bundle . Apply ( ctx , b , mutator . SetRunAs ( ) )
err := diags . Error ( )
configPath := filepath . FromSlash ( "run_as/not_allowed/both_sp_and_user/databricks.yml" )
assert . EqualError ( t , err , fmt . Sprintf ( "run_as section must specify exactly one identity. A service_principal_name \"my_service_principal\" is specified at %s:6:27. A user_name \"my_user_name\" is defined at %s:7:14" , configPath , configPath ) )
}
func TestRunAsErrorNeitherUserOrSpSpecified ( t * testing . T ) {
b := load ( t , "./run_as/not_allowed/neither_sp_nor_user" )
ctx := context . Background ( )
bundle . ApplyFunc ( ctx , b , func ( ctx context . Context , b * bundle . Bundle ) diag . Diagnostics {
b . Config . Workspace . CurrentUser = & config . User {
User : & iam . User {
UserName : "my_service_principal" ,
} ,
}
return nil
} )
diags := bundle . Apply ( ctx , b , mutator . SetRunAs ( ) )
err := diags . Error ( )
configPath := filepath . FromSlash ( "run_as/not_allowed/neither_sp_nor_user/databricks.yml" )
assert . EqualError ( t , err , fmt . Sprintf ( "run_as section must specify exactly one identity. Neither service_principal_name nor user_name is specified at %s:4:8" , configPath ) )
}
func TestRunAsErrorNeitherUserOrSpSpecifiedAtTargetOverride ( t * testing . T ) {
b := loadTarget ( t , "./run_as/not_allowed/neither_sp_nor_user_override" , "development" )
ctx := context . Background ( )
bundle . ApplyFunc ( ctx , b , func ( ctx context . Context , b * bundle . Bundle ) diag . Diagnostics {
b . Config . Workspace . CurrentUser = & config . User {
User : & iam . User {
UserName : "my_service_principal" ,
} ,
}
return nil
} )
diags := bundle . Apply ( ctx , b , mutator . SetRunAs ( ) )
err := diags . Error ( )
2023-08-23 16:47:07 +00:00
2024-03-27 16:13:53 +00:00
configPath := filepath . FromSlash ( "run_as/not_allowed/neither_sp_nor_user_override/override.yml" )
assert . EqualError ( t , err , fmt . Sprintf ( "run_as section must specify exactly one identity. Neither service_principal_name nor user_name is specified at %s:4:12" , configPath ) )
2023-08-23 16:47:07 +00:00
}
