mirror of https://github.com/databricks/cli.git
Merge 4233a7c292 into 14fe03dcb9
This commit is contained in:
shreyas-goenka
GitHub
commit
eb236a3258
|
|
@ -0,0 +1,135 @@
|
||||||
|
package validate
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/databricks/cli/bundle"
|
||||||
|
"github.com/databricks/cli/libs/diag"
|
||||||
|
"github.com/databricks/cli/libs/dyn"
|
||||||
|
"github.com/databricks/cli/libs/dyn/convert"
|
||||||
|
"github.com/databricks/cli/libs/log"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Validates that any single node clusters defined in the bundle are correctly configured.
|
||||||
|
func SingleNodeCluster() bundle.ReadOnlyMutator {
|
||||||
|
return &singleNodeCluster{}
|
||||||
|
}
|
||||||
|
|
||||||
|
type singleNodeCluster struct{}
|
||||||
|
|
||||||
|
func (m *singleNodeCluster) Name() string {
|
||||||
|
return "validate:SingleNodeCluster"
|
||||||
|
}
|
||||||
|
|
||||||
|
const singleNodeWarningDetail = `num_workers should be 0 only for single-node clusters. To create a
|
||||||
|
valid single node cluster please ensure that the following properties
|
||||||
|
are correctly set in the cluster specification:
|
||||||
|
|
||||||
|
spark_conf:
|
||||||
|
spark.databricks.cluster.profile: singleNode
|
||||||
|
spark.master: local[*]
|
||||||
|
|
||||||
|
custom_tags:
|
||||||
|
ResourceClass: SingleNode
|
||||||
|
`
|
||||||
|
|
||||||
|
const singleNodeWarningSummary = `Single node cluster is not correctly configured`
|
||||||
|
|
||||||
|
func showSingleNodeClusterWarning(ctx context.Context, v dyn.Value) bool {
|
||||||
|
// Check if the user has explicitly set the num_workers to 0. Skip the warning
|
||||||
|
// if that's not the case.
|
||||||
|
numWorkers, ok := v.Get("num_workers").AsInt()
|
||||||
|
if !ok || numWorkers > 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// Convenient type that contains the common fields from compute.ClusterSpec and
|
||||||
|
// pipelines.PipelineCluster that we are interested in.
|
||||||
|
type ClusterConf struct {
|
||||||
|
SparkConf map[string]string `json:"spark_conf"`
|
||||||
|
CustomTags map[string]string `json:"custom_tags"`
|
||||||
|
PolicyId string `json:"policy_id"`
|
||||||
|
}
|
||||||
|
|
||||||
|
conf := &ClusterConf{}
|
||||||
|
err := convert.ToTyped(conf, v)
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// If the policy id is set, we don't want to show the warning. This is because
|
||||||
|
// the user might have configured `spark_conf` and `custom_tags` correctly
|
||||||
|
// in their cluster policy.
|
||||||
|
if conf.PolicyId != "" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
profile, ok := conf.SparkConf["spark.databricks.cluster.profile"]
|
||||||
|
if !ok {
|
||||||
|
log.Warnf(ctx, "spark_conf spark.databricks.cluster.profile not found in single-node cluster spec")
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if profile != "singleNode" {
|
||||||
|
log.Warnf(ctx, "spark_conf spark.databricks.cluster.profile is not singleNode in single-node cluster spec: %s", profile)
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
master, ok := conf.SparkConf["spark.master"]
|
||||||
|
if !ok {
|
||||||
|
log.Warnf(ctx, "spark_conf spark.master not found in single-node cluster spec")
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if !strings.HasPrefix(master, "local") {
|
||||||
|
log.Warnf(ctx, "spark_conf spark.master does not start with local in single-node cluster spec: %s", master)
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
resourceClass, ok := conf.CustomTags["ResourceClass"]
|
||||||
|
if !ok {
|
||||||
|
log.Warnf(ctx, "custom_tag ResourceClass not found in single-node cluster spec")
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if resourceClass != "SingleNode" {
|
||||||
|
log.Warnf(ctx, "custom_tag ResourceClass is not SingleNode in single-node cluster spec: %s", resourceClass)
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *singleNodeCluster) Apply(ctx context.Context, rb bundle.ReadOnlyBundle) diag.Diagnostics {
|
||||||
|
diags := diag.Diagnostics{}
|
||||||
|
|
||||||
|
patterns := []dyn.Pattern{
|
||||||
|
// Interactive clusters
|
||||||
|
dyn.NewPattern(dyn.Key("resources"), dyn.Key("clusters"), dyn.AnyKey()),
|
||||||
|
// Job clusters
|
||||||
|
dyn.NewPattern(dyn.Key("resources"), dyn.Key("jobs"), dyn.AnyKey(), dyn.Key("job_clusters"), dyn.AnyIndex(), dyn.Key("new_cluster")),
|
||||||
|
// Job task clusters
|
||||||
|
dyn.NewPattern(dyn.Key("resources"), dyn.Key("jobs"), dyn.AnyKey(), dyn.Key("tasks"), dyn.AnyIndex(), dyn.Key("new_cluster")),
|
||||||
|
// Pipeline clusters
|
||||||
|
dyn.NewPattern(dyn.Key("resources"), dyn.Key("pipelines"), dyn.AnyKey(), dyn.Key("clusters"), dyn.AnyIndex()),
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, p := range patterns {
|
||||||
|
_, err := dyn.MapByPattern(rb.Config().Value(), p, func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
|
||||||
|
warning := diag.Diagnostic{
|
||||||
|
Severity: diag.Warning,
|
||||||
|
Summary: singleNodeWarningSummary,
|
||||||
|
Detail: singleNodeWarningDetail,
|
||||||
|
Locations: v.Locations(),
|
||||||
|
Paths: []dyn.Path{p},
|
||||||
|
}
|
||||||
|
|
||||||
|
if showSingleNodeClusterWarning(ctx, v) {
|
||||||
|
diags = append(diags, warning)
|
||||||
|
}
|
||||||
|
return v, nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
log.Debugf(ctx, "Error while applying single node cluster validation: %s", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return diags
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,440 @@
|
||||||
|
package validate
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/databricks/cli/bundle"
|
||||||
|
"github.com/databricks/cli/bundle/config"
|
||||||
|
"github.com/databricks/cli/bundle/config/resources"
|
||||||
|
"github.com/databricks/cli/bundle/internal/bundletest"
|
||||||
|
"github.com/databricks/cli/libs/diag"
|
||||||
|
"github.com/databricks/cli/libs/dyn"
|
||||||
|
"github.com/databricks/databricks-sdk-go/service/compute"
|
||||||
|
"github.com/databricks/databricks-sdk-go/service/jobs"
|
||||||
|
"github.com/databricks/databricks-sdk-go/service/pipelines"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestValidateSingleNodeClusterFail(t *testing.T) {
|
||||||
|
failCases := []struct {
|
||||||
|
name string
|
||||||
|
sparkConf map[string]string
|
||||||
|
customTags map[string]string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "no tags or conf",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "no tags",
|
||||||
|
sparkConf: map[string]string{
|
||||||
|
"spark.databricks.cluster.profile": "singleNode",
|
||||||
|
"spark.master": "local[*]",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "no conf",
|
||||||
|
customTags: map[string]string{"ResourceClass": "SingleNode"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid spark cluster profile",
|
||||||
|
sparkConf: map[string]string{
|
||||||
|
"spark.databricks.cluster.profile": "invalid",
|
||||||
|
"spark.master": "local[*]",
|
||||||
|
},
|
||||||
|
customTags: map[string]string{"ResourceClass": "SingleNode"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid spark.master",
|
||||||
|
sparkConf: map[string]string{
|
||||||
|
"spark.databricks.cluster.profile": "singleNode",
|
||||||
|
"spark.master": "invalid",
|
||||||
|
},
|
||||||
|
customTags: map[string]string{"ResourceClass": "SingleNode"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid tags",
|
||||||
|
sparkConf: map[string]string{
|
||||||
|
"spark.databricks.cluster.profile": "singleNode",
|
||||||
|
"spark.master": "local[*]",
|
||||||
|
},
|
||||||
|
customTags: map[string]string{"ResourceClass": "invalid"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "missing ResourceClass tag",
|
||||||
|
sparkConf: map[string]string{
|
||||||
|
"spark.databricks.cluster.profile": "singleNode",
|
||||||
|
"spark.master": "local[*]",
|
||||||
|
},
|
||||||
|
customTags: map[string]string{"what": "ever"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "missing spark.master",
|
||||||
|
sparkConf: map[string]string{
|
||||||
|
"spark.databricks.cluster.profile": "singleNode",
|
||||||
|
},
|
||||||
|
customTags: map[string]string{"ResourceClass": "SingleNode"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "missing spark.databricks.cluster.profile",
|
||||||
|
sparkConf: map[string]string{
|
||||||
|
"spark.master": "local[*]",
|
||||||
|
},
|
||||||
|
customTags: map[string]string{"ResourceClass": "SingleNode"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
// Interactive clusters.
|
||||||
|
for _, tc := range failCases {
|
||||||
|
t.Run("interactive_"+tc.name, func(t *testing.T) {
|
||||||
|
b := &bundle.Bundle{
|
||||||
|
Config: config.Root{
|
||||||
|
Resources: config.Resources{
|
||||||
|
Clusters: map[string]*resources.Cluster{
|
||||||
|
"foo": {
|
||||||
|
ClusterSpec: &compute.ClusterSpec{
|
||||||
|
SparkConf: tc.sparkConf,
|
||||||
|
CustomTags: tc.customTags,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
bundletest.SetLocation(b, "resources.clusters.foo", []dyn.Location{{File: "a.yml", Line: 1, Column: 1}})
|
||||||
|
|
||||||
|
// We can't set num_workers to 0 explicitly in the typed configuration.
|
||||||
|
// Do it on the dyn.Value directly.
|
||||||
|
bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) {
|
||||||
|
return dyn.Set(v, "resources.clusters.foo.num_workers", dyn.V(0))
|
||||||
|
})
|
||||||
|
diags := bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), SingleNodeCluster())
|
||||||
|
assert.Equal(t, diag.Diagnostics{
|
||||||
|
{
|
||||||
|
Severity: diag.Warning,
|
||||||
|
Summary: singleNodeWarningSummary,
|
||||||
|
Detail: singleNodeWarningDetail,
|
||||||
|
Locations: []dyn.Location{{File: "a.yml", Line: 1, Column: 1}},
|
||||||
|
Paths: []dyn.Path{dyn.NewPath(dyn.Key("resources"), dyn.Key("clusters"), dyn.Key("foo"))},
|
||||||
|
},
|
||||||
|
}, diags)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Job clusters.
|
||||||
|
for _, tc := range failCases {
|
||||||
|
t.Run("job_"+tc.name, func(t *testing.T) {
|
||||||
|
b := &bundle.Bundle{
|
||||||
|
Config: config.Root{
|
||||||
|
Resources: config.Resources{
|
||||||
|
Jobs: map[string]*resources.Job{
|
||||||
|
"foo": {
|
||||||
|
JobSettings: &jobs.JobSettings{
|
||||||
|
JobClusters: []jobs.JobCluster{
|
||||||
|
{
|
||||||
|
NewCluster: compute.ClusterSpec{
|
||||||
|
ClusterName: "my_cluster",
|
||||||
|
SparkConf: tc.sparkConf,
|
||||||
|
CustomTags: tc.customTags,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
bundletest.SetLocation(b, "resources.jobs.foo.job_clusters[0].new_cluster", []dyn.Location{{File: "b.yml", Line: 1, Column: 1}})
|
||||||
|
|
||||||
|
// We can't set num_workers to 0 explicitly in the typed configuration.
|
||||||
|
// Do it on the dyn.Value directly.
|
||||||
|
bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) {
|
||||||
|
return dyn.Set(v, "resources.jobs.foo.job_clusters[0].new_cluster.num_workers", dyn.V(0))
|
||||||
|
})
|
||||||
|
|
||||||
|
diags := bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), SingleNodeCluster())
|
||||||
|
assert.Equal(t, diag.Diagnostics{
|
||||||
|
{
|
||||||
|
Severity: diag.Warning,
|
||||||
|
Summary: singleNodeWarningSummary,
|
||||||
|
Detail: singleNodeWarningDetail,
|
||||||
|
Locations: []dyn.Location{{File: "b.yml", Line: 1, Column: 1}},
|
||||||
|
Paths: []dyn.Path{dyn.MustPathFromString("resources.jobs.foo.job_clusters[0].new_cluster")},
|
||||||
|
},
|
||||||
|
}, diags)
|
||||||
|
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Job task clusters.
|
||||||
|
for _, tc := range failCases {
|
||||||
|
t.Run("task_"+tc.name, func(t *testing.T) {
|
||||||
|
b := &bundle.Bundle{
|
||||||
|
Config: config.Root{
|
||||||
|
Resources: config.Resources{
|
||||||
|
Jobs: map[string]*resources.Job{
|
||||||
|
"foo": {
|
||||||
|
JobSettings: &jobs.JobSettings{
|
||||||
|
Tasks: []jobs.Task{
|
||||||
|
{
|
||||||
|
NewCluster: &compute.ClusterSpec{
|
||||||
|
ClusterName: "my_cluster",
|
||||||
|
SparkConf: tc.sparkConf,
|
||||||
|
CustomTags: tc.customTags,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
bundletest.SetLocation(b, "resources.jobs.foo.tasks[0].new_cluster", []dyn.Location{{File: "c.yml", Line: 1, Column: 1}})
|
||||||
|
|
||||||
|
// We can't set num_workers to 0 explicitly in the typed configuration.
|
||||||
|
// Do it on the dyn.Value directly.
|
||||||
|
bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) {
|
||||||
|
return dyn.Set(v, "resources.jobs.foo.tasks[0].new_cluster.num_workers", dyn.V(0))
|
||||||
|
})
|
||||||
|
|
||||||
|
diags := bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), SingleNodeCluster())
|
||||||
|
assert.Equal(t, diag.Diagnostics{
|
||||||
|
{
|
||||||
|
Severity: diag.Warning,
|
||||||
|
Summary: singleNodeWarningSummary,
|
||||||
|
Detail: singleNodeWarningDetail,
|
||||||
|
Locations: []dyn.Location{{File: "c.yml", Line: 1, Column: 1}},
|
||||||
|
Paths: []dyn.Path{dyn.MustPathFromString("resources.jobs.foo.tasks[0].new_cluster")},
|
||||||
|
},
|
||||||
|
}, diags)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Pipeline clusters.
|
||||||
|
for _, tc := range failCases {
|
||||||
|
t.Run("pipeline_"+tc.name, func(t *testing.T) {
|
||||||
|
b := &bundle.Bundle{
|
||||||
|
Config: config.Root{
|
||||||
|
Resources: config.Resources{
|
||||||
|
Pipelines: map[string]*resources.Pipeline{
|
||||||
|
"foo": {
|
||||||
|
PipelineSpec: &pipelines.PipelineSpec{
|
||||||
|
Clusters: []pipelines.PipelineCluster{
|
||||||
|
{
|
||||||
|
SparkConf: tc.sparkConf,
|
||||||
|
CustomTags: tc.customTags,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
bundletest.SetLocation(b, "resources.pipelines.foo.clusters[0]", []dyn.Location{{File: "d.yml", Line: 1, Column: 1}})
|
||||||
|
|
||||||
|
// We can't set num_workers to 0 explicitly in the typed configuration.
|
||||||
|
// Do it on the dyn.Value directly.
|
||||||
|
bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) {
|
||||||
|
return dyn.Set(v, "resources.pipelines.foo.clusters[0].num_workers", dyn.V(0))
|
||||||
|
})
|
||||||
|
|
||||||
|
diags := bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), SingleNodeCluster())
|
||||||
|
assert.Equal(t, diag.Diagnostics{
|
||||||
|
{
|
||||||
|
Severity: diag.Warning,
|
||||||
|
Summary: singleNodeWarningSummary,
|
||||||
|
Detail: singleNodeWarningDetail,
|
||||||
|
Locations: []dyn.Location{{File: "d.yml", Line: 1, Column: 1}},
|
||||||
|
Paths: []dyn.Path{dyn.MustPathFromString("resources.pipelines.foo.clusters[0]")},
|
||||||
|
},
|
||||||
|
}, diags)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateSingleNodeClusterPass(t *testing.T) {
|
||||||
|
zero := 0
|
||||||
|
one := 1
|
||||||
|
|
||||||
|
passCases := []struct {
|
||||||
|
name string
|
||||||
|
numWorkers *int
|
||||||
|
sparkConf map[string]string
|
||||||
|
customTags map[string]string
|
||||||
|
policyId string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "single node cluster",
|
||||||
|
sparkConf: map[string]string{
|
||||||
|
"spark.databricks.cluster.profile": "singleNode",
|
||||||
|
"spark.master": "local[*]",
|
||||||
|
},
|
||||||
|
customTags: map[string]string{
|
||||||
|
"ResourceClass": "SingleNode",
|
||||||
|
},
|
||||||
|
numWorkers: &zero,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "num workers is not zero",
|
||||||
|
numWorkers: &one,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "num workers is not set",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "policy id is not empty",
|
||||||
|
policyId: "policy-abc",
|
||||||
|
numWorkers: &zero,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
// Interactive clusters.
|
||||||
|
for _, tc := range passCases {
|
||||||
|
t.Run("interactive_"+tc.name, func(t *testing.T) {
|
||||||
|
b := &bundle.Bundle{
|
||||||
|
Config: config.Root{
|
||||||
|
Resources: config.Resources{
|
||||||
|
Clusters: map[string]*resources.Cluster{
|
||||||
|
"foo": {
|
||||||
|
ClusterSpec: &compute.ClusterSpec{
|
||||||
|
SparkConf: tc.sparkConf,
|
||||||
|
CustomTags: tc.customTags,
|
||||||
|
PolicyId: tc.policyId,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
if tc.numWorkers != nil {
|
||||||
|
bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) {
|
||||||
|
return dyn.Set(v, "resources.clusters.foo.num_workers", dyn.V(*tc.numWorkers))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
diags := bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), SingleNodeCluster())
|
||||||
|
assert.Empty(t, diags)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Job clusters.
|
||||||
|
for _, tc := range passCases {
|
||||||
|
t.Run("job_"+tc.name, func(t *testing.T) {
|
||||||
|
b := &bundle.Bundle{
|
||||||
|
Config: config.Root{
|
||||||
|
Resources: config.Resources{
|
||||||
|
Jobs: map[string]*resources.Job{
|
||||||
|
"foo": {
|
||||||
|
JobSettings: &jobs.JobSettings{
|
||||||
|
JobClusters: []jobs.JobCluster{
|
||||||
|
{
|
||||||
|
NewCluster: compute.ClusterSpec{
|
||||||
|
ClusterName: "my_cluster",
|
||||||
|
SparkConf: tc.sparkConf,
|
||||||
|
CustomTags: tc.customTags,
|
||||||
|
PolicyId: tc.policyId,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
if tc.numWorkers != nil {
|
||||||
|
bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) {
|
||||||
|
return dyn.Set(v, "resources.jobs.foo.job_clusters[0].new_cluster.num_workers", dyn.V(*tc.numWorkers))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
diags := bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), SingleNodeCluster())
|
||||||
|
assert.Empty(t, diags)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Job task clusters.
|
||||||
|
for _, tc := range passCases {
|
||||||
|
t.Run("task_"+tc.name, func(t *testing.T) {
|
||||||
|
b := &bundle.Bundle{
|
||||||
|
Config: config.Root{
|
||||||
|
Resources: config.Resources{
|
||||||
|
Jobs: map[string]*resources.Job{
|
||||||
|
"foo": {
|
||||||
|
JobSettings: &jobs.JobSettings{
|
||||||
|
Tasks: []jobs.Task{
|
||||||
|
{
|
||||||
|
NewCluster: &compute.ClusterSpec{
|
||||||
|
ClusterName: "my_cluster",
|
||||||
|
SparkConf: tc.sparkConf,
|
||||||
|
CustomTags: tc.customTags,
|
||||||
|
PolicyId: tc.policyId,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
if tc.numWorkers != nil {
|
||||||
|
bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) {
|
||||||
|
return dyn.Set(v, "resources.jobs.foo.tasks[0].new_cluster.num_workers", dyn.V(*tc.numWorkers))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
diags := bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), SingleNodeCluster())
|
||||||
|
assert.Empty(t, diags)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Pipeline clusters.
|
||||||
|
for _, tc := range passCases {
|
||||||
|
t.Run("pipeline_"+tc.name, func(t *testing.T) {
|
||||||
|
b := &bundle.Bundle{
|
||||||
|
Config: config.Root{
|
||||||
|
Resources: config.Resources{
|
||||||
|
Pipelines: map[string]*resources.Pipeline{
|
||||||
|
"foo": {
|
||||||
|
PipelineSpec: &pipelines.PipelineSpec{
|
||||||
|
Clusters: []pipelines.PipelineCluster{
|
||||||
|
{
|
||||||
|
SparkConf: tc.sparkConf,
|
||||||
|
CustomTags: tc.customTags,
|
||||||
|
PolicyId: tc.policyId,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
if tc.numWorkers != nil {
|
||||||
|
bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) {
|
||||||
|
return dyn.Set(v, "resources.pipelines.foo.clusters[0].num_workers", dyn.V(*tc.numWorkers))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
diags := bundle.ApplyReadOnly(ctx, bundle.ReadOnly(b), SingleNodeCluster())
|
||||||
|
assert.Empty(t, diags)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -36,6 +36,7 @@ func (v *validate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics
|
||||||
ValidateSyncPatterns(),
|
ValidateSyncPatterns(),
|
||||||
JobTaskClusterSpec(),
|
JobTaskClusterSpec(),
|
||||||
ValidateFolderPermissions(),
|
ValidateFolderPermissions(),
|
||||||
|
SingleNodeCluster(),
|
||||||
))
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue