steve/databricks-cli
steve
/
databricks-cli
mirror of https://github.com/databricks/cli.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Cleanup

This commit is contained in:
Lennart Kats 2024-12-20 09:28:38 +01:00
parent feb23ddefb
commit 65f10d187d
No known key found for this signature in database
GPG Key ID: 1EB8B57673197023
1 changed files with 101 additions and 116 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

217
bundle/config/mutator/apply_presets_catalog_schema.go
View File

@ -28,34 +28,40 @@ func (m *applyPresetsCatalogSchema) Name() string {
} }
func (m *applyPresetsCatalogSchema) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { func (m *applyPresetsCatalogSchema) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
diags := validateCatalogAndSchema(b) diags := diag.Diagnostics{}
if diags.HasError() { p := b.Config.Presets
r := b.Config.Resources
if p.Catalog == "" && p.Schema == "" {
return diags return diags
} }
if (p.Schema == "") || (p.Catalog == "" && p.Schema != "") {
r := b.Config.Resources return diag.Diagnostics{{
p := b.Config.Presets Summary: "presets.catalog and presets.schema must always be set together",
Severity: diag.Error,
Locations: []dyn.Location{b.Config.GetLocation("presets")},
}}
}
// Jobs // Jobs
for key, j := range r.Jobs { for key, j := range r.Jobs {
if j.JobSettings == nil { if j.JobSettings == nil {
continue continue
} }
if p.Catalog != "" || p.Schema != "" {
for _, task := range j.Tasks { for _, task := range j.Tasks {
if task.DbtTask != nil { if task.DbtTask != nil {
if task.DbtTask.Catalog == "" { if task.DbtTask.Catalog == "" {
task.DbtTask.Catalog = p.Catalog task.DbtTask.Catalog = p.Catalog
} }
if task.DbtTask.Schema == "" { if task.DbtTask.Schema == "" {
task.DbtTask.Schema = p.Schema task.DbtTask.Schema = p.Schema
}
} }
} }
diags = diags.Extend(addCatalogSchemaParameters(b, key, j, p))
diags = diags.Extend(recommendCatalogSchemaUsage(b, ctx, key, j))
} }
diags = diags.Extend(addCatalogSchemaParameters(b, key, j, p))
diags = diags.Extend(recommendCatalogSchemaUsage(b, ctx, key, j))
} }
// Pipelines // Pipelines
@ -63,58 +69,53 @@ func (m *applyPresetsCatalogSchema) Apply(ctx context.Context, b *bundle.Bundle)
if pl.PipelineSpec == nil { if pl.PipelineSpec == nil {
continue continue
} }
if p.Catalog != "" && p.Schema != "" { if pl.Catalog == "" {
if pl.Catalog == "" { pl.Catalog = p.Catalog
pl.Catalog = p.Catalog }
if pl.GatewayDefinition != nil {
if pl.GatewayDefinition.GatewayStorageCatalog == "" {
pl.GatewayDefinition.GatewayStorageCatalog = p.Catalog
} }
if pl.Target == "" { if pl.GatewayDefinition.GatewayStorageSchema == "" {
pl.Target = p.Schema pl.GatewayDefinition.GatewayStorageSchema = p.Schema
} }
if pl.GatewayDefinition != nil { }
if pl.GatewayDefinition.GatewayStorageCatalog == "" { if pl.IngestionDefinition != nil {
pl.GatewayDefinition.GatewayStorageCatalog = p.Catalog for _, obj := range pl.IngestionDefinition.Objects {
} if obj.Report != nil {
if pl.GatewayDefinition.GatewayStorageSchema == "" { if obj.Report.DestinationCatalog == "" {
pl.GatewayDefinition.GatewayStorageSchema = p.Schema obj.Report.DestinationCatalog = p.Catalog
}
}
if pl.IngestionDefinition != nil {
for _, obj := range pl.IngestionDefinition.Objects {
if obj.Report != nil {
if obj.Report.DestinationCatalog == "" {
obj.Report.DestinationCatalog = p.Catalog
}
if obj.Report.DestinationSchema == "" {
obj.Report.DestinationSchema = p.Schema
}
} }
if obj.Schema != nil { if obj.Report.DestinationSchema == "" {
if obj.Schema.SourceCatalog == "" { obj.Report.DestinationSchema = p.Schema
obj.Schema.SourceCatalog = p.Catalog
}
if obj.Schema.SourceSchema == "" {
obj.Schema.SourceSchema = p.Schema
}
if obj.Schema.DestinationCatalog == "" {
obj.Schema.DestinationCatalog = p.Catalog
}
if obj.Schema.DestinationSchema == "" {
obj.Schema.DestinationSchema = p.Schema
}
} }
if obj.Table != nil { }
if obj.Table.SourceCatalog == "" { if obj.Schema != nil {
obj.Table.SourceCatalog = p.Catalog if obj.Schema.SourceCatalog == "" {
} obj.Schema.SourceCatalog = p.Catalog
if obj.Table.SourceSchema == "" { }
obj.Table.SourceSchema = p.Schema if obj.Schema.SourceSchema == "" {
} obj.Schema.SourceSchema = p.Schema
if obj.Table.DestinationCatalog == "" { }
obj.Table.DestinationCatalog = p.Catalog if obj.Schema.DestinationCatalog == "" {
} obj.Schema.DestinationCatalog = p.Catalog
if obj.Table.DestinationSchema == "" { }
obj.Table.DestinationSchema = p.Schema if obj.Schema.DestinationSchema == "" {
} obj.Schema.DestinationSchema = p.Schema
}
}
if obj.Table != nil {
if obj.Table.SourceCatalog == "" {
obj.Table.SourceCatalog = p.Catalog
}
if obj.Table.SourceSchema == "" {
obj.Table.SourceSchema = p.Schema
}
if obj.Table.DestinationCatalog == "" {
obj.Table.DestinationCatalog = p.Catalog
}
if obj.Table.DestinationSchema == "" {
obj.Table.DestinationSchema = p.Schema
} }
} }
} }
@ -127,35 +128,33 @@ func (m *applyPresetsCatalogSchema) Apply(ctx context.Context, b *bundle.Bundle)
continue continue
} }
if p.Catalog != "" || p.Schema != "" { if e.CreateServingEndpoint.AiGateway != nil && e.CreateServingEndpoint.AiGateway.InferenceTableConfig != nil {
if e.CreateServingEndpoint.AiGateway != nil && e.CreateServingEndpoint.AiGateway.InferenceTableConfig != nil { if e.CreateServingEndpoint.AiGateway.InferenceTableConfig.CatalogName == "" {
if p.Catalog != "" && e.CreateServingEndpoint.AiGateway.InferenceTableConfig.CatalogName == "" { e.CreateServingEndpoint.AiGateway.InferenceTableConfig.CatalogName = p.Catalog
e.CreateServingEndpoint.AiGateway.InferenceTableConfig.CatalogName = p.Catalog
}
if p.Schema != "" && e.CreateServingEndpoint.AiGateway.InferenceTableConfig.SchemaName == "" {
e.CreateServingEndpoint.AiGateway.InferenceTableConfig.SchemaName = p.Schema
}
} }
if e.CreateServingEndpoint.AiGateway.InferenceTableConfig.SchemaName == "" {
e.CreateServingEndpoint.AiGateway.InferenceTableConfig.SchemaName = p.Schema
}
}
if e.CreateServingEndpoint.Config.AutoCaptureConfig != nil { if e.CreateServingEndpoint.Config.AutoCaptureConfig != nil {
if p.Catalog != "" && e.CreateServingEndpoint.Config.AutoCaptureConfig.CatalogName == "" { if e.CreateServingEndpoint.Config.AutoCaptureConfig.CatalogName == "" {
e.CreateServingEndpoint.Config.AutoCaptureConfig.CatalogName = p.Catalog e.CreateServingEndpoint.Config.AutoCaptureConfig.CatalogName = p.Catalog
}
if p.Schema != "" && e.CreateServingEndpoint.Config.AutoCaptureConfig.SchemaName == "" {
e.CreateServingEndpoint.Config.AutoCaptureConfig.SchemaName = p.Schema
}
} }
if e.CreateServingEndpoint.Config.AutoCaptureConfig.SchemaName == "" {
e.CreateServingEndpoint.Config.AutoCaptureConfig.SchemaName = p.Schema
}
}
for i := range e.CreateServingEndpoint.Config.ServedEntities { for i := range e.CreateServingEndpoint.Config.ServedEntities {
e.CreateServingEndpoint.Config.ServedEntities[i].EntityName = fullyQualifyName( e.CreateServingEndpoint.Config.ServedEntities[i].EntityName = fullyQualifyName(
e.CreateServingEndpoint.Config.ServedEntities[i].EntityName, p.Catalog, p.Schema, e.CreateServingEndpoint.Config.ServedEntities[i].EntityName, p.Catalog, p.Schema,
) )
} }
for i := range e.CreateServingEndpoint.Config.ServedModels { for i := range e.CreateServingEndpoint.Config.ServedModels {
e.CreateServingEndpoint.Config.ServedModels[i].ModelName = fullyQualifyName( e.CreateServingEndpoint.Config.ServedModels[i].ModelName = fullyQualifyName(
e.CreateServingEndpoint.Config.ServedModels[i].ModelName, p.Catalog, p.Schema, e.CreateServingEndpoint.Config.ServedModels[i].ModelName, p.Catalog, p.Schema,
) )
}
} }
} }
@ -164,10 +163,10 @@ func (m *applyPresetsCatalogSchema) Apply(ctx context.Context, b *bundle.Bundle)
if m.CreateRegisteredModelRequest == nil { if m.CreateRegisteredModelRequest == nil {
continue continue
} }
if p.Catalog != "" && m.CatalogName == "" { if m.CatalogName == "" {
m.CatalogName = p.Catalog m.CatalogName = p.Catalog
} }
if p.Schema != "" && m.SchemaName == "" { if m.SchemaName == "" {
m.SchemaName = p.Schema m.SchemaName = p.Schema
} }
} }
@ -177,11 +176,9 @@ func (m *applyPresetsCatalogSchema) Apply(ctx context.Context, b *bundle.Bundle)
if q.CreateMonitor == nil { if q.CreateMonitor == nil {
continue continue
} }
if p.Catalog != "" && p.Schema != "" { q.TableName = fullyQualifyName(q.TableName, p.Catalog, p.Schema)
q.TableName = fullyQualifyName(q.TableName, p.Catalog, p.Schema) if q.OutputSchemaName == "" {
if q.OutputSchemaName == "" { q.OutputSchemaName = p.Catalog + "." + p.Schema
q.OutputSchemaName = p.Catalog + "." + p.Schema
}
} }
} }
@ -190,10 +187,10 @@ func (m *applyPresetsCatalogSchema) Apply(ctx context.Context, b *bundle.Bundle)
if s.CreateSchema == nil { if s.CreateSchema == nil {
continue continue
} }
if p.Catalog != "" && s.CatalogName == "" { if s.CatalogName == "" {
s.CatalogName = p.Catalog s.CatalogName = p.Catalog
} }
if p.Schema != "" && s.Name == "" { if s.Name == "" {
// If there is a schema preset such as 'dev', we directly // If there is a schema preset such as 'dev', we directly
// use that name and don't add any prefix (which might result in dev_dev). // use that name and don't add any prefix (which might result in dev_dev).
s.Name = p.Schema s.Name = p.Schema
@ -203,21 +200,9 @@ func (m *applyPresetsCatalogSchema) Apply(ctx context.Context, b *bundle.Bundle)
return diags return diags
} }
func validateCatalogAndSchema(b *bundle.Bundle) diag.Diagnostics {
p := b.Config.Presets
if (p.Catalog != "" && p.Schema == "") || (p.Catalog == "" && p.Schema != "") {
return diag.Diagnostics{{
Summary: "presets.catalog and presets.schema must always be set together",
Severity: diag.Error,
Locations: []dyn.Location{b.Config.GetLocation("presets")},
}}
}
return diag.Diagnostics{}
}
// addCatalogSchemaParameters adds catalog and schema parameters to a job if they don't already exist. // addCatalogSchemaParameters adds catalog and schema parameters to a job if they don't already exist.
// Returns any warning diagnostics for existing parameters. // Returns any warning diagnostics for existing parameters.
func addCatalogSchemaParameters(b *bundle.Bundle, key string, job *resources.Job, t config.Presets) diag.Diagnostics { func addCatalogSchemaParameters(b *bundle.Bundle, key string, job *resources.Job, p config.Presets) diag.Diagnostics {
var diags diag.Diagnostics var diags diag.Diagnostics
// Check for existing catalog/schema parameters // Check for existing catalog/schema parameters
@ -250,18 +235,18 @@ func addCatalogSchemaParameters(b *bundle.Bundle, key string, job *resources.Job
} }
// Add catalog parameter if not already present // Add catalog parameter if not already present
if !hasCatalog && t.Catalog != "" { if !hasCatalog {
job.Parameters = append(job.Parameters, jobs.JobParameterDefinition{ job.Parameters = append(job.Parameters, jobs.JobParameterDefinition{
Name: "catalog", Name: "catalog",
Default: t.Catalog, Default: p.Catalog,
}) })
} }
// Add schema parameter if not already present // Add schema parameter if not already present
if !hasSchema && t.Schema != "" { if !hasSchema {
job.Parameters = append(job.Parameters, jobs.JobParameterDefinition{ job.Parameters = append(job.Parameters, jobs.JobParameterDefinition{
Name: "schema", Name: "schema",
Default: t.Schema, Default: p.Schema,
}) })
} }