From ee4a4b4c248ebdc1db93fe854adc5b26d12c8ddf Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 20 Jan 2025 17:33:03 +0100 Subject: [PATCH] Migrate quality_monitor_test.go to acceptance test (#2192) --- .../bundle}/quality_monitor/databricks.yml | 0 acceptance/bundle/quality_monitor/output.txt | 73 +++++++++++++++++++ acceptance/bundle/quality_monitor/script | 3 + bundle/tests/quality_monitor_test.go | 59 --------------- 4 files changed, 76 insertions(+), 59 deletions(-) rename {bundle/tests => acceptance/bundle}/quality_monitor/databricks.yml (100%) create mode 100644 acceptance/bundle/quality_monitor/output.txt create mode 100644 acceptance/bundle/quality_monitor/script delete mode 100644 bundle/tests/quality_monitor_test.go diff --git a/bundle/tests/quality_monitor/databricks.yml b/acceptance/bundle/quality_monitor/databricks.yml similarity index 100% rename from bundle/tests/quality_monitor/databricks.yml rename to acceptance/bundle/quality_monitor/databricks.yml diff --git a/acceptance/bundle/quality_monitor/output.txt b/acceptance/bundle/quality_monitor/output.txt new file mode 100644 index 000000000..b3718c802 --- /dev/null +++ b/acceptance/bundle/quality_monitor/output.txt @@ -0,0 +1,73 @@ + +>>> $CLI bundle validate -o json -t development +{ + "mode": "development", + "quality_monitors": { + "my_monitor": { + "assets_dir": "/Shared/provider-test/databricks_monitoring/main.test.thing1", + "inference_log": { + "granularities": [ + "1 day" + ], + "model_id_col": "model_id", + "prediction_col": "prediction", + "problem_type": "PROBLEM_TYPE_REGRESSION", + "timestamp_col": "timestamp" + }, + "output_schema_name": "main.dev", + "schedule": null, + "table_name": "main.test.dev" + } + } +} + +>>> $CLI bundle validate -o json -t staging +{ + "mode": null, + "quality_monitors": { + "my_monitor": { + "assets_dir": "/Shared/provider-test/databricks_monitoring/main.test.thing1", + "inference_log": { + "granularities": [ + "1 day" + ], + "model_id_col": "model_id", + "prediction_col": "prediction", + "problem_type": "PROBLEM_TYPE_REGRESSION", + "timestamp_col": "timestamp" + }, + "output_schema_name": "main.staging", + "schedule": { + "quartz_cron_expression": "0 0 12 * * ?", + "timezone_id": "UTC" + }, + "table_name": "main.test.staging" + } + } +} + +>>> $CLI bundle validate -o json -t production +{ + "mode": null, + "quality_monitors": { + "my_monitor": { + "assets_dir": "/Shared/provider-test/databricks_monitoring/main.test.thing1", + "inference_log": { + "granularities": [ + "1 day", + "1 hour" + ], + "model_id_col": "model_id_prod", + "prediction_col": "prediction_prod", + "problem_type": "PROBLEM_TYPE_REGRESSION", + "timestamp_col": "timestamp_prod" + }, + "output_schema_name": "main.prod", + "schedule": { + "quartz_cron_expression": "0 0 12 * * ?", + "timezone_id": "UTC" + }, + "table_name": "main.test.prod" + } + } +} diff --git a/acceptance/bundle/quality_monitor/script b/acceptance/bundle/quality_monitor/script new file mode 100644 index 000000000..85a69d5e7 --- /dev/null +++ b/acceptance/bundle/quality_monitor/script @@ -0,0 +1,3 @@ +trace $CLI bundle validate -o json -t development | jq '{ mode: .bundle.mode, quality_monitors: .resources.quality_monitors }' +trace $CLI bundle validate -o json -t staging | jq '{ mode: .bundle.mode, quality_monitors: .resources.quality_monitors }' +trace $CLI bundle validate -o json -t production | jq '{ mode: .bundle.mode, quality_monitors: .resources.quality_monitors }' diff --git a/bundle/tests/quality_monitor_test.go b/bundle/tests/quality_monitor_test.go deleted file mode 100644 index e95c7b7c1..000000000 --- a/bundle/tests/quality_monitor_test.go +++ /dev/null @@ -1,59 +0,0 @@ -package config_tests - -import ( - "testing" - - "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/databricks-sdk-go/service/catalog" - "github.com/stretchr/testify/assert" -) - -func assertExpectedMonitor(t *testing.T, p *resources.QualityMonitor) { - assert.Equal(t, "timestamp", p.InferenceLog.TimestampCol) - assert.Equal(t, "prediction", p.InferenceLog.PredictionCol) - assert.Equal(t, "model_id", p.InferenceLog.ModelIdCol) - assert.Equal(t, catalog.MonitorInferenceLogProblemType("PROBLEM_TYPE_REGRESSION"), p.InferenceLog.ProblemType) -} - -func TestMonitorTableNames(t *testing.T) { - b := loadTarget(t, "./quality_monitor", "development") - assert.Len(t, b.Config.Resources.QualityMonitors, 1) - assert.Equal(t, config.Development, b.Config.Bundle.Mode) - - p := b.Config.Resources.QualityMonitors["my_monitor"] - assert.Equal(t, "main.test.dev", p.TableName) - assert.Equal(t, "/Shared/provider-test/databricks_monitoring/main.test.thing1", p.AssetsDir) - assert.Equal(t, "main.dev", p.OutputSchemaName) - - assertExpectedMonitor(t, p) -} - -func TestMonitorStaging(t *testing.T) { - b := loadTarget(t, "./quality_monitor", "staging") - assert.Len(t, b.Config.Resources.QualityMonitors, 1) - - p := b.Config.Resources.QualityMonitors["my_monitor"] - assert.Equal(t, "main.test.staging", p.TableName) - assert.Equal(t, "/Shared/provider-test/databricks_monitoring/main.test.thing1", p.AssetsDir) - assert.Equal(t, "main.staging", p.OutputSchemaName) - - assertExpectedMonitor(t, p) -} - -func TestMonitorProduction(t *testing.T) { - b := loadTarget(t, "./quality_monitor", "production") - assert.Len(t, b.Config.Resources.QualityMonitors, 1) - - p := b.Config.Resources.QualityMonitors["my_monitor"] - assert.Equal(t, "main.test.prod", p.TableName) - assert.Equal(t, "/Shared/provider-test/databricks_monitoring/main.test.thing1", p.AssetsDir) - assert.Equal(t, "main.prod", p.OutputSchemaName) - - inferenceLog := p.InferenceLog - assert.Equal(t, []string{"1 day", "1 hour"}, inferenceLog.Granularities) - assert.Equal(t, "timestamp_prod", p.InferenceLog.TimestampCol) - assert.Equal(t, "prediction_prod", p.InferenceLog.PredictionCol) - assert.Equal(t, "model_id_prod", p.InferenceLog.ModelIdCol) - assert.Equal(t, catalog.MonitorInferenceLogProblemType("PROBLEM_TYPE_REGRESSION"), p.InferenceLog.ProblemType) -}