From 3894d5796d57f0677611a2038cc8e38397e42a24 Mon Sep 17 00:00:00 2001 From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> Date: Fri, 14 Apr 2023 11:11:30 +0200 Subject: [PATCH] Add progress logging event for pipeline update URLs (#331) ## Changes Output now: ``` shreyas.goenka@THW32HFW6T pipeline-progress % bricks bundle run foo The update can be found at https://e2-dogfood.staging.cloud.databricks.com/#joblist/pipelines/1cc605db-daab-4218-b38a-a63030e3eb03/updates/f92f2159-1141-47de-b1e2-1ca854b7238f 2023-04-12T20:41:19.813Z update_progress INFO "Update f92f21 is INITIALIZING." 2023-04-12T20:41:19.841Z update_progress INFO "Update f92f21 is SETTING_UP_TABLES." 2023-04-12T20:41:21.270Z update_progress INFO "Update f92f21 is RUNNING." 2023-04-12T20:41:21.271Z flow_progress INFO "Flow 'sales_orders_raw' is QUEUED." 2023-04-12T20:41:21.349Z flow_progress INFO "Flow 'sales_orders_raw' is STARTING." 2023-04-12T20:41:21.480Z flow_progress INFO "Flow 'sales_orders_raw' is RUNNING." 2023-04-12T20:41:23.493Z flow_progress INFO "Flow 'sales_orders_raw' has COMPLETED." 2023-04-12T20:41:25.484Z update_progress INFO "Update f92f21 is COMPLETED." ``` ## Tests --- bundle/run/pipeline.go | 3 +-- bundle/run/pipeline/events.go | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 bundle/run/pipeline/events.go diff --git a/bundle/run/pipeline.go b/bundle/run/pipeline.go index 070666a1f..f276737a1 100644 --- a/bundle/run/pipeline.go +++ b/bundle/run/pipeline.go @@ -172,8 +172,7 @@ func (r *pipelineRunner) Run(ctx context.Context, opts *Options) (RunOutput, err } // Log the pipeline update URL as soon as it is available. - updateUrl := fmt.Sprintf("%s/#joblist/pipelines/%s/updates/%s", w.Config.Host, pipelineID, updateID) - log.Infof(ctx, "Update available at %s", updateUrl) + progressLogger.Log(pipeline.NewUpdateUrlEvent(w.Config.Host, updateID, pipelineID)) // Poll update for completion and post status. // Note: there is no "StartUpdateAndWait" wrapper for this API. diff --git a/bundle/run/pipeline/events.go b/bundle/run/pipeline/events.go new file mode 100644 index 000000000..1ee234c4a --- /dev/null +++ b/bundle/run/pipeline/events.go @@ -0,0 +1,23 @@ +package pipeline + +import "fmt" + +type UpdateUrlEvent struct { + Type string `json:"type"` + UpdateId string `json:"update_id"` + PipelineId string `json:"pipeline_id"` + Url string `json:"url"` +} + +func NewUpdateUrlEvent(host, updateId, pipelineId string) *UpdateUrlEvent { + return &UpdateUrlEvent{ + Type: "update_url", + UpdateId: updateId, + PipelineId: pipelineId, + Url: fmt.Sprintf("%s/#joblist/pipelines/%s/updates/%s", host, pipelineId, updateId), + } +} + +func (event *UpdateUrlEvent) String() string { + return fmt.Sprintf("The update can be found at %s\n", event.Url) +}