2022-12-12 11:49:25 +00:00
package phases
import (
2024-07-31 12:16:28 +00:00
"context"
"fmt"
2023-05-16 16:35:39 +00:00
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/artifacts"
2023-09-14 10:14:13 +00:00
"github.com/databricks/cli/bundle/config"
2023-07-30 12:44:33 +00:00
"github.com/databricks/cli/bundle/config/mutator"
2024-02-07 11:17:17 +00:00
"github.com/databricks/cli/bundle/deploy"
2023-05-16 16:35:39 +00:00
"github.com/databricks/cli/bundle/deploy/files"
"github.com/databricks/cli/bundle/deploy/lock"
Persist deployment metadata in WSFS (#845)
## Changes
This PR introduces a metadata struct that stores a subset of bundle
configuration that we wish to expose to other Databricks services that
wish to integrate with bundles.
This metadata file is uploaded to a file
`${bundle.workspace.state_path}/metadata.json` in the WSFS destination
of the bundle deployment.
Documentation for emitted metadata fields:
* `version`: Version for the metadata file schema
* `config.bundle.git.branch`: Name of the git branch the bundle was
deployed from.
* `config.bundle.git.origin_url`: URL for git remote "origin"
* `config.bundle.git.bundle_root_path`: Relative path of the bundle root
from the root of the git repository. Is set to "." if they are the same.
* `config.bundle.git.commit`: SHA-1 commit hash of the exact commit this
bundle was deployed from. Note, the deployment might not exactly match
this commit version if there are changes that have not been committed to
git at deploy time,
* `file_path`: Path in workspace where we sync bundle files to.
* `resources.jobs.[job-ref].id`: Id of the job
* `resources.jobs.[job-ref].relative_path`: Relative path of the yaml
config file from the bundle root where this job was defined.
Example metadata object when bundle root and git root are the same:
```json
{
"version": 1,
"config": {
"bundle": {
"lock": {},
"git": {
"branch": "master",
"origin_url": "www.host.com",
"commit": "7af8e5d3f5dceffff9295d42d21606ccf056dce0",
"bundle_root_path": "."
}
},
"workspace": {
"file_path": "/Users/shreyas.goenka@databricks.com/.bundle/pipeline-progress/default/files"
},
"resources": {
"jobs": {
"bar": {
"id": "245921165354846",
"relative_path": "databricks.yml"
}
}
},
"sync": {}
}
}
```
Example metadata when the git root is one level above the bundle repo:
```json
{
"version": 1,
"config": {
"bundle": {
"lock": {},
"git": {
"branch": "dev-branch",
"origin_url": "www.my-repo.com",
"commit": "3db46ef750998952b00a2b3e7991e31787e4b98b",
"bundle_root_path": "pipeline-progress"
}
},
"workspace": {
"file_path": "/Users/shreyas.goenka@databricks.com/.bundle/pipeline-progress/default/files"
},
"resources": {
"jobs": {
"bar": {
"id": "245921165354846",
"relative_path": "databricks.yml"
}
}
},
"sync": {}
}
}
```
This unblocks integration to the jobs break glass UI for bundles.
## Tests
Unit tests and integration tests.
2023-10-27 12:55:43 +00:00
"github.com/databricks/cli/bundle/deploy/metadata"
2023-05-16 16:35:39 +00:00
"github.com/databricks/cli/bundle/deploy/terraform"
2023-07-25 11:35:08 +00:00
"github.com/databricks/cli/bundle/libraries"
2023-11-13 11:29:40 +00:00
"github.com/databricks/cli/bundle/permissions"
2023-08-30 12:21:39 +00:00
"github.com/databricks/cli/bundle/python"
2023-09-14 10:14:13 +00:00
"github.com/databricks/cli/bundle/scripts"
2024-07-31 12:16:28 +00:00
"github.com/databricks/cli/libs/cmdio"
terraformlib "github.com/databricks/cli/libs/terraform"
2024-08-12 14:00:20 +00:00
tfjson "github.com/hashicorp/terraform-json"
2022-12-12 11:49:25 +00:00
)
2024-08-12 14:00:20 +00:00
func parseTerraformActions ( changes [ ] * tfjson . ResourceChange , toInclude func ( typ string , actions tfjson . Actions ) bool ) [ ] terraformlib . Action {
res := make ( [ ] terraformlib . Action , 0 )
for _ , rc := range changes {
if ! toInclude ( rc . Type , rc . Change . Actions ) {
2024-07-31 12:16:28 +00:00
continue
}
var actionType terraformlib . ActionType
switch {
case rc . Change . Actions . Delete ( ) :
actionType = terraformlib . ActionTypeDelete
case rc . Change . Actions . Replace ( ) :
actionType = terraformlib . ActionTypeRecreate
default :
2024-08-12 14:00:20 +00:00
// No use case for other action types yet.
2024-07-31 12:16:28 +00:00
continue
}
2024-08-12 14:00:20 +00:00
res = append ( res , terraformlib . Action {
2024-07-31 12:16:28 +00:00
Action : actionType ,
ResourceType : rc . Type ,
ResourceName : rc . Name ,
} )
}
2024-08-12 14:00:20 +00:00
return res
}
2024-08-12 14:16:25 +00:00
func approvalForDeploy ( ctx context . Context , b * bundle . Bundle ) ( bool , error ) {
2024-08-12 14:00:20 +00:00
tf := b . Terraform
if tf == nil {
return false , fmt . Errorf ( "terraform not initialized" )
}
// read plan file
plan , err := tf . ShowPlanFile ( ctx , b . Plan . Path )
if err != nil {
return false , err
}
schemaActions := parseTerraformActions ( plan . ResourceChanges , func ( typ string , actions tfjson . Actions ) bool {
// Filter in only UC schema resources.
if typ != "databricks_schema" {
return false
}
// We only display prompts for destructive actions like deleting or
// recreating a schema.
return actions . Delete ( ) || actions . Replace ( )
} )
dltActions := parseTerraformActions ( plan . ResourceChanges , func ( typ string , actions tfjson . Actions ) bool {
// Filter in only DLT pipeline resources.
if typ != "databricks_pipeline" {
return false
}
// Recreating DLT pipeline leads to metadata loss and for a transient period
// the underling tables will be unavailable.
return actions . Replace ( )
} )
// We don't need to display any prompts in this case.
if len ( dltActions ) == 0 && len ( schemaActions ) == 0 {
2024-07-31 12:16:28 +00:00
return true , nil
}
2024-08-12 14:00:20 +00:00
// One or more UC schema resources will be deleted or recreated.
if len ( schemaActions ) != 0 {
cmdio . LogString ( ctx , "The following UC schemas will be deleted or recreated. Any underlying data may be lost:" )
for _ , action := range schemaActions {
cmdio . Log ( ctx , action )
}
}
// One or more DLT pipelines is being recreated.
if len ( dltActions ) != 0 {
2024-08-12 14:13:18 +00:00
cmdio . LogString ( ctx , "The following DLT pipelines will be recreated. Underlying tables will be unavailable for a transient period until the newly recreated pipelines are run once successfully. History of previous pipeline update runs will be lost because of recreation:" )
2024-08-12 14:00:20 +00:00
for _ , action := range dltActions {
cmdio . Log ( ctx , action )
}
2024-07-31 12:16:28 +00:00
}
if b . AutoApprove {
return true , nil
}
if ! cmdio . IsPromptSupported ( ctx ) {
return false , fmt . Errorf ( "the deployment requires destructive actions, but current console does not support prompting. Please specify --auto-approve if you would like to skip prompts and proceed" )
}
cmdio . LogString ( ctx , "" )
approved , err := cmdio . AskYesOrNo ( ctx , "Would you like to proceed?" )
if err != nil {
return false , err
}
return approved , nil
}
2022-12-12 11:49:25 +00:00
// The deploy phase deploys artifacts and resources.
func Deploy ( ) bundle . Mutator {
2024-07-31 12:16:28 +00:00
// Core mutators that CRUD resources and modify deployment state. These
// mutators need informed consent if they are potentially destructive.
deployCore := bundle . Defer (
2024-07-31 14:07:25 +00:00
bundle . Seq (
bundle . LogString ( "Deploying resources..." ) ,
terraform . Apply ( ) ,
) ,
2024-07-31 12:16:28 +00:00
bundle . Seq (
terraform . StatePush ( ) ,
terraform . Load ( ) ,
metadata . Compute ( ) ,
metadata . Upload ( ) ,
bundle . LogString ( "Deployment complete!" ) ,
) ,
)
2023-05-24 12:45:19 +00:00
deployMutator := bundle . Seq (
2023-09-14 10:14:13 +00:00
scripts . Execute ( config . ScriptPreDeploy ) ,
2023-05-16 16:01:50 +00:00
lock . Acquire ( ) ,
2023-05-24 12:45:19 +00:00
bundle . Defer (
bundle . Seq (
2024-02-07 11:17:17 +00:00
terraform . StatePull ( ) ,
2024-03-18 14:41:58 +00:00
deploy . StatePull ( ) ,
2023-07-30 12:44:33 +00:00
mutator . ValidateGitDetails ( ) ,
2023-07-25 11:35:08 +00:00
artifacts . CleanUp ( ) ,
2024-08-14 09:03:44 +00:00
libraries . ExpandGlobReferences ( ) ,
libraries . Upload ( ) ,
2023-08-30 12:21:39 +00:00
python . TransformWheelTask ( ) ,
files . Upload ( ) ,
2024-03-18 14:41:58 +00:00
deploy . StateUpdate ( ) ,
2024-03-19 09:47:41 +00:00
deploy . StatePush ( ) ,
2023-11-13 11:29:40 +00:00
permissions . ApplyWorkspaceRootPermissions ( ) ,
2023-05-24 12:45:19 +00:00
terraform . Interpolate ( ) ,
terraform . Write ( ) ,
2024-05-15 12:41:44 +00:00
terraform . CheckRunningResource ( ) ,
2024-07-31 12:16:28 +00:00
terraform . Plan ( terraform . PlanGoal ( "deploy" ) ) ,
bundle . If (
2024-08-12 14:16:25 +00:00
approvalForDeploy ,
2024-07-31 12:16:28 +00:00
deployCore ,
bundle . LogString ( "Deployment cancelled!" ) ,
2023-10-26 14:38:01 +00:00
) ,
2023-05-24 12:45:19 +00:00
) ,
2023-06-19 13:57:25 +00:00
lock . Release ( lock . GoalDeploy ) ,
2023-05-24 12:45:19 +00:00
) ,
2023-09-14 10:14:13 +00:00
scripts . Execute ( config . ScriptPostDeploy ) ,
2023-05-24 12:45:19 +00:00
)
2023-05-16 16:01:50 +00:00
2022-12-12 11:49:25 +00:00
return newPhase (
"deploy" ,
2023-05-24 12:45:19 +00:00
[ ] bundle . Mutator { deployMutator } ,
2022-12-12 11:49:25 +00:00
)
}