mirror of https://github.com/databricks/cli.git
Bundle path rewrites for dbt and SQL file tasks (#962)
## Changes Support path rewrites for Dbt and SQL file job taks. <!-- Summary of your changes that are easy to understand --> ## Tests * Added unit test <!-- How is this tested? -->
This commit is contained in:
parent
56bcb6f833
commit
10291b0e13
|
@ -135,6 +135,17 @@ func translateFilePath(literal, localFullPath, localRelPath, remotePath string)
|
|||
return remotePath, nil
|
||||
}
|
||||
|
||||
func translateDirectoryPath(literal, localFullPath, localRelPath, remotePath string) (string, error) {
|
||||
info, err := os.Stat(localFullPath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return "", fmt.Errorf("%s is not a directory", localFullPath)
|
||||
}
|
||||
return remotePath, nil
|
||||
}
|
||||
|
||||
func translateNoOp(literal, localFullPath, localRelPath, remotePath string) (string, error) {
|
||||
return localRelPath, nil
|
||||
}
|
||||
|
|
|
@ -50,6 +50,34 @@ func transformWhlLibrary(resource any, dir string) *transformer {
|
|||
}
|
||||
}
|
||||
|
||||
func transformDbtTask(resource any, dir string) *transformer {
|
||||
task, ok := resource.(*jobs.Task)
|
||||
if !ok || task.DbtTask == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
return &transformer{
|
||||
dir,
|
||||
&task.DbtTask.ProjectDirectory,
|
||||
"tasks.dbt_task.project_directory",
|
||||
translateDirectoryPath,
|
||||
}
|
||||
}
|
||||
|
||||
func transformSqlFileTask(resource any, dir string) *transformer {
|
||||
task, ok := resource.(*jobs.Task)
|
||||
if !ok || task.SqlTask == nil || task.SqlTask.File == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
return &transformer{
|
||||
dir,
|
||||
&task.SqlTask.File.Path,
|
||||
"tasks.sql_task.file.path",
|
||||
translateFilePath,
|
||||
}
|
||||
}
|
||||
|
||||
func transformJarLibrary(resource any, dir string) *transformer {
|
||||
library, ok := resource.(*compute.Library)
|
||||
if !ok || library.Jar == "" {
|
||||
|
@ -70,6 +98,8 @@ func applyJobTransformers(m *translatePaths, b *bundle.Bundle) error {
|
|||
transformSparkTask,
|
||||
transformWhlLibrary,
|
||||
transformJarLibrary,
|
||||
transformDbtTask,
|
||||
transformSqlFileTask,
|
||||
}
|
||||
|
||||
for key, job := range b.Config.Resources.Jobs {
|
||||
|
|
|
@ -275,6 +275,8 @@ func TestTranslatePathsInSubdirectories(t *testing.T) {
|
|||
touchEmptyFile(t, filepath.Join(dir, "job", "my_python_file.py"))
|
||||
touchEmptyFile(t, filepath.Join(dir, "job", "dist", "task.jar"))
|
||||
touchEmptyFile(t, filepath.Join(dir, "pipeline", "my_python_file.py"))
|
||||
touchEmptyFile(t, filepath.Join(dir, "job", "my_sql_file.sql"))
|
||||
touchEmptyFile(t, filepath.Join(dir, "job", "my_dbt_project", "dbt_project.yml"))
|
||||
|
||||
bundle := &bundle.Bundle{
|
||||
Config: config.Root{
|
||||
|
@ -303,6 +305,18 @@ func TestTranslatePathsInSubdirectories(t *testing.T) {
|
|||
{Jar: "./dist/task.jar"},
|
||||
},
|
||||
},
|
||||
{
|
||||
SqlTask: &jobs.SqlTask{
|
||||
File: &jobs.SqlTaskFile{
|
||||
Path: "./my_sql_file.sql",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
DbtTask: &jobs.DbtTask{
|
||||
ProjectDirectory: "./my_dbt_project",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
|
@ -341,6 +355,16 @@ func TestTranslatePathsInSubdirectories(t *testing.T) {
|
|||
"/bundle/job/dist/task.jar",
|
||||
bundle.Config.Resources.Jobs["job"].Tasks[1].Libraries[0].Jar,
|
||||
)
|
||||
assert.Equal(
|
||||
t,
|
||||
"/bundle/job/my_sql_file.sql",
|
||||
bundle.Config.Resources.Jobs["job"].Tasks[2].SqlTask.File.Path,
|
||||
)
|
||||
assert.Equal(
|
||||
t,
|
||||
"/bundle/job/my_dbt_project",
|
||||
bundle.Config.Resources.Jobs["job"].Tasks[3].DbtTask.ProjectDirectory,
|
||||
)
|
||||
|
||||
assert.Equal(
|
||||
t,
|
||||
|
|
Loading…
Reference in New Issue