Bundle path rewrites for dbt and SQL file tasks (#962)

## Changes
Support path rewrites for Dbt and SQL file job taks.
<!-- Summary of your changes that are easy to understand -->

## Tests
* Added unit test
<!-- How is this tested? -->
This commit is contained in:
Michał Szafrański 2023-11-07 21:00:09 +01:00 committed by GitHub
parent 56bcb6f833
commit 10291b0e13
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 65 additions and 0 deletions

View File

@ -135,6 +135,17 @@ func translateFilePath(literal, localFullPath, localRelPath, remotePath string)
return remotePath, nil
}
func translateDirectoryPath(literal, localFullPath, localRelPath, remotePath string) (string, error) {
info, err := os.Stat(localFullPath)
if err != nil {
return "", err
}
if !info.IsDir() {
return "", fmt.Errorf("%s is not a directory", localFullPath)
}
return remotePath, nil
}
func translateNoOp(literal, localFullPath, localRelPath, remotePath string) (string, error) {
return localRelPath, nil
}

View File

@ -50,6 +50,34 @@ func transformWhlLibrary(resource any, dir string) *transformer {
}
}
func transformDbtTask(resource any, dir string) *transformer {
task, ok := resource.(*jobs.Task)
if !ok || task.DbtTask == nil {
return nil
}
return &transformer{
dir,
&task.DbtTask.ProjectDirectory,
"tasks.dbt_task.project_directory",
translateDirectoryPath,
}
}
func transformSqlFileTask(resource any, dir string) *transformer {
task, ok := resource.(*jobs.Task)
if !ok || task.SqlTask == nil || task.SqlTask.File == nil {
return nil
}
return &transformer{
dir,
&task.SqlTask.File.Path,
"tasks.sql_task.file.path",
translateFilePath,
}
}
func transformJarLibrary(resource any, dir string) *transformer {
library, ok := resource.(*compute.Library)
if !ok || library.Jar == "" {
@ -70,6 +98,8 @@ func applyJobTransformers(m *translatePaths, b *bundle.Bundle) error {
transformSparkTask,
transformWhlLibrary,
transformJarLibrary,
transformDbtTask,
transformSqlFileTask,
}
for key, job := range b.Config.Resources.Jobs {

View File

@ -275,6 +275,8 @@ func TestTranslatePathsInSubdirectories(t *testing.T) {
touchEmptyFile(t, filepath.Join(dir, "job", "my_python_file.py"))
touchEmptyFile(t, filepath.Join(dir, "job", "dist", "task.jar"))
touchEmptyFile(t, filepath.Join(dir, "pipeline", "my_python_file.py"))
touchEmptyFile(t, filepath.Join(dir, "job", "my_sql_file.sql"))
touchEmptyFile(t, filepath.Join(dir, "job", "my_dbt_project", "dbt_project.yml"))
bundle := &bundle.Bundle{
Config: config.Root{
@ -303,6 +305,18 @@ func TestTranslatePathsInSubdirectories(t *testing.T) {
{Jar: "./dist/task.jar"},
},
},
{
SqlTask: &jobs.SqlTask{
File: &jobs.SqlTaskFile{
Path: "./my_sql_file.sql",
},
},
},
{
DbtTask: &jobs.DbtTask{
ProjectDirectory: "./my_dbt_project",
},
},
},
},
},
@ -341,6 +355,16 @@ func TestTranslatePathsInSubdirectories(t *testing.T) {
"/bundle/job/dist/task.jar",
bundle.Config.Resources.Jobs["job"].Tasks[1].Libraries[0].Jar,
)
assert.Equal(
t,
"/bundle/job/my_sql_file.sql",
bundle.Config.Resources.Jobs["job"].Tasks[2].SqlTask.File.Path,
)
assert.Equal(
t,
"/bundle/job/my_dbt_project",
bundle.Config.Resources.Jobs["job"].Tasks[3].DbtTask.ProjectDirectory,
)
assert.Equal(
t,