[DAB] Add support for requirements libraries in Job Tasks (#1543)

## Changes
While experimenting with DAB I discovered that requirements libraries
are being ignored.

One thing worth mentioning is that `bundle validate` runs successfully,
but `bundle deploy` fails. This PR only covers the second part.


## Tests
<!-- How is this tested? -->
Added a unit test
This commit is contained in:
Witold Czaplewski 2024-08-21 12:03:56 +02:00 committed by GitHub
parent c775d251ed
commit 192f33bb13
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 18 additions and 0 deletions

View File

@ -50,6 +50,11 @@ func rewritePatterns(t *translateContext, base dyn.Pattern) []jobRewritePattern
t.translateNoOp,
noSkipRewrite,
},
{
base.Append(dyn.Key("libraries"), dyn.AnyIndex(), dyn.Key("requirements")),
t.translateFilePath,
noSkipRewrite,
},
}
}

View File

@ -110,6 +110,7 @@ func TestTranslatePaths(t *testing.T) {
touchNotebookFile(t, filepath.Join(dir, "my_pipeline_notebook.py"))
touchEmptyFile(t, filepath.Join(dir, "my_python_file.py"))
touchEmptyFile(t, filepath.Join(dir, "dist", "task.jar"))
touchEmptyFile(t, filepath.Join(dir, "requirements.txt"))
b := &bundle.Bundle{
RootPath: dir,
@ -140,6 +141,9 @@ func TestTranslatePaths(t *testing.T) {
NotebookTask: &jobs.NotebookTask{
NotebookPath: "./my_job_notebook.py",
},
Libraries: []compute.Library{
{Requirements: "./requirements.txt"},
},
},
{
PythonWheelTask: &jobs.PythonWheelTask{
@ -232,6 +236,11 @@ func TestTranslatePaths(t *testing.T) {
"/bundle/my_job_notebook",
b.Config.Resources.Jobs["job"].Tasks[2].NotebookTask.NotebookPath,
)
assert.Equal(
t,
"/bundle/requirements.txt",
b.Config.Resources.Jobs["job"].Tasks[2].Libraries[0].Requirements,
)
assert.Equal(
t,
"/bundle/my_python_file.py",

View File

@ -12,5 +12,8 @@ func libraryPath(library *compute.Library) string {
if library.Egg != "" {
return library.Egg
}
if library.Requirements != "" {
return library.Requirements
}
return ""
}

View File

@ -13,5 +13,6 @@ func TestLibraryPath(t *testing.T) {
assert.Equal(t, path, libraryPath(&compute.Library{Whl: path}))
assert.Equal(t, path, libraryPath(&compute.Library{Jar: path}))
assert.Equal(t, path, libraryPath(&compute.Library{Egg: path}))
assert.Equal(t, path, libraryPath(&compute.Library{Requirements: path}))
assert.Equal(t, "", libraryPath(&compute.Library{}))
}