mirror of https://github.com/databricks/cli.git
[DAB] Add support for requirements libraries in Job Tasks (#1543)
## Changes While experimenting with DAB I discovered that requirements libraries are being ignored. One thing worth mentioning is that `bundle validate` runs successfully, but `bundle deploy` fails. This PR only covers the second part. ## Tests <!-- How is this tested? --> Added a unit test
This commit is contained in:
parent
c775d251ed
commit
192f33bb13
|
@ -50,6 +50,11 @@ func rewritePatterns(t *translateContext, base dyn.Pattern) []jobRewritePattern
|
|||
t.translateNoOp,
|
||||
noSkipRewrite,
|
||||
},
|
||||
{
|
||||
base.Append(dyn.Key("libraries"), dyn.AnyIndex(), dyn.Key("requirements")),
|
||||
t.translateFilePath,
|
||||
noSkipRewrite,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -110,6 +110,7 @@ func TestTranslatePaths(t *testing.T) {
|
|||
touchNotebookFile(t, filepath.Join(dir, "my_pipeline_notebook.py"))
|
||||
touchEmptyFile(t, filepath.Join(dir, "my_python_file.py"))
|
||||
touchEmptyFile(t, filepath.Join(dir, "dist", "task.jar"))
|
||||
touchEmptyFile(t, filepath.Join(dir, "requirements.txt"))
|
||||
|
||||
b := &bundle.Bundle{
|
||||
RootPath: dir,
|
||||
|
@ -140,6 +141,9 @@ func TestTranslatePaths(t *testing.T) {
|
|||
NotebookTask: &jobs.NotebookTask{
|
||||
NotebookPath: "./my_job_notebook.py",
|
||||
},
|
||||
Libraries: []compute.Library{
|
||||
{Requirements: "./requirements.txt"},
|
||||
},
|
||||
},
|
||||
{
|
||||
PythonWheelTask: &jobs.PythonWheelTask{
|
||||
|
@ -232,6 +236,11 @@ func TestTranslatePaths(t *testing.T) {
|
|||
"/bundle/my_job_notebook",
|
||||
b.Config.Resources.Jobs["job"].Tasks[2].NotebookTask.NotebookPath,
|
||||
)
|
||||
assert.Equal(
|
||||
t,
|
||||
"/bundle/requirements.txt",
|
||||
b.Config.Resources.Jobs["job"].Tasks[2].Libraries[0].Requirements,
|
||||
)
|
||||
assert.Equal(
|
||||
t,
|
||||
"/bundle/my_python_file.py",
|
||||
|
|
|
@ -12,5 +12,8 @@ func libraryPath(library *compute.Library) string {
|
|||
if library.Egg != "" {
|
||||
return library.Egg
|
||||
}
|
||||
if library.Requirements != "" {
|
||||
return library.Requirements
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
|
|
@ -13,5 +13,6 @@ func TestLibraryPath(t *testing.T) {
|
|||
assert.Equal(t, path, libraryPath(&compute.Library{Whl: path}))
|
||||
assert.Equal(t, path, libraryPath(&compute.Library{Jar: path}))
|
||||
assert.Equal(t, path, libraryPath(&compute.Library{Egg: path}))
|
||||
assert.Equal(t, path, libraryPath(&compute.Library{Requirements: path}))
|
||||
assert.Equal(t, "", libraryPath(&compute.Library{}))
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue