mirror of https://github.com/databricks/cli.git
[DAB] Add support for requirements libraries in Job Tasks (#1543)
## Changes While experimenting with DAB I discovered that requirements libraries are being ignored. One thing worth mentioning is that `bundle validate` runs successfully, but `bundle deploy` fails. This PR only covers the second part. ## Tests <!-- How is this tested? --> Added a unit test
This commit is contained in:
parent
c775d251ed
commit
192f33bb13
|
@ -50,6 +50,11 @@ func rewritePatterns(t *translateContext, base dyn.Pattern) []jobRewritePattern
|
||||||
t.translateNoOp,
|
t.translateNoOp,
|
||||||
noSkipRewrite,
|
noSkipRewrite,
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
base.Append(dyn.Key("libraries"), dyn.AnyIndex(), dyn.Key("requirements")),
|
||||||
|
t.translateFilePath,
|
||||||
|
noSkipRewrite,
|
||||||
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -110,6 +110,7 @@ func TestTranslatePaths(t *testing.T) {
|
||||||
touchNotebookFile(t, filepath.Join(dir, "my_pipeline_notebook.py"))
|
touchNotebookFile(t, filepath.Join(dir, "my_pipeline_notebook.py"))
|
||||||
touchEmptyFile(t, filepath.Join(dir, "my_python_file.py"))
|
touchEmptyFile(t, filepath.Join(dir, "my_python_file.py"))
|
||||||
touchEmptyFile(t, filepath.Join(dir, "dist", "task.jar"))
|
touchEmptyFile(t, filepath.Join(dir, "dist", "task.jar"))
|
||||||
|
touchEmptyFile(t, filepath.Join(dir, "requirements.txt"))
|
||||||
|
|
||||||
b := &bundle.Bundle{
|
b := &bundle.Bundle{
|
||||||
RootPath: dir,
|
RootPath: dir,
|
||||||
|
@ -140,6 +141,9 @@ func TestTranslatePaths(t *testing.T) {
|
||||||
NotebookTask: &jobs.NotebookTask{
|
NotebookTask: &jobs.NotebookTask{
|
||||||
NotebookPath: "./my_job_notebook.py",
|
NotebookPath: "./my_job_notebook.py",
|
||||||
},
|
},
|
||||||
|
Libraries: []compute.Library{
|
||||||
|
{Requirements: "./requirements.txt"},
|
||||||
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
PythonWheelTask: &jobs.PythonWheelTask{
|
PythonWheelTask: &jobs.PythonWheelTask{
|
||||||
|
@ -232,6 +236,11 @@ func TestTranslatePaths(t *testing.T) {
|
||||||
"/bundle/my_job_notebook",
|
"/bundle/my_job_notebook",
|
||||||
b.Config.Resources.Jobs["job"].Tasks[2].NotebookTask.NotebookPath,
|
b.Config.Resources.Jobs["job"].Tasks[2].NotebookTask.NotebookPath,
|
||||||
)
|
)
|
||||||
|
assert.Equal(
|
||||||
|
t,
|
||||||
|
"/bundle/requirements.txt",
|
||||||
|
b.Config.Resources.Jobs["job"].Tasks[2].Libraries[0].Requirements,
|
||||||
|
)
|
||||||
assert.Equal(
|
assert.Equal(
|
||||||
t,
|
t,
|
||||||
"/bundle/my_python_file.py",
|
"/bundle/my_python_file.py",
|
||||||
|
|
|
@ -12,5 +12,8 @@ func libraryPath(library *compute.Library) string {
|
||||||
if library.Egg != "" {
|
if library.Egg != "" {
|
||||||
return library.Egg
|
return library.Egg
|
||||||
}
|
}
|
||||||
|
if library.Requirements != "" {
|
||||||
|
return library.Requirements
|
||||||
|
}
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|
|
@ -13,5 +13,6 @@ func TestLibraryPath(t *testing.T) {
|
||||||
assert.Equal(t, path, libraryPath(&compute.Library{Whl: path}))
|
assert.Equal(t, path, libraryPath(&compute.Library{Whl: path}))
|
||||||
assert.Equal(t, path, libraryPath(&compute.Library{Jar: path}))
|
assert.Equal(t, path, libraryPath(&compute.Library{Jar: path}))
|
||||||
assert.Equal(t, path, libraryPath(&compute.Library{Egg: path}))
|
assert.Equal(t, path, libraryPath(&compute.Library{Egg: path}))
|
||||||
|
assert.Equal(t, path, libraryPath(&compute.Library{Requirements: path}))
|
||||||
assert.Equal(t, "", libraryPath(&compute.Library{}))
|
assert.Equal(t, "", libraryPath(&compute.Library{}))
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue