mirror of https://github.com/databricks/cli.git
Allow use of file library in pipeline (#308)
## Changes This requires databricks/databricks-sdk-go#359. ## Tests Tests pass and ran manual verification of deployment with files.
This commit is contained in:
parent
4e4c0658db
commit
d7ac265536
|
@ -115,6 +115,13 @@ func (m *translatePaths) translatePipelineLibrary(b *bundle.Bundle, library *pip
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if library.File != nil {
|
||||||
|
err = m.rewritePath(b, &library.File.Path, m.translateFilePath)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -99,6 +99,11 @@ func TestTranslatePaths(t *testing.T) {
|
||||||
{
|
{
|
||||||
Jar: "foo",
|
Jar: "foo",
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
File: &pipelines.FileLibrary{
|
||||||
|
Path: "./my_python_file.py",
|
||||||
|
},
|
||||||
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
@ -148,6 +153,11 @@ func TestTranslatePaths(t *testing.T) {
|
||||||
"/bundle/my_pipeline_notebook",
|
"/bundle/my_pipeline_notebook",
|
||||||
bundle.Config.Resources.Pipelines["pipeline"].Libraries[2].Notebook.Path,
|
bundle.Config.Resources.Pipelines["pipeline"].Libraries[2].Notebook.Path,
|
||||||
)
|
)
|
||||||
|
assert.Equal(
|
||||||
|
t,
|
||||||
|
"/bundle/my_python_file.py",
|
||||||
|
bundle.Config.Resources.Pipelines["pipeline"].Libraries[4].File.Path,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestJobNotebookDoesNotExistError(t *testing.T) {
|
func TestJobNotebookDoesNotExistError(t *testing.T) {
|
||||||
|
|
|
@ -9,6 +9,7 @@ import (
|
||||||
"github.com/databricks/databricks-sdk-go/service/jobs"
|
"github.com/databricks/databricks-sdk-go/service/jobs"
|
||||||
"github.com/databricks/databricks-sdk-go/service/libraries"
|
"github.com/databricks/databricks-sdk-go/service/libraries"
|
||||||
"github.com/databricks/databricks-sdk-go/service/mlflow"
|
"github.com/databricks/databricks-sdk-go/service/mlflow"
|
||||||
|
"github.com/databricks/databricks-sdk-go/service/pipelines"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
@ -108,6 +109,39 @@ func TestConvertJobTaskLibraries(t *testing.T) {
|
||||||
assert.Equal(t, "mlflow", out.Resource.Job["my_job"].Task[0].Library[0].Pypi.Package)
|
assert.Equal(t, "mlflow", out.Resource.Job["my_job"].Task[0].Library[0].Pypi.Package)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestConvertPipeline(t *testing.T) {
|
||||||
|
var src = resources.Pipeline{
|
||||||
|
PipelineSpec: &pipelines.PipelineSpec{
|
||||||
|
Name: "my pipeline",
|
||||||
|
Libraries: []pipelines.PipelineLibrary{
|
||||||
|
{
|
||||||
|
Notebook: &pipelines.NotebookLibrary{
|
||||||
|
Path: "notebook path",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
File: &pipelines.FileLibrary{
|
||||||
|
Path: "file path",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
var config = config.Root{
|
||||||
|
Resources: config.Resources{
|
||||||
|
Pipelines: map[string]*resources.Pipeline{
|
||||||
|
"my_pipeline": &src,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
out := BundleToTerraform(&config)
|
||||||
|
assert.Equal(t, "my pipeline", out.Resource.Pipeline["my_pipeline"].Name)
|
||||||
|
assert.Len(t, out.Resource.Pipeline["my_pipeline"].Library, 2)
|
||||||
|
assert.Nil(t, out.Data)
|
||||||
|
}
|
||||||
|
|
||||||
func TestConvertPipelinePermissions(t *testing.T) {
|
func TestConvertPipelinePermissions(t *testing.T) {
|
||||||
var src = resources.Pipeline{
|
var src = resources.Pipeline{
|
||||||
Permissions: []resources.Permission{
|
Permissions: []resources.Permission{
|
||||||
|
|
Loading…
Reference in New Issue