Allow use of file library in pipeline (#308)

## Changes

This requires databricks/databricks-sdk-go#359.

## Tests

Tests pass and ran manual verification of deployment with files.
This commit is contained in:
Pieter Noordhuis 2023-04-05 16:29:42 +02:00 committed by GitHub
parent 4e4c0658db
commit d7ac265536
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 51 additions and 0 deletions

View File

@ -115,6 +115,13 @@ func (m *translatePaths) translatePipelineLibrary(b *bundle.Bundle, library *pip
}
}
if library.File != nil {
err = m.rewritePath(b, &library.File.Path, m.translateFilePath)
if err != nil {
return err
}
}
return nil
}

View File

@ -99,6 +99,11 @@ func TestTranslatePaths(t *testing.T) {
{
Jar: "foo",
},
{
File: &pipelines.FileLibrary{
Path: "./my_python_file.py",
},
},
},
},
},
@ -148,6 +153,11 @@ func TestTranslatePaths(t *testing.T) {
"/bundle/my_pipeline_notebook",
bundle.Config.Resources.Pipelines["pipeline"].Libraries[2].Notebook.Path,
)
assert.Equal(
t,
"/bundle/my_python_file.py",
bundle.Config.Resources.Pipelines["pipeline"].Libraries[4].File.Path,
)
}
func TestJobNotebookDoesNotExistError(t *testing.T) {

View File

@ -9,6 +9,7 @@ import (
"github.com/databricks/databricks-sdk-go/service/jobs"
"github.com/databricks/databricks-sdk-go/service/libraries"
"github.com/databricks/databricks-sdk-go/service/mlflow"
"github.com/databricks/databricks-sdk-go/service/pipelines"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@ -108,6 +109,39 @@ func TestConvertJobTaskLibraries(t *testing.T) {
assert.Equal(t, "mlflow", out.Resource.Job["my_job"].Task[0].Library[0].Pypi.Package)
}
func TestConvertPipeline(t *testing.T) {
var src = resources.Pipeline{
PipelineSpec: &pipelines.PipelineSpec{
Name: "my pipeline",
Libraries: []pipelines.PipelineLibrary{
{
Notebook: &pipelines.NotebookLibrary{
Path: "notebook path",
},
},
{
File: &pipelines.FileLibrary{
Path: "file path",
},
},
},
},
}
var config = config.Root{
Resources: config.Resources{
Pipelines: map[string]*resources.Pipeline{
"my_pipeline": &src,
},
},
}
out := BundleToTerraform(&config)
assert.Equal(t, "my pipeline", out.Resource.Pipeline["my_pipeline"].Name)
assert.Len(t, out.Resource.Pipeline["my_pipeline"].Library, 2)
assert.Nil(t, out.Data)
}
func TestConvertPipelinePermissions(t *testing.T) {
var src = resources.Pipeline{
Permissions: []resources.Permission{