Do not replace pipeline libraries if there are no matches for pattern (#1021)

## Changes
If there are no matches when doing Glob call for pipeline library
defined, leave the entry as is.
The next mutators in the chain will detect that file is missing and the
error will be more user friendly.


Before the change

```
Starting resource deployment
Error: terraform apply: exit status 1

Error: cannot create pipeline: libraries must contain at least one element
```

After

```
Error: notebook ./non-existent not found
```


## Tests
Added regression unit tests
This commit is contained in:
Andrew Nester 2023-11-29 14:20:13 +01:00 committed by GitHub
parent 5431174302
commit 833746cbdd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 67 additions and 1 deletions

View File

@ -38,6 +38,11 @@ func (m *expandPipelineGlobPaths) Apply(_ context.Context, b *bundle.Bundle) err
return err return err
} }
if len(matches) == 0 {
expandedLibraries = append(expandedLibraries, *library)
continue
}
for _, match := range matches { for _, match := range matches {
m, err := filepath.Rel(dir, match) m, err := filepath.Rel(dir, match)
if err != nil { if err != nil {

View File

@ -85,6 +85,11 @@ func TestExpandGlobPathsInPipelines(t *testing.T) {
Path: "/Repos/somerepo/test.ipynb", Path: "/Repos/somerepo/test.ipynb",
}, },
}, },
{
Notebook: &pipelines.NotebookLibrary{
Path: "./non-existent.ipynb",
},
},
}, },
}, },
}, },
@ -98,7 +103,7 @@ func TestExpandGlobPathsInPipelines(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
libraries := b.Config.Resources.Pipelines["pipeline"].Libraries libraries := b.Config.Resources.Pipelines["pipeline"].Libraries
require.Len(t, libraries, 10) require.Len(t, libraries, 11)
// Making sure glob patterns are expanded correctly // Making sure glob patterns are expanded correctly
require.True(t, containsNotebook(libraries, filepath.Join("test", "test2.ipynb"))) require.True(t, containsNotebook(libraries, filepath.Join("test", "test2.ipynb")))
@ -117,6 +122,7 @@ func TestExpandGlobPathsInPipelines(t *testing.T) {
// Making sure other libraries are not replaced // Making sure other libraries are not replaced
require.True(t, containsJar(libraries, "./*.jar")) require.True(t, containsJar(libraries, "./*.jar"))
require.True(t, containsMaven(libraries, "org.jsoup:jsoup:1.7.2")) require.True(t, containsMaven(libraries, "org.jsoup:jsoup:1.7.2"))
require.True(t, containsNotebook(libraries, "./non-existent.ipynb"))
} }
func containsNotebook(libraries []pipelines.PipelineLibrary, path string) bool { func containsNotebook(libraries []pipelines.PipelineLibrary, path string) bool {

View File

@ -0,0 +1,12 @@
bundle:
name: pipeline_glob_paths
resources:
pipelines:
nyc_taxi_pipeline:
name: "nyc taxi loader"
libraries:
- notebook:
path: ./dlt/*
- notebook:
path: ./non-existent

View File

@ -0,0 +1,3 @@
# Databricks notebook source
print("Hello from notebook!")

View File

@ -0,0 +1,40 @@
package bundle
import (
"context"
"testing"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/config/mutator"
"github.com/databricks/cli/bundle/phases"
"github.com/databricks/cli/cmd/root"
"github.com/databricks/databricks-sdk-go/service/iam"
"github.com/stretchr/testify/require"
)
func TestExpandPipelineGlobPathsWithNonExistent(t *testing.T) {
ctx := context.Background()
ctx = root.SetWorkspaceClient(ctx, nil)
b, err := bundle.Load(ctx, "./pipeline_glob_paths")
require.NoError(t, err)
err = bundle.Apply(ctx, b, bundle.Seq(mutator.DefaultMutators()...))
require.NoError(t, err)
b.Config.Bundle.Target = "default"
b.Config.Workspace.CurrentUser = &config.User{User: &iam.User{UserName: "user@domain.com"}}
b.WorkspaceClient()
m := phases.Initialize()
err = bundle.Apply(ctx, b, m)
require.Error(t, err)
require.ErrorContains(t, err, "notebook ./non-existent not found")
require.Equal(
t,
b.Config.Resources.Pipelines["nyc_taxi_pipeline"].Libraries[0].Notebook.Path,
"/Users/user@domain.com/.bundle/pipeline_glob_paths/default/files/dlt/nyc_taxi_loader",
)
}