Fixed using repo files as pipeline libraries (#847)

## Changes
Fixed using repo files as pipeline libraries

## Tests
Added regression test
This commit is contained in:
Andrew Nester 2023-10-09 12:10:28 +02:00 committed by GitHub
parent 054df2b58b
commit 8d8de3f509
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 15 additions and 3 deletions

View File

@ -80,6 +80,11 @@ func TestExpandGlobPathsInPipelines(t *testing.T) {
Path: "dbfs:/me@company.com/test.ipynb", Path: "dbfs:/me@company.com/test.ipynb",
}, },
}, },
{
Notebook: &pipelines.NotebookLibrary{
Path: "/Repos/somerepo/test.ipynb",
},
},
}, },
}, },
}, },
@ -93,7 +98,7 @@ func TestExpandGlobPathsInPipelines(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
libraries := b.Config.Resources.Pipelines["pipeline"].Libraries libraries := b.Config.Resources.Pipelines["pipeline"].Libraries
require.Len(t, libraries, 9) require.Len(t, libraries, 10)
// Making sure glob patterns are expanded correctly // Making sure glob patterns are expanded correctly
require.True(t, containsNotebook(libraries, filepath.Join("test", "test2.ipynb"))) require.True(t, containsNotebook(libraries, filepath.Join("test", "test2.ipynb")))
@ -107,6 +112,7 @@ func TestExpandGlobPathsInPipelines(t *testing.T) {
// Making sure absolute pass to remote FS file references work as well // Making sure absolute pass to remote FS file references work as well
require.True(t, containsNotebook(libraries, "/Workspace/Users/me@company.com/test.ipynb")) require.True(t, containsNotebook(libraries, "/Workspace/Users/me@company.com/test.ipynb"))
require.True(t, containsNotebook(libraries, "dbfs:/me@company.com/test.ipynb")) require.True(t, containsNotebook(libraries, "dbfs:/me@company.com/test.ipynb"))
require.True(t, containsNotebook(libraries, "/Repos/somerepo/test.ipynb"))
// Making sure other libraries are not replaced // Making sure other libraries are not replaced
require.True(t, containsJar(libraries, "./*.jar")) require.True(t, containsJar(libraries, "./*.jar"))

View File

@ -4,6 +4,7 @@ import (
"context" "context"
"fmt" "fmt"
"net/url" "net/url"
"path"
"path/filepath" "path/filepath"
"strings" "strings"
@ -173,7 +174,7 @@ func IsLocalPath(path string) bool {
return false return false
} }
return !isWorkspacePath(path) return !isAbsoluteRemotePath(path)
} }
func isExplicitFileScheme(path string) bool { func isExplicitFileScheme(path string) bool {
@ -200,3 +201,8 @@ func isWorkspacePath(path string) bool {
strings.HasPrefix(path, "/Users/") || strings.HasPrefix(path, "/Users/") ||
strings.HasPrefix(path, "/Shared/") strings.HasPrefix(path, "/Shared/")
} }
func isAbsoluteRemotePath(p string) bool {
// If path for library starts with /, it's a remote absolute path
return path.IsAbs(p)
}

View File

@ -10,7 +10,7 @@ import (
var testCases map[string]bool = map[string]bool{ var testCases map[string]bool = map[string]bool{
"./some/local/path": true, "./some/local/path": true,
"/some/full/path": true, "/some/full/path": false,
"/Workspace/path/to/package": false, "/Workspace/path/to/package": false,
"/Users/path/to/package": false, "/Users/path/to/package": false,
"file://path/to/package": true, "file://path/to/package": true,