Correctly identify local paths in libraries section (#702)

## Changes
Fixes #699

## Tests
Added unit test
This commit is contained in:
Andrew Nester 2023-08-29 10:26:09 +02:00 committed by GitHub
parent 5477afe4f4
commit 842cd8b7ae
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 60 additions and 6 deletions

View File

@ -3,6 +3,7 @@ package libraries
import (
"context"
"fmt"
"net/url"
"path/filepath"
"strings"
@ -92,13 +93,13 @@ func findArtifactsAndMarkForUpload(ctx context.Context, lib *compute.Library, b
}
if len(matches) == 0 && isLocalLibrary(lib) {
return fmt.Errorf("no library found for %s", libPath(lib))
return fmt.Errorf("file %s is referenced in libraries section but doesn't exist on the local file system", libPath(lib))
}
for _, match := range matches {
af, err := findArtifactFileByLocalPath(match, b)
if err != nil {
cmdio.LogString(ctx, fmt.Sprintf("%s. Skipping %s. In order to use the library upload it manually", err.Error(), match))
cmdio.LogString(ctx, fmt.Sprintf("%s. Skipping uploading. In order to use the define 'artifacts' section", err.Error()))
} else {
af.Libraries = append(af.Libraries, lib)
}
@ -116,7 +117,7 @@ func findArtifactFileByLocalPath(path string, b *bundle.Bundle) (*config.Artifac
}
}
return nil, fmt.Errorf("artifact file is not found for path %s", path)
return nil, fmt.Errorf("artifact section is not defined for file at %s", path)
}
func libPath(library *compute.Library) string {
@ -139,11 +140,34 @@ func isLocalLibrary(library *compute.Library) bool {
return false
}
return !isDbfsPath(path) && !isWorkspacePath(path)
if isExplicitFileScheme(path) {
return true
}
if isRemoteStorageScheme(path) {
return false
}
return !isWorkspacePath(path)
}
func isDbfsPath(path string) bool {
return strings.HasPrefix(path, "dbfs:/")
func isExplicitFileScheme(path string) bool {
return strings.HasPrefix(path, "file://")
}
func isRemoteStorageScheme(path string) bool {
url, err := url.Parse(path)
if err != nil {
return false
}
if url.Scheme == "" {
return false
}
// If the path starts with scheme:// format, it's a correct remote storage scheme
return strings.HasPrefix(path, url.Scheme+"://")
}
func isWorkspacePath(path string) bool {

View File

@ -0,0 +1,30 @@
package libraries
import (
"fmt"
"testing"
"github.com/databricks/databricks-sdk-go/service/compute"
"github.com/stretchr/testify/require"
)
var testCases map[string]bool = map[string]bool{
"./some/local/path": true,
"/some/full/path": true,
"/Workspace/path/to/package": false,
"/Users/path/to/package": false,
"file://path/to/package": true,
"C:\\path\\to\\package": true,
"dbfs://path/to/package": false,
"s3://path/to/package": false,
"abfss://path/to/package": false,
}
func TestIsLocalLbrary(t *testing.T) {
for p, result := range testCases {
lib := compute.Library{
Whl: p,
}
require.Equal(t, result, isLocalLibrary(&lib), fmt.Sprintf("isLocalLibrary must return %t for path %s ", result, p))
}
}