Fixed marking libraries from DBFS as remote (#750)

## Changes
Fixed marking libraries from DBFS as remote

## Tests
Updated unit tests to catch the regression
This commit is contained in:
Andrew Nester 2023-09-08 11:53:57 +02:00 committed by GitHub
parent e08f419ef6
commit e64463ba47
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 6 additions and 5 deletions

View File

@ -162,7 +162,7 @@ func TestTranslatePaths(t *testing.T) {
MainClassName: "HelloWorldRemote", MainClassName: "HelloWorldRemote",
}, },
Libraries: []compute.Library{ Libraries: []compute.Library{
{Jar: "dbfs:///bundle/dist/task_remote.jar"}, {Jar: "dbfs:/bundle/dist/task_remote.jar"},
}, },
}, },
}, },
@ -243,7 +243,7 @@ func TestTranslatePaths(t *testing.T) {
) )
assert.Equal( assert.Equal(
t, t,
"dbfs:///bundle/dist/task_remote.jar", "dbfs:/bundle/dist/task_remote.jar",
bundle.Config.Resources.Jobs["job"].Tasks[6].Libraries[0].Jar, bundle.Config.Resources.Jobs["job"].Tasks[6].Libraries[0].Jar,
) )

View File

@ -165,8 +165,8 @@ func isRemoteStorageScheme(path string) bool {
return false return false
} }
// If the path starts with scheme:// format, it's a correct remote storage scheme // If the path starts with scheme:/ format, it's a correct remote storage scheme
return strings.HasPrefix(path, url.Scheme+"://") return strings.HasPrefix(path, url.Scheme+":/")
} }

View File

@ -16,6 +16,7 @@ var testCases map[string]bool = map[string]bool{
"file://path/to/package": true, "file://path/to/package": true,
"C:\\path\\to\\package": true, "C:\\path\\to\\package": true,
"dbfs://path/to/package": false, "dbfs://path/to/package": false,
"dbfs:/path/to/package": false,
"s3://path/to/package": false, "s3://path/to/package": false,
"abfss://path/to/package": false, "abfss://path/to/package": false,
} }

View File

@ -12,4 +12,4 @@ resources:
package_name: "my_test_code" package_name: "my_test_code"
entry_point: "run" entry_point: "run"
libraries: libraries:
- whl: dbfs://path/to/dist/mywheel.whl - whl: dbfs:/path/to/dist/mywheel.whl