steve/databricks-cli
steve
/
databricks-cli
mirror of https://github.com/databricks/cli.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity
databricks-cli/libs/notebook/detect_test.go

120 lines
2.8 KiB
Go
Raw Permalink Normal View History

Move notebook detection logic to package (#206)
2023-02-15 16:14:59 +00:00
package notebook
import (
Update error checks that use the `os` package to use `errors.Is` (#1461) ## Changes From the [documentation](https://pkg.go.dev/os#IsNotExist) on the functions in the `os` package: > This function predates errors.Is. It only supports errors returned by the os package. > New code should use errors.Is(err, fs.ErrNotExist). This issue surfaced while working on using a different `vfs.Path` implementation that uses errors from the `fs` package. Calls to `os.IsNotExist` didn't return true for errors that wrap `fs.ErrNotExist`. ## Tests n/a
2024-06-03 12:39:36 +00:00
"errors"
"io/fs"
Move notebook detection logic to package (#206)
2023-02-15 16:14:59 +00:00
"os"
"path/filepath"
"testing"
"github.com/databricks/databricks-sdk-go/service/workspace"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
Detect Jupyter notebook files (#219) Files with extension `.ipynb` are imported are Jupyter notebooks. This code detects 1) if the file is a valid Jupyter notebook and 2) the Databricks specific language it contains.
2023-02-21 12:49:01 +00:00
func TestDetectSource(t *testing.T) {
Move notebook detection logic to package (#206)
2023-02-15 16:14:59 +00:00
var nb bool
var lang workspace.Language
var err error
Detect Jupyter notebook files (#219) Files with extension `.ipynb` are imported are Jupyter notebooks. This code detects 1) if the file is a valid Jupyter notebook and 2) the Databricks specific language it contains.
2023-02-21 12:49:01 +00:00
nb, lang, err = Detect("./testdata/py_source.py")
Move notebook detection logic to package (#206)
2023-02-15 16:14:59 +00:00
require.NoError(t, err)
assert.True(t, nb)
assert.Equal(t, workspace.LanguagePython, lang)
Detect Jupyter notebook files (#219) Files with extension `.ipynb` are imported are Jupyter notebooks. This code detects 1) if the file is a valid Jupyter notebook and 2) the Databricks specific language it contains.
2023-02-21 12:49:01 +00:00
nb, lang, err = Detect("./testdata/r_source.r")
Move notebook detection logic to package (#206)
2023-02-15 16:14:59 +00:00
require.NoError(t, err)
assert.True(t, nb)
assert.Equal(t, workspace.LanguageR, lang)
Detect Jupyter notebook files (#219) Files with extension `.ipynb` are imported are Jupyter notebooks. This code detects 1) if the file is a valid Jupyter notebook and 2) the Databricks specific language it contains.
2023-02-21 12:49:01 +00:00
nb, lang, err = Detect("./testdata/scala_source.scala")
Move notebook detection logic to package (#206)
2023-02-15 16:14:59 +00:00
require.NoError(t, err)
assert.True(t, nb)
assert.Equal(t, workspace.LanguageScala, lang)
Detect Jupyter notebook files (#219) Files with extension `.ipynb` are imported are Jupyter notebooks. This code detects 1) if the file is a valid Jupyter notebook and 2) the Databricks specific language it contains.
2023-02-21 12:49:01 +00:00
nb, lang, err = Detect("./testdata/sql_source.sql")
Move notebook detection logic to package (#206)
2023-02-15 16:14:59 +00:00
require.NoError(t, err)
assert.True(t, nb)
assert.Equal(t, workspace.LanguageSql, lang)
nb, lang, err = Detect("./testdata/txt.txt")
require.NoError(t, err)
assert.False(t, nb)
assert.Equal(t, workspace.Language(""), lang)
}
Detect Jupyter notebook files (#219) Files with extension `.ipynb` are imported are Jupyter notebooks. This code detects 1) if the file is a valid Jupyter notebook and 2) the Databricks specific language it contains.
2023-02-21 12:49:01 +00:00
func TestDetectCallsDetectJupyter(t *testing.T) {
nb, lang, err := Detect("./testdata/py_ipynb.ipynb")
require.NoError(t, err)
assert.True(t, nb)
assert.Equal(t, workspace.LanguagePython, lang)
}
Move notebook detection logic to package (#206)
2023-02-15 16:14:59 +00:00
func TestDetectUnknownExtension(t *testing.T) {
Error out if notebook file does not exist locally (#261) Adds check for whether file exists locally case 1: local (relative) file does not exist ``` foo: name: "[job-output] test-job by shreyas" tasks: - task_key: my_notebook_task existing_cluster_id: *** notebook_task: notebook_path: "./doesnotexist" ``` output: ``` shreyas.goenka@THW32HFW6T job-output % bricks bundle deploy Error: notebook ./doesnotexist not found. Error: open /Users/shreyas.goenka/projects/job-output/doesnotexist: no such file or directory ``` case 2: remote (absolute) file does not exist ``` foo: name: "[job-output] test-job by shreyas" tasks: - task_key: my_notebook_task existing_cluster_id: *** notebook_task: notebook_path: "/Users/shreyas.goenka@databricks.com/doesnotexist" ``` output: ``` shreyas.goenka@THW32HFW6T job-output % bricks bundle deploy shreyas.goenka@THW32HFW6T job-output % bricks bundle run foo Error: failed to reach TERMINATED or SKIPPED, got INTERNAL_ERROR: Task my_notebook_task failed with message: Notebook not found: /Users/shreyas.goenka@databricks.com/doesnotexist. This caused all downstream tasks to get skipped. ``` case 3: remote exists Successful deploy and run
2023-03-21 17:13:16 +00:00
_, _, err := Detect("./testdata/doesntexist.foobar")
Update error checks that use the `os` package to use `errors.Is` (#1461) ## Changes From the [documentation](https://pkg.go.dev/os#IsNotExist) on the functions in the `os` package: > This function predates errors.Is. It only supports errors returned by the os package. > New code should use errors.Is(err, fs.ErrNotExist). This issue surfaced while working on using a different `vfs.Path` implementation that uses errors from the `fs` package. Calls to `os.IsNotExist` didn't return true for errors that wrap `fs.ErrNotExist`. ## Tests n/a
2024-06-03 12:39:36 +00:00
assert.True(t, errors.Is(err, fs.ErrNotExist))
Error out if notebook file does not exist locally (#261) Adds check for whether file exists locally case 1: local (relative) file does not exist ``` foo: name: "[job-output] test-job by shreyas" tasks: - task_key: my_notebook_task existing_cluster_id: *** notebook_task: notebook_path: "./doesnotexist" ``` output: ``` shreyas.goenka@THW32HFW6T job-output % bricks bundle deploy Error: notebook ./doesnotexist not found. Error: open /Users/shreyas.goenka/projects/job-output/doesnotexist: no such file or directory ``` case 2: remote (absolute) file does not exist ``` foo: name: "[job-output] test-job by shreyas" tasks: - task_key: my_notebook_task existing_cluster_id: *** notebook_task: notebook_path: "/Users/shreyas.goenka@databricks.com/doesnotexist" ``` output: ``` shreyas.goenka@THW32HFW6T job-output % bricks bundle deploy shreyas.goenka@THW32HFW6T job-output % bricks bundle run foo Error: failed to reach TERMINATED or SKIPPED, got INTERNAL_ERROR: Task my_notebook_task failed with message: Notebook not found: /Users/shreyas.goenka@databricks.com/doesnotexist. This caused all downstream tasks to get skipped. ``` case 3: remote exists Successful deploy and run
2023-03-21 17:13:16 +00:00
nb, _, err := Detect("./testdata/unknown_extension.foobar")
Move notebook detection logic to package (#206)
2023-02-15 16:14:59 +00:00
require.NoError(t, err)
assert.False(t, nb)
}
func TestDetectNoExtension(t *testing.T) {
Error out if notebook file does not exist locally (#261) Adds check for whether file exists locally case 1: local (relative) file does not exist ``` foo: name: "[job-output] test-job by shreyas" tasks: - task_key: my_notebook_task existing_cluster_id: *** notebook_task: notebook_path: "./doesnotexist" ``` output: ``` shreyas.goenka@THW32HFW6T job-output % bricks bundle deploy Error: notebook ./doesnotexist not found. Error: open /Users/shreyas.goenka/projects/job-output/doesnotexist: no such file or directory ``` case 2: remote (absolute) file does not exist ``` foo: name: "[job-output] test-job by shreyas" tasks: - task_key: my_notebook_task existing_cluster_id: *** notebook_task: notebook_path: "/Users/shreyas.goenka@databricks.com/doesnotexist" ``` output: ``` shreyas.goenka@THW32HFW6T job-output % bricks bundle deploy shreyas.goenka@THW32HFW6T job-output % bricks bundle run foo Error: failed to reach TERMINATED or SKIPPED, got INTERNAL_ERROR: Task my_notebook_task failed with message: Notebook not found: /Users/shreyas.goenka@databricks.com/doesnotexist. This caused all downstream tasks to get skipped. ``` case 3: remote exists Successful deploy and run
2023-03-21 17:13:16 +00:00
_, _, err := Detect("./testdata/doesntexist")
Update error checks that use the `os` package to use `errors.Is` (#1461) ## Changes From the [documentation](https://pkg.go.dev/os#IsNotExist) on the functions in the `os` package: > This function predates errors.Is. It only supports errors returned by the os package. > New code should use errors.Is(err, fs.ErrNotExist). This issue surfaced while working on using a different `vfs.Path` implementation that uses errors from the `fs` package. Calls to `os.IsNotExist` didn't return true for errors that wrap `fs.ErrNotExist`. ## Tests n/a
2024-06-03 12:39:36 +00:00
assert.True(t, errors.Is(err, fs.ErrNotExist))
Error out if notebook file does not exist locally (#261) Adds check for whether file exists locally case 1: local (relative) file does not exist ``` foo: name: "[job-output] test-job by shreyas" tasks: - task_key: my_notebook_task existing_cluster_id: *** notebook_task: notebook_path: "./doesnotexist" ``` output: ``` shreyas.goenka@THW32HFW6T job-output % bricks bundle deploy Error: notebook ./doesnotexist not found. Error: open /Users/shreyas.goenka/projects/job-output/doesnotexist: no such file or directory ``` case 2: remote (absolute) file does not exist ``` foo: name: "[job-output] test-job by shreyas" tasks: - task_key: my_notebook_task existing_cluster_id: *** notebook_task: notebook_path: "/Users/shreyas.goenka@databricks.com/doesnotexist" ``` output: ``` shreyas.goenka@THW32HFW6T job-output % bricks bundle deploy shreyas.goenka@THW32HFW6T job-output % bricks bundle run foo Error: failed to reach TERMINATED or SKIPPED, got INTERNAL_ERROR: Task my_notebook_task failed with message: Notebook not found: /Users/shreyas.goenka@databricks.com/doesnotexist. This caused all downstream tasks to get skipped. ``` case 3: remote exists Successful deploy and run
2023-03-21 17:13:16 +00:00
nb, _, err := Detect("./testdata/no_extension")
Move notebook detection logic to package (#206)
2023-02-15 16:14:59 +00:00
require.NoError(t, err)
assert.False(t, nb)
}
func TestDetectFileDoesNotExists(t *testing.T) {
_, _, err := Detect("./testdata/doesntexist.py")
require.Error(t, err)
}
func TestDetectEmptyFile(t *testing.T) {
// Create empty file.
dir := t.TempDir()
path := filepath.Join(dir, "file.py")
err := os.WriteFile(path, nil, 0644)
require.NoError(t, err)
// No contents means not a notebook.
nb, _, err := Detect(path)
require.NoError(t, err)
assert.False(t, nb)
}
func TestDetectFileWithLongHeader(t *testing.T) {
// Create 128kb garbage file.
dir := t.TempDir()
path := filepath.Join(dir, "file.py")
buf := make([]byte, 128*1024)
err := os.WriteFile(path, buf, 0644)
require.NoError(t, err)
// Garbage contents means not a notebook.
nb, _, err := Detect(path)
require.NoError(t, err)
assert.False(t, nb)
}
Let notebook detection code use underlying metadata if available (#1574) ## Changes If we're using a `vfs.Path` backed by a workspace filesystem filer, we have access to the `workspace.ObjectInfo` value for every file. By providing access to this value we can use it directly and avoid reading the first line of the underlying file. A follow-up change will implement the interface defined in this change for the workspace filesystem filer. ## Tests Unit tests.
2024-07-10 06:37:47 +00:00
func TestDetectWithObjectInfo(t *testing.T) {
fakeFS := &fakeFS{
fakeFile{
fakeFileInfo{
workspace.ObjectInfo{
ObjectType: workspace.ObjectTypeNotebook,
Language: workspace.LanguagePython,
},
},
},
}
nb, lang, err := DetectWithFS(fakeFS, "doesntmatter")
require.NoError(t, err)
assert.True(t, nb)
assert.Equal(t, workspace.LanguagePython, lang)
}