mirror of https://github.com/databricks/cli.git
Compare commits
7 Commits
d1f804d23c
...
796e3e5553
Author | SHA1 | Date |
---|---|---|
Gleb Kanterov | 796e3e5553 | |
shreyas-goenka | cc112961ce | |
Gleb Kanterov | bbcbffd7f4 | |
Gleb Kanterov | 6b4641c530 | |
Gleb Kanterov | 55c36bc2fa | |
Gleb Kanterov | 6fd701ec84 | |
Gleb Kanterov | 37d677c079 |
|
@ -38,7 +38,7 @@ jobs:
|
||||||
- name: Setup Python
|
- name: Setup Python
|
||||||
uses: actions/setup-python@v5
|
uses: actions/setup-python@v5
|
||||||
with:
|
with:
|
||||||
python-version: '3.9'
|
python-version: '3.10'
|
||||||
|
|
||||||
- name: Set go env
|
- name: Set go env
|
||||||
run: |
|
run: |
|
||||||
|
|
|
@ -0,0 +1,15 @@
|
||||||
|
bundle:
|
||||||
|
name: python-import-dataclass-no-wheel
|
||||||
|
|
||||||
|
experimental:
|
||||||
|
pydabs:
|
||||||
|
enabled: true
|
||||||
|
import:
|
||||||
|
- "my_job"
|
||||||
|
|
||||||
|
variables:
|
||||||
|
default_cluster_spec:
|
||||||
|
type: complex
|
||||||
|
value:
|
||||||
|
num_workers: 1
|
||||||
|
spark_version: "15.4.x-scala2.12"
|
|
@ -0,0 +1,2 @@
|
||||||
|
# Databricks Notebook Source
|
||||||
|
1+1
|
|
@ -0,0 +1,22 @@
|
||||||
|
from databricks.bundles.jobs import Job, Task, NotebookTask, JobCluster
|
||||||
|
from databricks.bundles.variables import Bundle
|
||||||
|
|
||||||
|
my_job = Job(
|
||||||
|
name="Test Job",
|
||||||
|
resource_name="my_job",
|
||||||
|
job_clusters=[
|
||||||
|
JobCluster(
|
||||||
|
job_cluster_key="my_cluster",
|
||||||
|
new_cluster=Bundle.variables.default_cluster_spec,
|
||||||
|
),
|
||||||
|
],
|
||||||
|
tasks=[
|
||||||
|
Task(
|
||||||
|
task_key="my_notebook_task",
|
||||||
|
job_cluster_key="my_cluster",
|
||||||
|
notebook_task=NotebookTask(
|
||||||
|
notebook_path="notebooks/my_notebook.py",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
],
|
||||||
|
)
|
|
@ -0,0 +1,95 @@
|
||||||
|
package config_tests
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
pathlib "path"
|
||||||
|
"runtime"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/databricks/cli/bundle/config/resources"
|
||||||
|
"github.com/databricks/cli/libs/dyn"
|
||||||
|
"github.com/databricks/databricks-sdk-go/service/jobs"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
"golang.org/x/exp/maps"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestPythonImport_dataclass_no_wheel(t *testing.T) {
|
||||||
|
activateVEnv(t)
|
||||||
|
setPythonPath(t, "python_import/dataclass_no_wheel/src")
|
||||||
|
|
||||||
|
expected := &resources.Job{
|
||||||
|
JobSettings: &jobs.JobSettings{
|
||||||
|
Name: "Test Job",
|
||||||
|
JobClusters: []jobs.JobCluster{
|
||||||
|
{
|
||||||
|
JobClusterKey: "my_cluster",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
Tasks: []jobs.Task{
|
||||||
|
{
|
||||||
|
NotebookTask: &jobs.NotebookTask{
|
||||||
|
NotebookPath: "notebooks/my_notebook.py",
|
||||||
|
},
|
||||||
|
JobClusterKey: "my_cluster",
|
||||||
|
TaskKey: "my_notebook_task",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
b := load(t, "./python_import/dataclass_no_wheel")
|
||||||
|
|
||||||
|
assert.Equal(t, []string{"my_job"}, maps.Keys(b.Config.Resources.Jobs))
|
||||||
|
|
||||||
|
myJob := b.Config.Resources.Jobs["my_job"]
|
||||||
|
assert.Equal(t, expected, myJob)
|
||||||
|
|
||||||
|
// NewCluster is reference to a variable and needs to be checked separately
|
||||||
|
err := b.Config.Mutate(func(value dyn.Value) (dyn.Value, error) {
|
||||||
|
path := dyn.MustPathFromString("resources.jobs.my_job.job_clusters[0].new_cluster")
|
||||||
|
value, err := dyn.GetByPath(value, path)
|
||||||
|
if err != nil {
|
||||||
|
return dyn.InvalidValue, err
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.Equal(t, "${var.default_cluster_spec}", value.AsAny())
|
||||||
|
|
||||||
|
return value, nil
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func setPythonPath(t *testing.T, path string) {
|
||||||
|
wd, err := os.Getwd()
|
||||||
|
require.NoError(t, err)
|
||||||
|
t.Setenv("PYTHONPATH", pathlib.Join(wd, path))
|
||||||
|
}
|
||||||
|
|
||||||
|
func activateVEnv(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
venvDir := pathlib.Join(dir, "venv")
|
||||||
|
|
||||||
|
err := exec.Command("python3", "-m", "venv", venvDir).Run()
|
||||||
|
require.NoError(t, err, "failed to create venv")
|
||||||
|
|
||||||
|
// we don't have shell to activate venv, updating PATH is enough
|
||||||
|
|
||||||
|
var venvBinDir string
|
||||||
|
if runtime.GOOS == "windows" {
|
||||||
|
venvBinDir = pathlib.Join(venvDir, "Scripts")
|
||||||
|
t.Setenv("PATH", venvBinDir+";"+os.Getenv("PATH"))
|
||||||
|
} else {
|
||||||
|
venvBinDir = pathlib.Join(venvDir, "bin")
|
||||||
|
t.Setenv("PATH", venvBinDir+":"+os.Getenv("PATH"))
|
||||||
|
}
|
||||||
|
|
||||||
|
err = exec.Command(
|
||||||
|
pathlib.Join(venvBinDir, "pip"),
|
||||||
|
"install",
|
||||||
|
"databricks-pydabs==0.5.1",
|
||||||
|
).Run()
|
||||||
|
require.NoError(t, err, "failed to install databricks-pydabs")
|
||||||
|
}
|
|
@ -112,8 +112,8 @@ func TestAccFsMkdirWhenFileExistsAtPath(t *testing.T) {
|
||||||
// assert mkdir fails
|
// assert mkdir fails
|
||||||
_, _, err = RequireErrorRun(t, "fs", "mkdir", path.Join(tmpDir, "hello"))
|
_, _, err = RequireErrorRun(t, "fs", "mkdir", path.Join(tmpDir, "hello"))
|
||||||
|
|
||||||
// Different cloud providers return different errors.
|
// Different cloud providers or cloud configurations return different errors.
|
||||||
regex := regexp.MustCompile(`(^|: )Path is a file: .*$|(^|: )Cannot create directory .* because .* is an existing file\.$|(^|: )mkdirs\(hadoopPath: .*, permission: rwxrwxrwx\): failed$`)
|
regex := regexp.MustCompile(`(^|: )Path is a file: .*$|(^|: )Cannot create directory .* because .* is an existing file\.$|(^|: )mkdirs\(hadoopPath: .*, permission: rwxrwxrwx\): failed$|(^|: )"The specified path already exists.".*$`)
|
||||||
assert.Regexp(t, regex, err.Error())
|
assert.Regexp(t, regex, err.Error())
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
|
@ -20,6 +20,7 @@ import (
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/databricks/cli/cmd/root"
|
"github.com/databricks/cli/cmd/root"
|
||||||
|
"github.com/databricks/cli/internal/acc"
|
||||||
"github.com/databricks/cli/libs/flags"
|
"github.com/databricks/cli/libs/flags"
|
||||||
|
|
||||||
"github.com/databricks/cli/cmd"
|
"github.com/databricks/cli/cmd"
|
||||||
|
@ -591,13 +592,10 @@ func setupWsfsExtensionsFiler(t *testing.T) (filer.Filer, string) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func setupDbfsFiler(t *testing.T) (filer.Filer, string) {
|
func setupDbfsFiler(t *testing.T) (filer.Filer, string) {
|
||||||
t.Log(GetEnvOrSkipTest(t, "CLOUD_ENV"))
|
_, wt := acc.WorkspaceTest(t)
|
||||||
|
|
||||||
w, err := databricks.NewWorkspaceClient()
|
tmpDir := TemporaryDbfsDir(t, wt.W)
|
||||||
require.NoError(t, err)
|
f, err := filer.NewDbfsClient(wt.W, tmpDir)
|
||||||
|
|
||||||
tmpDir := TemporaryDbfsDir(t, w)
|
|
||||||
f, err := filer.NewDbfsClient(w, tmpDir)
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
return f, path.Join("dbfs:/", tmpDir)
|
return f, path.Join("dbfs:/", tmpDir)
|
||||||
|
|
Loading…
Reference in New Issue