mirror of https://github.com/databricks/cli.git
Allow referencing local Python wheels without artifacts section defined (#703)
## Changes Now if the user reference local Python wheel files and do not specify "artifacts" section, this file will be automatically uploaded by CLI. Fixes #693 ## Tests Added unit tests Ran bundle deploy for this configuration ``` resources: jobs: some_other_job: name: "[${bundle.environment}] My Wheel Job" tasks: - task_key: TestTask existing_cluster_id: ${var.job_existing_cluster} python_wheel_task: package_name: "my_test_code" entry_point: "run" libraries: - whl: ./dist/*.whl ``` Result ``` andrew.nester@HFW9Y94129 wheel % databricks bundle deploy artifacts.whl.AutoDetect: Detecting Python wheel project... artifacts.whl.AutoDetect: No Python wheel project found at bundle root folder Starting upload of bundle files Uploaded bundle files at /Users/andrew.nester@databricks.com/.bundle/wheel-task/default/files! artifacts.Upload(my_test_code-0.0.1-py3-none-any.whl): Uploading... artifacts.Upload(my_test_code-0.0.1-py3-none-any.whl): Upload succeeded ```
This commit is contained in:
parent
861f33d376
commit
5f6289e3a7
|
@ -28,5 +28,6 @@ func (m *autodetect) Apply(ctx context.Context, b *bundle.Bundle) error {
|
||||||
|
|
||||||
return bundle.Apply(ctx, b, bundle.Seq(
|
return bundle.Apply(ctx, b, bundle.Seq(
|
||||||
whl.DetectPackage(),
|
whl.DetectPackage(),
|
||||||
|
whl.DefineArtifactsFromLibraries(),
|
||||||
))
|
))
|
||||||
}
|
}
|
||||||
|
|
|
@ -47,7 +47,11 @@ func (m *infer) Apply(ctx context.Context, b *bundle.Bundle) error {
|
||||||
return fmt.Errorf("artifact doesn't exist: %s", m.name)
|
return fmt.Errorf("artifact doesn't exist: %s", m.name)
|
||||||
}
|
}
|
||||||
|
|
||||||
if artifact.BuildCommand != "" {
|
// only try to infer command if it's not already defined
|
||||||
|
// and there is no explicitly files defined which means
|
||||||
|
// that the package is built outside of bundle cycles
|
||||||
|
// manually by customer
|
||||||
|
if artifact.BuildCommand != "" || len(artifact.Files) > 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,56 @@
|
||||||
|
package whl
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"path/filepath"
|
||||||
|
|
||||||
|
"github.com/databricks/cli/bundle"
|
||||||
|
"github.com/databricks/cli/bundle/config"
|
||||||
|
"github.com/databricks/cli/bundle/libraries"
|
||||||
|
"github.com/databricks/cli/libs/log"
|
||||||
|
)
|
||||||
|
|
||||||
|
type fromLibraries struct{}
|
||||||
|
|
||||||
|
func DefineArtifactsFromLibraries() bundle.Mutator {
|
||||||
|
return &fromLibraries{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *fromLibraries) Name() string {
|
||||||
|
return "artifacts.whl.DefineArtifactsFromLibraries"
|
||||||
|
}
|
||||||
|
|
||||||
|
func (*fromLibraries) Apply(ctx context.Context, b *bundle.Bundle) error {
|
||||||
|
if len(b.Config.Artifacts) != 0 {
|
||||||
|
log.Debugf(ctx, "Skipping defining artifacts from libraries because artifacts section is explicitly defined")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
tasks := libraries.FindAllWheelTasks(b)
|
||||||
|
for _, task := range tasks {
|
||||||
|
for _, lib := range task.Libraries {
|
||||||
|
matches, err := filepath.Glob(filepath.Join(b.Config.Path, lib.Whl))
|
||||||
|
// File referenced from libraries section does not exists, skipping
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, match := range matches {
|
||||||
|
name := filepath.Base(match)
|
||||||
|
if b.Config.Artifacts == nil {
|
||||||
|
b.Config.Artifacts = make(map[string]*config.Artifact)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Debugf(ctx, "Adding an artifact block for %s", match)
|
||||||
|
b.Config.Artifacts[name] = &config.Artifact{
|
||||||
|
Files: []config.ArtifactFile{
|
||||||
|
{Source: match},
|
||||||
|
},
|
||||||
|
Type: config.ArtifactPythonWheel,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
|
@ -0,0 +1,3 @@
|
||||||
|
build/
|
||||||
|
*.egg-info
|
||||||
|
.databricks
|
|
@ -0,0 +1,22 @@
|
||||||
|
bundle:
|
||||||
|
name: python-wheel-local
|
||||||
|
|
||||||
|
resources:
|
||||||
|
jobs:
|
||||||
|
test_job:
|
||||||
|
name: "[${bundle.environment}] My Wheel Job"
|
||||||
|
tasks:
|
||||||
|
- task_key: TestTask
|
||||||
|
existing_cluster_id: "0717-aaaaa-bbbbbb"
|
||||||
|
python_wheel_task:
|
||||||
|
package_name: "my_test_code"
|
||||||
|
entry_point: "run"
|
||||||
|
libraries:
|
||||||
|
- whl: ./package/*.whl
|
||||||
|
- task_key: TestTask2
|
||||||
|
existing_cluster_id: "0717-aaaaa-bbbbbb"
|
||||||
|
python_wheel_task:
|
||||||
|
package_name: "my_test_code"
|
||||||
|
entry_point: "run"
|
||||||
|
libraries:
|
||||||
|
- whl: ./non-existing/*.whl
|
Binary file not shown.
|
@ -60,3 +60,29 @@ func TestBundlePythonWheelWithDBFSLib(t *testing.T) {
|
||||||
err = match.Apply(ctx, b)
|
err = match.Apply(ctx, b)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestBundlePythonWheelBuildNoBuildJustUpload(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
b, err := bundle.Load(ctx, "./python_wheel_no_artifact_no_setup")
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
m := phases.Build()
|
||||||
|
err = m.Apply(ctx, b)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
match := libraries.MatchWithArtifacts()
|
||||||
|
err = match.Apply(ctx, b)
|
||||||
|
require.ErrorContains(t, err, "./non-existing/*.whl")
|
||||||
|
|
||||||
|
require.NotZero(t, len(b.Config.Artifacts))
|
||||||
|
|
||||||
|
artifact := b.Config.Artifacts["my_test_code-0.0.1-py3-none-any.whl"]
|
||||||
|
require.NotNil(t, artifact)
|
||||||
|
require.Empty(t, artifact.BuildCommand)
|
||||||
|
require.Contains(t, artifact.Files[0].Source, filepath.Join(
|
||||||
|
b.Config.Path,
|
||||||
|
"package",
|
||||||
|
"my_test_code-0.0.1-py3-none-any.whl",
|
||||||
|
))
|
||||||
|
require.True(t, artifact.Files[0].NeedsUpload())
|
||||||
|
}
|
||||||
|
|
Loading…
Reference in New Issue