mirror of https://github.com/databricks/cli.git
Allow referencing local Python wheels without artifacts section defined (#703)
## Changes Now if the user reference local Python wheel files and do not specify "artifacts" section, this file will be automatically uploaded by CLI. Fixes #693 ## Tests Added unit tests Ran bundle deploy for this configuration ``` resources: jobs: some_other_job: name: "[${bundle.environment}] My Wheel Job" tasks: - task_key: TestTask existing_cluster_id: ${var.job_existing_cluster} python_wheel_task: package_name: "my_test_code" entry_point: "run" libraries: - whl: ./dist/*.whl ``` Result ``` andrew.nester@HFW9Y94129 wheel % databricks bundle deploy artifacts.whl.AutoDetect: Detecting Python wheel project... artifacts.whl.AutoDetect: No Python wheel project found at bundle root folder Starting upload of bundle files Uploaded bundle files at /Users/andrew.nester@databricks.com/.bundle/wheel-task/default/files! artifacts.Upload(my_test_code-0.0.1-py3-none-any.whl): Uploading... artifacts.Upload(my_test_code-0.0.1-py3-none-any.whl): Upload succeeded ```
This commit is contained in:
parent
861f33d376
commit
5f6289e3a7
|
@ -28,5 +28,6 @@ func (m *autodetect) Apply(ctx context.Context, b *bundle.Bundle) error {
|
|||
|
||||
return bundle.Apply(ctx, b, bundle.Seq(
|
||||
whl.DetectPackage(),
|
||||
whl.DefineArtifactsFromLibraries(),
|
||||
))
|
||||
}
|
||||
|
|
|
@ -47,7 +47,11 @@ func (m *infer) Apply(ctx context.Context, b *bundle.Bundle) error {
|
|||
return fmt.Errorf("artifact doesn't exist: %s", m.name)
|
||||
}
|
||||
|
||||
if artifact.BuildCommand != "" {
|
||||
// only try to infer command if it's not already defined
|
||||
// and there is no explicitly files defined which means
|
||||
// that the package is built outside of bundle cycles
|
||||
// manually by customer
|
||||
if artifact.BuildCommand != "" || len(artifact.Files) > 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
|
@ -0,0 +1,56 @@
|
|||
package whl
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/databricks/cli/bundle"
|
||||
"github.com/databricks/cli/bundle/config"
|
||||
"github.com/databricks/cli/bundle/libraries"
|
||||
"github.com/databricks/cli/libs/log"
|
||||
)
|
||||
|
||||
type fromLibraries struct{}
|
||||
|
||||
func DefineArtifactsFromLibraries() bundle.Mutator {
|
||||
return &fromLibraries{}
|
||||
}
|
||||
|
||||
func (m *fromLibraries) Name() string {
|
||||
return "artifacts.whl.DefineArtifactsFromLibraries"
|
||||
}
|
||||
|
||||
func (*fromLibraries) Apply(ctx context.Context, b *bundle.Bundle) error {
|
||||
if len(b.Config.Artifacts) != 0 {
|
||||
log.Debugf(ctx, "Skipping defining artifacts from libraries because artifacts section is explicitly defined")
|
||||
return nil
|
||||
}
|
||||
|
||||
tasks := libraries.FindAllWheelTasks(b)
|
||||
for _, task := range tasks {
|
||||
for _, lib := range task.Libraries {
|
||||
matches, err := filepath.Glob(filepath.Join(b.Config.Path, lib.Whl))
|
||||
// File referenced from libraries section does not exists, skipping
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
for _, match := range matches {
|
||||
name := filepath.Base(match)
|
||||
if b.Config.Artifacts == nil {
|
||||
b.Config.Artifacts = make(map[string]*config.Artifact)
|
||||
}
|
||||
|
||||
log.Debugf(ctx, "Adding an artifact block for %s", match)
|
||||
b.Config.Artifacts[name] = &config.Artifact{
|
||||
Files: []config.ArtifactFile{
|
||||
{Source: match},
|
||||
},
|
||||
Type: config.ArtifactPythonWheel,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
|
@ -0,0 +1,3 @@
|
|||
build/
|
||||
*.egg-info
|
||||
.databricks
|
|
@ -0,0 +1,22 @@
|
|||
bundle:
|
||||
name: python-wheel-local
|
||||
|
||||
resources:
|
||||
jobs:
|
||||
test_job:
|
||||
name: "[${bundle.environment}] My Wheel Job"
|
||||
tasks:
|
||||
- task_key: TestTask
|
||||
existing_cluster_id: "0717-aaaaa-bbbbbb"
|
||||
python_wheel_task:
|
||||
package_name: "my_test_code"
|
||||
entry_point: "run"
|
||||
libraries:
|
||||
- whl: ./package/*.whl
|
||||
- task_key: TestTask2
|
||||
existing_cluster_id: "0717-aaaaa-bbbbbb"
|
||||
python_wheel_task:
|
||||
package_name: "my_test_code"
|
||||
entry_point: "run"
|
||||
libraries:
|
||||
- whl: ./non-existing/*.whl
|
Binary file not shown.
|
@ -60,3 +60,29 @@ func TestBundlePythonWheelWithDBFSLib(t *testing.T) {
|
|||
err = match.Apply(ctx, b)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestBundlePythonWheelBuildNoBuildJustUpload(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
b, err := bundle.Load(ctx, "./python_wheel_no_artifact_no_setup")
|
||||
require.NoError(t, err)
|
||||
|
||||
m := phases.Build()
|
||||
err = m.Apply(ctx, b)
|
||||
require.NoError(t, err)
|
||||
|
||||
match := libraries.MatchWithArtifacts()
|
||||
err = match.Apply(ctx, b)
|
||||
require.ErrorContains(t, err, "./non-existing/*.whl")
|
||||
|
||||
require.NotZero(t, len(b.Config.Artifacts))
|
||||
|
||||
artifact := b.Config.Artifacts["my_test_code-0.0.1-py3-none-any.whl"]
|
||||
require.NotNil(t, artifact)
|
||||
require.Empty(t, artifact.BuildCommand)
|
||||
require.Contains(t, artifact.Files[0].Source, filepath.Join(
|
||||
b.Config.Path,
|
||||
"package",
|
||||
"my_test_code-0.0.1-py3-none-any.whl",
|
||||
))
|
||||
require.True(t, artifact.Files[0].NeedsUpload())
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue