Allow referencing local Python wheels without artifacts section defined (#703)

## Changes
Now if the user reference local Python wheel files and do not specify
"artifacts" section, this file will be automatically uploaded by CLI.

Fixes #693 

## Tests
Added unit tests

Ran bundle deploy for this configuration
```
resources:
  jobs:
    some_other_job:
      name: "[${bundle.environment}] My Wheel Job"
      tasks:
        - task_key: TestTask
          existing_cluster_id: ${var.job_existing_cluster}
          python_wheel_task:
            package_name: "my_test_code"
            entry_point: "run"
          libraries:
          - whl: ./dist/*.whl
 ```
 
 Result
 
 ```
andrew.nester@HFW9Y94129 wheel % databricks bundle deploy
artifacts.whl.AutoDetect: Detecting Python wheel project...
artifacts.whl.AutoDetect: No Python wheel project found at bundle root folder
Starting upload of bundle files
Uploaded bundle files at /Users/andrew.nester@databricks.com/.bundle/wheel-task/default/files!

artifacts.Upload(my_test_code-0.0.1-py3-none-any.whl): Uploading...
artifacts.Upload(my_test_code-0.0.1-py3-none-any.whl): Upload succeeded
 
 ```
This commit is contained in:
Andrew Nester 2023-08-28 18:29:04 +02:00 committed by GitHub
parent 861f33d376
commit 5f6289e3a7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 113 additions and 1 deletions

View File

@ -28,5 +28,6 @@ func (m *autodetect) Apply(ctx context.Context, b *bundle.Bundle) error {
return bundle.Apply(ctx, b, bundle.Seq(
whl.DetectPackage(),
whl.DefineArtifactsFromLibraries(),
))
}

View File

@ -47,7 +47,11 @@ func (m *infer) Apply(ctx context.Context, b *bundle.Bundle) error {
return fmt.Errorf("artifact doesn't exist: %s", m.name)
}
if artifact.BuildCommand != "" {
// only try to infer command if it's not already defined
// and there is no explicitly files defined which means
// that the package is built outside of bundle cycles
// manually by customer
if artifact.BuildCommand != "" || len(artifact.Files) > 0 {
return nil
}

View File

@ -0,0 +1,56 @@
package whl
import (
"context"
"path/filepath"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/libraries"
"github.com/databricks/cli/libs/log"
)
type fromLibraries struct{}
func DefineArtifactsFromLibraries() bundle.Mutator {
return &fromLibraries{}
}
func (m *fromLibraries) Name() string {
return "artifacts.whl.DefineArtifactsFromLibraries"
}
func (*fromLibraries) Apply(ctx context.Context, b *bundle.Bundle) error {
if len(b.Config.Artifacts) != 0 {
log.Debugf(ctx, "Skipping defining artifacts from libraries because artifacts section is explicitly defined")
return nil
}
tasks := libraries.FindAllWheelTasks(b)
for _, task := range tasks {
for _, lib := range task.Libraries {
matches, err := filepath.Glob(filepath.Join(b.Config.Path, lib.Whl))
// File referenced from libraries section does not exists, skipping
if err != nil {
continue
}
for _, match := range matches {
name := filepath.Base(match)
if b.Config.Artifacts == nil {
b.Config.Artifacts = make(map[string]*config.Artifact)
}
log.Debugf(ctx, "Adding an artifact block for %s", match)
b.Config.Artifacts[name] = &config.Artifact{
Files: []config.ArtifactFile{
{Source: match},
},
Type: config.ArtifactPythonWheel,
}
}
}
}
return nil
}

View File

@ -0,0 +1,3 @@
build/
*.egg-info
.databricks

View File

@ -0,0 +1,22 @@
bundle:
name: python-wheel-local
resources:
jobs:
test_job:
name: "[${bundle.environment}] My Wheel Job"
tasks:
- task_key: TestTask
existing_cluster_id: "0717-aaaaa-bbbbbb"
python_wheel_task:
package_name: "my_test_code"
entry_point: "run"
libraries:
- whl: ./package/*.whl
- task_key: TestTask2
existing_cluster_id: "0717-aaaaa-bbbbbb"
python_wheel_task:
package_name: "my_test_code"
entry_point: "run"
libraries:
- whl: ./non-existing/*.whl

View File

@ -60,3 +60,29 @@ func TestBundlePythonWheelWithDBFSLib(t *testing.T) {
err = match.Apply(ctx, b)
require.NoError(t, err)
}
func TestBundlePythonWheelBuildNoBuildJustUpload(t *testing.T) {
ctx := context.Background()
b, err := bundle.Load(ctx, "./python_wheel_no_artifact_no_setup")
require.NoError(t, err)
m := phases.Build()
err = m.Apply(ctx, b)
require.NoError(t, err)
match := libraries.MatchWithArtifacts()
err = match.Apply(ctx, b)
require.ErrorContains(t, err, "./non-existing/*.whl")
require.NotZero(t, len(b.Config.Artifacts))
artifact := b.Config.Artifacts["my_test_code-0.0.1-py3-none-any.whl"]
require.NotNil(t, artifact)
require.Empty(t, artifact.BuildCommand)
require.Contains(t, artifact.Files[0].Source, filepath.Join(
b.Config.Path,
"package",
"my_test_code-0.0.1-py3-none-any.whl",
))
require.True(t, artifact.Files[0].NeedsUpload())
}