diff --git a/bundle/artifacts/whl/autodetect.go b/bundle/artifacts/whl/autodetect.go index 41d80bb7..29031e86 100644 --- a/bundle/artifacts/whl/autodetect.go +++ b/bundle/artifacts/whl/autodetect.go @@ -27,9 +27,9 @@ func (m *detectPkg) Name() string { } func (m *detectPkg) Apply(ctx context.Context, b *bundle.Bundle) error { - wheelTasks := libraries.FindAllWheelTasks(b) + wheelTasks := libraries.FindAllWheelTasksWithLocalLibraries(b) if len(wheelTasks) == 0 { - log.Infof(ctx, "No wheel tasks in databricks.yml config, skipping auto detect") + log.Infof(ctx, "No local wheel tasks in databricks.yml config, skipping auto detect") return nil } cmdio.LogString(ctx, "artifacts.whl.AutoDetect: Detecting Python wheel project...") diff --git a/bundle/artifacts/whl/from_libraries.go b/bundle/artifacts/whl/from_libraries.go index 855e5b94..9d35f631 100644 --- a/bundle/artifacts/whl/from_libraries.go +++ b/bundle/artifacts/whl/from_libraries.go @@ -26,7 +26,7 @@ func (*fromLibraries) Apply(ctx context.Context, b *bundle.Bundle) error { return nil } - tasks := libraries.FindAllWheelTasks(b) + tasks := libraries.FindAllWheelTasksWithLocalLibraries(b) for _, task := range tasks { for _, lib := range task.Libraries { matches, err := filepath.Glob(filepath.Join(b.Config.Path, lib.Whl)) diff --git a/bundle/libraries/libraries.go b/bundle/libraries/libraries.go index 076180f4..d9a257db 100644 --- a/bundle/libraries/libraries.go +++ b/bundle/libraries/libraries.go @@ -56,11 +56,11 @@ func findAllTasks(b *bundle.Bundle) []*jobs.Task { return result } -func FindAllWheelTasks(b *bundle.Bundle) []*jobs.Task { +func FindAllWheelTasksWithLocalLibraries(b *bundle.Bundle) []*jobs.Task { tasks := findAllTasks(b) wheelTasks := make([]*jobs.Task, 0) for _, task := range tasks { - if task.PythonWheelTask != nil { + if task.PythonWheelTask != nil && IsTaskWithLocalLibraries(task) { wheelTasks = append(wheelTasks, task) } } @@ -68,6 +68,16 @@ func FindAllWheelTasks(b *bundle.Bundle) []*jobs.Task { return wheelTasks } +func IsTaskWithLocalLibraries(task *jobs.Task) bool { + for _, l := range task.Libraries { + if isLocalLibrary(&l) { + return true + } + } + + return false +} + func isMissingRequiredLibraries(task *jobs.Task) bool { if task.Libraries != nil { return false diff --git a/bundle/python/transform.go b/bundle/python/transform.go index 53db450b..3d744df9 100644 --- a/bundle/python/transform.go +++ b/bundle/python/transform.go @@ -7,6 +7,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/mutator" + "github.com/databricks/cli/bundle/libraries" "github.com/databricks/databricks-sdk-go/service/jobs" ) @@ -72,8 +73,8 @@ func (t *pythonTrampoline) GetTasks(b *bundle.Bundle) []mutator.TaskWithJobKey { for i := range tasks { task := &tasks[i] - // Keep only Python wheel tasks - if task.PythonWheelTask == nil { + // Keep only Python wheel tasks with local libraries referenced + if task.PythonWheelTask == nil || !libraries.IsTaskWithLocalLibraries(task) { continue } diff --git a/bundle/python/transform_test.go b/bundle/python/transform_test.go index a9f57db8..99d3129d 100644 --- a/bundle/python/transform_test.go +++ b/bundle/python/transform_test.go @@ -9,6 +9,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/paths" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/require" ) @@ -82,11 +83,21 @@ func TestTransformFiltersWheelTasksOnly(t *testing.T) { { TaskKey: "key1", PythonWheelTask: &jobs.PythonWheelTask{}, + Libraries: []compute.Library{ + {Whl: "./dist/test.whl"}, + }, }, { TaskKey: "key2", NotebookTask: &jobs.NotebookTask{}, }, + { + TaskKey: "key3", + PythonWheelTask: &jobs.PythonWheelTask{}, + Libraries: []compute.Library{ + {Whl: "dbfs:/FileStore/dist/test.whl"}, + }, + }, }, }, },