Skip path translation of job task for jobs with a Git source (#404)

## Changes

Added skipping of translating paths for notebook path in notebook tasks
and python file path in spark python tasks if the git source is not null.

Resolves: #402

## Tests

There is a unit test and also tested with a sample bundle:

```
resources:
  jobs:
    demo:
      git_source:
        git_branch: master
        git_provider: github
        git_url: https://github.com/test/dummy
   ....
```

---------

Co-authored-by: Pieter Noordhuis <pieter.noordhuis@databricks.com>
This commit is contained in:
stikkireddy 2023-06-07 06:34:59 -04:00 committed by GitHub
parent be10ff9a75
commit 402fcdd62c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 72 additions and 0 deletions

View File

@ -154,6 +154,11 @@ func (m *translatePaths) Apply(_ context.Context, b *bundle.Bundle) error {
return fmt.Errorf("unable to determine directory for job %s: %w", key, err)
}
// Do not translate job task paths if using git source
if job.GitSource != nil {
continue
}
for i := 0; i < len(job.Tasks); i++ {
err := m.translateJobTask(dir, b, &job.Tasks[i])
if err != nil {

View File

@ -31,6 +31,73 @@ func touchEmptyFile(t *testing.T, path string) {
f.Close()
}
func TestTranslatePathsSkippedWithGitSource(t *testing.T) {
dir := t.TempDir()
bundle := &bundle.Bundle{
Config: config.Root{
Path: dir,
Workspace: config.Workspace{
FilesPath: "/bundle",
},
Resources: config.Resources{
Jobs: map[string]*resources.Job{
"job": {
Paths: resources.Paths{
ConfigFilePath: filepath.Join(dir, "resource.yml"),
},
JobSettings: &jobs.JobSettings{
GitSource: &jobs.GitSource{
GitBranch: "somebranch",
GitCommit: "somecommit",
GitProvider: "github",
GitTag: "sometag",
GitUrl: "https://github.com/someuser/somerepo",
},
Tasks: []jobs.JobTaskSettings{
{
NotebookTask: &jobs.NotebookTask{
NotebookPath: "my_job_notebook.py",
},
},
{
PythonWheelTask: &jobs.PythonWheelTask{
PackageName: "foo",
},
},
{
SparkPythonTask: &jobs.SparkPythonTask{
PythonFile: "my_python_file.py",
},
},
},
},
},
},
},
},
}
err := mutator.TranslatePaths().Apply(context.Background(), bundle)
require.NoError(t, err)
assert.Equal(
t,
"my_job_notebook.py",
bundle.Config.Resources.Jobs["job"].Tasks[0].NotebookTask.NotebookPath,
)
assert.Equal(
t,
"foo",
bundle.Config.Resources.Jobs["job"].Tasks[1].PythonWheelTask.PackageName,
)
assert.Equal(
t,
"my_python_file.py",
bundle.Config.Resources.Jobs["job"].Tasks[2].SparkPythonTask.PythonFile,
)
}
func TestTranslatePaths(t *testing.T) {
dir := t.TempDir()
touchNotebookFile(t, filepath.Join(dir, "my_job_notebook.py"))