Fix folder syncing on windows (#149)

Tested by running the unit and integration tests locally

Tested manually on windows

Screenshot from windows sync logs indicating that the correct slashed
for paths were used:
<img width="623" alt="Screenshot 2022-12-21 at 9 09 13 PM"
src="https://user-images.githubusercontent.com/88374338/208943937-146670b2-1afd-4e0b-8f4e-6091c8c7e17a.png">

@pietern with this the state machine for syncing becomes slightly more
complicated, indicating a stronger need for a tree based approach herre

Co-authored-by: Pieter Noordhuis <pieter.noordhuis@databricks.com>
This commit is contained in:
shreyas-goenka 2022-12-22 11:35:32 +01:00 committed by GitHub
parent f3112d90ad
commit 198eefcf39
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 45 additions and 4 deletions

View File

@ -226,17 +226,24 @@ func (s *Snapshot) diff(all []git.File) (change diff, err error) {
lastSeenModified, seen := lastModifiedTimes[f.Relative]
if !seen || modified.After(lastSeenModified) {
change.put = append(change.put, f.Relative)
lastModifiedTimes[f.Relative] = modified
// Keep track of remote names of local files so we can delete them later
// change separators to '/' for file paths in remote store
unixFileName := filepath.ToSlash(f.Relative)
// put file in databricks workspace
change.put = append(change.put, unixFileName)
// get file metadata about whether it's a notebook
isNotebook, typeOfNotebook, err := getNotebookDetails(f.Absolute)
if err != nil {
return change, err
}
remoteName := f.Relative
// strip `.py` for python notebooks
remoteName := unixFileName
if isNotebook && typeOfNotebook == "PYTHON" {
remoteName = strings.TrimSuffix(f.Relative, `.py`)
remoteName = strings.TrimSuffix(remoteName, `.py`)
}
// If the remote handle of a file changes, we want to delete the old

View File

@ -136,6 +136,40 @@ func TestDiff(t *testing.T) {
assert.Equal(t, map[string]string{"world.txt": "world.txt"}, state.RemoteToLocalNames)
}
func TestFolderDiff(t *testing.T) {
// Create temp project dir
projectDir := t.TempDir()
fileSet := git.NewFileSet(projectDir)
state := Snapshot{
LastUpdatedTimes: make(map[string]time.Time),
LocalToRemoteNames: make(map[string]string),
RemoteToLocalNames: make(map[string]string),
}
err := os.Mkdir(filepath.Join(projectDir, "foo"), os.ModePerm)
assert.NoError(t, err)
f1 := createFile(t, filepath.Join(projectDir, "foo", "bar.py"))
defer f1.close(t)
f1.overwrite(t, "# Databricks notebook source\nprint(\"abc\")")
files, err := fileSet.All()
assert.NoError(t, err)
change, err := state.diff(files)
assert.NoError(t, err)
assert.Len(t, change.delete, 0)
assert.Len(t, change.put, 1)
assert.Contains(t, change.put, "foo/bar.py")
f1.remove(t)
files, err = fileSet.All()
assert.NoError(t, err)
change, err = state.diff(files)
assert.NoError(t, err)
assert.Len(t, change.delete, 1)
assert.Len(t, change.put, 0)
assert.Contains(t, change.delete, "foo/bar")
}
func TestPythonNotebookDiff(t *testing.T) {
// Create temp project dir
projectDir := t.TempDir()