databricks-cli/cmd/sync/sync.go

181 lines
4.7 KiB
Go
Raw Normal View History

2022-07-07 18:56:59 +00:00
package sync
import (
"context"
"flag"
2022-07-07 18:56:59 +00:00
"fmt"
"io"
"path/filepath"
stdsync "sync"
2022-07-07 18:56:59 +00:00
"time"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/cmd/root"
"github.com/databricks/cli/libs/flags"
"github.com/databricks/cli/libs/sync"
"github.com/databricks/databricks-sdk-go"
2022-07-07 18:56:59 +00:00
"github.com/spf13/cobra"
)
type syncFlags struct {
// project files polling interval
interval time.Duration
full bool
watch bool
output flags.Output
}
func (f *syncFlags) syncOptionsFromBundle(cmd *cobra.Command, args []string, b *bundle.Bundle) (*sync.SyncOptions, error) {
if len(args) > 0 {
return nil, fmt.Errorf("SRC and DST are not configurable in the context of a bundle")
}
cacheDir, err := b.CacheDir(cmd.Context())
if err != nil {
return nil, fmt.Errorf("cannot get bundle cache directory: %w", err)
}
includes, err := b.GetSyncIncludePatterns(cmd.Context())
Added support for sync.include and sync.exclude sections (#671) ## Changes Added support for `sync.include` and `sync.exclude` sections ## Tests Added `sample-java` folder to gitignore ``` bundle: name: wheel-task sync: include: - "./sample-java/*.kts" ``` Kotlin files were correctly synced. ``` [DEBUG] Test execution command: /opt/homebrew/opt/go@1.21/bin/go test ./... -json -timeout 1h -coverpkg=./... -coverprofile=coverage.txt -run ^TestAcc [DEBUG] Test execution directory: /Users/andrew.nester/cli 2023/08/17 17:12:10 [INFO] ✅ TestAccAlertsCreateErrWhenNoArguments (2.320s) 2023/08/17 17:12:10 [INFO] ✅ TestAccApiGet (0.650s) 2023/08/17 17:12:12 [INFO] ✅ TestAccClustersList (1.060s) 2023/08/17 17:12:12 [INFO] ✅ TestAccClustersGet (0.760s) 2023/08/17 17:12:26 [INFO] ✅ TestAccFilerWorkspaceFilesReadWrite (13.270s) 2023/08/17 17:12:32 [INFO] ✅ TestAccFilerWorkspaceFilesReadDir (6.860s) 2023/08/17 17:12:46 [INFO] ✅ TestAccFilerDbfsReadWrite (13.380s) 2023/08/17 17:12:53 [INFO] ✅ TestAccFilerDbfsReadDir (7.460s) 2023/08/17 17:13:01 [INFO] ✅ TestAccFilerWorkspaceNotebookConflict (7.920s) 2023/08/17 17:13:10 [INFO] ✅ TestAccFilerWorkspaceNotebookWithOverwriteFlag (9.290s) 2023/08/17 17:13:10 [INFO] ✅ TestAccFilerLocalReadWrite (0.010s) 2023/08/17 17:13:11 [INFO] ✅ TestAccFilerLocalReadDir (0.010s) 2023/08/17 17:13:14 [INFO] ✅ TestAccFsCatForDbfs (3.180s) 2023/08/17 17:13:15 [INFO] ✅ TestAccFsCatForDbfsOnNonExistentFile (0.940s) 2023/08/17 17:13:15 [INFO] ✅ TestAccFsCatForDbfsInvalidScheme (0.560s) 2023/08/17 17:13:18 [INFO] ✅ TestAccFsCatDoesNotSupportOutputModeJson (2.910s) 2023/08/17 17:13:51 [INFO] ✅ TestAccFsCpDir (32.730s) 2023/08/17 17:14:06 [INFO] ✅ TestAccFsCpFileToFile (14.740s) 2023/08/17 17:14:20 [INFO] ✅ TestAccFsCpFileToDir (14.340s) 2023/08/17 17:14:53 [INFO] ✅ TestAccFsCpDirToDirFileNotOverwritten (32.710s) 2023/08/17 17:15:12 [INFO] ✅ TestAccFsCpFileToDirFileNotOverwritten (19.590s) 2023/08/17 17:15:32 [INFO] ✅ TestAccFsCpFileToFileFileNotOverwritten (19.950s) 2023/08/17 17:16:11 [INFO] ✅ TestAccFsCpDirToDirWithOverwriteFlag (38.970s) 2023/08/17 17:16:32 [INFO] ✅ TestAccFsCpFileToFileWithOverwriteFlag (21.040s) 2023/08/17 17:16:52 [INFO] ✅ TestAccFsCpFileToDirWithOverwriteFlag (19.670s) 2023/08/17 17:16:54 [INFO] ✅ TestAccFsCpErrorsWhenSourceIsDirWithoutRecursiveFlag (1.890s) 2023/08/17 17:16:54 [INFO] ✅ TestAccFsCpErrorsOnInvalidScheme (0.690s) 2023/08/17 17:17:10 [INFO] ✅ TestAccFsCpSourceIsDirectoryButTargetIsFile (15.810s) 2023/08/17 17:17:14 [INFO] ✅ TestAccFsLsForDbfs (4.000s) 2023/08/17 17:17:18 [INFO] ✅ TestAccFsLsForDbfsWithAbsolutePaths (4.000s) 2023/08/17 17:17:21 [INFO] ✅ TestAccFsLsForDbfsOnFile (3.140s) 2023/08/17 17:17:23 [INFO] ✅ TestAccFsLsForDbfsOnEmptyDir (2.030s) 2023/08/17 17:17:24 [INFO] ✅ TestAccFsLsForDbfsForNonexistingDir (0.840s) 2023/08/17 17:17:25 [INFO] ✅ TestAccFsLsWithoutScheme (0.590s) 2023/08/17 17:17:27 [INFO] ✅ TestAccFsMkdirCreatesDirectory (2.310s) 2023/08/17 17:17:30 [INFO] ✅ TestAccFsMkdirCreatesMultipleDirectories (2.800s) 2023/08/17 17:17:33 [INFO] ✅ TestAccFsMkdirWhenDirectoryAlreadyExists (2.700s) 2023/08/17 17:17:35 [INFO] ✅ TestAccFsMkdirWhenFileExistsAtPath (2.870s) 2023/08/17 17:17:40 [INFO] ✅ TestAccFsRmForFile (4.030s) 2023/08/17 17:17:43 [INFO] ✅ TestAccFsRmForEmptyDirectory (3.470s) 2023/08/17 17:17:46 [INFO] ✅ TestAccFsRmForNonEmptyDirectory (3.350s) 2023/08/17 17:17:47 [INFO] ✅ TestAccFsRmForNonExistentFile (0.940s) 2023/08/17 17:17:51 [INFO] ✅ TestAccFsRmForNonEmptyDirectoryWithRecursiveFlag (3.570s) 2023/08/17 17:17:52 [INFO] ✅ TestAccGitClone (0.890s) 2023/08/17 17:17:52 [INFO] ✅ TestAccGitCloneWithOnlyRepoNameOnAlternateBranch (0.730s) 2023/08/17 17:17:53 [INFO] ✅ TestAccGitCloneErrorsWhenRepositoryDoesNotExist (0.540s) 2023/08/17 17:18:02 [INFO] ✅ TestAccLock (8.800s) 2023/08/17 17:18:06 [INFO] ✅ TestAccLockUnlockWithoutAllowsLockFileNotExist (3.930s) 2023/08/17 17:18:09 [INFO] ✅ TestAccLockUnlockWithAllowsLockFileNotExist (3.320s) 2023/08/17 17:18:20 [INFO] ✅ TestAccSyncFullFileSync (10.570s) 2023/08/17 17:18:31 [INFO] ✅ TestAccSyncIncrementalFileSync (11.460s) 2023/08/17 17:18:42 [INFO] ✅ TestAccSyncNestedFolderSync (10.850s) 2023/08/17 17:18:53 [INFO] ✅ TestAccSyncNestedFolderDoesntFailOnNonEmptyDirectory (10.650s) 2023/08/17 17:19:04 [INFO] ✅ TestAccSyncNestedSpacePlusAndHashAreEscapedSync (10.930s) 2023/08/17 17:19:11 [INFO] ✅ TestAccSyncIncrementalFileOverwritesFolder (7.010s) 2023/08/17 17:19:18 [INFO] ✅ TestAccSyncIncrementalSyncPythonNotebookToFile (7.380s) 2023/08/17 17:19:24 [INFO] ✅ TestAccSyncIncrementalSyncFileToPythonNotebook (6.220s) 2023/08/17 17:19:30 [INFO] ✅ TestAccSyncIncrementalSyncPythonNotebookDelete (5.530s) 2023/08/17 17:19:32 [INFO] ✅ TestAccSyncEnsureRemotePathIsUsableIfRepoDoesntExist (2.620s) 2023/08/17 17:19:38 [INFO] ✅ TestAccSyncEnsureRemotePathIsUsableIfRepoExists (5.460s) 2023/08/17 17:19:40 [INFO] ✅ TestAccSyncEnsureRemotePathIsUsableInWorkspace (1.850s) 2023/08/17 17:19:40 [INFO] ✅ TestAccWorkspaceList (0.780s) 2023/08/17 17:19:51 [INFO] ✅ TestAccExportDir (10.350s) 2023/08/17 17:19:54 [INFO] ✅ TestAccExportDirDoesNotOverwrite (3.330s) 2023/08/17 17:19:58 [INFO] ✅ TestAccExportDirWithOverwriteFlag (3.770s) 2023/08/17 17:20:07 [INFO] ✅ TestAccImportDir (9.320s) 2023/08/17 17:20:24 [INFO] ✅ TestAccImportDirDoesNotOverwrite (16.950s) 2023/08/17 17:20:35 [INFO] ✅ TestAccImportDirWithOverwriteFlag (10.620s) 2023/08/17 17:20:35 [INFO] ✅ 68/68 passed, 0 failed, 3 skipped ```
2023-08-18 08:07:25 +00:00
if err != nil {
return nil, fmt.Errorf("cannot get list of sync includes: %w", err)
}
opts := sync.SyncOptions{
LocalPath: b.Config.Path,
RemotePath: b.Config.Workspace.FilesPath,
Added support for sync.include and sync.exclude sections (#671) ## Changes Added support for `sync.include` and `sync.exclude` sections ## Tests Added `sample-java` folder to gitignore ``` bundle: name: wheel-task sync: include: - "./sample-java/*.kts" ``` Kotlin files were correctly synced. ``` [DEBUG] Test execution command: /opt/homebrew/opt/go@1.21/bin/go test ./... -json -timeout 1h -coverpkg=./... -coverprofile=coverage.txt -run ^TestAcc [DEBUG] Test execution directory: /Users/andrew.nester/cli 2023/08/17 17:12:10 [INFO] ✅ TestAccAlertsCreateErrWhenNoArguments (2.320s) 2023/08/17 17:12:10 [INFO] ✅ TestAccApiGet (0.650s) 2023/08/17 17:12:12 [INFO] ✅ TestAccClustersList (1.060s) 2023/08/17 17:12:12 [INFO] ✅ TestAccClustersGet (0.760s) 2023/08/17 17:12:26 [INFO] ✅ TestAccFilerWorkspaceFilesReadWrite (13.270s) 2023/08/17 17:12:32 [INFO] ✅ TestAccFilerWorkspaceFilesReadDir (6.860s) 2023/08/17 17:12:46 [INFO] ✅ TestAccFilerDbfsReadWrite (13.380s) 2023/08/17 17:12:53 [INFO] ✅ TestAccFilerDbfsReadDir (7.460s) 2023/08/17 17:13:01 [INFO] ✅ TestAccFilerWorkspaceNotebookConflict (7.920s) 2023/08/17 17:13:10 [INFO] ✅ TestAccFilerWorkspaceNotebookWithOverwriteFlag (9.290s) 2023/08/17 17:13:10 [INFO] ✅ TestAccFilerLocalReadWrite (0.010s) 2023/08/17 17:13:11 [INFO] ✅ TestAccFilerLocalReadDir (0.010s) 2023/08/17 17:13:14 [INFO] ✅ TestAccFsCatForDbfs (3.180s) 2023/08/17 17:13:15 [INFO] ✅ TestAccFsCatForDbfsOnNonExistentFile (0.940s) 2023/08/17 17:13:15 [INFO] ✅ TestAccFsCatForDbfsInvalidScheme (0.560s) 2023/08/17 17:13:18 [INFO] ✅ TestAccFsCatDoesNotSupportOutputModeJson (2.910s) 2023/08/17 17:13:51 [INFO] ✅ TestAccFsCpDir (32.730s) 2023/08/17 17:14:06 [INFO] ✅ TestAccFsCpFileToFile (14.740s) 2023/08/17 17:14:20 [INFO] ✅ TestAccFsCpFileToDir (14.340s) 2023/08/17 17:14:53 [INFO] ✅ TestAccFsCpDirToDirFileNotOverwritten (32.710s) 2023/08/17 17:15:12 [INFO] ✅ TestAccFsCpFileToDirFileNotOverwritten (19.590s) 2023/08/17 17:15:32 [INFO] ✅ TestAccFsCpFileToFileFileNotOverwritten (19.950s) 2023/08/17 17:16:11 [INFO] ✅ TestAccFsCpDirToDirWithOverwriteFlag (38.970s) 2023/08/17 17:16:32 [INFO] ✅ TestAccFsCpFileToFileWithOverwriteFlag (21.040s) 2023/08/17 17:16:52 [INFO] ✅ TestAccFsCpFileToDirWithOverwriteFlag (19.670s) 2023/08/17 17:16:54 [INFO] ✅ TestAccFsCpErrorsWhenSourceIsDirWithoutRecursiveFlag (1.890s) 2023/08/17 17:16:54 [INFO] ✅ TestAccFsCpErrorsOnInvalidScheme (0.690s) 2023/08/17 17:17:10 [INFO] ✅ TestAccFsCpSourceIsDirectoryButTargetIsFile (15.810s) 2023/08/17 17:17:14 [INFO] ✅ TestAccFsLsForDbfs (4.000s) 2023/08/17 17:17:18 [INFO] ✅ TestAccFsLsForDbfsWithAbsolutePaths (4.000s) 2023/08/17 17:17:21 [INFO] ✅ TestAccFsLsForDbfsOnFile (3.140s) 2023/08/17 17:17:23 [INFO] ✅ TestAccFsLsForDbfsOnEmptyDir (2.030s) 2023/08/17 17:17:24 [INFO] ✅ TestAccFsLsForDbfsForNonexistingDir (0.840s) 2023/08/17 17:17:25 [INFO] ✅ TestAccFsLsWithoutScheme (0.590s) 2023/08/17 17:17:27 [INFO] ✅ TestAccFsMkdirCreatesDirectory (2.310s) 2023/08/17 17:17:30 [INFO] ✅ TestAccFsMkdirCreatesMultipleDirectories (2.800s) 2023/08/17 17:17:33 [INFO] ✅ TestAccFsMkdirWhenDirectoryAlreadyExists (2.700s) 2023/08/17 17:17:35 [INFO] ✅ TestAccFsMkdirWhenFileExistsAtPath (2.870s) 2023/08/17 17:17:40 [INFO] ✅ TestAccFsRmForFile (4.030s) 2023/08/17 17:17:43 [INFO] ✅ TestAccFsRmForEmptyDirectory (3.470s) 2023/08/17 17:17:46 [INFO] ✅ TestAccFsRmForNonEmptyDirectory (3.350s) 2023/08/17 17:17:47 [INFO] ✅ TestAccFsRmForNonExistentFile (0.940s) 2023/08/17 17:17:51 [INFO] ✅ TestAccFsRmForNonEmptyDirectoryWithRecursiveFlag (3.570s) 2023/08/17 17:17:52 [INFO] ✅ TestAccGitClone (0.890s) 2023/08/17 17:17:52 [INFO] ✅ TestAccGitCloneWithOnlyRepoNameOnAlternateBranch (0.730s) 2023/08/17 17:17:53 [INFO] ✅ TestAccGitCloneErrorsWhenRepositoryDoesNotExist (0.540s) 2023/08/17 17:18:02 [INFO] ✅ TestAccLock (8.800s) 2023/08/17 17:18:06 [INFO] ✅ TestAccLockUnlockWithoutAllowsLockFileNotExist (3.930s) 2023/08/17 17:18:09 [INFO] ✅ TestAccLockUnlockWithAllowsLockFileNotExist (3.320s) 2023/08/17 17:18:20 [INFO] ✅ TestAccSyncFullFileSync (10.570s) 2023/08/17 17:18:31 [INFO] ✅ TestAccSyncIncrementalFileSync (11.460s) 2023/08/17 17:18:42 [INFO] ✅ TestAccSyncNestedFolderSync (10.850s) 2023/08/17 17:18:53 [INFO] ✅ TestAccSyncNestedFolderDoesntFailOnNonEmptyDirectory (10.650s) 2023/08/17 17:19:04 [INFO] ✅ TestAccSyncNestedSpacePlusAndHashAreEscapedSync (10.930s) 2023/08/17 17:19:11 [INFO] ✅ TestAccSyncIncrementalFileOverwritesFolder (7.010s) 2023/08/17 17:19:18 [INFO] ✅ TestAccSyncIncrementalSyncPythonNotebookToFile (7.380s) 2023/08/17 17:19:24 [INFO] ✅ TestAccSyncIncrementalSyncFileToPythonNotebook (6.220s) 2023/08/17 17:19:30 [INFO] ✅ TestAccSyncIncrementalSyncPythonNotebookDelete (5.530s) 2023/08/17 17:19:32 [INFO] ✅ TestAccSyncEnsureRemotePathIsUsableIfRepoDoesntExist (2.620s) 2023/08/17 17:19:38 [INFO] ✅ TestAccSyncEnsureRemotePathIsUsableIfRepoExists (5.460s) 2023/08/17 17:19:40 [INFO] ✅ TestAccSyncEnsureRemotePathIsUsableInWorkspace (1.850s) 2023/08/17 17:19:40 [INFO] ✅ TestAccWorkspaceList (0.780s) 2023/08/17 17:19:51 [INFO] ✅ TestAccExportDir (10.350s) 2023/08/17 17:19:54 [INFO] ✅ TestAccExportDirDoesNotOverwrite (3.330s) 2023/08/17 17:19:58 [INFO] ✅ TestAccExportDirWithOverwriteFlag (3.770s) 2023/08/17 17:20:07 [INFO] ✅ TestAccImportDir (9.320s) 2023/08/17 17:20:24 [INFO] ✅ TestAccImportDirDoesNotOverwrite (16.950s) 2023/08/17 17:20:35 [INFO] ✅ TestAccImportDirWithOverwriteFlag (10.620s) 2023/08/17 17:20:35 [INFO] ✅ 68/68 passed, 0 failed, 3 skipped ```
2023-08-18 08:07:25 +00:00
Include: includes,
Exclude: b.Config.Sync.Exclude,
Full: f.full,
PollInterval: f.interval,
SnapshotBasePath: cacheDir,
WorkspaceClient: b.WorkspaceClient(),
}
return &opts, nil
}
func (f *syncFlags) syncOptionsFromArgs(cmd *cobra.Command, args []string) (*sync.SyncOptions, error) {
if len(args) != 2 {
return nil, flag.ErrHelp
}
opts := sync.SyncOptions{
LocalPath: args[0],
RemotePath: args[1],
Full: f.full,
PollInterval: f.interval,
// We keep existing behavior for VS Code extension where if there is
// no bundle defined, we store the snapshots in `.databricks`.
// The sync code will automatically create this directory if it doesn't
// exist and add it to the `.gitignore` file in the root.
SnapshotBasePath: filepath.Join(args[0], ".databricks"),
WorkspaceClient: databricks.Must(databricks.NewWorkspaceClient()),
}
return &opts, nil
}
func New() *cobra.Command {
cmd := &cobra.Command{
Use: "sync [flags] SRC DST",
Short: "Synchronize a local directory to a workspace directory",
Args: cobra.MaximumNArgs(2),
}
f := syncFlags{
output: flags.OutputText,
}
cmd.Flags().DurationVar(&f.interval, "interval", 1*time.Second, "file system polling interval (for --watch)")
cmd.Flags().BoolVar(&f.full, "full", false, "perform full synchronization (default is incremental)")
cmd.Flags().BoolVar(&f.watch, "watch", false, "watch local file system for changes")
cmd.Flags().Var(&f.output, "output", "type of output format")
cmd.PreRunE = root.MustWorkspaceClient
cmd.RunE = func(cmd *cobra.Command, args []string) error {
var opts *sync.SyncOptions
var err error
//
// To be uncommented and used once our VS Code extension is bundle aware.
// Until then, this could interfere with extension usage where a `databricks.yml` file is present.
// See https://github.com/databricks/cli/pull/207.
//
// b := bundle.GetOrNil(cmd.Context())
// if b != nil {
// // Run initialize phase to make sure paths are set.
// err = bundle.Apply(cmd.Context(), b, phases.Initialize())
// if err != nil {
// return err
// }
// opts, err = syncOptionsFromBundle(cmd, args, b)
// } else {
opts, err = f.syncOptionsFromArgs(cmd, args)
// }
if err != nil {
return err
}
ctx := cmd.Context()
s, err := sync.New(ctx, *opts)
if err != nil {
return err
}
var outputFunc func(context.Context, <-chan sync.Event, io.Writer)
switch f.output {
Add optional JSON output for sync command (#230) JSON output makes it easy to process synchronization progress information in downstream tools (e.g. the vscode extension). This changes introduces a `sync.Event` interface type for progress events as well as an `sync.EventNotifier` that lets the sync code pass along progress events to calling code. Example output in text mode (default, this uses the existing logger calls): ```text 2023/03/03 14:07:17 [INFO] Remote file sync location: /Repos/pieter.noordhuis@databricks.com/... 2023/03/03 14:07:18 [INFO] Initial Sync Complete 2023/03/03 14:07:22 [INFO] Action: PUT: foo 2023/03/03 14:07:23 [INFO] Uploaded foo 2023/03/03 14:07:23 [INFO] Complete 2023/03/03 14:07:25 [INFO] Action: DELETE: foo 2023/03/03 14:07:25 [INFO] Deleted foo 2023/03/03 14:07:25 [INFO] Complete ``` Example output in JSON mode: ```json {"timestamp":"2023-03-03T14:08:15.459439+01:00","seq":0,"type":"start"} {"timestamp":"2023-03-03T14:08:15.459461+01:00","seq":0,"type":"complete"} {"timestamp":"2023-03-03T14:08:18.459821+01:00","seq":1,"type":"start","put":["foo"]} {"timestamp":"2023-03-03T14:08:18.459867+01:00","seq":1,"type":"progress","action":"put","path":"foo","progress":0} {"timestamp":"2023-03-03T14:08:19.418696+01:00","seq":1,"type":"progress","action":"put","path":"foo","progress":1} {"timestamp":"2023-03-03T14:08:19.421397+01:00","seq":1,"type":"complete","put":["foo"]} {"timestamp":"2023-03-03T14:08:22.459238+01:00","seq":2,"type":"start","delete":["foo"]} {"timestamp":"2023-03-03T14:08:22.459268+01:00","seq":2,"type":"progress","action":"delete","path":"foo","progress":0} {"timestamp":"2023-03-03T14:08:22.686413+01:00","seq":2,"type":"progress","action":"delete","path":"foo","progress":1} {"timestamp":"2023-03-03T14:08:22.688989+01:00","seq":2,"type":"complete","delete":["foo"]} ``` --------- Co-authored-by: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com>
2023-03-08 09:27:19 +00:00
case flags.OutputText:
outputFunc = textOutput
Add optional JSON output for sync command (#230) JSON output makes it easy to process synchronization progress information in downstream tools (e.g. the vscode extension). This changes introduces a `sync.Event` interface type for progress events as well as an `sync.EventNotifier` that lets the sync code pass along progress events to calling code. Example output in text mode (default, this uses the existing logger calls): ```text 2023/03/03 14:07:17 [INFO] Remote file sync location: /Repos/pieter.noordhuis@databricks.com/... 2023/03/03 14:07:18 [INFO] Initial Sync Complete 2023/03/03 14:07:22 [INFO] Action: PUT: foo 2023/03/03 14:07:23 [INFO] Uploaded foo 2023/03/03 14:07:23 [INFO] Complete 2023/03/03 14:07:25 [INFO] Action: DELETE: foo 2023/03/03 14:07:25 [INFO] Deleted foo 2023/03/03 14:07:25 [INFO] Complete ``` Example output in JSON mode: ```json {"timestamp":"2023-03-03T14:08:15.459439+01:00","seq":0,"type":"start"} {"timestamp":"2023-03-03T14:08:15.459461+01:00","seq":0,"type":"complete"} {"timestamp":"2023-03-03T14:08:18.459821+01:00","seq":1,"type":"start","put":["foo"]} {"timestamp":"2023-03-03T14:08:18.459867+01:00","seq":1,"type":"progress","action":"put","path":"foo","progress":0} {"timestamp":"2023-03-03T14:08:19.418696+01:00","seq":1,"type":"progress","action":"put","path":"foo","progress":1} {"timestamp":"2023-03-03T14:08:19.421397+01:00","seq":1,"type":"complete","put":["foo"]} {"timestamp":"2023-03-03T14:08:22.459238+01:00","seq":2,"type":"start","delete":["foo"]} {"timestamp":"2023-03-03T14:08:22.459268+01:00","seq":2,"type":"progress","action":"delete","path":"foo","progress":0} {"timestamp":"2023-03-03T14:08:22.686413+01:00","seq":2,"type":"progress","action":"delete","path":"foo","progress":1} {"timestamp":"2023-03-03T14:08:22.688989+01:00","seq":2,"type":"complete","delete":["foo"]} ``` --------- Co-authored-by: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com>
2023-03-08 09:27:19 +00:00
case flags.OutputJSON:
outputFunc = jsonOutput
}
var wg stdsync.WaitGroup
if outputFunc != nil {
wg.Add(1)
go func() {
defer wg.Done()
outputFunc(ctx, s.Events(), cmd.OutOrStdout())
}()
Add optional JSON output for sync command (#230) JSON output makes it easy to process synchronization progress information in downstream tools (e.g. the vscode extension). This changes introduces a `sync.Event` interface type for progress events as well as an `sync.EventNotifier` that lets the sync code pass along progress events to calling code. Example output in text mode (default, this uses the existing logger calls): ```text 2023/03/03 14:07:17 [INFO] Remote file sync location: /Repos/pieter.noordhuis@databricks.com/... 2023/03/03 14:07:18 [INFO] Initial Sync Complete 2023/03/03 14:07:22 [INFO] Action: PUT: foo 2023/03/03 14:07:23 [INFO] Uploaded foo 2023/03/03 14:07:23 [INFO] Complete 2023/03/03 14:07:25 [INFO] Action: DELETE: foo 2023/03/03 14:07:25 [INFO] Deleted foo 2023/03/03 14:07:25 [INFO] Complete ``` Example output in JSON mode: ```json {"timestamp":"2023-03-03T14:08:15.459439+01:00","seq":0,"type":"start"} {"timestamp":"2023-03-03T14:08:15.459461+01:00","seq":0,"type":"complete"} {"timestamp":"2023-03-03T14:08:18.459821+01:00","seq":1,"type":"start","put":["foo"]} {"timestamp":"2023-03-03T14:08:18.459867+01:00","seq":1,"type":"progress","action":"put","path":"foo","progress":0} {"timestamp":"2023-03-03T14:08:19.418696+01:00","seq":1,"type":"progress","action":"put","path":"foo","progress":1} {"timestamp":"2023-03-03T14:08:19.421397+01:00","seq":1,"type":"complete","put":["foo"]} {"timestamp":"2023-03-03T14:08:22.459238+01:00","seq":2,"type":"start","delete":["foo"]} {"timestamp":"2023-03-03T14:08:22.459268+01:00","seq":2,"type":"progress","action":"delete","path":"foo","progress":0} {"timestamp":"2023-03-03T14:08:22.686413+01:00","seq":2,"type":"progress","action":"delete","path":"foo","progress":1} {"timestamp":"2023-03-03T14:08:22.688989+01:00","seq":2,"type":"complete","delete":["foo"]} ``` --------- Co-authored-by: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com>
2023-03-08 09:27:19 +00:00
}
if f.watch {
err = s.RunContinuous(ctx)
} else {
err = s.RunOnce(ctx)
}
s.Close()
wg.Wait()
return err
}
cmd.ValidArgsFunction = func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
ctx := cmd.Context()
cmd.SetContext(root.NoPrompt(ctx))
err := root.MustWorkspaceClient(cmd, args)
if err != nil {
return nil, cobra.ShellCompDirectiveError
}
// No completion in the context of a bundle.
// Source and destination paths are taken from bundle configuration.
b := bundle.GetOrNil(cmd.Context())
if b != nil {
return nil, cobra.ShellCompDirectiveNoFileComp
}
switch len(args) {
case 0:
return nil, cobra.ShellCompDirectiveFilterDirs
case 1:
wsc := root.WorkspaceClient(cmd.Context())
return completeRemotePath(cmd.Context(), wsc, toComplete)
default:
return nil, cobra.ShellCompDirectiveNoFileComp
}
}
2022-07-07 18:56:59 +00:00
return cmd
2022-07-07 18:56:59 +00:00
}