steve/databricks-cli
steve
/
databricks-cli
mirror of https://github.com/databricks/cli.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

simplify copy

This commit is contained in:
Shreyas Goenka 2025-01-02 18:53:14 +05:30
parent 9552131a2a
commit 7ab9fb7cec
No known key found for this signature in database
GPG Key ID: 92A07DF49CCB0622
1 changed files with 1 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
libs/filer/dbfs_client.go
View File

@ -167,18 +167,7 @@ func (w *DbfsClient) putFile(ctx context.Context, path string, overwrite bool, f
pw.CloseWithError(fmt.Errorf("failed to write contents field in multipart form: %w", err)) pw.CloseWithError(fmt.Errorf("failed to write contents field in multipart form: %w", err))
return return
} }
for { _, err = io.Copy(contents, file)
// Copy the file in 10 MB chunks. This reduces the memory usage of the
// program when copying large files.
_, err := io.CopyN(contents, file, 10*1024*1024)
if err == io.EOF {
break
}
if err != nil {
pw.CloseWithError(fmt.Errorf("failed to copy file in multipart form: %w", err))
return
}
}
err = writer.Close() err = writer.Close()
if err != nil { if err != nil {
pw.CloseWithError(fmt.Errorf("failed to close multipart form writer: %w", err)) pw.CloseWithError(fmt.Errorf("failed to close multipart form writer: %w", err))