package python import ( "context" "fmt" "io" "os" "path" "strings" "github.com/databricks/bricks/libs/log" "github.com/databricks/bricks/project" "github.com/databricks/databricks-sdk-go/service/dbfs" ) func BuildWheel(ctx context.Context, dir string) (string, error) { defer chdirAndBack(dir)() // remove previous dist leak os.RemoveAll("dist") // remove all other irrelevant traces silentlyCleanupWheelFolder(".") // call simple wheel builder. we may need to pip install wheel as well out, err := Py(ctx, "setup.py", "bdist_wheel") if err != nil { return "", err } log.Debugf(ctx, "Built wheel: %s", out) // and cleanup afterwards silentlyCleanupWheelFolder(".") wheel := silentChildWithSuffix("dist", ".whl") if wheel == "" { return "", fmt.Errorf("cannot find built wheel in %s", dir) } return path.Join(dir, wheel), nil } const DBFSWheelLocation = "dbfs:/FileStore/wheels/simple" // TODO: research deeper if we make new data resource for terraform, like `databricks_latest_wheel` (preferred), // or do we bypass the environment variable into terraform deployer. And make a decision. // // Whatever this method gets refactored to is intended to be used for two purposes: // - uploading project's wheel archives: one per project or one per project/developer, depending on isolation // - synchronising enterprise artifactories, jfrogs, azdo feeds, so that we fix the gap of private code artifact // repository integration. func UploadWheelToDBFSWithPEP503(ctx context.Context, dir string) (string, error) { wheel, err := BuildWheel(ctx, dir) if err != nil { return "", err } defer chdirAndBack(dir)() dist, err := ReadDistribution(ctx) if err != nil { return "", err } // TODO: figure out wheel naming criteria for Soft project isolation to allow multiple // people workin on the same project to upload wheels and let them be deployed as independent jobs. // we should also consider multiple PEP503 index stacking: per enterprise, per project, per developer. // PEP503 indexes can be rolled out to clusters via checksummed global init script, that creates // a driver/worker `/etc/pip.conf` with FUSE-mounted file:///dbfs/FileStore/wheels/simple/.. // extra index URLs. See more pointers at https://stackoverflow.com/q/30889494/277035 dbfsLoc := fmt.Sprintf("%s/%s/%s", DBFSWheelLocation, dist.NormalizedName(), path.Base(wheel)) wsc := project.Get(ctx).WorkspacesClient() wf, err := os.Open(wheel) if err != nil { return "", err } defer wf.Close() h, err := wsc.Dbfs.Open(ctx, dbfsLoc, dbfs.FileModeOverwrite|dbfs.FileModeWrite) if err != nil { return "", err } _, err = io.Copy(h, wf) // TODO: maintain PEP503 compliance and update meta-files: // ${DBFSWheelLocation}/index.html and ${DBFSWheelLocation}/${NormalizedName}/index.html return dbfsLoc, err } func silentlyCleanupWheelFolder(dir string) { // there or not there - we don't care os.RemoveAll(path.Join(dir, "__pycache__")) os.RemoveAll(path.Join(dir, "build")) eggInfo := silentChildWithSuffix(dir, ".egg-info") if eggInfo == "" { return } os.RemoveAll(eggInfo) } func silentChildWithSuffix(dir, suffix string) string { f, err := os.Open(dir) if err != nil { log.Debugf(context.Background(), "open dir %s: %s", dir, err) return "" } entries, err := f.ReadDir(0) if err != nil { log.Debugf(context.Background(), "read dir %s: %s", dir, err) // todo: log return "" } for _, child := range entries { if !strings.HasSuffix(child.Name(), suffix) { continue } return path.Join(dir, child.Name()) } return "" } func chdirAndBack(dir string) func() { wd, _ := os.Getwd() os.Chdir(dir) return func() { os.Chdir(wd) } }