aou/data-maker
aou
/
data-maker
1
1
Fork 0
Code Issues 4 Pull Requests Packages Projects 1 Releases Wiki Activity

bug fix: @TODO: split partition data so we always process a decent sized data

This commit is contained in:
Steve Nyemba 2020-04-02 07:52:09 -05:00
parent 8418208da0
commit 00e640df21
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
pipeline.py
View File

@ -152,6 +152,7 @@ class Components :
# reader = args['reader'] # reader = args['reader']
# df = reader() # df = reader()
df = args['reader']() if 'reader' in args else args['data'] df = args['reader']() if 'reader' in args else args['data']
# bounds = Components.split(df,MAX_ROWS,PART_SIZE) # bounds = Components.split(df,MAX_ROWS,PART_SIZE)
# if partition != '' : # if partition != '' :
# columns = args['columns'] # columns = args['columns']
@ -168,7 +169,7 @@ class Components :
# -- Let us tray assessing # -- Let us tray assessing
df = np.array_split(df,N)
_dc = pd.DataFrame() _dc = pd.DataFrame()
# for mdf in df : # for mdf in df :
_args['data'] = df _args['data'] = df