aou/data-maker
aou
/
data-maker
1
1
Fork 0
Code Issues 4 Pull Requests Packages Projects 1 Releases Wiki Activity

bug fix: continuous values

This commit is contained in:
Steve Nyemba 2020-03-12 09:55:29 -05:00
parent fc08a8f643
commit 60cbf2dd3f
2 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
data/maker/__init__.py
View File

@ -112,6 +112,7 @@ def train (**args) :
if col in CONTINUOUS: if col in CONTINUOUS:
BIN_SIZE = 4 if 'bin_size' not in args else int(args['bin_size']) BIN_SIZE = 4 if 'bin_size' not in args else int(args['bin_size'])
args['real'] = ContinuousToDiscrete.binary(df[col],BIN_SIZE).astype(np.float32) args['real'] = ContinuousToDiscrete.binary(df[col],BIN_SIZE).astype(np.float32)
# print ( pd.DataFrame(args['real']).head() )
else: else:
# df.to_csv('tmp-'+args['logs'].replace('/','_')+'-'+col+'.csv',index=False) # df.to_csv('tmp-'+args['logs'].replace('/','_')+'-'+col+'.csv',index=False)
# print (df[col].dtypes) # print (df[col].dtypes)

4
pipeline.py
View File

@ -143,7 +143,7 @@ class Components :
# columns = args['columns'] # columns = args['columns']
# df = np.array_split(df[columns].values,PART_SIZE) # df = np.array_split(df[columns].values,PART_SIZE)
# df = pd.DataFrame(df[ int (partition) ],columns = columns) # df = pd.DataFrame(df[ int (partition) ],columns = columns)
info = {"parition":int(partition),"gpu":_args["gpu"],"rows":df.shape[0],"cols":df.shape[1],"part_size":PART_SIZE} info = {"parition":int(partition),"gpu":_args["gpu"],"rows":str(df.shape[0]),"cols":str(df.shape[1]),"part_size":int(PART_SIZE)}
logger.write({"module":"generate","action":"partition","input":info}) logger.write({"module":"generate","action":"partition","input":info})
_args['partition'] = int(partition) _args['partition'] = int(partition)
_args['continuous']= args['continuous'] if 'continuous' in args else [] _args['continuous']= args['continuous'] if 'continuous' in args else []
@ -352,7 +352,7 @@ if __name__ == '__main__' :
args['partition'] = index args['partition'] = index
# _df = pd.DataFrame(DATA[index],columns=args['columns']) # _df = pd.DataFrame(DATA[index],columns=args['columns'])
args['data'] = DATA[index] args['data'] = DATA[index]
args['data'].to_csv('aou-'+str(index)+'csv',index=False) # args['data'].to_csv('aou-'+str(index)+'csv',index=False)
# args['reader'] = lambda: _df # args['reader'] = lambda: _df
if int(args['num_gpu']) > 1 : if int(args['num_gpu']) > 1 :
args['gpu'] = index args['gpu'] = index