aou/data-maker
aou
/
data-maker
1
1
Fork 0
Code Issues 4 Pull Requests Packages Projects 1 Releases Wiki Activity

...

This commit is contained in:
Steve Nyemba 2021-04-04 11:45:20 -05:00
parent efd2fd6a9a
commit 88d602de1c
1 changed files with 6 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
pipeline.py
View File

@ -79,7 +79,8 @@ class Components :
if 'columns' in _args and 'data' in _args : if 'columns' in _args and 'data' in _args :
_df = _args['data'] _df = _args['data']
terms = _args['columns'] terms = _args['columns']
return [name for name in _df.columns if np.sum( [int(field in name )for field in terms ]) > 0 ] return [name for name in _df.columns if np.sum( [int(field in name )for field in terms ]) ]
return [] return []
def set_gpu(self,**_args) : def set_gpu(self,**_args) :
if 'gpu' in _args : if 'gpu' in _args :
@ -247,7 +248,10 @@ class Components :
args['gpu'] = self.set_gpu(gpu=args['gpu']) args['gpu'] = self.set_gpu(gpu=args['gpu'])
_info = {"module":"gan-prep","action":"prune","shape":{"rows":args['data'].shape[0],"columns":args['data'].shape[1]}} _info = {"module":"gan-prep","action":"prune","shape":{"rows":args['data'].shape[0],"columns":args['data'].shape[1]}}
logger.write(_info) logger.write(_info)
candidates = (data.maker.generate(**args)) if args['data'].shape[0] > 0 and args['data'].shape[1] > 0 :
candidates = (data.maker.generate(**args))
else:
candidate = [df]
if 'sql.BQWriter' in ostore['type'] : if 'sql.BQWriter' in ostore['type'] :
#table = ".".join([ostore['['dataset'],args['context']]) #table = ".".join([ostore['['dataset'],args['context']])
# writer = factory.instance(**ostore) # writer = factory.instance(**ostore)