aou/data-maker
aou
/
data-maker
1
1
Fork 0
Code Issues 4 Pull Requests Packages Projects 1 Releases Wiki Activity

bug fixes, added logger (not yet using though)

This commit is contained in:
Steve Nyemba 2022-04-12 13:16:48 -05:00
parent a35c0ed6a2
commit 260f102186
2 changed files with 6 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
data/__init__.py
View File

@ -3,15 +3,3 @@ from data.params import SYS_ARGS
import transport import transport
from multiprocessing import Process, Queue from multiprocessing import Process, Queue
from data.maker import prepare from data.maker import prepare
class Trainer (Process) :
pass
class Maker(Process):
pass
if __name__ == '__main__' :
logger = transport.factory.instance(SYS_ARGS['store']['logger'])

6
data/maker/__init__.py
View File

@ -207,9 +207,11 @@ class Learner(Process):
self.gpu = int(_args['gpu']) self.gpu = int(_args['gpu'])
else: else:
self.gpu = None self.gpu = None
self.info = _args['info'] self.info = _args['info']
self.columns = self.info['columns'] if 'columns' in self.info else None self.columns = self.info['columns'] if 'columns' in self.info else None
self.store = _args['store'] self.store = _args['store']
self.logger = transport.factory.instance(_args['logger']) if 'logger' in self.store else transport.factory.instance(provider='console',context='write',lock=True)
if 'network_args' not in _args : if 'network_args' not in _args :
self.network_args ={ self.network_args ={
'context':_args['context'] if 'context' in _args else 'GENERAL', 'context':_args['context'] if 'context' in _args else 'GENERAL',
@ -379,11 +381,15 @@ class Generator (Learner):
_store = self.store['target'] if 'target' in self.store else {'provider':'console'} _store = self.store['target'] if 'target' in self.store else {'provider':'console'}
_store['lock'] = True _store['lock'] = True
_store['context'] = 'write' #-- Just in case
writer = transport.factory.instance(**_store) writer = transport.factory.instance(**_store)
for _iodf in _candidates : for _iodf in _candidates :
_df = self._df.copy() _df = self._df.copy()
_df[self.columns] = _iodf[self.columns] _df[self.columns] = _iodf[self.columns]
#
#@TODO:
# Improve formatting with better post-processing pipeline
if 'approximate' in self.info : if 'approximate' in self.info :
_df = self.approximate(_df) _df = self.approximate(_df)
if 'make_date' in self.info : if 'make_date' in self.info :