hiplab/data-transport
hiplab
/
data-transport
mirror of https://github.com/lnyemba/data-transport
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

bug fixes: etl, mongodb lexicon

This commit is contained in:
Steve Nyemba 2023-05-30 15:47:10 -05:00
parent 78ad8f3ad8
commit 66f43a98c1
2 changed files with 12 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
transport/etl.py
View File

@ -106,8 +106,10 @@ class ETL (Process):
# self.sql = _args['source']['sql'] if 'sql' in _args['source'] else None # self.sql = _args['source']['sql'] if 'sql' in _args['source'] else None
# self.cmd = _args['source']['cmd'] if 'cmd' in _args['source'] else None # self.cmd = _args['source']['cmd'] if 'cmd' in _args['source'] else None
# self._oargs = _args['target'] #transport.factory.instance(**_args['target']) # self._oargs = _args['target'] #transport.factory.instance(**_args['target'])
self._source = _args['source'] #dict(_args ['source'],**{'context':'write'}) self._source = _args ['source']
self._target = _args['target'] #dict(_args['target'],**{'context':'read','lock':True}) self._target = _args['target']
self._source['context'] = 'read'
self._target['context'] = 'write'
self.JOB_COUNT = _args['jobs'] self.JOB_COUNT = _args['jobs']
self.jobs = [] self.jobs = []
@ -131,6 +133,8 @@ class ETL (Process):
# writing the data to a designated data source # writing the data to a designated data source
# #
try: try:
_log = {"name":self.name,"rows":{"input":0,"output":0}} _log = {"name":self.name,"rows":{"input":0,"output":0}}
_reader = transport.factory.instance(**self._source) _reader = transport.factory.instance(**self._source)
if 'table' in self._source : if 'table' in self._source :

5
transport/mongo.py
View File

@ -37,7 +37,7 @@ class Mongo :
self.mechanism= 'SCRAM-SHA-256' if 'mechanism' not in args else args['mechanism'] self.mechanism= 'SCRAM-SHA-256' if 'mechanism' not in args else args['mechanism']
# authSource=(args['authSource'] if 'authSource' in args else self.dbname) # authSource=(args['authSource'] if 'authSource' in args else self.dbname)
self._lock = False if 'lock' not in args else args['lock'] self._lock = False if 'lock' not in args else args['lock']
self.dbname = None
username = password = None username = password = None
if 'auth_file' in args : if 'auth_file' in args :
_info = json.loads((open(args['auth_file'])).read()) _info = json.loads((open(args['auth_file'])).read())
@ -46,12 +46,15 @@ class Mongo :
else: else:
_info = {} _info = {}
_args = dict(args,**_info) _args = dict(args,**_info)
_map = {'dbname':'db','database':'db','table':'uid','collection':'uid','col':'uid','doc':'uid'}
for key in _args : for key in _args :
if key in ['username','password'] : if key in ['username','password'] :
username = _args['username'] if key=='username' else username username = _args['username'] if key=='username' else username
password = _args['password'] if key == 'password' else password password = _args['password'] if key == 'password' else password
continue continue
value = _args[key] value = _args[key]
if key in _map :
key = _map[key]
self.setattr(key,value) self.setattr(key,value)
# #