This commit is contained in:
Steve Nyemba 2021-02-06 12:08:43 -06:00
parent 7b5367e4b1
commit 2a641b3c83
4 changed files with 13 additions and 2 deletions

View File

@ -216,6 +216,7 @@ class Factory:
for row in job_args: for row in job_args:
# _store = json.loads(json.dumps(wstore)) # _store = json.loads(json.dumps(wstore))
_store = copy.deepcopy(wstore) _store = copy.deepcopy(wstore)
_store['args']['table'] = row['table'] _store['args']['table'] = row['table']
_pipe = [ _pipe = [
workers.CreateSQL(prefix=PREFIX,schema=SCHEMA,store=_store,sql=row['sql']), workers.CreateSQL(prefix=PREFIX,schema=SCHEMA,store=_store,sql=row['sql']),

View File

@ -17,6 +17,7 @@ class Subject (Process):
self.observers = _args['observers'] self.observers = _args['observers']
self.index = 0 self.index = 0
self.name = _args['name'] self.name = _args['name']
self.table = self.observers[1].table
pass pass
def run(self): def run(self):
self.notify() self.notify()
@ -58,6 +59,10 @@ class Worker :
def execute(self): def execute(self):
try: try:
self._apply() self._apply()
except Exception as error:
print ()
print (error)
print ()
finally: finally:
self.caller.notify() self.caller.notify()
@ -137,7 +142,10 @@ class Reader(Worker):
def _apply(self): def _apply(self):
self.reader = transport.factory.instance(**self._info) ; self.reader = transport.factory.instance(**self._info) ;
print()
print (self.table)
print (json.dumps(self.pipeline))
print ()
self.rows = self.reader.read(mongo=self.pipeline) self.rows = self.reader.read(mongo=self.pipeline)
N = len(self.rows) / self.MAX_ROWS if len(self.rows) > self.MAX_ROWS else 1 N = len(self.rows) / self.MAX_ROWS if len(self.rows) > self.MAX_ROWS else 1

View File

@ -392,6 +392,8 @@ if __name__ == '__main__' :
pipes = export.Factory.instance(type=TYPE,write_store=_store) #"inspect":0,"cast":0}}) pipes = export.Factory.instance(type=TYPE,write_store=_store) #"inspect":0,"cast":0}})
# pipes[0].run() # pipes[0].run()
for thread in pipes: for thread in pipes:
if 'table' in SYS_ARGS and SYS_ARGS['table'] != thread.table :
continue
thread.start() thread.start()
time.sleep(1) time.sleep(1)
while pipes : while pipes :

View File

@ -8,7 +8,7 @@ import sys
def read(fname): def read(fname):
return open(os.path.join(os.path.dirname(__file__), fname)).read() return open(os.path.join(os.path.dirname(__file__), fname)).read()
args = { args = {
"name":"healthcareio","version":"1.5.9", "name":"healthcareio","version":"1.5.9.1",
"author":"Vanderbilt University Medical Center", "author":"Vanderbilt University Medical Center",
"author_email":"steve.l.nyemba@vumc.org", "author_email":"steve.l.nyemba@vumc.org",
"include_package_data":True, "include_package_data":True,