diff --git a/data/maker/__init__.py b/data/maker/__init__.py index 803590a..4867bf6 100644 --- a/data/maker/__init__.py +++ b/data/maker/__init__.py @@ -144,9 +144,12 @@ def generate(**_args): """ partition = _args['partition'] if 'partition' in _args else None if not partition : - f = open(os.sep.join([_args['logs'],'output',_args['context'],'map.json'])) + LOG_DIR = os.sep.join([_args['logs'],'output',_args['context']]) + # f = open(os.sep.join([_args['logs'],'output',_args['context'],'map.json'])) else: - f = open(os.sep.join([_args['logs'],'output',_args['context'],str(partition),'map.json'])) + LOG_DIR = os.sep.join([_args['logs'],'output',_args['context'],str(partition)]) + # f = open(os.sep.join([_args['logs'],'output',_args['context'],str(partition),'map.json'])) + f = open(os.sep.join([LOG_DIR,'map.json'])) _map = json.loads(f.read()) f.close() # if 'file' in _args : @@ -154,7 +157,7 @@ def generate(**_args): # else: # df = _args['data'] if not isinstance(_args['data'],str) else pd.read_csv(_args['data']) args = {"context":_args['context'],"max_epochs":_args['max_epochs'],"candidates":_args['candidates']} - args['logs'] = _args['logs'] if 'logs' in _args else 'logs' + args['logs'] = LOG_DIR if 'logs' in _args else 'logs' args ['max_epochs'] = _args['max_epochs'] # args['matrix_size'] = _matrix.shape[0] args['batch_size'] = 2000 diff --git a/pipeline.py b/pipeline.py index 2311007..a958bb8 100644 --- a/pipeline.py +++ b/pipeline.py @@ -583,10 +583,10 @@ if __name__ == '__main__' : if 'all-chips' in SYS_ARGS and GPU_CHIPS: index = 0 jobs = [] - for _id in GPU_CHIPS : + for _gpu in GPU_CHIPS : _args = copy.deepcopy(args) _args['gpu'] = [int(_gpu)] - _args['partition'] = index + _args['partition'] = int(_gpu) #index index += 1 make = lambda _params: (Components()).generate(_params) job = Process(target=make,args=( dict(_args),)) @@ -624,7 +624,7 @@ if __name__ == '__main__' : for _gpu in GPU_CHIPS : _args = copy.deepcopy(args) _args['gpu'] = [int(_gpu)] - _args['partition'] = index + _args['partition'] = int(_gpu) #index index += 1 make = lambda _params: (Components()).train(**_params) job = Process(target=make,args=( _args,)) diff --git a/setup.py b/setup.py index 1efc05e..1c126f5 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ import sys def read(fname): return open(os.path.join(os.path.dirname(__file__), fname)).read() args = {"name":"data-maker", - "version":"1.4.7", + "version":"1.4.7.1", "author":"Vanderbilt University Medical Center","author_email":"steve.l.nyemba@vanderbilt.edu","license":"MIT", "packages":find_packages(),"keywords":["healthcare","data","transport","protocol"]} args["install_requires"] = ['data-transport@git+https://dev.the-phi.com/git/steve/data-transport.git','tensorflow==1.15','pandas','pandas-gbq','pymongo']