aou/data-maker
aou
/
data-maker
1
1
Fork 0
Code Issues 4 Pull Requests Packages Projects 1 Releases Wiki Activity

bug fix ...

This commit is contained in:
Steve Nyemba 2021-05-10 14:33:18 -05:00
parent 79c5f3ff25
commit b10296246d
3 changed files with 10 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
data/maker/__init__.py
View File

@ -144,9 +144,12 @@ def generate(**_args):
""" """
partition = _args['partition'] if 'partition' in _args else None partition = _args['partition'] if 'partition' in _args else None
if not partition : if not partition :
f = open(os.sep.join([_args['logs'],'output',_args['context'],'map.json'])) LOG_DIR = os.sep.join([_args['logs'],'output',_args['context']])
# f = open(os.sep.join([_args['logs'],'output',_args['context'],'map.json']))
else: else:
f = open(os.sep.join([_args['logs'],'output',_args['context'],str(partition),'map.json'])) LOG_DIR = os.sep.join([_args['logs'],'output',_args['context'],str(partition)])
# f = open(os.sep.join([_args['logs'],'output',_args['context'],str(partition),'map.json']))
f = open(os.sep.join([LOG_DIR,'map.json']))
_map = json.loads(f.read()) _map = json.loads(f.read())
f.close() f.close()
# if 'file' in _args : # if 'file' in _args :
@ -154,7 +157,7 @@ def generate(**_args):
# else: # else:
# df = _args['data'] if not isinstance(_args['data'],str) else pd.read_csv(_args['data']) # df = _args['data'] if not isinstance(_args['data'],str) else pd.read_csv(_args['data'])
args = {"context":_args['context'],"max_epochs":_args['max_epochs'],"candidates":_args['candidates']} args = {"context":_args['context'],"max_epochs":_args['max_epochs'],"candidates":_args['candidates']}
args['logs'] = _args['logs'] if 'logs' in _args else 'logs' args['logs'] = LOG_DIR if 'logs' in _args else 'logs'
args ['max_epochs'] = _args['max_epochs'] args ['max_epochs'] = _args['max_epochs']
# args['matrix_size'] = _matrix.shape[0] # args['matrix_size'] = _matrix.shape[0]
args['batch_size'] = 2000 args['batch_size'] = 2000

6
pipeline.py
View File

@ -583,10 +583,10 @@ if __name__ == '__main__' :
if 'all-chips' in SYS_ARGS and GPU_CHIPS: if 'all-chips' in SYS_ARGS and GPU_CHIPS:
index = 0 index = 0
jobs = [] jobs = []
for _id in GPU_CHIPS : for _gpu in GPU_CHIPS :
_args = copy.deepcopy(args) _args = copy.deepcopy(args)
_args['gpu'] = [int(_gpu)] _args['gpu'] = [int(_gpu)]
_args['partition'] = index _args['partition'] = int(_gpu) #index
index += 1 index += 1
make = lambda _params: (Components()).generate(_params) make = lambda _params: (Components()).generate(_params)
job = Process(target=make,args=( dict(_args),)) job = Process(target=make,args=( dict(_args),))
@ -624,7 +624,7 @@ if __name__ == '__main__' :
for _gpu in GPU_CHIPS : for _gpu in GPU_CHIPS :
_args = copy.deepcopy(args) _args = copy.deepcopy(args)
_args['gpu'] = [int(_gpu)] _args['gpu'] = [int(_gpu)]
_args['partition'] = index _args['partition'] = int(_gpu) #index
index += 1 index += 1
make = lambda _params: (Components()).train(**_params) make = lambda _params: (Components()).train(**_params)
job = Process(target=make,args=( _args,)) job = Process(target=make,args=( _args,))

2
setup.py
View File

@ -5,7 +5,7 @@ import sys
def read(fname): def read(fname):
return open(os.path.join(os.path.dirname(__file__), fname)).read() return open(os.path.join(os.path.dirname(__file__), fname)).read()
args = {"name":"data-maker", args = {"name":"data-maker",
"version":"1.4.7", "version":"1.4.7.1",
"author":"Vanderbilt University Medical Center","author_email":"steve.l.nyemba@vanderbilt.edu","license":"MIT", "author":"Vanderbilt University Medical Center","author_email":"steve.l.nyemba@vanderbilt.edu","license":"MIT",
"packages":find_packages(),"keywords":["healthcare","data","transport","protocol"]} "packages":find_packages(),"keywords":["healthcare","data","transport","protocol"]}
args["install_requires"] = ['data-transport@git+https://dev.the-phi.com/git/steve/data-transport.git','tensorflow==1.15','pandas','pandas-gbq','pymongo'] args["install_requires"] = ['data-transport@git+https://dev.the-phi.com/git/steve/data-transport.git','tensorflow==1.15','pandas','pandas-gbq','pymongo']