data-maker/pipeline.py

384 lines
14 KiB
Python
Raw Normal View History

2020-03-04 18:16:50 +00:00
#!/usr/bin/env python3
2020-03-01 18:07:02 +00:00
import json
from transport import factory
import numpy as np
import time
2020-03-01 18:07:02 +00:00
import os
from multiprocessing import Process
import pandas as pd
from google.oauth2 import service_account
import data.maker
from data.params import SYS_ARGS
#
# The configuration array is now loaded and we will execute the pipe line as follows
2020-03-04 20:30:40 +00:00
DATASET='combined20191004v2_deid'
2020-03-01 18:07:02 +00:00
class Components :
@staticmethod
def get(args):
"""
This function returns a data-frame provided a bigquery sql statement with conditions (and limits for testing purposes)
The function must be wrapped around a lambda this makes testing easier and changing data stores transparent to the rest of the code. (Vital when testing)
:sql basic sql statement
:condition optional condition and filters
"""
SQL = args['sql']
if 'condition' in args :
condition = ' '.join([args['condition']['field'],args['condition']['qualifier'],'(',args['condition']['value'],')'])
SQL = " ".join([SQL,'WHERE',condition])
2020-03-01 18:07:02 +00:00
2020-03-08 13:48:38 +00:00
SQL = SQL.replace(':dataset',args['dataset']) #+ " LI "
if 'limit' in args :
2020-03-08 13:48:38 +00:00
SQL = SQL + ' LIMIT ' + args['limit']
credentials = service_account.Credentials.from_service_account_file('/home/steve/dev/aou/accounts/curation-prod.json')
2020-03-08 13:48:38 +00:00
df = pd.read_gbq(SQL,credentials=credentials,dialect='standard').astype(object)
return df
# return lambda: pd.read_gbq(SQL,credentials=credentials,dialect='standard')[args['columns']].dropna()
@staticmethod
def split(X,MAX_ROWS=3,PART_SIZE=3):
return list(pd.cut( np.arange(X.shape[0]+1),PART_SIZE).categories)
2020-03-01 18:07:02 +00:00
def train(self,**args):
"""
This function will perform training on the basis of a given pointer that reads data
2020-03-01 18:07:02 +00:00
"""
#
# @TODO: we need to log something here about the parameters being passed
2020-03-07 15:16:17 +00:00
# pointer = args['reader'] if 'reader' in args else lambda: Components.get(**args)
2020-03-08 13:48:38 +00:00
df = args['data']
2020-03-07 15:16:17 +00:00
# if df.shape[0] == 0 :
# print ("CAN NOT TRAIN EMPTY DATASET ")
# return
#
# Now we can parse the arguments and submit the entire thing to training
#
logger = factory.instance(type='mongo.MongoWriter',args={'dbname':'aou','doc':args['context']})
log_folder = args['logs'] if 'logs' in args else 'logs'
2020-03-08 13:48:38 +00:00
# _args = {"batch_size":10000,"logs":log_folder,"context":args['context'],"max_epochs":150,"column":args['columns'],"id":"person_id","logger":logger}
2020-03-08 13:48:38 +00:00
# _args['max_epochs'] = 150 if 'max_epochs' not in args else int(args['max_epochs'])
# _args['num_gpu'] = int(args['num_gpu']) if 'num_gpu' in args else 1
# _args['gpu'] = args['gpu'] if 'gpu' in args else 0
2020-03-08 13:48:38 +00:00
# # MAX_ROWS = args['max_rows'] if 'max_rows' in args else 0
PART_SIZE = int(args['part_size'])
2020-03-08 13:48:38 +00:00
partition = args['partition']
log_folder = os.sep.join([log_folder,args['context'],str(partition)])
_args = {"batch_size":10000,"logs":log_folder,"context":args['context'],"max_epochs":150,"column":args['columns'],"id":"person_id","logger":logger}
_args['max_epochs'] = 150 if 'max_epochs' not in args else int(args['max_epochs'])
2020-03-08 20:00:26 +00:00
if 'batch_size' in args :
_args['batch_size'] = int(args['batch_size'])
2020-03-08 13:48:38 +00:00
#
# We ask the process to assume 1 gpu given the system number of GPU and that these tasks can run in parallel
#
if int(args['num_gpu']) > 1 :
2020-03-09 18:10:26 +00:00
_args['gpu'] = int(args['gpu']) if int(args['gpu']) < 8 else np.random.choice(np.arange(8)).astype(int)
else:
2020-03-08 13:48:38 +00:00
_args['gpu'] = 0
_args['num_gpu'] = 1
os.environ['CUDA_VISIBLE_DEVICES'] = str(args['gpu'])
2020-03-09 00:33:08 +00:00
_args['partition'] = int(partition)
_args['continuous']= args['continuous'] if 'continuous' in args else []
2020-03-08 13:48:38 +00:00
_args['store'] = {'type':'mongo.MongoWriter','args':{'dbname':'aou','doc':args['context']}}
_args['data'] = args['data']
# print (['partition ',partition,df.value_source_concept_id.unique()])
#
# @log :
# Logging information about the training process for this partition (or not)
#
info = {"rows":df.shape[0],"cols":df.shape[1], "partition":int(partition),"logs":_args['logs']}
logger.write({"module":"train","action":"train","input":info})
data.maker.train(**_args)
pass
# @staticmethod
def generate(self,args):
"""
This function will generate data and store it to a given,
"""
logger = factory.instance(type='mongo.MongoWriter',args={'dbname':'aou','doc':args['context']})
log_folder = args['logs'] if 'logs' in args else 'logs'
partition = args['partition'] if 'partition' in args else ''
2020-03-05 18:03:04 +00:00
log_folder = os.sep.join([log_folder,args['context'],str(partition)])
2020-03-06 17:17:29 +00:00
_args = {"batch_size":2000,"logs":log_folder,"context":args['context'],"max_epochs":150,"column":args['columns'],"id":"person_id","logger":logger}
_args['max_epochs'] = 150 if 'max_epochs' not in args else int(args['max_epochs'])
# _args['num_gpu'] = int(args['num_gpu']) if 'num_gpu' in args else 1
2020-03-08 20:00:26 +00:00
if 'batch_size' in args :
_args['batch_size'] = int(args['batch_size'])
2020-03-06 20:56:28 +00:00
if int(args['num_gpu']) > 1 :
2020-03-09 18:10:26 +00:00
_args['gpu'] = int(args['gpu']) if int(args['gpu']) < 8 else np.random.choice(np.arange(8)).astype(int)
else:
_args['gpu'] = 0
_args['num_gpu'] = 1
2020-03-06 20:42:40 +00:00
os.environ['CUDA_VISIBLE_DEVICES'] = str(args['gpu'])
_args['no_value']= args['no_value']
# MAX_ROWS = args['max_rows'] if 'max_rows' in args else 0
PART_SIZE = int(args['part_size']) if 'part_size' in args else 8
# credentials = service_account.Credentials.from_service_account_file('/home/steve/dev/aou/accounts/curation-prod.json')
# _args['data'] = pd.read_gbq(SQL,credentials=credentials,dialect='standard').dropna()
# reader = args['reader']
# df = reader()
df = args['reader']() if 'reader' in args else args['data']
# bounds = Components.split(df,MAX_ROWS,PART_SIZE)
# if partition != '' :
# columns = args['columns']
# df = np.array_split(df[columns].values,PART_SIZE)
# df = pd.DataFrame(df[ int (partition) ],columns = columns)
2020-03-12 14:55:29 +00:00
info = {"parition":int(partition),"gpu":_args["gpu"],"rows":str(df.shape[0]),"cols":str(df.shape[1]),"part_size":int(PART_SIZE)}
logger.write({"module":"generate","action":"partition","input":info})
2020-03-09 00:33:08 +00:00
_args['partition'] = int(partition)
_args['continuous']= args['continuous'] if 'continuous' in args else []
_args['data'] = df
# _args['data'] = reader()
#_args['data'] = _args['data'].astype(object)
# _args['num_gpu'] = 1
_dc = data.maker.generate(**_args)
#
# We need to post the generate the data in order to :
# 1. compare immediately
# 2. synthetic copy
#
cols = _dc.columns.tolist()
data_comp = _args['data'][args['columns']].join(_dc[args['columns']],rsuffix='_io') #-- will be used for comparison (store this in big query)
base_cols = list(set(_args['data'].columns) - set(args['columns'])) #-- rebuilt the dataset (and store it)
for name in cols :
_args['data'][name] = _dc[name]
info = {"module":"generate","action":"io","input":{"rows":_dc[name].shape[0],"name":name}}
if partition != '' :
info['partition'] = int(partition)
logger.write(info)
# filename = os.sep.join([log_folder,'output',name+'.csv'])
# data_comp[[name]].to_csv(filename,index=False)
#
#-- Let us store all of this into bigquery
prefix = args['notify']+'.'+_args['context']
2020-03-08 13:48:38 +00:00
partition = str(partition)
table = '_'.join([prefix,partition,'io']).replace('__','_')
folder = os.sep.join([args['logs'],args['context'],partition,'output'])
if 'file' in args :
_fname = os.sep.join([folder,table.replace('_io','_full_io.csv')])
_pname = os.sep.join([folder,table])+'.csv'
data_comp.to_csv( _pname,index=False)
_args['data'].to_csv(_fname,index=False)
2020-03-08 13:48:38 +00:00
_id = 'path'
else:
credentials = service_account.Credentials.from_service_account_file('/home/steve/dev/aou/accounts/curation-prod.json')
_pname = os.sep.join([folder,table+'.csv'])
_fname = table.replace('_io','_full_io')
2020-03-08 13:48:38 +00:00
partial = '.'.join(['io',args['context']+'_partial_io'])
complete= '.'.join(['io',args['context']+'_full_io'])
data_comp.to_gbq(if_exists='append',destination_table=partial,credentials=credentials,chunksize=50000)
data_comp.to_csv(_pname,index=False)
INSERT_FLAG = 'replace' if 'partition' not in args or 'segment' not in args else 'append'
2020-03-09 00:33:08 +00:00
_args['data'].to_gbq(if_exists='append',destination_table=complete,credentials=credentials,chunksize=50000)
2020-03-08 13:48:38 +00:00
_id = 'dataset'
info = {"full":{_id:_fname,"rows":_args['data'].shape[0]},"partial":{"path":_pname,"rows":data_comp.shape[0]} }
if partition :
info ['partition'] = int(partition)
logger.write({"module":"generate","action":"write","input":info} )
@staticmethod
def callback(channel,method,header,stream):
2020-03-06 17:17:29 +00:00
if stream.decode('utf8') in ['QUIT','EXIT','END'] :
channel.close()
channel.connection.close()
info = json.loads(stream)
logger = factory.instance(type='mongo.MongoWriter',args={'dbname':'aou','doc':SYS_ARGS['context']})
logger.write({'module':'process','action':'read-partition','input':info['input']})
df = pd.DataFrame(info['data'])
args = info['args']
if args['num_gpu'] > 1 :
2020-03-09 18:10:26 +00:00
args['gpu'] = int(info['input']['partition']) if info['input']['partition'] < 8 else np.random.choice(np.arange(8)).astype(int)
2020-03-06 17:17:29 +00:00
else:
args['gpu'] = 0
2020-03-06 17:17:29 +00:00
args['num_gpu'] = 1
2020-03-04 20:27:54 +00:00
# if int(args['num_gpu']) > 1 and args['gpu'] > 0:
# args['gpu'] = args['gpu'] + args['num_gpu'] if args['gpu'] + args['num_gpu'] < 8 else args['gpu'] #-- 8 max gpus
args['reader'] = lambda: df
#
# @TODO: Fix
# There is an inconsistency in column/columns ... fix this shit!
#
channel.close()
channel.connection.close()
args['columns'] = args['column']
(Components()).train(**args)
logger.write({"module":"process","action":"exit","input":info["input"]})
pass
2020-03-01 18:07:02 +00:00
if __name__ == '__main__' :
filename = SYS_ARGS['config'] if 'config' in SYS_ARGS else 'config.json'
f = open (filename)
PIPELINE = json.loads(f.read())
f.close()
index = int(SYS_ARGS['index']) if 'index' in SYS_ARGS else 0
args = (PIPELINE[index])
2020-03-04 20:30:40 +00:00
args = dict(args,**SYS_ARGS)
2020-03-06 17:17:29 +00:00
args['logs'] = args['logs'] if 'logs' in args else 'logs'
2020-03-04 20:30:40 +00:00
if 'dataset' not in args :
args['dataset'] = 'combined20191004v2_deid'
2020-03-08 13:48:38 +00:00
PART_SIZE = int(args['part_size']) if 'part_size' in args else 8
#
# @TODO:
# Log what was initiated so we have context of this processing ...
#
if 'listen' not in SYS_ARGS :
if 'file' in args :
2020-03-08 13:48:38 +00:00
DATA = pd.read_csv(args['file']) ;
else:
DATA = Components().get(args)
2020-03-08 13:48:38 +00:00
COLUMNS = DATA.columns
DATA = np.array_split(DATA,PART_SIZE)
if 'generate' in SYS_ARGS :
#
# Let us see if we have partitions given the log folder
content = os.listdir( os.sep.join([args['logs'],args['context']]))
generator = Components()
2020-03-08 13:48:38 +00:00
if ''.join(content).isnumeric() :
#
# we have partitions we are working with
2020-03-07 15:16:17 +00:00
2020-03-06 18:11:22 +00:00
jobs = []
2020-03-08 13:48:38 +00:00
# columns = DATA.columns.tolist()
# DATA = np.array_split(DATA,PART_SIZE)
2020-03-06 20:56:28 +00:00
2020-03-08 13:48:38 +00:00
for index in range(0,PART_SIZE) :
if 'focus' in args and int(args['focus']) != index :
2020-03-06 20:56:28 +00:00
#
# This handles failures/recoveries for whatever reason
# If we are only interested in generating data for a given partition
continue
2020-03-08 13:48:38 +00:00
# index = id.index(id)
args['partition'] = index
args['data'] = DATA[index]
2020-03-06 20:56:28 +00:00
if int(args['num_gpu']) > 1 :
2020-03-08 13:48:38 +00:00
args['gpu'] = index
else:
args['gpu']=0
2020-03-07 15:16:17 +00:00
make = lambda _args: (Components()).generate(_args)
job = Process(target=make,args=(args,))
2020-03-08 13:48:38 +00:00
job.name = 'generator # '+str(index)
2020-03-06 18:11:22 +00:00
job.start()
jobs.append(job)
2020-03-07 15:16:17 +00:00
print (["Started ",len(jobs),"generators" if len(jobs)>1 else "generator" ])
while len(jobs)> 0 :
jobs = [job for job in jobs if job.is_alive()]
2020-03-06 18:11:22 +00:00
time.sleep(2)
2020-03-06 18:11:22 +00:00
# generator.generate(args)
else:
generator.generate(args)
# Components.generate(args)
elif 'listen' in args :
#
# This will start a worker just in case to listen to a queue
2020-03-04 19:40:26 +00:00
SYS_ARGS = dict(args) #-- things get lost in context
if 'read' in SYS_ARGS :
QUEUE_TYPE = 'queue.QueueReader'
pointer = lambda qreader: qreader.read()
else:
QUEUE_TYPE = 'queue.QueueListener'
pointer = lambda qlistener: qlistener.listen()
N = int(SYS_ARGS['jobs']) if 'jobs' in SYS_ARGS else 1
qhandlers = [factory.instance(type=QUEUE_TYPE,args={'queue':'aou.io'}) for i in np.arange(N)]
jobs = []
for qhandler in qhandlers :
qhandler.callback = Components.callback
job = Process(target=pointer,args=(qhandler,))
job.start()
jobs.append(job)
#
# let us wait for the jobs
print (["Started ",len(jobs)," trainers"])
while len(jobs) > 0 :
jobs = [job for job in jobs if job.is_alive()]
time.sleep(2)
# pointer(qhandler)
2020-03-01 18:07:02 +00:00
# qreader.read(1)
pass
else:
2020-03-08 13:48:38 +00:00
# DATA = np.array_split(DATA,PART_SIZE)
2020-03-07 15:16:17 +00:00
jobs = []
2020-03-08 13:48:38 +00:00
for index in range(0,PART_SIZE) :
2020-03-07 15:16:17 +00:00
if 'focus' in args and int(args['focus']) != index :
continue
2020-03-08 13:48:38 +00:00
args['part_size'] = PART_SIZE
2020-03-07 15:16:17 +00:00
args['partition'] = index
2020-03-08 13:48:38 +00:00
# _df = pd.DataFrame(DATA[index],columns=args['columns'])
args['data'] = DATA[index]
2020-03-12 14:55:29 +00:00
# args['data'].to_csv('aou-'+str(index)+'csv',index=False)
2020-03-08 13:48:38 +00:00
# args['reader'] = lambda: _df
if int(args['num_gpu']) > 1 :
args['gpu'] = index
else:
args['gpu']=0
2020-03-07 15:16:17 +00:00
make = lambda _args: (Components()).train(**_args)
2020-03-08 13:48:38 +00:00
job = Process(target=make,args=( dict(args),))
2020-03-07 15:16:17 +00:00
job.name = 'Trainer # ' + str(index)
job.start()
jobs.append(job)
# args['gpu']
print (["Started ",len(jobs),"trainers" if len(jobs)>1 else "trainer" ])
while len(jobs)> 0 :
jobs = [job for job in jobs if job.is_alive()]
time.sleep(2)
# trainer = Components()
# trainer.train(**args)
# Components.train(**args)
2020-03-01 18:07:02 +00:00
#for args in PIPELINE :
#args['dataset'] = 'combined20190510'
#process = Process(target=Components.train,args=(args,))
#process.name = args['context']
#process.start()
# Components.train(args)