aou/data-maker
aou
/
data-maker
1
1
Fork 0
Code Issues 4 Pull Requests Packages Projects 1 Releases Wiki Activity

bug fix

This commit is contained in:
Steve Nyemba 2020-03-04 13:22:25 -06:00
parent eb1abdf989
commit 076db1ec2c
1 changed files with 4 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
pipeline.py
View File

@ -33,7 +33,7 @@ class Components :
if 'limit' in args : if 'limit' in args :
SQL = SQL + 'LIMIT ' + args['limit'] SQL = SQL + 'LIMIT ' + args['limit']
credentials = service_account.Credentials.from_service_account_file('/home/steve/dev/aou/accounts/curation-prod.json') credentials = service_account.Credentials.from_service_account_file('/home/steve/dev/aou/accounts/curation-prod.json')
df = pd.read_gbq(SQL,credentials=credentials,dialect='standard').dropna() df = pd.read_gbq(SQL,credentials=credentials,dialect='standard')
return df return df
# return lambda: pd.read_gbq(SQL,credentials=credentials,dialect='standard')[args['columns']].dropna() # return lambda: pd.read_gbq(SQL,credentials=credentials,dialect='standard')[args['columns']].dropna()
@ -51,7 +51,9 @@ class Components :
# @TODO: we need to log something here about the parameters being passed # @TODO: we need to log something here about the parameters being passed
pointer = args['reader'] if 'reader' in args else lambda: Components.get(**args) pointer = args['reader'] if 'reader' in args else lambda: Components.get(**args)
df = pointer() df = pointer()
if df.shape[0] == 0 :
print ("CAN NOT TRAIN EMPTY DATASET ")
return
# #
# Now we can parse the arguments and submit the entire thing to training # Now we can parse the arguments and submit the entire thing to training
# #