aou/data-maker
aou
/
data-maker
1
1
Fork 0
Code Issues 4 Pull Requests Packages Projects 1 Releases Wiki Activity

bug fix: date conversions

This commit is contained in:
Steve Nyemba 2022-05-17 19:10:33 -05:00
parent 377e84daea
commit f3598efa0d
1 changed files with 25 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
data/maker/__init__.py
View File

@ -296,35 +296,35 @@ class Generator (Learner):
name = _item['name'] name = _item['name']
if _item['type'].upper() in ['DATE','DATETIME','TIMESTAMP'] : if _item['type'].upper() in ['DATE','DATETIME','TIMESTAMP'] :
FORMAT = '%m-%d-%Y' FORMAT = '%Y-%m-%d'
# try: try:
# # #
# #-- Sometimes data isn't all it's meant to be #-- Sometimes data isn't all it's meant to be
# SIZE = -1 SIZE = -1
# if 'format' in self.info and name in self.info['format'] : if 'format' in self.info and name in self.info['format'] :
# FORMAT = self.info['format'][name] FORMAT = self.info['format'][name]
# SIZE = 10 SIZE = 10
# elif _item['type'] in ['DATETIME','TIMESTAMP'] : elif _item['type'] in ['DATETIME','TIMESTAMP'] :
# FORMAT = '%m-%d-%Y %H:%M:%S' FORMAT = '%Y-%m-%-d %H:%M:%S'
# SIZE = 19 SIZE = 19
# if SIZE > 0 : if SIZE > 0 :
# values = pd.to_datetime(_df[name], format=FORMAT).astype(str) values = pd.to_datetime(_df[name], format=FORMAT).astype(np.datetime64)
# _df[name] = [_date[:SIZE].strip() for _date in values] # _df[name] = [_date[:SIZE].strip() for _date in values]
# # _df[name] = _df[name].astype(str) # _df[name] = _df[name].astype(str)
# r[name] = FORMAT r[name] = FORMAT
# # _df[name] = pd.to_datetime(_df[name], format=FORMAT) #.astype('datetime64[ns]') # _df[name] = pd.to_datetime(_df[name], format=FORMAT) #.astype('datetime64[ns]')
# if _item['type'] in ['DATETIME','TIMESTAMP']: if _item['type'] in ['DATETIME','TIMESTAMP']:
# pass #;_df[name] = _df[name].fillna('').astype('datetime64[ns]') pass #;_df[name] = _df[name].fillna('').astype('datetime64[ns]')
# except Exception as e: except Exception as e:
# pass pass
# finally: finally:
# pass pass
else: else:
# #
@ -387,10 +387,11 @@ class Generator (Learner):
# w = transport.factory.instance(doc='observation',provider='mongodb',context='write',db='IOV01_LOGS',auth_file='/home/steve/dev/transport/mongo.json') # w = transport.factory.instance(doc='observation',provider='mongodb',context='write',db='IOV01_LOGS',auth_file='/home/steve/dev/transport/mongo.json')
# w.write(_df) # w.write(_df)
cols = [name for name in _df.columns if name.endswith('datetime')]
# print (_df[cols]) # print (_df[cols])
writer = transport.factory.instance(**_store) writer = transport.factory.instance(**_store)
writer.write(_df,schema=_schema) writer.write(_df[cols],schema=[_item for _item in _schema if _item['name'] in cols])