aou/data-maker
aou
/
data-maker
1
1
Fork 0
Code Issues 4 Pull Requests Packages Projects 1 Releases Wiki Activity

bug fix: date conversions

This commit is contained in:
Steve Nyemba 2022-05-17 19:10:33 -05:00
parent 377e84daea
commit f3598efa0d
1 changed files with 25 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
data/maker/__init__.py
View File

@ -296,35 +296,35 @@ class Generator (Learner):
name = _item['name']
if _item['type'].upper() in ['DATE','DATETIME','TIMESTAMP'] :
FORMAT = '%m-%d-%Y'
FORMAT = '%Y-%m-%d'
# try:
# #
# #-- Sometimes data isn't all it's meant to be
# SIZE = -1
# if 'format' in self.info and name in self.info['format'] :
# FORMAT = self.info['format'][name]
# SIZE = 10
# elif _item['type'] in ['DATETIME','TIMESTAMP'] :
# FORMAT = '%m-%d-%Y %H:%M:%S'
# SIZE = 19
try:
#
#-- Sometimes data isn't all it's meant to be
SIZE = -1
if 'format' in self.info and name in self.info['format'] :
FORMAT = self.info['format'][name]
SIZE = 10
elif _item['type'] in ['DATETIME','TIMESTAMP'] :
FORMAT = '%Y-%m-%-d %H:%M:%S'
SIZE = 19
# if SIZE > 0 :
if SIZE > 0 :
# values = pd.to_datetime(_df[name], format=FORMAT).astype(str)
# _df[name] = [_date[:SIZE].strip() for _date in values]
values = pd.to_datetime(_df[name], format=FORMAT).astype(np.datetime64)
# _df[name] = [_date[:SIZE].strip() for _date in values]
# # _df[name] = _df[name].astype(str)
# r[name] = FORMAT
# # _df[name] = pd.to_datetime(_df[name], format=FORMAT) #.astype('datetime64[ns]')
# if _item['type'] in ['DATETIME','TIMESTAMP']:
# pass #;_df[name] = _df[name].fillna('').astype('datetime64[ns]')
# _df[name] = _df[name].astype(str)
r[name] = FORMAT
# _df[name] = pd.to_datetime(_df[name], format=FORMAT) #.astype('datetime64[ns]')
if _item['type'] in ['DATETIME','TIMESTAMP']:
pass #;_df[name] = _df[name].fillna('').astype('datetime64[ns]')
# except Exception as e:
# pass
# finally:
# pass
except Exception as e:
pass
finally:
pass
else:
#
@ -387,10 +387,11 @@ class Generator (Learner):
# w = transport.factory.instance(doc='observation',provider='mongodb',context='write',db='IOV01_LOGS',auth_file='/home/steve/dev/transport/mongo.json')
# w.write(_df)
cols = [name for name in _df.columns if name.endswith('datetime')]
# print (_df[cols])
writer = transport.factory.instance(**_store)
writer.write(_df,schema=_schema)
writer.write(_df[cols],schema=[_item for _item in _schema if _item['name'] in cols])