aou/data-maker
aou
/
data-maker
1
1
Fork 0
Code Issues 4 Pull Requests Packages Projects 1 Releases Wiki Activity

bug fix: date, hack put in place

This commit is contained in:
Steve Nyemba 2021-05-12 10:14:53 -05:00
parent 776a1103f2
commit 79b83c71d5
1 changed files with 10 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
pipeline.py
View File

@ -251,13 +251,16 @@ class Components :
_value = 0 _value = 0
if _item['type'] in ['DATE','TIMESTAMP','DATETIMESTAMP','DATETIME'] : if _item['type'] in ['DATE','TIMESTAMP','DATETIMESTAMP','DATETIME'] :
if _item['type'] == 'DATE' : if _item['type'] == 'DATE' :
_df[name] = _df[name].dt.date #
_df[name] = pd.to_datetime(_df[name],errors='coerce') # There is an issue with missing dates that needs to be resolved.
# for some reason a missing date/time here will cause the types to turn into timestamp (problem)
# The following is a hack to address the issue (alas) assuming 10 digit dates and 'NaT' replaces missing date values (pandas specifications)
#
_df[name] = _df[name].apply(lambda value: '' if str(value) == 'NaT' else str(value)[:10])
#_df[name] = _df[name].dt.date
# _df[name] = pd.to_datetime(_df[name].fillna(''),errors='coerce')
else:
print ([' ** ',name,_item['type']])
else: else:
if _item['type'] == 'INTEGER' : if _item['type'] == 'INTEGER' :
_type = np.int64 _type = np.int64