aou/data-maker
aou
/
data-maker
1
1
Fork 0
Code Issues 4 Pull Requests Packages Projects 1 Releases Wiki Activity

..

This commit is contained in:
Steve Nyemba 2021-04-01 13:53:29 -05:00
parent db496f9983
commit e56254000e
1 changed files with 7 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
pipeline.py
View File

@ -231,7 +231,7 @@ class Components :
if 'ignore' in args and 'columns' in args['ignore'] : if 'ignore' in args and 'columns' in args['ignore'] :
_cols = self.get_ignore(data=df,columns=args['ignore']['columns']) _cols = self.get_ignore(data=df,columns=args['ignore']['columns'])
args['data'] = df[ list(set(df.columns)- set(_cols))] args['data'] = args['data'][ list(set(df.columns)- set(_cols))]
# #
# We need to remove the continuous columns from the data-frame # We need to remove the continuous columns from the data-frame
# @TODO: Abstract this !! # @TODO: Abstract this !!
@ -267,12 +267,6 @@ class Components :
# for _name in _df.columns: # for _name in _df.columns:
# if _name in name: # if _name in name:
# skip_columns.append(_name) # skip_columns.append(_name)
if x_cols :
for _col in x_cols :
if df[_col].unique().size > 0 :
_df[_col] = self.approximate(df[_col].fillna(-1))
else:
_df[_col] = -1
# #
# We perform a series of set operations to insure that the following conditions are met: # We perform a series of set operations to insure that the following conditions are met:
# - the synthetic dataset only has fields that need to be synthesized # - the synthetic dataset only has fields that need to be synthesized
@ -284,6 +278,12 @@ class Components :
if set(df.columns) & set(_df.columns) : if set(df.columns) & set(_df.columns) :
_columns = set(df.columns) - set(_df.columns) _columns = set(df.columns) - set(_df.columns)
df = df[_columns] df = df[_columns]
if x_cols :
for _col in x_cols :
if df[_col].unique().size > 0 :
_df[_col] = self.approximate(df[_col].fillna(-1))
else:
_df[_col] = -1
# #
# Let us merge the dataset here and and have a comprehensive dataset # Let us merge the dataset here and and have a comprehensive dataset