aou/data-maker
aou
/
data-maker
1
1
Fork 0
Code Issues 4 Pull Requests Packages Projects 1 Releases Wiki Activity

bug fix ...

This commit is contained in:
Steve Nyemba 2020-03-12 14:37:01 -05:00
parent d30d2233c8
commit 772d841ee8
1 changed files with 20 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
data/maker/__init__.py
View File

@ -24,24 +24,25 @@ class ContinuousToDiscrete :
# BOUNDS = np.repeat(np.divide(X.max(),n),n).cumsum().tolist()
# print ( X.values.astype(np.float32))
# print ("___________________________")
values = X.values.astype(np.float32)
values = np.array(X).astype(np.float32)
BOUNDS = ContinuousToDiscrete.bounds(values,n)
# _map = [{"index":BOUNDS.index(i),"ubound":i} for i in BOUNDS]
_matrix = []
m = []
for value in X :
x_ = np.zeros(n)
_matrix.append(x_)
for row in BOUNDS :
if value>= row.left and value <= row.right :
index = BOUNDS.index(row)
x_[index] = 1
break
_matrix += x_.tolist()
#
# for items in BOUNDS :
# index = BOUNDS.index(items)
return np.array(_matrix)
return np.array(_matrix).reshape(len(X),n)
@staticmethod
def bounds(x,n):
@ -92,7 +93,7 @@ def train (**args) :
:context label of what we are synthesizing
"""
column = args['column'] if (isinstance(args['column'],list)) else [args['column']]
CONTINUOUS = args['continuous'] if 'continuous' in args else []
# CONTINUOUS = args['continuous'] if 'continuous' in args else []
# column_id = args['id']
df = args['data'] if not isinstance(args['data'],str) else pd.read_csv(args['data'])
df.columns = [name.lower() for name in df.columns]
@ -109,15 +110,16 @@ def train (**args) :
# args['real'] = pd.get_dummies(df[col].fillna('')).astype(np.float32).values
# if 'float' not in df[col].dtypes.name :
# args['real'] = pd.get_dummies(df[col].fillna('')).astype(np.float32).values
if col in CONTINUOUS:
BIN_SIZE = 4 if 'bin_size' not in args else int(args['bin_size'])
args['real'] = ContinuousToDiscrete.binary(df[col],BIN_SIZE).astype(np.float32)
# print ( pd.DataFrame(args['real']).head() )
else:
# if col in CONTINUOUS:
# BIN_SIZE = 4 if 'bin_size' not in args else int(args['bin_size'])
# args['real'] = ContinuousToDiscrete.binary(df[col],BIN_SIZE).astype(np.float32)
# # args['real'] = args['real'].reshape(df.shape[0],BIN_SIZE)
# else:
# df.to_csv('tmp-'+args['logs'].replace('/','_')+'-'+col+'.csv',index=False)
# print (df[col].dtypes)
# print (df[col].dropna/(axis=1).unique())
args['real'] = pd.get_dummies(df[col].dropna()).astype(np.float32).values
args['real'] = pd.get_dummies(df[col].dropna()).astype(np.float32).values
@ -170,6 +172,7 @@ def generate(**args):
#@TODO:
# If the identifier is not present, we should fine a way to determine or make one
#
BIN_SIZE = 4 if 'bin_size' not in args else int(args['bin_size'])
_df = df.copy()
for col in column :
args['context'] = col
@ -181,10 +184,15 @@ def generate(**args):
# BIN_SIZE = 4 if 'bin_size' not in args else int(args['bin_size'])
# values = ContinuousToDiscrete.continuous(df[col].values,BIN_SIZE)
# # values = np.unique(values).tolist()
# else:
# if col in CONTINUOUS :
# values = ContinuousToDiscrete.binary(df[col],BIN_SIZE).astype(np.float32).T
# else:
values = df[col].dropna().unique().tolist()
args['values'] = values
args['row_count'] = df.shape[0]
#
@ -192,10 +200,9 @@ def generate(**args):
handler = gan.Predict (**args)
handler.load_meta(col)
r = handler.apply()
BIN_SIZE = 4 if 'bin_size' not in args else int(args['bin_size'])
# _df[col] = ContinuousToDiscrete.continuous(r[col],BIN_SIZE) if col in CONTINUOUS else r[col]
_df[col] = r[col]
_df[col] = ContinuousToDiscrete.continuous(r[col],BIN_SIZE) if col in CONTINUOUS else r[col]
# _df[col] = r[col]
#
# @TODO: log basic stats about the synthetic attribute
#