aou/data-maker
aou
/
data-maker
1
1
Fork 0
Code Issues 4 Pull Requests Packages Projects 1 Releases Wiki Activity

bug fix ...

This commit is contained in:
Steve Nyemba 2020-03-12 14:37:01 -05:00
parent d30d2233c8
commit 772d841ee8
1 changed files with 20 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
data/maker/__init__.py
View File

@ -24,24 +24,25 @@ class ContinuousToDiscrete :
# BOUNDS = np.repeat(np.divide(X.max(),n),n).cumsum().tolist() # BOUNDS = np.repeat(np.divide(X.max(),n),n).cumsum().tolist()
# print ( X.values.astype(np.float32)) # print ( X.values.astype(np.float32))
# print ("___________________________") # print ("___________________________")
values = X.values.astype(np.float32) values = np.array(X).astype(np.float32)
BOUNDS = ContinuousToDiscrete.bounds(values,n) BOUNDS = ContinuousToDiscrete.bounds(values,n)
# _map = [{"index":BOUNDS.index(i),"ubound":i} for i in BOUNDS] # _map = [{"index":BOUNDS.index(i),"ubound":i} for i in BOUNDS]
_matrix = [] _matrix = []
m = [] m = []
for value in X : for value in X :
x_ = np.zeros(n) x_ = np.zeros(n)
_matrix.append(x_)
for row in BOUNDS : for row in BOUNDS :
if value>= row.left and value <= row.right : if value>= row.left and value <= row.right :
index = BOUNDS.index(row) index = BOUNDS.index(row)
x_[index] = 1 x_[index] = 1
break break
_matrix += x_.tolist()
# #
# for items in BOUNDS : # for items in BOUNDS :
# index = BOUNDS.index(items) # index = BOUNDS.index(items)
return np.array(_matrix) return np.array(_matrix).reshape(len(X),n)
@staticmethod @staticmethod
def bounds(x,n): def bounds(x,n):
@ -92,7 +93,7 @@ def train (**args) :
:context label of what we are synthesizing :context label of what we are synthesizing
""" """
column = args['column'] if (isinstance(args['column'],list)) else [args['column']] column = args['column'] if (isinstance(args['column'],list)) else [args['column']]
CONTINUOUS = args['continuous'] if 'continuous' in args else [] # CONTINUOUS = args['continuous'] if 'continuous' in args else []
# column_id = args['id'] # column_id = args['id']
df = args['data'] if not isinstance(args['data'],str) else pd.read_csv(args['data']) df = args['data'] if not isinstance(args['data'],str) else pd.read_csv(args['data'])
df.columns = [name.lower() for name in df.columns] df.columns = [name.lower() for name in df.columns]
@ -109,15 +110,16 @@ def train (**args) :
# args['real'] = pd.get_dummies(df[col].fillna('')).astype(np.float32).values # args['real'] = pd.get_dummies(df[col].fillna('')).astype(np.float32).values
# if 'float' not in df[col].dtypes.name : # if 'float' not in df[col].dtypes.name :
# args['real'] = pd.get_dummies(df[col].fillna('')).astype(np.float32).values # args['real'] = pd.get_dummies(df[col].fillna('')).astype(np.float32).values
if col in CONTINUOUS: # if col in CONTINUOUS:
BIN_SIZE = 4 if 'bin_size' not in args else int(args['bin_size']) # BIN_SIZE = 4 if 'bin_size' not in args else int(args['bin_size'])
args['real'] = ContinuousToDiscrete.binary(df[col],BIN_SIZE).astype(np.float32) # args['real'] = ContinuousToDiscrete.binary(df[col],BIN_SIZE).astype(np.float32)
# print ( pd.DataFrame(args['real']).head() ) # # args['real'] = args['real'].reshape(df.shape[0],BIN_SIZE)
else:
# else:
# df.to_csv('tmp-'+args['logs'].replace('/','_')+'-'+col+'.csv',index=False) # df.to_csv('tmp-'+args['logs'].replace('/','_')+'-'+col+'.csv',index=False)
# print (df[col].dtypes) # print (df[col].dtypes)
# print (df[col].dropna/(axis=1).unique()) # print (df[col].dropna/(axis=1).unique())
args['real'] = pd.get_dummies(df[col].dropna()).astype(np.float32).values args['real'] = pd.get_dummies(df[col].dropna()).astype(np.float32).values
@ -170,6 +172,7 @@ def generate(**args):
#@TODO: #@TODO:
# If the identifier is not present, we should fine a way to determine or make one # If the identifier is not present, we should fine a way to determine or make one
# #
BIN_SIZE = 4 if 'bin_size' not in args else int(args['bin_size'])
_df = df.copy() _df = df.copy()
for col in column : for col in column :
args['context'] = col args['context'] = col
@ -181,10 +184,15 @@ def generate(**args):
# BIN_SIZE = 4 if 'bin_size' not in args else int(args['bin_size']) # BIN_SIZE = 4 if 'bin_size' not in args else int(args['bin_size'])
# values = ContinuousToDiscrete.continuous(df[col].values,BIN_SIZE) # values = ContinuousToDiscrete.continuous(df[col].values,BIN_SIZE)
# # values = np.unique(values).tolist() # # values = np.unique(values).tolist()
# else:
# if col in CONTINUOUS :
# values = ContinuousToDiscrete.binary(df[col],BIN_SIZE).astype(np.float32).T
# else: # else:
values = df[col].dropna().unique().tolist() values = df[col].dropna().unique().tolist()
args['values'] = values args['values'] = values
args['row_count'] = df.shape[0] args['row_count'] = df.shape[0]
# #
@ -192,10 +200,9 @@ def generate(**args):
handler = gan.Predict (**args) handler = gan.Predict (**args)
handler.load_meta(col) handler.load_meta(col)
r = handler.apply() r = handler.apply()
BIN_SIZE = 4 if 'bin_size' not in args else int(args['bin_size'])
# _df[col] = ContinuousToDiscrete.continuous(r[col],BIN_SIZE) if col in CONTINUOUS else r[col] _df[col] = ContinuousToDiscrete.continuous(r[col],BIN_SIZE) if col in CONTINUOUS else r[col]
_df[col] = r[col] # _df[col] = r[col]
# #
# @TODO: log basic stats about the synthetic attribute # @TODO: log basic stats about the synthetic attribute
# #