diff --git a/pipeline.py b/pipeline.py index acb4f6c..78ffefe 100644 --- a/pipeline.py +++ b/pipeline.py @@ -159,7 +159,7 @@ class Components : # df = pd.DataFrame(df[ int (partition) ],columns = columns) # max_rows = int(args['partition_max_rows']) if 'partition_max_rows' in args else 1000000 # N = np.divide(df.shape[0],max_rows).astype(int) + 1 - info = {"parition":int(partition),"gpu":_args["gpu"],"rows":int(df.shape[0]),"cols":int(df.shape[1]),"part_size":int(PART_SIZE),"partition-info":{"count":int(N),"max_rows":max_rows}} + info = {"parition":int(partition),"gpu":_args["gpu"],"rows":int(df.shape[0]),"cols":int(df.shape[1]),"part_size":int(PART_SIZE)} logger.write({"module":"generate","action":"partition","input":info}) _args['partition'] = int(partition) _args['continuous']= args['continuous'] if 'continuous' in args else []