hiplab/data-transport
hiplab
/
data-transport
mirror of https://github.com/lnyemba/data-transport
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #7 from lnyemba/dev 

bug fix: etl job and segments
This commit is contained in:
Steve L. Nyemba 2023-12-26 14:58:51 -06:00 committed by GitHub
parent 01d92c48b4 88b0437bbc
commit 98964a623d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 6 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.gitignore vendored
View File

@ -1 +1,6 @@
build
*.pyc *.pyc
*.csv
*.json
*.swp
*.egg-info

2
transport/etl.py
View File

@ -109,7 +109,7 @@ class Transporter(Process):
SEGMENT_COUNT = 6 SEGMENT_COUNT = 6
MAX_ROWS = 1000000 MAX_ROWS = 1000000
# _df = self.read() # _df = self.read()
_segments = np.array_split(np.range(_df.shape[0]),SEGMENT_COUNT) if _df.shape[0] > MAX_ROWS else np.array( [np.arange(_df.shape[0])]) _segments = np.array_split(np.arange(_df.shape[0]),SEGMENT_COUNT) if _df.shape[0] > MAX_ROWS else np.array( [np.arange(_df.shape[0])])
# _index = 0 # _index = 0