hiplab/data-transport
hiplab
/
data-transport
mirror of https://github.com/lnyemba/data-transport
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

bug fix: filereader

This commit is contained in:
Steve Nyemba 2021-07-19 19:07:06 -05:00
parent b21ddd6945
commit 835e1253a5
2 changed files with 6 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
setup.py
View File

@ -8,12 +8,12 @@ def read(fname):
return open(os.path.join(os.path.dirname(__file__), fname)).read() return open(os.path.join(os.path.dirname(__file__), fname)).read()
args = { args = {
"name":"data-transport", "name":"data-transport",
"version":"1.3.8.8", "version":"1.3.9.0",
"author":"The Phi Technology LLC","author_email":"info@the-phi.com", "author":"The Phi Technology LLC","author_email":"info@the-phi.com",
"license":"MIT", "license":"MIT",
"packages":["transport"]} "packages":["transport"]}
args["keywords"]=['mongodb','couchdb','rabbitmq','file','read','write','s3','sqlite'] args["keywords"]=['mongodb','couchdb','rabbitmq','file','read','write','s3','sqlite']
args["install_requires"] = ['pymongo','numpy','cloudant','pika','nzpy','boto3','boto','pyarrow','google-cloud-bigquery','google-cloud-bigquery-storage','flask-session','smart_open','botocore','psycopg2-binary','mysql-connector-python'] args["install_requires"] = ['pymongo','pandas','numpy','cloudant','pika','nzpy','boto3','boto','pyarrow','google-cloud-bigquery','google-cloud-bigquery-storage','flask-session','smart_open','botocore','psycopg2-binary','mysql-connector-python']
args["url"] = "https://healthcareio.the-phi.com/git/code/transport.git" args["url"] = "https://healthcareio.the-phi.com/git/code/transport.git"
args['scripts'] = ['bin/transport'] args['scripts'] = ['bin/transport']
if sys.version_info[0] == 2 : if sys.version_info[0] == 2 :

5
transport/disk.py
View File

@ -26,10 +26,13 @@ class DiskReader(Reader) :
def isready(self): def isready(self):
return os.path.exists(self.path) return os.path.exists(self.path)
def read(self,**args): def read(self,**args):
return pd.read_csv(self.path,delimiter=self.delimiter)
def stream(self,**args):
""" """
This function reads the rows from a designated location on disk This function reads the rows from a designated location on disk
@param size number of rows to be read, -1 suggests all rows @param size number of rows to be read, -1 suggests all rows
""" """
size = -1 if 'size' not in args else int(args['size']) size = -1 if 'size' not in args else int(args['size'])
f = open(self.path,'rU') f = open(self.path,'rU')
i = 1 i = 1
@ -39,7 +42,7 @@ class DiskReader(Reader) :
if size == i: if size == i:
break break
if self.delimiter : if self.delimiter :
yield row.split(self.char) yield row.split(self.delimiter)
yield row yield row
f.close() f.close()
class DiskWriter(Writer): class DiskWriter(Writer):