hiplab/data-transport
hiplab
/
data-transport
mirror of https://github.com/lnyemba/data-transport
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

bug fixes with sqlite and provider

This commit is contained in:
Steve Nyemba 2023-11-30 12:05:04 -06:00
parent 7d29a69a23
commit 9da2894b07
2 changed files with 11 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
transport/disk.py
View File

@ -209,17 +209,20 @@ class SQLiteWriter(SQLite,DiskWriter) :
""" """
""" """
if not self.fields : #if not self.fields :
if type(info) == pd.DataFrame : # #if type(info) == pd.DataFrame :
_columns = list(info.columns) # # _columns = list(info.columns)
self.init(list(info.keys())) # #self.init(list(info.keys()))
if type(info) == dict : if type(info) == dict :
info = [info] info = [info]
elif type(info) == pd.DataFrame : elif type(info) == pd.DataFrame :
info = info.fillna('') info = info.fillna('')
info = info.to_dict(orient='records') info = info.to_dict(orient='records')
if not self.fields :
_rec = info[0]
self.init(list(_rec.keys()))
SQLiteWriter.LOCK.acquire() SQLiteWriter.LOCK.acquire()
try: try:
@ -238,4 +241,4 @@ class SQLiteWriter(SQLite,DiskWriter) :
except Exception as e : except Exception as e :
print (e) print (e)
pass pass
SQLiteWriter.LOCK.release() SQLiteWriter.LOCK.release()

3
transport/providers.py
View File

@ -72,6 +72,7 @@ WRITE = {'sql':sql.SQLWriter,'nosql':{MONGODB:mongo.MongoWriter,COUCHDB:couch.Co
PROVIDERS = { PROVIDERS = {
FILE:{'read':disk.DiskReader,'write':disk.DiskWriter}, FILE:{'read':disk.DiskReader,'write':disk.DiskWriter},
SQLITE:{'read':disk.SQLiteReader,'write':disk.SQLiteWriter,'driver':sqlite3}, SQLITE:{'read':disk.SQLiteReader,'write':disk.SQLiteWriter,'driver':sqlite3},
'sqlite3':{'read':disk.SQLiteReader,'write':disk.SQLiteWriter,'driver':sqlite3},
POSTGRESQL:{'read':sql.SQLReader,'write':sql.SQLWriter,'driver':pg,'default':{'host':'localhost','port':5432}}, POSTGRESQL:{'read':sql.SQLReader,'write':sql.SQLWriter,'driver':pg,'default':{'host':'localhost','port':5432}},
NETEZZA:{'read':sql.SQLReader,'write':sql.SQLWriter,'driver':nz,'default':{'port':5480}}, NETEZZA:{'read':sql.SQLReader,'write':sql.SQLWriter,'driver':nz,'default':{'port':5480}},
@ -98,4 +99,4 @@ DEFAULT = {PG:{'host':'localhost','port':5432},MYSQL:{'host':'localhost','port':
DEFAULT[MONGODB] = {'port':27017,'host':'localhost'} DEFAULT[MONGODB] = {'port':27017,'host':'localhost'}
DEFAULT[REDSHIFT] = DEFAULT[PG] DEFAULT[REDSHIFT] = DEFAULT[PG]
DEFAULT[MARIADB] = DEFAULT[MYSQL] DEFAULT[MARIADB] = DEFAULT[MYSQL]
DEFAULT[NETEZZA] = {'port':5480} DEFAULT[NETEZZA] = {'port':5480}