hiplab/data-transport
hiplab
/
data-transport
mirror of https://github.com/lnyemba/data-transport
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

bug fix mongodb,bigquery

This commit is contained in:
Steve Nyemba 2022-11-13 15:45:21 -06:00
parent 2c675d5656
commit b8fb538ec7
3 changed files with 9 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
setup.py
View File

@ -8,7 +8,7 @@ def read(fname):
return open(os.path.join(os.path.dirname(__file__), fname)).read() return open(os.path.join(os.path.dirname(__file__), fname)).read()
args = { args = {
"name":"data-transport", "name":"data-transport",
"version":"1.6.0", "version":"1.6.1",
"author":"The Phi Technology LLC","author_email":"info@the-phi.com", "author":"The Phi Technology LLC","author_email":"info@the-phi.com",
"license":"MIT", "license":"MIT",
"packages":["transport"]} "packages":["transport"]}

10
transport/mongo.py
View File

@ -34,7 +34,7 @@ class Mongo :
:password password for current user :password password for current user
""" """
self.authMechanism= 'SCRAM-SHA-256' if 'mechanism' not in args else args['mechanism'] self.mechanism= 'SCRAM-SHA-256' if 'mechanism' not in args else args['mechanism']
# authSource=(args['authSource'] if 'authSource' in args else self.dbname) # authSource=(args['authSource'] if 'authSource' in args else self.dbname)
self._lock = False if 'lock' not in args else args['lock'] self._lock = False if 'lock' not in args else args['lock']
@ -64,7 +64,7 @@ class Mongo :
username=username, username=username,
password=password , password=password ,
authSource=self.authSource, authSource=self.authSource,
authMechanism=self.authMechanism) authMechanism=self.mechanism)
else: else:
self.client = MongoClient(self.host,maxPoolSize=10000) self.client = MongoClient(self.host,maxPoolSize=10000)
@ -76,7 +76,7 @@ class Mongo :
q = self.uid in self.client[self.dbname].list_collection_names() q = self.uid in self.client[self.dbname].list_collection_names()
return p and q return p and q
def setattr(self,key,value): def setattr(self,key,value):
_allowed = ['host','port','db','doc','authSource'] _allowed = ['host','port','db','doc','authSource','mechanism']
if key in _allowed : if key in _allowed :
setattr(self,key,value) setattr(self,key,value)
pass pass
@ -92,10 +92,10 @@ class MongoReader(Mongo,Reader):
Mongo.__init__(self,**args) Mongo.__init__(self,**args)
def read(self,**args): def read(self,**args):
if 'mongo' in args : if 'mongo' in args or 'cmd' :
# #
# @TODO: # @TODO:
cmd = args['mongo'] cmd = args['mongo'] if 'mongo' in args else args['cmd']
if "aggregate" in cmd : if "aggregate" in cmd :
if "allowDiskUse" not in cmd : if "allowDiskUse" not in cmd :
cmd["allowDiskUse"] = True cmd["allowDiskUse"] = True

3
transport/sql.py
View File

@ -430,6 +430,9 @@ class BQWriter(BigQuery,Writer):
self.mode['destination_table'] = _args['table'].strip() self.mode['destination_table'] = _args['table'].strip()
if 'schema' in _args : if 'schema' in _args :
self.mode['table_schema'] = _args['schema'] self.mode['table_schema'] = _args['schema']
#
# Let us insure that the types are somewhat compatible ...
# _map = {'INTEGER':np.int64,'DATETIME':'datetime64[ns]','TIMESTAMP':'datetime64[ns]','FLOAT':np.float64,'DOUBLE':np.float64,'STRING':str}
# _mode = copy.deepcopy(self.mode) # _mode = copy.deepcopy(self.mode)
_mode = self.mode _mode = self.mode
# _df.to_gbq(**self.mode) #if_exists='append',destination_table=partial,credentials=credentials,chunksize=90000) # _df.to_gbq(**self.mode) #if_exists='append',destination_table=partial,credentials=credentials,chunksize=90000)