bug fix mongodb,bigquery
This commit is contained in:
parent
2c675d5656
commit
b8fb538ec7
2
setup.py
2
setup.py
|
@ -8,7 +8,7 @@ def read(fname):
|
|||
return open(os.path.join(os.path.dirname(__file__), fname)).read()
|
||||
args = {
|
||||
"name":"data-transport",
|
||||
"version":"1.6.0",
|
||||
"version":"1.6.1",
|
||||
"author":"The Phi Technology LLC","author_email":"info@the-phi.com",
|
||||
"license":"MIT",
|
||||
"packages":["transport"]}
|
||||
|
|
|
@ -34,7 +34,7 @@ class Mongo :
|
|||
:password password for current user
|
||||
"""
|
||||
|
||||
self.authMechanism= 'SCRAM-SHA-256' if 'mechanism' not in args else args['mechanism']
|
||||
self.mechanism= 'SCRAM-SHA-256' if 'mechanism' not in args else args['mechanism']
|
||||
# authSource=(args['authSource'] if 'authSource' in args else self.dbname)
|
||||
self._lock = False if 'lock' not in args else args['lock']
|
||||
|
||||
|
@ -64,7 +64,7 @@ class Mongo :
|
|||
username=username,
|
||||
password=password ,
|
||||
authSource=self.authSource,
|
||||
authMechanism=self.authMechanism)
|
||||
authMechanism=self.mechanism)
|
||||
|
||||
else:
|
||||
self.client = MongoClient(self.host,maxPoolSize=10000)
|
||||
|
@ -76,7 +76,7 @@ class Mongo :
|
|||
q = self.uid in self.client[self.dbname].list_collection_names()
|
||||
return p and q
|
||||
def setattr(self,key,value):
|
||||
_allowed = ['host','port','db','doc','authSource']
|
||||
_allowed = ['host','port','db','doc','authSource','mechanism']
|
||||
if key in _allowed :
|
||||
setattr(self,key,value)
|
||||
pass
|
||||
|
@ -92,10 +92,10 @@ class MongoReader(Mongo,Reader):
|
|||
Mongo.__init__(self,**args)
|
||||
def read(self,**args):
|
||||
|
||||
if 'mongo' in args :
|
||||
if 'mongo' in args or 'cmd' :
|
||||
#
|
||||
# @TODO:
|
||||
cmd = args['mongo']
|
||||
cmd = args['mongo'] if 'mongo' in args else args['cmd']
|
||||
if "aggregate" in cmd :
|
||||
if "allowDiskUse" not in cmd :
|
||||
cmd["allowDiskUse"] = True
|
||||
|
|
|
@ -430,6 +430,9 @@ class BQWriter(BigQuery,Writer):
|
|||
self.mode['destination_table'] = _args['table'].strip()
|
||||
if 'schema' in _args :
|
||||
self.mode['table_schema'] = _args['schema']
|
||||
#
|
||||
# Let us insure that the types are somewhat compatible ...
|
||||
# _map = {'INTEGER':np.int64,'DATETIME':'datetime64[ns]','TIMESTAMP':'datetime64[ns]','FLOAT':np.float64,'DOUBLE':np.float64,'STRING':str}
|
||||
# _mode = copy.deepcopy(self.mode)
|
||||
_mode = self.mode
|
||||
# _df.to_gbq(**self.mode) #if_exists='append',destination_table=partial,credentials=credentials,chunksize=90000)
|
||||
|
|
Loading…
Reference in New Issue