From b8fb538ec7f243aef1c3e764b6c34367b574f556 Mon Sep 17 00:00:00 2001 From: Steve Nyemba Date: Sun, 13 Nov 2022 15:45:21 -0600 Subject: [PATCH] bug fix mongodb,bigquery --- setup.py | 2 +- transport/mongo.py | 10 +++++----- transport/sql.py | 3 +++ 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index 2e6991e..4c79ec7 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ def read(fname): return open(os.path.join(os.path.dirname(__file__), fname)).read() args = { "name":"data-transport", - "version":"1.6.0", + "version":"1.6.1", "author":"The Phi Technology LLC","author_email":"info@the-phi.com", "license":"MIT", "packages":["transport"]} diff --git a/transport/mongo.py b/transport/mongo.py index eb5f2da..e3dbaff 100644 --- a/transport/mongo.py +++ b/transport/mongo.py @@ -34,7 +34,7 @@ class Mongo : :password password for current user """ - self.authMechanism= 'SCRAM-SHA-256' if 'mechanism' not in args else args['mechanism'] + self.mechanism= 'SCRAM-SHA-256' if 'mechanism' not in args else args['mechanism'] # authSource=(args['authSource'] if 'authSource' in args else self.dbname) self._lock = False if 'lock' not in args else args['lock'] @@ -64,7 +64,7 @@ class Mongo : username=username, password=password , authSource=self.authSource, - authMechanism=self.authMechanism) + authMechanism=self.mechanism) else: self.client = MongoClient(self.host,maxPoolSize=10000) @@ -76,7 +76,7 @@ class Mongo : q = self.uid in self.client[self.dbname].list_collection_names() return p and q def setattr(self,key,value): - _allowed = ['host','port','db','doc','authSource'] + _allowed = ['host','port','db','doc','authSource','mechanism'] if key in _allowed : setattr(self,key,value) pass @@ -92,10 +92,10 @@ class MongoReader(Mongo,Reader): Mongo.__init__(self,**args) def read(self,**args): - if 'mongo' in args : + if 'mongo' in args or 'cmd' : # # @TODO: - cmd = args['mongo'] + cmd = args['mongo'] if 'mongo' in args else args['cmd'] if "aggregate" in cmd : if "allowDiskUse" not in cmd : cmd["allowDiskUse"] = True diff --git a/transport/sql.py b/transport/sql.py index 2f4d96f..9d278a3 100644 --- a/transport/sql.py +++ b/transport/sql.py @@ -430,6 +430,9 @@ class BQWriter(BigQuery,Writer): self.mode['destination_table'] = _args['table'].strip() if 'schema' in _args : self.mode['table_schema'] = _args['schema'] + # + # Let us insure that the types are somewhat compatible ... + # _map = {'INTEGER':np.int64,'DATETIME':'datetime64[ns]','TIMESTAMP':'datetime64[ns]','FLOAT':np.float64,'DOUBLE':np.float64,'STRING':str} # _mode = copy.deepcopy(self.mode) _mode = self.mode # _df.to_gbq(**self.mode) #if_exists='append',destination_table=partial,credentials=credentials,chunksize=90000)