hiplab/data-transport
hiplab
/
data-transport
mirror of https://github.com/lnyemba/data-transport
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

bug fix: lock, parallel processing bigquery

This commit is contained in:
Steve Nyemba 2021-04-13 17:27:23 -05:00
parent afcc5ed690
commit 84e212d5a3
2 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
setup.py
View File

@ -8,7 +8,7 @@ def read(fname):
return open(os.path.join(os.path.dirname(__file__), fname)).read() return open(os.path.join(os.path.dirname(__file__), fname)).read()
args = { args = {
"name":"data-transport", "name":"data-transport",
"version":"1.3.8.1", "version":"1.3.8.2",
"author":"The Phi Technology LLC","author_email":"info@the-phi.com", "author":"The Phi Technology LLC","author_email":"info@the-phi.com",
"license":"MIT", "license":"MIT",
"packages":["transport"]} "packages":["transport"]}

4
transport/sql.py
View File

@ -238,11 +238,11 @@ class BQWriter(BigQuery,Writer):
def write(self,_info,**_args) : def write(self,_info,**_args) :
try: try:
if self.parallel : if self.parallel :
self.lock.acquire() BQWriter.lock.acquire()
self._write(_info,**_args) self._write(_info,**_args)
finally: finally:
if self.parallel: if self.parallel:
self.lock.release() BQWriter.lock.release()
def _write(self,_info,**_args) : def _write(self,_info,**_args) :
_df = None _df = None
if type(_info) in [list,pd.DataFrame] : if type(_info) in [list,pd.DataFrame] :