hiplab/data-transport
hiplab
/
data-transport
mirror of https://github.com/lnyemba/data-transport
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

bug fix: close & etl

This commit is contained in:
Steve Nyemba 2025-02-02 22:36:28 -06:00
parent c3627586b3
commit 49ebd4a432
2 changed files with 32 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
transport/iowrapper.py
View File

@ -109,8 +109,10 @@ class IETL(IReader) :
self._hasParentProcess = False if 'hasParentProcess' not in _args else _args['hasParentProcess'] self._hasParentProcess = False if 'hasParentProcess' not in _args else _args['hasParentProcess']
def read(self,**_args): def read(self,**_args):
_data = super().read(**_args) _data = super().read(**_args)
_schema = super().meta()
for _kwargs in self._targets : for _kwargs in self._targets :
if _schema :
_kwargs['schema'] = _schema
self.post(_data,**_kwargs) self.post(_data,**_kwargs)
return _data return _data
@ -122,5 +124,8 @@ class IETL(IReader) :
:_args parameters associated with writer object :_args parameters associated with writer object
""" """
writer = transport.get.writer(**_args) writer = transport.get.writer(**_args)
writer.write(_data) if 'schema' in _args :
writer.write(_data,schema=_args['schema'])
else:
writer.write(_data)
writer.close() writer.close()

40
transport/sql/common.py
View File

@ -3,7 +3,7 @@ This file encapsulates common operations associated with SQL databases via SQLAl
""" """
import sqlalchemy as sqa import sqlalchemy as sqa
from sqlalchemy import text from sqlalchemy import text , MetaData, inspect
import pandas as pd import pandas as pd
@ -34,20 +34,26 @@ class Base:
:table optional name of the table (can be fully qualified) :table optional name of the table (can be fully qualified)
""" """
_table = self._table if 'table' not in _args else _args['table'] _table = self._table if 'table' not in _args else _args['table']
_map = {'TINYINT':'INTEGER','BIGINT':'INTEGER','TEXT':'STRING','DOUBLE_PRECISION':'FLOAT','NUMERIC':'FLOAT','DECIMAL':'FLOAT','REAL':'FLOAT'}
_schema = [] _schema = []
if _table : # if _table :
if sqa.__version__.startswith('1.') : # if sqa.__version__.startswith('1.') :
_handler = sqa.MetaData(bind=self._engine) # _handler = sqa.MetaData(bind=self._engine)
_handler.reflect() # _handler.reflect()
else: # else:
# # #
# sqlalchemy's version 2.+ # # sqlalchemy's version 2.+
_handler = sqa.MetaData() # _handler = sqa.MetaData()
_handler.reflect(bind=self._engine) # _handler.reflect(bind=self._engine)
# # #
# Let us extract the schema with the native types # # Let us extract the schema with the native types
_map = {'BIGINT':'INTEGER','TEXT':'STRING','DOUBLE_PRECISION':'FLOAT','NUMERIC':'FLOAT','DECIMAL':'FLOAT','REAL':'FLOAT'} # _map = {'BIGINT':'INTEGER','TEXT':'STRING','DOUBLE_PRECISION':'FLOAT','NUMERIC':'FLOAT','DECIMAL':'FLOAT','REAL':'FLOAT'}
_schema = [{"name":_attr.name,"type":_map.get(str(_attr.type),str(_attr.type))} for _attr in _handler.tables[_table].columns] # _schema = [{"name":_attr.name,"type":_map.get(str(_attr.type),str(_attr.type))} for _attr in _handler.tables[_table].columns]
#
_inspector = inspect(self._engine)
_columns = _inspector.get_columns(_table)
_schema = [{'name':column['name'],'type':_map.get(str(column['type']),str(column['type'])) } for column in _columns]
return _schema return _schema
def has(self,**_args): def has(self,**_args):
return self.meta(**_args) return self.meta(**_args)
@ -94,7 +100,11 @@ class SQLBase(Base):
# _uri = [_item.strip() for _item in _uri if _item.strip()] # _uri = [_item.strip() for _item in _uri if _item.strip()]
# return '/'.join(_uri) # return '/'.join(_uri)
return f'{_provider}://{_host}/{_database}' if _account == '' else f'{_provider}://{_account}{_host}/{_database}' return f'{_provider}://{_host}/{_database}' if _account == '' else f'{_provider}://{_account}{_host}/{_database}'
def close(self,) :
try:
self._engine.dispose()
except :
pass
class BaseReader(SQLBase): class BaseReader(SQLBase):
def __init__(self,**_args): def __init__(self,**_args):
super().__init__(**_args) super().__init__(**_args)