hiplab/data-transport
hiplab
/
data-transport
mirror of https://github.com/lnyemba/data-transport
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

bug fixes and documentation

This commit is contained in:
Steve Nyemba 2024-04-24 13:00:03 -05:00
parent 67b91b43ab
commit 5adbb5a61e
2 changed files with 10 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
README.md
View File

@ -4,12 +4,13 @@ This project implements an abstraction of objects that can have access to a vari
# Why Use Data-Transport ? # Why Use Data-Transport ?
Mostly data scientists that don't really care about the underlying database and would like a simple and consistent way to read/write data and have will be well served. Additionally we implemented lightweight Extract Transform Loading API and command line (CLI) tool. Mostly data scientists that don't really care about the underlying database and would like a simple and consistent way to read/write and move data are well served. Additionally we implemented lightweight Extract Transform Loading API and command line (CLI) tool. Finally it is possible to add pre/post processing pipeline functions to read/write
1. Familiarity with **pandas data-frames** 1. Familiarity with **pandas data-frames**
2. Connectivity **drivers** are included 2. Connectivity **drivers** are included
3. Mining data from various sources 3. Mining data from various sources
4. Useful for data migrations or ETL 4. Useful for data migrations or **ETL**
## Installation ## Installation

14
transport/iowrapper.py
View File

@ -6,9 +6,9 @@ class IO:
""" """
Base wrapper class for read/write Base wrapper class for read/write
""" """
def __init__(self,_agent,loader): def __init__(self,_agent,plugins):
self._agent = _agent self._agent = _agent
self._loader = loader self._plugins = plugins
def meta (self,**_args): def meta (self,**_args):
if hasattr(self._agent,'meta') : if hasattr(self._agent,'meta') :
return self._agent.meta(**_args) return self._agent.meta(**_args)
@ -21,7 +21,7 @@ class IO:
""" """
applying pre/post conditions given a pipeline expression applying pre/post conditions given a pipeline expression
""" """
for _pointer in self._loader : for _pointer in self._plugins :
_data = _pointer(_data) _data = _pointer(_data)
def apply(self,_query): def apply(self,_query):
if hasattr(self._agent,'apply') : if hasattr(self._agent,'apply') :
@ -32,8 +32,8 @@ class IReader(IO):
super().__init__(_agent,pipeline) super().__init__(_agent,pipeline)
def read(self,**_args): def read(self,**_args):
_data = self._agent.read(**_args) _data = self._agent.read(**_args)
if self._loader and self._loader.ratio() > 0 : if self._plugins and self._plugins.ratio() > 0 :
_data = self._loader.apply(_data) _data = self._plugins.apply(_data)
# #
# output data # output data
return _data return _data
@ -41,7 +41,7 @@ class IWriter(IO):
def __init__(self,_agent,pipeline=None): def __init__(self,_agent,pipeline=None):
super().__init__(_agent,pipeline) super().__init__(_agent,pipeline)
def write(self,_data,**_args): def write(self,_data,**_args):
if self._loader and self._loader.ratio() > 0 : if self._plugins and self._plugins.ratio() > 0 :
_data = self._loader.apply(_data) _data = self._plugins.apply(_data)
self._agent.write(_data,**_args) self._agent.write(_data,**_args)