data-transport/transport/__init__.py

"""
Data Transport, The Phi Technology LLC
Steve L. Nyemba, steve@the-phi.com

This library is designed to serve as a wrapper to a set of supported data stores :
    - couchdb
    - mongodb
    - Files (character delimited)
    - Queues (RabbmitMq)
    - Session (Flask)
    - s3
	- sqlite
The supported operations are read/write and providing meta data to the calling code
We separated reads from writes to mitigate accidents associated with writes.
Source Code is available under MIT License:    
    https://healthcareio.the-phi.com/data-transport
    https://hiplab.mc.vanderbilt.edu/git/hiplab/data-transport
"""
import numpy as np

import sql, nosql, cloud, other
import pandas as pd
import json
import os
from info import __version__,__author__
from transport.iowrapper import IWriter, IReader
from transport.plugins import PluginLoader
from transport import providers
PROVIDERS = {}
def init():
    global PROVIDERS
    for _module in [cloud,sql,nosql,other] :
        for _provider_name in dir(_module) :
            if _provider_name.startswith('__') :
                continue
            PROVIDERS[_provider_name] = {'module':getattr(_module,_provider_name),'type':_module.__name__}

def instance (**_args):
    """
    type:
    read: true|false (default true)
    auth_file
    """
    global PROVIDERS
    if 'auth_file' in _args:
        if os.path.exists(_args['auth_file']) :
            f = open(_args['auth_file'])
            _args = dict (_args,** json.loads(f.read()) )
            f.close()
        else:
            filename = _args['auth_file']
            raise Exception(f" {filename} was not found or is invalid")
    if _args['provider'] in PROVIDERS :
        _info = PROVIDERS[_args['provider']]
        _module = _info['module']
        if 'context' in _args :
            _context = _args['context']
        else:
            _context = 'read'
        _pointer    = getattr(_module,'Reader') if _context == 'read' else getattr(_module,'Writer')
        _agent      = _pointer (**_args)
        #
        loader = None
        if 'plugins' in _args :
            _params = _args['plugins']

            if 'path' in _params and 'names' in _params :
                loader = PluginLoader(**_params)
            elif type(_params) == list:
                loader = PluginLoader()
                for _delegate in _params :
                    loader.set(_delegate)


        return IReader(_agent,loader) if _context == 'read' else IWriter(_agent,loader)

    else:
        raise Exception ("Missing or Unknown provider")
    pass
def supported ():
    _info = {}
    for _provider in PROVIDERS :
        _item = PROVIDERS[_provider]
        if _item['type'] not in _info :
            _info[_item['type']] = []
        _info[_item['type']].append(_provider)
    _df  = pd.DataFrame()
    for _id in _info :
        if not _df.shape[0] :
            _df = pd.DataFrame(_info[_id],columns=[_id.replace('transport.','')])
        else:
            _df = pd.DataFrame(_info[_id],columns=[_id.replace('transport.','')]).join(_df, how='outer')
    return _df.fillna('')
class factory :
    pass
factory.instance = instance
init()