diff --git a/transport/version.py b/info/__init__.py similarity index 100% rename from transport/version.py rename to info/__init__.py diff --git a/setup.py b/setup.py index c322c38..9eb2c3e 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,10 @@ This is a build file for the from setuptools import setup, find_packages import os import sys -from version import __version__,__author__ +# from version import __version__,__author__ +from info import __version__, __author__ + + # __author__ = 'The Phi Technology' # __version__= '1.8.0' @@ -15,7 +18,7 @@ args = { "version":__version__, "author":__author__,"author_email":"info@the-phi.com", "license":"MIT", - "packages":["transport"]} + "packages":["transport","info"]} args["keywords"]=['mongodb','couchdb','rabbitmq','file','read','write','s3','sqlite'] args["install_requires"] = ['pyncclient','pymongo','sqlalchemy<2.0.0','pandas','typer','pandas-gbq','numpy','cloudant','pika','nzpy','boto3','boto','pyarrow','google-cloud-bigquery','google-cloud-bigquery-storage','flask-session','smart_open','botocore','psycopg2-binary','mysql-connector-python'] args["url"] = "https://healthcareio.the-phi.com/git/code/transport.git" diff --git a/transport/__init__.py b/transport/__init__.py index 234c418..7b37a12 100644 --- a/transport/__init__.py +++ b/transport/__init__.py @@ -21,8 +21,8 @@ The configuration for the data-store is as follows : provider:'mongodb',[port:27017],[host:localhost],db:,doc:<_name>,context: """ -import pandas as pd -import numpy as np +# import pandas as pd +# import numpy as np import json import importlib import sys @@ -38,7 +38,8 @@ if sys.version_info[0] > 2 : from transport import mongo as mongo from transport import sql as sql from transport import etl as etl - from transport.version import __version__ + # from transport.version import __version__ + from info import __version__,__author__ from transport import providers else: from common import Reader, Writer,Console #, factory @@ -51,10 +52,10 @@ else: import etl from version import __version__ import providers -import psycopg2 as pg -import mysql.connector as my -from google.cloud import bigquery as bq -import nzpy as nz #--- netezza drivers +# import psycopg2 as pg +# import mysql.connector as my +# from google.cloud import bigquery as bq +# import nzpy as nz #--- netezza drivers import os # class providers : @@ -85,42 +86,42 @@ import os # import providers # class IEncoder (json.JSONEncoder): -def IEncoder (self,object): - if type(object) == np.integer : - return int(object) - elif type(object) == np.floating: - return float(object) - elif type(object) == np.ndarray : - return object.tolist() - elif type(object) == datetime : - return o.isoformat() - else: - return super(IEncoder,self).default(object) +# def IEncoder (self,object): +# if type(object) == np.integer : +# return int(object) +# elif type(object) == np.floating: +# return float(object) +# elif type(object) == np.ndarray : +# return object.tolist() +# elif type(object) == datetime : +# return o.isoformat() +# else: +# return super(IEncoder,self).default(object) class factory : - TYPE = {"sql":{"providers":["postgresql","mysql","neteeza","bigquery","mariadb","redshift"]}} - PROVIDERS = { - "etl":{"class":{"read":etl.instance,"write":etl.instance}}, - # "console":{"class":{"write":Console,"read":Console}}, - "file":{"class":{"read":disk.DiskReader,"write":disk.DiskWriter}}, - "sqlite":{"class":{"read":disk.SQLiteReader,"write":disk.SQLiteWriter}}, - "postgresql":{"port":5432,"host":"localhost","database":None,"driver":pg,"default":{"type":"VARCHAR"},"class":{"read":sql.SQLReader,"write":sql.SQLWriter}}, - "redshift":{"port":5432,"host":"localhost","database":None,"driver":pg,"default":{"type":"VARCHAR"},"class":{"read":sql.SQLReader,"write":sql.SQLWriter}}, - "bigquery":{"class":{"read":sql.BQReader,"write":sql.BQWriter}}, - "mysql":{"port":3306,"host":"localhost","default":{"type":"VARCHAR(256)"},"driver":my,"class":{"read":sql.SQLReader,"write":sql.SQLWriter}}, - "mariadb":{"port":3306,"host":"localhost","default":{"type":"VARCHAR(256)"},"driver":my,"class":{"read":sql.SQLReader,"write":sql.SQLWriter}}, - "mongo":{"port":27017,"host":"localhost","class":{"read":mongo.MongoReader,"write":mongo.MongoWriter}}, - "couch":{"port":5984,"host":"localhost","class":{"read":couch.CouchReader,"write":couch.CouchWriter}}, - "netezza":{"port":5480,"driver":nz,"default":{"type":"VARCHAR(256)"},"class":{"read":sql.SQLReader,"write":sql.SQLWriter}}, - "rabbitmq":{"port":5672,"host":"localhost","class":{"read":queue.QueueReader,"write":queue.QueueWriter,"listen":queue.QueueListener,"listener":queue.QueueListener},"default":{"type":"application/json"}}} - # - # creating synonyms - PROVIDERS['mongodb'] = PROVIDERS['mongo'] - PROVIDERS['couchdb'] = PROVIDERS['couch'] - PROVIDERS['bq'] = PROVIDERS['bigquery'] - PROVIDERS['sqlite3'] = PROVIDERS['sqlite'] - PROVIDERS['rabbit'] = PROVIDERS['rabbitmq'] - PROVIDERS['rabbitmq-server'] = PROVIDERS['rabbitmq'] + # TYPE = {"sql":{"providers":["postgresql","mysql","neteeza","bigquery","mariadb","redshift"]}} + # PROVIDERS = { + # "etl":{"class":{"read":etl.instance,"write":etl.instance}}, + # # "console":{"class":{"write":Console,"read":Console}}, + # "file":{"class":{"read":disk.DiskReader,"write":disk.DiskWriter}}, + # "sqlite":{"class":{"read":disk.SQLiteReader,"write":disk.SQLiteWriter}}, + # "postgresql":{"port":5432,"host":"localhost","database":None,"driver":pg,"default":{"type":"VARCHAR"},"class":{"read":sql.SQLReader,"write":sql.SQLWriter}}, + # "redshift":{"port":5432,"host":"localhost","database":None,"driver":pg,"default":{"type":"VARCHAR"},"class":{"read":sql.SQLReader,"write":sql.SQLWriter}}, + # "bigquery":{"class":{"read":sql.BQReader,"write":sql.BQWriter}}, + # "mysql":{"port":3306,"host":"localhost","default":{"type":"VARCHAR(256)"},"driver":my,"class":{"read":sql.SQLReader,"write":sql.SQLWriter}}, + # "mariadb":{"port":3306,"host":"localhost","default":{"type":"VARCHAR(256)"},"driver":my,"class":{"read":sql.SQLReader,"write":sql.SQLWriter}}, + # "mongo":{"port":27017,"host":"localhost","class":{"read":mongo.MongoReader,"write":mongo.MongoWriter}}, + # "couch":{"port":5984,"host":"localhost","class":{"read":couch.CouchReader,"write":couch.CouchWriter}}, + # "netezza":{"port":5480,"driver":nz,"default":{"type":"VARCHAR(256)"},"class":{"read":sql.SQLReader,"write":sql.SQLWriter}}, + # "rabbitmq":{"port":5672,"host":"localhost","class":{"read":queue.QueueReader,"write":queue.QueueWriter,"listen":queue.QueueListener,"listener":queue.QueueListener},"default":{"type":"application/json"}}} + # # + # # creating synonyms + # PROVIDERS['mongodb'] = PROVIDERS['mongo'] + # PROVIDERS['couchdb'] = PROVIDERS['couch'] + # PROVIDERS['bq'] = PROVIDERS['bigquery'] + # PROVIDERS['sqlite3'] = PROVIDERS['sqlite'] + # PROVIDERS['rabbit'] = PROVIDERS['rabbitmq'] + # PROVIDERS['rabbitmq-server'] = PROVIDERS['rabbitmq'] @staticmethod def instance(**_args): diff --git a/transport/mongo.py b/transport/mongo.py index bac1780..b4338b4 100644 --- a/transport/mongo.py +++ b/transport/mongo.py @@ -15,12 +15,26 @@ import gridfs # from transport import Reader,Writer import sys if sys.version_info[0] > 2 : - from transport.common import Reader, Writer, IEncoder + from transport.common import Reader, Writer else: from common import Reader, Writer import json import re from multiprocessing import Lock, RLock + +def IEncoder (self,object): + if type(object) == np.integer : + return int(object) + elif type(object) == np.floating: + return float(object) + elif type(object) == np.ndarray : + return object.tolist() + elif type(object) == datetime : + return o.isoformat() + else: + return super(IEncoder,self).default(object) + + class Mongo : lock = RLock() """ diff --git a/version.py b/version.py deleted file mode 120000 index e666b28..0000000 --- a/version.py +++ /dev/null @@ -1 +0,0 @@ -transport/version.py \ No newline at end of file