hiplab/data-transport
hiplab
/
data-transport
mirror of https://github.com/lnyemba/data-transport
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

bug fix: environment variable on default database

This commit is contained in:
Steve Nyemba 2022-06-25 13:59:49 -05:00
parent 3494da8c6e
commit 8ca04e8c48
4 changed files with 57 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
setup.py
View File

@ -8,7 +8,7 @@ def read(fname):
return open(os.path.join(os.path.dirname(__file__), fname)).read()
args = {
"name":"data-transport",
"version":"1.5.5",
"version":"1.5.6",
"author":"The Phi Technology LLC","author_email":"info@the-phi.com",
"license":"MIT",
"packages":["transport"]}

4
transport/__init__.py
View File

@ -61,8 +61,8 @@ class factory :
"console":{"class":{"write":Console,"read":Console}},
"file":{"class":{"read":disk.DiskReader,"write":disk.DiskWriter}},
"sqlite":{"class":{"read":disk.SQLiteReader,"write":disk.SQLiteWriter}},
"postgresql":{"port":5432,"host":"localhost","database":os.environ['USER'],"driver":pg,"default":{"type":"VARCHAR"},"class":{"read":sql.SQLReader,"write":sql.SQLWriter}},
"redshift":{"port":5432,"host":"localhost","database":os.environ['USER'],"driver":pg,"default":{"type":"VARCHAR"},"class":{"read":sql.SQLReader,"write":sql.SQLWriter}},
"postgresql":{"port":5432,"host":"localhost","driver":pg,"default":{"type":"VARCHAR"},"class":{"read":sql.SQLReader,"write":sql.SQLWriter}},
"redshift":{"port":5432,"host":"localhost","driver":pg,"default":{"type":"VARCHAR"},"class":{"read":sql.SQLReader,"write":sql.SQLWriter}},
"bigquery":{"class":{"read":sql.BQReader,"write":sql.BQWriter}},
"mysql":{"port":3306,"host":"localhost","default":{"type":"VARCHAR(256)"},"driver":my,"class":{"read":sql.SQLReader,"write":sql.SQLWriter}},
"mariadb":{"port":3306,"host":"localhost","default":{"type":"VARCHAR(256)"},"driver":my,"class":{"read":sql.SQLReader,"write":sql.SQLWriter}},

2
transport/common.py
View File

@ -98,7 +98,7 @@ class Console(Writer):
self.debug = self.write
self.log = self.write
pass
def write (self,**_args):
def write (self,logs,**_args):
if self.lock :
Console.lock.acquire()
try:

73
transport/mongo.py
View File

@ -33,37 +33,59 @@ class Mongo :
:username username for authentication
:password password for current user
"""
port = str(args['port']) if 'port' in args else '27017'
host = args['host'] if 'host' in args else 'localhost'
host = ":".join([host,port]) #-- Formatting host information here
self.uid = args['doc'] if 'doc' in args else None #-- document identifier
self.dbname = args['dbname'] if 'dbname' in args else args['db']
authMechanism= 'SCRAM-SHA-256' if 'mechanism' not in args else args['mechanism']
authSource=(args['authSource'] if 'authSource' in args else self.dbname)
# port = str(args['port']) if 'port' in args else '27017'
# host = args['host'] if 'host' in args else 'localhost'
# host = ":".join([host,port]) #-- Formatting host information here
# self.uid = args['doc'] if 'doc' in args else None #-- document identifier
# self.dbname = args['dbname'] if 'dbname' in args else args['db']
self.authMechanism= 'SCRAM-SHA-256' if 'mechanism' not in args else args['mechanism']
# authSource=(args['authSource'] if 'authSource' in args else self.dbname)
self._lock = False if 'lock' not in args else args['lock']
username = password = None
if 'username' in args and 'password' in args:
username = args['username']
password=args['password']
# if 'username' in args and 'password' in args:
# username = args['username']
# password=args['password']
if 'auth_file' in args :
_info = json.loads((open(args['auth_file'])).read())
username = _info['username']
password = _info['password']
if 'mechanism' in _info:
authMechanism = _info['mechanism']
if 'authSource' in _info:
authSource = _info['authSource']
# username = _info['username']
# password = _info['password']
# if 'mechanism' in _info:
# authMechanism = _info['mechanism']
# if 'authSource' in _info:
# authSource = _info['authSource']
# #
# # We are allowing the authentication file to set collection and databases too
# if 'db' in _info :
# self.dbname = _info['db']
# if 'doc' in _info :
# self.uid = _info['doc']
else:
_info = {}
_args = dict(args,**_info)
for key in _args :
if key in ['username','password'] :
username = _args['username'] if key=='username' else username
password = _args['password'] if key == 'password' else password
continue
value = _args[key]
self.setattr(key,value)
#
# Let us perform aliasing in order to remain backwards compatible
self.dbname = self.db if hasattr(self,'db')else self.dbname
self.uid = _args['table'] if 'table' in _args else (_args['doc'] if 'doc' in _args else (_args['collection'] if 'collection' in _args else None))
if username and password :
self.client = MongoClient(host,
self.client = MongoClient(self.host,
username=username,
password=password ,
authSource=authSource,
authMechanism=authMechanism)
authSource=self.authSource,
authMechanism=self.authMechanism)
else:
self.client = MongoClient(host,maxPoolSize=10000)
self.client = MongoClient(self.host,maxPoolSize=10000)
self.db = self.client[self.dbname]
@ -71,6 +93,11 @@ class Mongo :
p = self.dbname in self.client.list_database_names()
q = self.uid in self.client[self.dbname].list_collection_names()
return p and q
def setattr(self,key,value):
_allowed = ['host','port','db','doc','authSource']
if key in _allowed :
setattr(self,key,value)
pass
def close(self):
self.client.close()
@ -110,7 +137,9 @@ class MongoReader(Mongo,Reader):
else:
collection = self.db[self.uid]
_filter = args['filter'] if 'filter' in args else {}
return collection.find(_filter)
_df = pd.DataFrame(collection.find(_filter))
columns = _df.columns.tolist()[1:]
return _df[columns]
def view(self,**args):
"""
This function is designed to execute a view (map/reduce) operation
@ -162,7 +191,7 @@ class MongoWriter(Mongo,Writer):
@param info new record in the collection to be added
"""
# document = self.db[self.uid].find()
collection = self.db[self.uid]
#collection = self.db[self.uid]
# if type(info) == list :
# self.db[self.uid].insert_many(info)
# else: