new: authentication using file on disk, misc bug fixes

This commit is contained in:
Steve Nyemba 2022-05-16 11:27:36 -05:00
parent 8cd34d902a
commit f64b945245
7 changed files with 106 additions and 81 deletions

View File

@ -8,7 +8,7 @@ def read(fname):
return open(os.path.join(os.path.dirname(__file__), fname)).read() return open(os.path.join(os.path.dirname(__file__), fname)).read()
args = { args = {
"name":"data-transport", "name":"data-transport",
"version":"1.5.0", "version":"1.5.2",
"author":"The Phi Technology LLC","author_email":"info@the-phi.com", "author":"The Phi Technology LLC","author_email":"info@the-phi.com",
"license":"MIT", "license":"MIT",
"packages":["transport"]} "packages":["transport"]}

View File

@ -57,25 +57,27 @@ import os
class factory : class factory :
TYPE = {"sql":{"providers":["postgresql","mysql","neteeza","bigquery","mariadb","redshift"]}} TYPE = {"sql":{"providers":["postgresql","mysql","neteeza","bigquery","mariadb","redshift"]}}
PROVIDERS = { PROVIDERS = {
"etl":{"class":{"read":etl.instance}}, "etl":{"class":{"read":etl.instance,"write":etl.instance}},
"console":{"class":{"write":Console,"read":Console}}, "console":{"class":{"write":Console,"read":Console}},
"file":{"class":{"read":disk.DiskReader,"write":disk.DiskWriter}}, "file":{"class":{"read":disk.DiskReader,"write":disk.DiskWriter}},
"sqlite":{"class":{"read":disk.SQLiteReader,"write":disk.SQLiteWriter}}, "sqlite":{"class":{"read":disk.SQLiteReader,"write":disk.SQLiteWriter}},
"postgresql":{"port":5432,"host":"localhost","database":os.environ['USER'],"driver":pg,"default":{"type":"VARCHAR"}}, "postgresql":{"port":5432,"host":"localhost","database":os.environ['USER'],"driver":pg,"default":{"type":"VARCHAR"},"class":{"read":sql.SQLReader,"write":sql.SQLWriter}},
"redshift":{"port":5432,"host":"localhost","database":os.environ['USER'],"driver":pg,"default":{"type":"VARCHAR"}}, "redshift":{"port":5432,"host":"localhost","database":os.environ['USER'],"driver":pg,"default":{"type":"VARCHAR"},"class":{"read":sql.SQLReader,"write":sql.SQLWriter}},
"bigquery":{"class":{"read":sql.BQReader,"write":sql.BQWriter}}, "bigquery":{"class":{"read":sql.BQReader,"write":sql.BQWriter}},
"mysql":{"port":3306,"host":"localhost","default":{"type":"VARCHAR(256)"},"driver":my}, "mysql":{"port":3306,"host":"localhost","default":{"type":"VARCHAR(256)"},"driver":my,"class":{"read":sql.SQLReader,"write":sql.SQLWriter}},
"mariadb":{"port":3306,"host":"localhost","default":{"type":"VARCHAR(256)"},"driver":my}, "mariadb":{"port":3306,"host":"localhost","default":{"type":"VARCHAR(256)"},"driver":my,"class":{"read":sql.SQLReader,"write":sql.SQLWriter}},
"mongo":{"port":27017,"host":"localhost","class":{"read":mongo.MongoReader,"write":mongo.MongoWriter}}, "mongo":{"port":27017,"host":"localhost","class":{"read":mongo.MongoReader,"write":mongo.MongoWriter}},
"couch":{"port":5984,"host":"localhost","class":{"read":couch.CouchReader,"write":couch.CouchWriter}}, "couch":{"port":5984,"host":"localhost","class":{"read":couch.CouchReader,"write":couch.CouchWriter}},
"netezza":{"port":5480,"driver":nz,"default":{"type":"VARCHAR(256)"}}, "netezza":{"port":5480,"driver":nz,"default":{"type":"VARCHAR(256)"},"class":{"read":sql.SQLReader,"write":sql.SQLWriter}},
"rabbitmq":{"port":5672,"host":"localhost","class":{"read":queue.QueueReader,"write":queue.QueueWriter,"listen":queue.QueueListener},"default":{"type":"application/json"}}} "rabbitmq":{"port":5672,"host":"localhost","class":{"read":queue.QueueReader,"write":queue.QueueWriter,"listen":queue.QueueListener,"listener":queue.QueueListener},"default":{"type":"application/json"}}}
# #
# creating synonyms # creating synonyms
PROVIDERS['mongodb'] = PROVIDERS['mongo'] PROVIDERS['mongodb'] = PROVIDERS['mongo']
PROVIDERS['couchdb'] = PROVIDERS['couch'] PROVIDERS['couchdb'] = PROVIDERS['couch']
PROVIDERS['bq'] = PROVIDERS['bigquery'] PROVIDERS['bq'] = PROVIDERS['bigquery']
PROVIDERS['sqlite3'] = PROVIDERS['sqlite'] PROVIDERS['sqlite3'] = PROVIDERS['sqlite']
PROVIDERS['rabbit'] = PROVIDERS['rabbitmq']
PROVIDERS['rabbitmq-server'] = PROVIDERS['rabbitmq']
@staticmethod @staticmethod
def instance(**_args): def instance(**_args):

View File

@ -113,32 +113,3 @@ class Console(Writer):
if self.lock : if self.lock :
Console.lock.release() Console.lock.release()
# class factory :
# @staticmethod
# def instance(**args):
# """
# This class will create an instance of a transport when providing
# :type name of the type we are trying to create
# :args The arguments needed to create the instance
# """
# source = args['type']
# params = args['args']
# anObject = None
# if source in ['HttpRequestReader','HttpSessionWriter']:
# #
# # @TODO: Make sure objects are serializable, be smart about them !!
# #
# aClassName = ''.join([source,'(**params)'])
# else:
# stream = json.dumps(params)
# aClassName = ''.join([source,'(**',stream,')'])
# try:
# anObject = eval( aClassName)
# #setattr(anObject,'name',source)
# except Exception,e:
# print ['Error ',e]
# return anObject

View File

@ -41,11 +41,21 @@ class Mongo :
self._lock = False if 'lock' not in args else args['lock'] self._lock = False if 'lock' not in args else args['lock']
if 'user' in args and 'password' in args: username = password = None
if 'username' in args and 'password' in args:
username = args['username']
password=args['password']
if 'auth_file' in args :
_info = json.loads((open(args['auth_file'])).read())
username = _info['username']
password = _info['password']
authSource=(args['authSource'] if 'authSource' in args else self.dbname)
if username and password :
self.client = MongoClient(host, self.client = MongoClient(host,
username=args['username'] , username=username,
password=args['password'] , password=password ,
authSource=(args['authSource'] if 'authSource' in args else self.dbname), authSource=authSource,
authMechanism='SCRAM-SHA-256') authMechanism='SCRAM-SHA-256')
else: else:
self.client = MongoClient(host,maxPoolSize=10000) self.client = MongoClient(host,maxPoolSize=10000)

View File

@ -16,7 +16,7 @@ if sys.version_info[0] > 2 :
else: else:
from common import Reader, Writer from common import Reader, Writer
import json import json
from multiprocessing import RLock
class MessageQueue: class MessageQueue:
""" """
This class hierarchy is designed to handle interactions with a queue server using pika framework (our tests are based on rabbitmq) This class hierarchy is designed to handle interactions with a queue server using pika framework (our tests are based on rabbitmq)
@ -29,12 +29,23 @@ class MessageQueue:
self.port= 5672 if 'port' not in params else params['port'] self.port= 5672 if 'port' not in params else params['port']
self.virtual_host = '/' if 'vhost' not in params else params['vhost'] self.virtual_host = '/' if 'vhost' not in params else params['vhost']
self.exchange = params['exchange'] if 'exchange' in params else 'amq.direct' #-- exchange self.exchange = params['exchange'] if 'exchange' in params else 'amq.direct' #-- exchange
self.queue = params['queue'] self.queue = params['queue'] if 'queue' in params else 'demo'
self.connection = None self.connection = None
self.channel = None self.channel = None
self.name = self.__class__.__name__.lower() if 'name' not in params else 'wtf' self.name = self.__class__.__name__.lower() if 'name' not in params else params['name']
username = password = None
if 'username' in params :
username = params['username']
password = params['password']
if 'auth_file' in params :
_info = json.loads((open(params['auth_file'])).read())
username=_info['username']
password=_info['password']
self.virtual_host = _info['virtual_host'] if 'virtual_host' in _info else self.virtual_host
self.exchange = _info['exchange'] if 'exchange' in _info else self.exchange
self.queue = _info['queue'] if 'queue' in _info else self.queue
self.credentials= pika.PlainCredentials('guest','guest') self.credentials= pika.PlainCredentials('guest','guest')
if 'username' in params : if 'username' in params :
@ -44,7 +55,9 @@ class MessageQueue:
) )
def init(self,label=None): def init(self,label=None):
properties = pika.ConnectionParameters(host=self.host,port=self.port,virtual_host=self.virtual_host,credentials=self.credentials) properties = pika.ConnectionParameters(host=self.host,port=self.port,virtual_host=self.virtual_host,
client_properties={'connection_name':self.name},
credentials=self.credentials)
self.connection = pika.BlockingConnection(properties) self.connection = pika.BlockingConnection(properties)
self.channel = self.connection.channel() self.channel = self.connection.channel()
self.info = self.channel.exchange_declare(exchange=self.exchange,exchange_type='direct',durable=True) self.info = self.channel.exchange_declare(exchange=self.exchange,exchange_type='direct',durable=True)
@ -93,23 +106,7 @@ class QueueWriter(MessageQueue,Writer):
@param object object to be written (will be converted to JSON) @param object object to be written (will be converted to JSON)
@TODO: make this less chatty @TODO: make this less chatty
""" """
# xchar = None
# if 'xchar' in params:
# xchar = params['xchar']
# object = self.format(params['row'],xchar)
# label = params['label']
# self.init(label)
# _mode = 2
# if isinstance(object,str):
# stream = object
# _type = 'text/plain'
# else:
# stream = json.dumps(object)
# if 'type' in params :
# _type = params['type']
# else:
# _type = 'application/json'
stream = json.dumps(data) if isinstance(data,dict) else data stream = json.dumps(data) if isinstance(data,dict) else data
self.channel.basic_publish( self.channel.basic_publish(
exchange=self.exchange, exchange=self.exchange,
@ -143,10 +140,11 @@ class QueueReader(MessageQueue,Reader):
#self.queue = params['qid'] #self.queue = params['qid']
MessageQueue.__init__(self,**params); MessageQueue.__init__(self,**params);
# self.init() # self.init()
if 'durable' in params : self.durable = False if 'durable' not in params else params['durable']
self.durable = True # if 'durable' in params :
else: # self.durable = True
self.durable = False # else:
# self.durable = False
self.size = -1 self.size = -1
self.data = {} self.data = {}
# def init(self,qid): # def init(self,qid):
@ -166,7 +164,8 @@ class QueueReader(MessageQueue,Reader):
""" """
r = [] r = []
if re.match("^\{|\[",stream) is not None: # if re.match("^\{|\[",stream) is not None:
if stream.startswith(b'{') or stream.startswith(b'['):
r = json.loads(stream) r = json.loads(stream)
else: else:
@ -215,6 +214,7 @@ class QueueReader(MessageQueue,Reader):
return self.data return self.data
class QueueListener(MessageQueue): class QueueListener(MessageQueue):
lock = RLock()
""" """
This class is designed to have an active listener (worker) against a specified Exchange/Queue This class is designed to have an active listener (worker) against a specified Exchange/Queue
It is initialized as would any other object and will require a callback function to address the objects returned. It is initialized as would any other object and will require a callback function to address the objects returned.
@ -223,6 +223,7 @@ class QueueListener(MessageQueue):
MessageQueue.__init__(self,**args) MessageQueue.__init__(self,**args)
self.listen = self.read self.listen = self.read
self.apply = args['apply'] if 'apply' in args else print self.apply = args['apply'] if 'apply' in args else print
self.lock = False if 'lock' not in args else args['lock']
def finalize(self,channel,ExceptionReason): def finalize(self,channel,ExceptionReason):
pass pass
@ -231,12 +232,30 @@ class QueueListener(MessageQueue):
_info= {} _info= {}
# if re.match("^\{|\[",stream) is not None: # if re.match("^\{|\[",stream) is not None:
if stream.startswith(b"[") or stream.startswith(b"{"): if stream.startswith(b"[") or stream.startswith(b"{"):
_info = json.loads(stream) _info = json.loads(stream)
else: else:
_info = stream _info = stream
self.apply(_info) #
# At this point we should invoke the apply function with a lock if need be
# @TODO: Establish a vocabulary
if stream == b'QUIT' :
# channel.exit()
self.close()
if self.lock == True :
QueueListener.lock.acquire()
try:
#
# In case the user has not specified a function to apply the data against, it will simply be printed
#
self.apply(_info)
except Exception as e:
pass
if self.lock == True :
QueueListener.lock.release()
def read(self): def read(self):
self.init(self.queue) self.init(self.queue)
@ -246,3 +265,15 @@ class QueueListener(MessageQueue):
class Factory :
@staticmethod
def instance(**_args):
"""
:param count number of workers
:param apply function workers
"""
_apply = _args['apply']
_count = _args['count']
for i in np.arange(_count) :
_name = _args['name'] if 'name' in _args else 'worker_'+str(i)
transport.factory.instance(provider="rabbit",context="listener",apply=_apply,auth_file=_args['auth_file'])

View File

@ -5,11 +5,11 @@ from common import Reader, Writer
import json import json
class HttpRequestReader(Reader): class HttpRequestReader(Reader):
""" """
This class is designed to read data from an Http request file handler provided to us by flask This class is designed to read data from an Http request file handler provided to us by flask
The file will be heald in memory and processed accordingly The file will be heald in memory and processed accordingly
NOTE: This is inefficient and can crash a micro-instance (becareful) NOTE: This is inefficient and can crash a micro-instance (becareful)
""" """
def __init__(self,**params): def __init__(self,**params):
self.file_length = 0 self.file_length = 0
@ -22,8 +22,8 @@ class HttpRequestReader(Reader):
#print 'size of file ',self.file_length #print 'size of file ',self.file_length
self.content = params['file'].readlines() self.content = params['file'].readlines()
self.file_length = len(self.content) self.file_length = len(self.content)
except Exception, e: except Exception as e:
print "Error ... ",e print ("Error ... ",e)
pass pass
def isready(self): def isready(self):
@ -37,13 +37,13 @@ class HttpRequestReader(Reader):
yield row yield row
class HttpSessionWriter(Writer): class HttpSessionWriter(Writer):
""" """
This class is designed to write data to a session/cookie This class is designed to write data to a session/cookie
""" """
def __init__(self,**params): def __init__(self,**params):
""" """
@param key required session key @param key required session key
""" """
self.session = params['queue'] self.session = params['queue']
self.session['sql'] = [] self.session['sql'] = []
self.session['csv'] = [] self.session['csv'] = []

View File

@ -64,6 +64,17 @@ class SQLRW :
key = 'username' if 'username' in _args else 'user' key = 'username' if 'username' in _args else 'user'
_info['user'] = _args[key] _info['user'] = _args[key]
_info['password'] = _args['password'] if 'password' in _args else '' _info['password'] = _args['password'] if 'password' in _args else ''
if 'auth_file' in _args :
_auth = json.loads( open(_args['auth_file']).read() )
key = 'username' if 'username' in _auth else 'user'
_info['user'] = _auth[key]
_info['password'] = _auth['password'] if 'password' in _auth else ''
_info['host'] = _auth['host'] if 'host' in _auth else _info['host']
_info['port'] = _auth['port'] if 'port' in _auth else _info['port']
if 'database' in _auth:
_info['dbname'] = _auth['database']
self.table = _auth['table'] if 'table' in _auth else self.table
# #
# We need to load the drivers here to see what we are dealing with ... # We need to load the drivers here to see what we are dealing with ...