hiplab/data-transport
hiplab
/
data-transport
mirror of https://github.com/lnyemba/data-transport
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

bug fix: registry (more usable) and added to factory method

This commit is contained in:
Steve Nyemba 2024-06-14 15:30:09 -05:00
parent 8edb764d11
commit b9bc898161
3 changed files with 66 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
bin/transport
View File

@ -47,7 +47,7 @@ def wait(jobs):
@app.command(name="apply")
def apply (path:Annotated[str,typer.Argument(help="path of the configuration file")],
index:int = typer.Option(help="index of the item of interest, otherwise everything in the file will be processed")):
index:int = typer.Option(default= None, help="index of the item of interest, otherwise everything in the file will be processed")):
"""
This function applies data transport ETL feature to read data from one source to write it one or several others
"""
@ -92,19 +92,23 @@ def version():
@app.command()
def generate (path:Annotated[str,typer.Argument(help="path of the ETL configuration file template (name included)")]):
"""
This function will generate a configuration template to give a sense of how to create one
"""
_config = [
{
"source":{"provider":"http","url":"https://raw.githubusercontent.com/codeforamerica/ohana-api/master/data/sample-csv/addresses.csv"},
"target":
"""
This function will generate a configuration template to give a sense of how to create one
"""
_config = [
{
"source":{"provider":"http","url":"https://raw.githubusercontent.com/codeforamerica/ohana-api/master/data/sample-csv/addresses.csv"},
"target":
[{"provider":"files","path":"addresses.csv","delimiter":","},{"provider":"sqlite","database":"sample.db3","table":"addresses"}]
}
]
file = open(path,'w')
file.write(json.dumps(_config))
file.close()
file = open(path,'w')
file.write(json.dumps(_config))
file.close()
print (f"""{CHECK_MARK} Successfully generated a template ETL file at {path}""" )
print ("""NOTE: Each line (source or target) is the content of an auth-file""")
@app.command(name="init")
def initregistry (email:Annotated[str,typer.Argument(help="email")],
@ -131,8 +135,11 @@ def register (label:Annotated[str,typer.Argument(help="unique label that will be
Learn more about auth-file at https://healthcareio.the-phi.com/data-transport
"""
try:
transport.registry.set(label=label,auth_file=auth_file, default=default, path=path)
_msg = f"""{CHECK_MARK} Successfully added label "{label}" to data-transport registry"""
if transport.registry.exists(path) :
transport.registry.set(label=label,auth_file=auth_file, default=default, path=path)
_msg = f"""{CHECK_MARK} Successfully added label "{label}" to data-transport registry"""
else:
_msg = f"""{TIMES_MARK} Registry is not initialized, please initialize the registry (check help)"""
except Exception as e:
_msg = f"""{TIMES_MARK} {e}"""
print (_msg)

29
transport/__init__.py
View File

@ -48,10 +48,16 @@ def instance (**_args):
kwargs These are arguments that are provider/vendor specific
"""
global PROVIDERS
if not registry.isloaded () :
registry.load() if 'path' not in _args else registry.load(_args['path'])
if 'label' in _args :
_info = registry.load(_args['label'])
# if not registry.isloaded () :
# if ('path' in _args and registry.exists(_args['path'] )) or registry.exists():
# registry.load() if 'path' not in _args else registry.load(_args['path'])
# print ([' GOT IT'])
# if 'label' in _args and registry.isloaded():
# _info = registry.get(_args['label'])
# if _info :
# #
# _args = dict(_args,**_info)
if 'auth_file' in _args:
if os.path.exists(_args['auth_file']) :
#
@ -67,6 +73,17 @@ def instance (**_args):
else:
filename = _args['auth_file']
raise Exception(f" {filename} was not found or is invalid")
if 'provider' not in _args and 'auth_file' not in _args :
if not registry.isloaded () :
if ('path' in _args and registry.exists(_args['path'] )) or registry.exists():
registry.load() if 'path' not in _args else registry.load(_args['path'])
if 'label' in _args and registry.isloaded():
_info = registry.get(_args['label'])
print(_info)
if _info :
#
_args = dict(_args,**_info)
if 'provider' in _args and _args['provider'] in PROVIDERS :
_info = PROVIDERS[_args['provider']]
_module = _info['module']
@ -110,6 +127,8 @@ class get :
"""
@staticmethod
def reader (**_args):
if not _args :
_args['label'] = 'default'
_args['context'] = 'read'
return instance(**_args)
@staticmethod
@ -117,6 +136,8 @@ class get :
"""
This function is a wrapper that will return a writer to a database. It disambiguates the interface
"""
if not _args :
_args['label'] = 'default'
_args['context'] = 'write'
return instance(**_args)
@staticmethod

26
transport/registry.py
View File

@ -16,11 +16,20 @@ DATA = {}
def isloaded ():
return DATA not in [{},None]
def exists (path=REGISTRY_PATH) :
"""
This function determines if there is a registry at all
"""
p = os.path.exists(path)
q = os.path.exists( os.sep.join([path,REGISTRY_FILE]))
print ([p,q, os.sep.join([path,REGISTRY_FILE])])
return p and q
def load (_path=REGISTRY_PATH):
global DATA
_path = os.sep.join([_path,REGISTRY_FILE])
if os.path.exists(_path) :
f = open(_path)
if exists(_path) :
path = os.sep.join([_path,REGISTRY_FILE])
f = open(path)
DATA = json.loads(f.read())
f.close()
def init (email,path=REGISTRY_PATH,override=False):
@ -45,12 +54,19 @@ def init (email,path=REGISTRY_PATH,override=False):
raise Exception (f"""Unable to write configuration, Please check parameters (or help) and try again""")
else:
raise Exception (f"""Invalid Input, {email} is not well formatted, provide an email with adequate format""")
def lookup (label):
global DATA
return label in DATA
def get (label='default') :
global DATA
return copy.copy(DATA[label]) if label in DATA else {}
def set (label, auth_file, default=False,path=REGISTRY_PATH) :
"""
This function will add a label (auth-file data) into the registry and can set it as the default
"""
if label == 'default' :
raise Exception ("""Invalid label name provided, please change the label name and use the switch""")
reg_file = os.sep.join([path,REGISTRY_FILE])
if os.path.exists (auth_file) and os.path.exists(path) and os.path.exists(reg_file):
f = open(auth_file)
@ -73,7 +89,7 @@ def set (label, auth_file, default=False,path=REGISTRY_PATH) :
f.write(json.dumps(_config))
f.close()
else:
_msg = f"""Unable to load file locate at {path},\nLearn how to generate auth-file with wizard found at https://healthcareio.the-phi.com/data-transport"""
raise Exception( f"""Unable to load file locate at {path},\nLearn how to generate auth-file with wizard found at https://healthcareio.the-phi.com/data-transport""")
pass
else:
pass