bug fix: export & sql doesn't always have objects casted

This commit is contained in:
Steve Nyemba 2024-02-06 14:32:59 -06:00
parent adcddf0654
commit a6b52719f9
4 changed files with 49 additions and 15 deletions

View File

@ -178,12 +178,28 @@ def publish (file_type:str,path:str):
path path to export configuration (data transport file)
file_type claims or remits
file_type claims or remits (835 or 837)
"""
if file_type in ['837','claims'] :
_type = None
if file_type.strip() in ['837','claims'] :
_type = 'claims'
elif file_type in ['835','remits']:
_x12 = '837'
elif file_type.strip() in ['835','remits']:
_type = 'remits'
_x12 = '835'
if _type :
print ([f"Exporting {_type}"])
_store = {'source':os.sep.join([CONFIG_FOLDER,'config.json']),'target':path}
for _key in _store :
f = open(_store[_key])
_store[_key] = json.loads(f.read())
f.close()
_store['source'] = _store['source']['store']
_plugins,_parents = x12.plugins.instance()
x12.publish.init(plugins=_plugins,x12=_x12,store=_store)
else:
print ("Can not determine type, (837 or 835)")
if __name__ == '__main__' :

View File

@ -17,6 +17,7 @@ from healthcareio.logger import X12Logger
import time
import pandas as pd
from transport import providers
class BasicParser (Process) :
def __init__(self,**_args):
@ -155,8 +156,10 @@ class X12Parser(BasicParser):
TABLE = 'claims' if _args['x12'] in ['837','claims'] else 'remits'
_store['table'] = TABLE
_store['cotnext'] = 'write'
_writer = transport.factory.instance(**_store)
_writer = transport.factory.instance(**_store)
# if _store['provider'] not in [providers.MONGODB, providers.COUCHDB] :
_writer.write(_documents,table=TABLE)
if getattr(_writer,'close') :
_writer.close()

View File

@ -5,6 +5,7 @@ import numpy as np
import time
import pandas as pd
from multiprocessing import Process
import json
def build (**_args):
"""
@ -56,19 +57,31 @@ def format(**_args) :
_pkvalue = None
if _primary_key in _claim :
_pkvalue = _claim[_primary_key]
for _attrName in _claim :
_item = _claim[_attrName]
_item = update(_item,_primary_key,_pkvalue)
#
# We have a casting problem, with relational data-store and JSON objects
#
if type(_item) == str and (_item.startswith("{") or _item.startswith("{")) :
try:
_item = json.loads(_item)
except Exception as ee :
# print (ee)
pass
if _attrName not in _tables and type(_item) in [dict,list]:
_tables[_attrName] = []
if type(_item) in [dict,list] :
_tables[_attrName] += _item if type(_item) == list else [_item]
pass
else:
#
# This section suggests we found a main table attribute
_main[_attrName] = _item
_tables[_mainTableName].append(_main)
return _tables
@ -100,20 +113,19 @@ def init(**_args):
_default = build(plugins=_plugins,x12=_file_type)
_df = read(store = _store['source'],x12=_file_type)
_pkey = util.getPrimaryKey(plugins = _plugins, x12=_file_type)
SEGMENTS = 4 # arbitrary choice
_indexes = np.array_split(np.arange(_df.shape[0]),SEGMENTS)
jobs = []
for _ii in _indexes :
try:
_data = format(rows= _df.iloc[_ii].to_dict(orient='records'),x12=_file_type,primary_key=_pkey)
_data = format(rows= _df.iloc[_ii].to_dict(orient='records'),x12=_file_type,primary_key=_pkey)
_thread = Process(target=post,args=({'store':_store['target'],'data':_data,'default':_default,'x12':_file_type},))
jobs.append(_thread)
except Exception as e:
#
# Log: sigment,
print (e)
pass
if jobs :
jobs[0].start()
@ -126,6 +138,7 @@ def read (**_args):
_store = copy.copy(_args['store'])
_x12 = _args['x12']
_store['table'] = getContext(_x12) #'claims' if _x12 == '837' else 'remits'
_store['context'] = 'read'
reader = transport.factory.instance(**_store)
#
# @TODO: reading should support streaming (for scalability)
@ -136,9 +149,14 @@ def read (**_args):
def post(_args):
_data = _args['data']
_store = _args['store']
_store['context'] = 'write'
_default = _args['default']
_prefix = 'clm_' if _args['x12'] == '837' else 'rem_'
# if 'claims' in _data or 'remits' in _data :
# _key = 'claims' if 'claims' in _data else 'remits'
# _data = _data[_key]
for _name in _data :
_tablename = _prefix+_name
_store['table'] = _tablename if _name not in ['remits','claims'] else _name
_store['context']='write'
@ -147,12 +165,11 @@ def post(_args):
_rows = [_default[_name]]
else:
_rows = _data[_name]
writer.write(pd.DataFrame(_rows).fillna(''))
if hasattr(writer,'close') :
writer.close()
#
# Have a logger here to log what's going on ...
# _xwriter = trasnport.factory.instance(**_store)
# _xwriter.write(_df)
# _info = format()

View File

@ -247,9 +247,7 @@ class Builder:
# The element has NOT been specified by the plugin (alas)
# For this case we would advise writing a user-defined plugin to handle this case
#
print (self._logger)
if self._logger :
print (['....................'])
self._logger.log(action='missing-plugin',module='build',data={'element':_row[0],'anchor':_row[1]})
return _document