From 00f80d92948f70e638fb171c3311f0d78641c05e Mon Sep 17 00:00:00 2001 From: Steve Nyemba Date: Mon, 5 Feb 2024 21:56:46 -0600 Subject: [PATCH] bug fix --- healthcareio/__main__.py | 81 +++++++++++++--- healthcareio/x12/parser.py | 107 ++++++--------------- healthcareio/x12/plugins/default/claims.py | 2 +- healthcareio/x12/plugins/default/remits.py | 6 +- healthcareio/x12/util/document.py | 53 ++++------ setup.py | 6 +- 6 files changed, 124 insertions(+), 131 deletions(-) diff --git a/healthcareio/__main__.py b/healthcareio/__main__.py index a5107d3..abe5d9a 100644 --- a/healthcareio/__main__.py +++ b/healthcareio/__main__.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python """ (c) 2019 EDI Parser Toolkit, Health Information Privacy Lab, Vanderbilt University Medical Center @@ -10,12 +11,12 @@ This code is intended to process and parse healthcare x12 837 (claims) and x12 8 The claims/outpout can be forwarded to a NoSQL Data store like couchdb and mongodb Usage : Commandline : - python x12parser + # parse {x12} + healthcare-io parse + + # export {x12} + healthcare-io export - action: - - parser - - create.plugin - - register.plugin - Embedded : @@ -26,11 +27,12 @@ from typing import Optional from typing_extensions import Annotated import uuid import os -import version +import meta import json import time from healthcareio import x12 from healthcareio.x12.parser import X12Parser +import requests # import healthcareio # import healthcareio.x12.util @@ -39,11 +41,12 @@ app = typer.Typer() CONFIG_FOLDER = os.sep.join([os.environ['HOME'],'.healthcareio']) @app.command(name='init') def config(email:str,provider:str='sqlite') : - """ + """\b Generate configuration file needed with default data store. For supported data-store providers visit https://hiplab.mc.vanderbilt.edu/git/hiplab/data-transport.git :email your email - \r:provider data store provider (visit https://hiplab.mc.vanderbilt.edu/git/hiplab/data-transport.git) + + :provider data store provider (visit https://hiplab.mc.vanderbilt.edu/git/hiplab/data-transport.git) """ _db = "healthcareio" # _PATH = os.sep.join([os.environ['HOME'],'.healthcareio']) @@ -62,27 +65,55 @@ def config(email:str,provider:str='sqlite') : "system":{ "uid":str(uuid.uuid4()), "email":email, - "version":version.__version__, - "copyright":version.__author__ + "version":meta.__version__, + "copyright":meta.__author__ } } # + # Let create or retrieve a user's key/token to make sure he/she has access to features they need + # This would also allow us to allow the users to be informed of new versions + # + try: + host = "https://healthcareio.the-phi.com" if 'PARSER_HOST_URL' not in os.environ else os.environ['PARSER_HOST'] + url = f"{host}/api/users/signup" + _body = {"email":email,"version":meta.__version__} + _headers = {"content-type":"application/json"} + resp = requests.post(url,headers=_headers,data=json.dumps(_body)) + if resp.ok : + _config['system'] = dict(_config['system'],**resp.json()) + except Exception as e: + print (e) + pass # store this on disk f = open(os.sep.join([CONFIG_FOLDER,'config.json']),'w') f.write(json.dumps(_config)) f.close() + _msg = f""" + Thank you for considering using our {{x12}} parser verion {meta.__version__} + The generatted configuration file found at {CONFIG_FOLDER} + The database provider is {provider} + + visit {host} to learn more about the features, + + """ + + print (_msg) @app.command(name='about') def copyright(): - for note in [version.__name__,version.__author__,version.__license__]: + for note in [meta.__name__,meta.__author__,meta.__license__]: print (note) pass @app.command() def parse (claim_folder:str,plugin_folder:str = None,config_path:str = None): """ - This function will parse 837 and or 835 claims given a location of parsing given claim folder and/or plugin folder + This function will parse 837 and or 835 claims given a location of parsing given claim folder and/or plugin folder. + + plugin_folder folder containing user defined plugins (default are loaded) + + config_path default configuration path """ _plugins,_parents = x12.plugins.instance(path=plugin_folder) @@ -94,7 +125,7 @@ def parse (claim_folder:str,plugin_folder:str = None,config_path:str = None): _config = json.loads(f.read()) f.close() _store = _config['store'] - # print (len(_files)) + # # print (len(_files)) jobs = [] for _chunks in _files: pthread = X12Parser(plugins=_plugins,parents=_parents,files=_chunks, store=_store) @@ -103,11 +134,29 @@ def parse (claim_folder:str,plugin_folder:str = None,config_path:str = None): while jobs : jobs = [pthread for pthread in jobs if pthread.is_alive()] time.sleep(1) - pass - else: - pass + # pass + # else: + # pass + print ("...................... FINISHED .........................") # # + +@app.command(name="export") +def publish (file_type:str,path:str): + """ + This function will export to a different database + file_type values are either claims or remits + + path path to export configuration (data transport file) + + file_type claims or remits + """ + if file_type in ['837','claims'] : + _type = 'claims' + elif file_type in ['835','remits']: + _type = 'remits' + + if __name__ == '__main__' : app() \ No newline at end of file diff --git a/healthcareio/x12/parser.py b/healthcareio/x12/parser.py index c047b5d..464ebe0 100644 --- a/healthcareio/x12/parser.py +++ b/healthcareio/x12/parser.py @@ -7,6 +7,7 @@ import os import json # from healthcareio.x12.util from healthcareio import x12 +from healthcareio.x12.util import file, document import numpy as np import transport import copy @@ -14,19 +15,21 @@ import copy from datetime import datetime from healthcareio.logger import X12Logger import time +import pandas as pd + + class BasicParser (Process) : def __init__(self,**_args): super().__init__() self._plugins = _args['plugins'] self._parents = _args['parents'] self._files = _args['files'] - self._store = _args['store'] + self._store = dict(_args['store'],**{'lock':True}) self._template = x12.util.template(plugins=self._plugins) - # self._logger = _args['logger'] if 'logger' in _args else None self._logger = X12Logger(store = self._store) if self._logger : _info = { key:len(self._plugins[key].keys())for key in self._plugins} - _data = {'plugins':_info,'files': len(self._files),'model':self._template} + _data = {'plugins':_info,'files': len(self._files),'model': self._template} self._logger.log(module='BasicParser',action='init',data=_data) def log (self,**_args): @@ -34,6 +37,8 @@ class BasicParser (Process) : This function logs data into a specified location in JSON format datetime,module,action,data """ + if self._logger : + self._logger.log(**_args) pass def apply(self,**_args): """ @@ -44,15 +49,17 @@ class BasicParser (Process) : _content = _args['content'] _filetype = _args['x12'] _doc = _args['document'] #{} - _documentHandler = x12.util.document.Builder(plugins = self._plugins,parents=self._parents) + + _documentHandler = x12.util.document.Builder(plugins = self._plugins,parents=self._parents, logger=self._logger) try: - + _tmp = {} for _row in _content : # _data = None _data,_meta = _documentHandler.bind(row=_row,x12=_filetype) if _data and _meta : + _doc = _documentHandler.build(data=_data,document=_doc,meta=_meta,row=_row) # print (['*** ',_doc]) pass @@ -61,15 +68,18 @@ class BasicParser (Process) : except Exception as e: # # Log something here .... - print (_row) + # print (_row) + print (e) # print (_row,_doc.keys()) pass return _doc def run(self): - _handleContent = x12.util.file.Content() - _handleDocument = x12.util.document.Builder(plugins = self._plugins,parents=self._parents) + _handleContent = file.Content() #x12.util.file.Content() + _handleDocument = document.Builder(plugins = self._plugins,parents=self._parents,logger=self._logger) + _template = self._template #x12.util.template(plugins=self._plugins) + # # @TODO: starting initializing parsing jobs : # - number of files, plugins meta data @@ -89,10 +99,14 @@ class BasicParser (Process) : _header = copy.deepcopy(_template[_filetype]) _header = self.apply(content=_content[0],x12=_filetype, document=_header) _docs = [] - + _ids = [] for _rawclaim in _content[1:] : _document = copy.deepcopy(_header) #copy.deepcopy(_template[_filetype]) + if 'claim_id' in _document : + # + # @TODO: Have a way to get the attribute for CLP or CLM + _ids.append(_document['claim_id']) # _document = dict(_document,**_header) if type(_absolute_path) == str: _document['filename'] = _absolute_path @@ -108,7 +122,9 @@ class BasicParser (Process) : _data = {'filename':_location, 'available':len(_content[1:]),'x12':_filetype} _args = {'module':'parse','action':'parse','data':_data} _data['parsed'] = len(_docs) - self._logger.log(**_args) + + self.log(**_args) + self.log(module='parse',action='file-count', data={'file_name':_absolute_path,'file_type':_filetype,'claims':_ids, 'claim_count':len(_ids)}) # # Let us submit the batch we have thus far # @@ -135,78 +151,17 @@ class X12Parser(BasicParser): _documents = _args['documents'] if _documents : - _store = copy.copy(self._store,**{}) + _store = copy.deepcopy(self._store) TABLE = 'claims' if _args['x12'] in ['837','claims'] else 'remits' _store['table'] = TABLE + _store['cotnext'] = 'write' _writer = transport.factory.instance(**_store) - _writer.write(_documents) + _writer.write(_documents,table=TABLE) if getattr(_writer,'close') : _writer.close() # # LOG: report what was written _data = {'x12':_args['x12'], 'documents':len(_documents),'filename':_args['filename']} - self._logger.log(module='write',action='write',data=_data) - -# def instance (**_args): -# """ -# :path -# """ -# # _files = x12.util.Files.get(_args['file']) - -# # # -# # # We can split these files (multi-processing) -# # # -# # _jobCount = 1 if 'jobs' not in _args else int (_args['jobs']) -# # _files = np.array_split(_files,_jobCount) -# # PATH = os.sep.join([os.environ['HOME'],'.healthcareio','config.json']) -# # if 'config' in _args : -# # PATH = _args['config'] -# # f = open(PATH) -# # _config = json.loads(f.read()) -# # f.close() -# # jobs = [] -# # for _batch in _files : -# # pthread = Parser(files=_batch,config=_config) -# # pthread.start() -# # jobs.append(pthread) -# # time.sleep(1) -# pass - - -# class parser (Process) : -# _CONFIGURATION = {} -# def __init__(self,path=None) : -# if not parser._CONFIGURATION : -# _path = path if path else os.sep.join([os.environ['HOME'],'.healthcareio/config.json']) -# # -# # @TODO: Load custom configuration just in case we need to do further processing -# config = json.loads(open(path).read()) -# parser._CONFIGURATION = config['parser'] -# # -# # do we have a custom configuration in this location -# # -# _custompath = _path.replace('config.json','') -# _custompath = _custompath if not _custompath.endswith(os.sep) else _custompath[:-1] -# _custompath = os.sep.join([_custompath,'custom']) -# if os.exists(_custompath) : -# files = os.listdir(_custompath) -# if files : -# _filename = os.sep.join([_custompath,files[0]]) -# _customconf = json.loads(open(_filename).read()) -# # -# # merge with existing configuration - - -# else: -# pass - -# # -# # -# class getter : -# def value(self,) : -# pass -# class setter : -# def files(self,files): -# pass - \ No newline at end of file + # self._logger.log(module='write',action='write',data=_data) + self.log(module='parse',action='write',data=_data) diff --git a/healthcareio/x12/plugins/default/claims.py b/healthcareio/x12/plugins/default/claims.py index c0ee5c2..7959965 100644 --- a/healthcareio/x12/plugins/default/claims.py +++ b/healthcareio/x12/plugins/default/claims.py @@ -1,7 +1,7 @@ import numpy as np from .. import parser from datetime import datetime -@parser(element='NM1',x12='*', anchor={'41':'submitter','40':'receiver','82':'rendering_provider','85':'billing_provider','87':'pay_to_provider','IL':'patient','PR':'payer','QC':'patient','DN':'referring_provider','77':'provider','2':'billing_provider'}, map={1:'type',3:'name',-1:'id'}) +@parser(element='NM1',x12='*', anchor={'41':'submitter','40':'receiver','82':'rendering_provider','85':'billing_provider','87':'pay_to_provider','IL':'patient','PR':'payer','QC':'patient','DN':'referring_provider','77':'provider','2':'billing_provider'}, map={1:'type',3:'name_1',4:'name_2',-1:'id'}) def NM1 (**_args): """ Expected Element NM1 diff --git a/healthcareio/x12/plugins/default/remits.py b/healthcareio/x12/plugins/default/remits.py index 0c6cbc7..d4b6523 100644 --- a/healthcareio/x12/plugins/default/remits.py +++ b/healthcareio/x12/plugins/default/remits.py @@ -57,9 +57,9 @@ def SVC (**_args): _data['paid_amount'] = np.float64(_data['paid_amount']) return _data pass -@parser(element='N1',x12='835',anchor={'PR':'provider'},map={1:'name'}) -def N1(**_args): - pass +# @parser(element='N1',x12='835',anchor={'PR':'provider'},map={1:'name'}) +# def N1(**_args): +# pass @parser(element='N3',x12='835',parent='N1',map={1:'address_line_1'}) def N3(**_args): pass diff --git a/healthcareio/x12/util/document.py b/healthcareio/x12/util/document.py index ffa9de1..c1ffdaa 100644 --- a/healthcareio/x12/util/document.py +++ b/healthcareio/x12/util/document.py @@ -15,7 +15,7 @@ class Builder: self._plugins = copy.deepcopy(_args['plugins']) self._parents = copy.deepcopy(_args['parents']) self._loop = {} - + self._logger = None if 'logger' not in _args else _args['logger'] def reset (self): self._last = {} @@ -32,34 +32,7 @@ class Builder: if _id : return self._last[_id] if _id in self._last else None return None - - # if _id in self._parents : - # self._last[_id] = - - # if 'parent' in _meta : #hasattr(_meta,'parent'): - # _hasField = 'field' in _meta - # _hasParent= _meta['element'] in self._parents - # if _hasField and _hasParent: #_meta.element in self._parents and hasattr(_meta,'field'): - - # self._last = _item - # pass - # else: - # for key in self._parents : - # if _meta['element'] in self._parents[key] : - - # _ikey = list(self._last.keys())[0] - # _oldinfo = self._last[_ikey] - # if type(_oldinfo) != dict : - # # - # # Only applicable against a dictionary not a list (sorry) - # pass - # else: - # _item = {_ikey: self.merge(_oldinfo,_item)} - - # break - # pass - - # return _item + def count(self,_element): if _element not in self._loop : self._loop[_element] = 0 @@ -251,10 +224,12 @@ class Builder: if _field : if 'container' in _meta and type(_document[_field]) != list : _document[_field] = [] - if _field and _document: + + if _field and _document : if _field not in _document : _document[_field] =_data + pass else: if 'container' in _meta : _document[_field].append(_data) @@ -263,9 +238,23 @@ class Builder: else: if not _field and 'anchor' in _meta : # - # This is an unusual situation ... + # We should determine if the element is either a parent or has a parent + # This would allow us to avoid having runaway attributes and undermine structural integrity + # + + + # + # The element has NOT been specified by the plugin (alas) + # For this case we would advise writing a user-defined plugin to handle this case + # + print (self._logger) + if self._logger : + print (['....................']) + self._logger.log(action='missing-plugin',module='build',data={'element':_row[0],'anchor':_row[1]}) + + return _document pass + # print ([_row[0],set(_data) - set(_document.keys())]) _document = self.merge(_document,_data) return _document - diff --git a/setup.py b/setup.py index 2ca912c..fe713fe 100644 --- a/setup.py +++ b/setup.py @@ -14,13 +14,13 @@ args = { "author":meta.__author__, "author_email":"steve.l.nyemba@vumc.org", "include_package_data":True, - "license":version.__license__, + "license":meta.__license__, "packages":find_packages(), "keywords":["healthcare","edi","x12","analytics","835","837","data","transport","protocol"] } -args["install_requires"] = ['typer','flask-socketio','seaborn','jinja2','jsonmerge', 'weasyprint','data-transport@git+https://healthcareio.the-phi.com/git/code/transport.git','pymongo','numpy','cloudant','pika','boto','botocore','flask-session','smart_open','smart-top@git+https://healthcareio.the-phi.com/git/code/smart-top.git@data-collector'] +args["install_requires"] = ['typer','flask-socketio','seaborn','jinja2','jsonmerge', 'weasyprint','data-transport@git+https://healthcareio.the-phi.com/git/code/transport.git','pymongo','numpy','cloudant','pika','boto','botocore','flask-session','smart_open'] args['url'] = 'https://hiplab.mc.vanderbilt.edu' -args['scripts']= ['healthcareio/healthcare-io.py'] +args['scripts']= ['bin/healthcare-io'] # args['entry_points'] = { # 'console_scripts' : ['healthcareio=healthcareio:register'] # }