""" This class refactors the default parsing class (better & streamlined implementation) The class will rely on the new plug/play architectural style perform parsing """ from multiprocessing import Process, RLock import os import json # from healthcareio.x12.util from healthcareio import x12 import numpy as np import transport import copy # from healthcareio.x12.util import file as File, document as Document from datetime import datetime from healthcareio.logger import X12Logger import time class BasicParser (Process) : def __init__(self,**_args): super().__init__() self._plugins = _args['plugins'] self._parents = _args['parents'] self._files = _args['files'] self._store = _args['store'] self._template = x12.util.template(plugins=self._plugins) # self._logger = _args['logger'] if 'logger' in _args else None self._logger = X12Logger(store = self._store) if self._logger : _info = { key:len(self._plugins[key].keys())for key in self._plugins} _data = {'plugins':_info,'files': len(self._files),'model':self._template} self._logger.log(module='BasicParser',action='init',data=_data) def log (self,**_args): """ This function logs data into a specified location in JSON format datetime,module,action,data """ pass def apply(self,**_args): """ :content raw claim i.e CLP/CLM Loops and related content :x12 file type 837|835 :document document template with attributes pre-populated """ _content = _args['content'] _filetype = _args['x12'] _doc = _args['document'] #{} _documentHandler = x12.util.document.Builder(plugins = self._plugins,parents=self._parents) try: for _row in _content : # _data = None _data,_meta = _documentHandler.bind(row=_row,x12=_filetype) if _data and _meta : _doc = _documentHandler.build(data=_data,document=_doc,meta=_meta,row=_row) # print (['*** ',_doc]) pass except Exception as e: # # Log something here .... print (_row) print (e) # print (_row,_doc.keys()) pass return _doc def run(self): _handleContent = x12.util.file.Content() _handleDocument = x12.util.document.Builder(plugins = self._plugins,parents=self._parents) _template = self._template #x12.util.template(plugins=self._plugins) # # @TODO: starting initializing parsing jobs : # - number of files, plugins meta data _log = {} for _absolute_path in self._files : try: _content = _handleContent.read(filename=_absolute_path) _content,_filetype = _handleContent.split(_content) # # LOG: filename with claims found in it # # The first row is the header (it will be common to all claims) _header = copy.deepcopy(_template[_filetype]) _header = self.apply(content=_content[0],x12=_filetype, document=_header) _docs = [] for _rawclaim in _content[1:] : _document = copy.deepcopy(_header) #copy.deepcopy(_template[_filetype]) # _document = dict(_document,**_header) if type(_absolute_path) == str: _document['filename'] = _absolute_path _doc = self.apply(content=_rawclaim,x12=_filetype, document=_document) if _doc : _docs.append(_doc) else: # print (['wtf ...',_rawclaim]) pass # # LOG: information abou the file that has just been processed. _location = _absolute_path if type(_absolute_path) == str else 'In-Memory' _data = {'filename':_location, 'available':len(_content[1:]),'x12':_filetype} _args = {'module':'parse','action':'parse','data':_data} _data['parsed'] = len(_docs) self._logger.log(**_args) # # Let us submit the batch we have thus far # self.post(documents=_docs,x12=_filetype,filename=_location) except Exception as e: # # LOG: We have filename and segment of the claim within filename # print (e) def post(self,**_args): pass class X12Parser(BasicParser): def __init__(self,**_args): super().__init__(**_args) self._store = _args['store'] def post(self,**_args): """ Writing the files to a persistent storage in JSON format (hopefully) """ _documents = _args['documents'] if _documents : _store = copy.copy(self._store,**{}) TABLE = 'claims' if _args['x12'] in ['837','claims'] else 'remits' _store['table'] = TABLE _writer = transport.factory.instance(**_store) _writer.write(_documents) if getattr(_writer,'close') : _writer.close() # # LOG: report what was written _data = {'x12':_args['x12'], 'documents':len(_documents),'filename':_args['filename']} self._logger.log(module='write',action='write',data=_data) # def instance (**_args): # """ # :path # """ # # _files = x12.util.Files.get(_args['file']) # # # # # # We can split these files (multi-processing) # # # # # _jobCount = 1 if 'jobs' not in _args else int (_args['jobs']) # # _files = np.array_split(_files,_jobCount) # # PATH = os.sep.join([os.environ['HOME'],'.healthcareio','config.json']) # # if 'config' in _args : # # PATH = _args['config'] # # f = open(PATH) # # _config = json.loads(f.read()) # # f.close() # # jobs = [] # # for _batch in _files : # # pthread = Parser(files=_batch,config=_config) # # pthread.start() # # jobs.append(pthread) # # time.sleep(1) # pass # class parser (Process) : # _CONFIGURATION = {} # def __init__(self,path=None) : # if not parser._CONFIGURATION : # _path = path if path else os.sep.join([os.environ['HOME'],'.healthcareio/config.json']) # # # # @TODO: Load custom configuration just in case we need to do further processing # config = json.loads(open(path).read()) # parser._CONFIGURATION = config['parser'] # # # # do we have a custom configuration in this location # # # _custompath = _path.replace('config.json','') # _custompath = _custompath if not _custompath.endswith(os.sep) else _custompath[:-1] # _custompath = os.sep.join([_custompath,'custom']) # if os.exists(_custompath) : # files = os.listdir(_custompath) # if files : # _filename = os.sep.join([_custompath,files[0]]) # _customconf = json.loads(open(_filename).read()) # # # # merge with existing configuration # else: # pass # # # # # class getter : # def value(self,) : # pass # class setter : # def files(self,files): # pass