Merge branch 'dev' of lab/parse-edi into master

healthcareio/Dockerfile

@@ -1,4 +1,8 @@
-FROM ubuntu:bionic-20200403
+#
+# Let us create an image for healthcareio
+#   The image will contain the {X12} Parser and the 
+# FROM ubuntu:bionic-20200403
+FROM ubuntu:focal
 RUN ["apt","update","--fix-missing"]
 RUN ["apt-get","upgrade","-y"]
 
@@ -6,9 +10,22 @@ RUN ["apt-get","-y","install","apt-utils"]
 
 RUN ["apt","update","--fix-missing"]
 RUN ["apt-get","upgrade","-y"]
-RUN ["apt-get","install","-y","sqlite3","sqlite3-pcre","libsqlite3-dev","python3-dev","python3","python3-pip","git","python3-virtualenv"]
+RUN ["apt-get","install","-y","mongo","sqlite3","sqlite3-pcre","libsqlite3-dev","python3-dev","python3","python3-pip","git","python3-virtualenv","wget"]
 #
 #
+RUN ["pip3","install","--upgrade","pip"]
+# RUN ["pip3","install","git+https://healthcare.the-phi.com/git/code/parser.git","botocore"]
 USER health-user
+#
+# This volume is where the data will be loaded from (otherwise it is assumed the user will have it in the container somehow)
+#
+VOLUME ["/data"]
+#
+# This is the port from which some degree of monitoring can/will happen
+EXPOSE 80
+# wget https://healthcareio.the-phi.com/git/code/parser.git/bootup.sh 
+COPY bootup.sh bootup.sh
+ENTRYPOINT ["bash","-C"]
+CMD ["bootup.sh"]
 # VOLUME ["/home/health-user/healthcare-io/","/home-healthuser/.healthcareio"]
 # RUN ["pip3","install","git+https://healthcareio.the-phi.com/git"]

healthcareio/analytics.py

@@ -11,7 +11,7 @@ import transport
 import matplotlib.pyplot as plt
 import re, base64
 # from weasyprint import HTML, CSS
-COLORS = ["#f79256","#7dcfb6","#fbd1a2","#00b2ca","#1d4e89","#4682B4","#c5c3c6","#4c5c68","#1985a1","#f72585","#7209b7","#3a0ca3","#4361ee","#4cc9f0","#ff595e","#ffca3a","#8ac926","#1982c4","#6a4c93"]
+COLORS = ["#fbd1a2","#00b2ca","#1d4e89","#4682B4","#c5c3c6","#4c5c68","#1985a1","#f72585","#7209b7","#3a0ca3","#4361ee","#4cc9f0","#ff595e","#ffca3a","#8ac926","#1982c4","#6a4c93"]
 class stdev :
     def __init__(self) :
         self.values = []
@@ -149,11 +149,16 @@ class Apex :
     This class will format a data-frame to work with Apex charting engine
     """
     @staticmethod
-    def apply(item):
+    def apply(item,theme={'mode':'light','palette':'palette6'}):
         pointer = item['chart']['type']
         if hasattr(Apex,pointer) :
             pointer = getattr(Apex,pointer)
+            
             options = pointer(item)
+            if 'apex' in options and 'colors' in options['apex'] :
+                del options['apex']['colors']
+            if 'apex' in options :    
+                options['apex']['theme'] = theme
             options['responsive']= [
                 {
                 'breakpoint': 1,
@@ -168,6 +173,18 @@ class Apex :
             print ("Oops")
         pass
     @staticmethod
+    def radial(item):
+        df = item['data']
+        x = item['chart']['axis']['x']
+        y  = item['chart']['axis']['y']
+        
+        labels = df[y].tolist()
+        values = [float(np.round(value,2)) for value in df[x].tolist()]
+        chart = {"type":"radialBar","height":200}
+        option = {"chart":chart,"series":values,"labels":labels,"plotOptions":{"radialBar":{"hollow":{"size":"70%"}}}}
+        return {'apex':option}
+
+    @staticmethod
     def scatter(item):
         options = Apex.spline(item)
         options['apex']['chart']['type'] = 'scatter'
@@ -175,7 +192,7 @@ class Apex :
     @staticmethod
     def scalar(item):
         _df = item['data']
-        print (_df)
+        
         name = _df.columns.tolist()[0]
         value = _df[name].values.round(2)[0]
         html = '<div class="scalar"><div class="value">:value</div><div class="label">:label</div></div>'
@@ -235,16 +252,17 @@ class Apex :
         @TODO: alias this with bar (!= column)
         """
         df = item['data']
+        
         N = df.shape[0] if df.shape[0] < 10 else 10
         axis = item['chart']['axis']
         y = axis['y']
         if type(y) == list :
             y = y[0]
         axis['x'] = [axis['x']] if type(axis['x']) != list else axis['x']
-        if not set(axis['x']) & set(df.columns.tolist()) :
+        # if not set(axis['x']) & set(df.columns.tolist()) :
-            print (set(axis['x']) & set(df.columns.tolist()))
+            # print (set(axis['x']) & set(df.columns.tolist()))
-            print (axis['x'])
+            # print (axis['x'])
-            print (df.columns)
+            # print (df.columns)
             # df.columns = axis['x']
         series = []
         _min=_max = 0
@@ -294,7 +312,6 @@ class Apex :
         values are x-axis
         """
         df = item['data']
-        
         if df.shape [0]> 1 :
             y_cols,x_cols = item['chart']['axis']['y'],item['chart']['axis']['x']
             labels = df[y_cols].values.tolist()
@@ -302,10 +319,11 @@ class Apex :
             values = df[x_cols].values.round(2).tolist()
         else:
             labels = [name.upper().replace('_',' ') for name in df.columns.tolist()]
+            df = df.astype(float)
             values = df.values.round(2).tolist()[0] if df.shape[1] > 1 else df.values.round(2).tolist()
             
         colors  = COLORS[:len(values)]
-        options = {"series":values,"colors":colors,"labels":labels,"chart":{"type":"donut"},"plotOptions":{"pie":{"customScale":.8}},"legend":{"position":"right"}}
+        options = {"series":values,"colors":colors,"labels":labels,"dataLabels":{"enabled":True,"style":{"colors":["#000000"]},"dropShadow":{"enabled":False}},"chart":{"type":"donut","width":200},"plotOptions":{"pie":{"customScale":.9}},"legend":{"position":"right"}}
         return {"apex":options}
 
         pass
@@ -329,43 +347,117 @@ class engine :
             _args['type'] = 'mongo.MongoReader'
         else:
             _args['type'] = 'disk.SQLiteReader'
-        self.reader = transport.factory.instance(**_args)
+        self.store_config = _args ;
+    
+    def filter (self,**args):
+        """
+            type: claims or remits
+            filter  optional identifier claims, procedures, taxonomy, ...
+        """
+        
+        
+        _m = {'claim':'837','claims':'837','remits':'835','remit':'835'}
+        table = _m[ args['type']]
+        _analytics = self.info[table]
+        if 'index' in args :
+            index = int(args['index'])
+            _analytics = [_analytics[index]]
+            
+        _info = list(_analytics) #if 'filter' not in args else [item for item in analytics if args['filter'] == item['id']]
+        # conn = lite.connect(self.store_config['args']['path'],isolation_level=None)
+        # conn.create_aggregate("stdev",1,stdev)
+        DB_TYPE = 'mongo' if (type(self.reader) == transport.mongo.MongoReader) else 'sql'
+        if DB_TYPE == 'mongo' :
+            self.store_config['args']['doc'] = args['type']
+        
+        self.reader = transport.factory.instance(**self.store_config)
+        r = []
+        for row in _info :
+            pipeline = row['pipeline']
+            
+            index = 0
+            for item in pipeline:
+                if not item[DB_TYPE] :
+                    continue
+                query = {DB_TYPE:item[DB_TYPE]}
+                
+                df = pd.DataFrame(self.reader.read(**query)) #item)
+                df = df.fillna('N/A')
+                # item['data'] = df
+                chart = item['chart']
+                pipe = {"data":df,"chart":chart}
+                for key in list(item.keys()) :
+                    if key not in ["chart","data","mongo","sql","couch"] :
+                        pipe[key] = item[key]
+                
+                
+                
+
+                r.append(pipe)
+        self.reader.close()
+        return {"id":_info[0]['id'],'pipeline':r}
     
     def apply (self,**args) :
         """
             type: claims or remits
             filter  optional identifier claims, procedures, taxonomy, ...
         """
+        
+        
         _m = {'claim':'837','claims':'837','remits':'835','remit':'835'}
         # key = '837' if args['type'] == 'claims' else '835'
         table = _m[ args['type']]
-        analytics = self.info[table]
+        
+        _analytics = self.info[table]
         if 'index' in args :
             index = int(args['index'])
-            analytics = [analytics[index]]
+            _analytics = [_analytics[index]]
             
-        _info = list(analytics) if 'filter' not in args else [item for item in analytics if args['filter'] == item['id']]
+        _info = list(_analytics) if 'filter' not in args else [item for item in analytics if args['filter'] == item['id']]
         # conn = lite.connect(self.store_config['args']['path'],isolation_level=None)
         # conn.create_aggregate("stdev",1,stdev)
-        DB_TYPE = 'mongo' if (type(self.reader) == transport.mongo.MongoReader) else 'sql'
+        #
+        # @TODO: Find a better way to handle database variance
+        #
+        # DB_TYPE = 'mongo' if (type(self.reader) == transport.mongo.MongoReader) else 'sql'
+        
+        if 'mongo' in self.store_config['type'] :
+            DB_TYPE='mongo'
+        else:
+            DB_TYPE='sql'
+            self.store_config['args']['table'] = args['type']
+        
+        self.reader = transport.factory.instance(**self.store_config)
         r = []
         for row in _info :
-            
+            pipeline = row['pipeline']
-            for item in row['pipeline'] :
+            index = 0
+            for item in pipeline:
                 # item['data'] = pd.read_sql(item['sql'],conn)
-                query = {DB_TYPE:item[DB_TYPE]}
+                # query = {DB_TYPE:item[DB_TYPE]}
-                item['data'] = self.reader.read(**item)
+                query = item[DB_TYPE]
+                if not query :
+                    continue
+                if DB_TYPE == 'sql' :
+                    query = {"sql":query}
+                
+                item['data'] = self.reader.read(**query) #item)
                 if 'serialize' in args :
                     
-                    item['data'] = json.dumps(item['data'].to_dict(orient='record')) if type(item['data']) == pd.DataFrame else item['data']
+                    # item['data'] = json.dumps(item['data'].to_dict(orient='record')) if type(item['data']) == pd.DataFrame else item['data']
+                    item['data'] = json.dumps(item['data'].to_dict('record')) if type(item['data']) == pd.DataFrame else item['data']
                 else:
                     item['data'] = (pd.DataFrame(item['data']))
-                    
+                pipeline[index] = item
+                index += 1
+            #
+            #
+            row['pipeline']= pipeline
                     
                 # if 'info' in item:
                 #     item['info'] = item['info'].replace(":rows",str(item["data"].shape[0]))
         # conn.close()
-        
+        self.reader.close()
         return _info
 
     def _html(self,item) :

healthcareio/docker/Dockerfile

@@ -0,0 +1,37 @@
+#
+# Let us create an image for healthcareio
+#   The image will contain the {X12} Parser and the 
+# FROM ubuntu:bionic-20200403
+FROM ubuntu:focal
+RUN ["apt-get","update","--fix-missing"]
+RUN ["apt-get","upgrade","-y"]
+
+RUN ["apt-get","-y","install","apt-utils"]
+
+RUN ["apt-get","update","--fix-missing"]
+RUN ["apt-get","upgrade","-y"]
+RUN ["apt-get","install","-y","mongodb","sqlite3","sqlite3-pcre","libsqlite3-dev","python3-dev","python3","python3-pip","git","python3-virtualenv","wget"]
+#
+#
+RUN ["pip3","install","--upgrade","pip"]
+RUN ["pip3","install","numpy","pandas","git+https://dev.the-phi.com/git/steve/data-transport","botocore","matplotlib"]
+# RUN ["pip3","install","git+https://healthcare.the-phi.com/git/code/parser.git","botocore"]
+# RUN ["useradd", "-ms", "/bin/bash", "health-user"]
+# USER health-user
+#
+# This volume is where the data will be loaded from (otherwise it is assumed the user will have it in the container somehow)
+#
+VOLUME ["/data","/app/healthcareio"]
+WORKDIR /app
+ENV PYTHONPATH="/app"
+
+#
+# This is the port from which some degree of monitoring can/will happen
+EXPOSE 80
+EXPOSE 27017    
+# wget https://healthcareio.the-phi.com/git/code/parser.git/bootup.sh 
+COPY bootup.sh bootup.sh
+ENTRYPOINT ["bash","-C"]
+CMD ["bootup.sh"]
+# VOLUME ["/home/health-user/healthcare-io/","/home-healthuser/.healthcareio"]
+# RUN ["pip3","install","git+https://healthcareio.the-phi.com/git"]

healthcareio/docker/bootup.sh

@@ -0,0 +1,10 @@
+set -e
+/etc/init.d/mongodb start
+cd /app
+export
+export PYTHONPATH=$PWD
+ls
+# python3 healthcareio/healthcare-io.py --signup $EMAIL --store mongo 
+# python3 healthcareio/healthcare-io.py --analytics --port 80 --debug 
+
+bash

healthcareio/healthcare-io.py

@@ -43,6 +43,8 @@ import numpy as np
 from multiprocessing import Process
 import time
 from healthcareio import x12
+import smart
+import pandas as pd
 
 PATH = os.sep.join([os.environ['HOME'],'.healthcareio'])
 OUTPUT_FOLDER = os.sep.join([os.environ['HOME'],'healthcare-io'])
@@ -337,9 +339,19 @@ if __name__ == '__main__' :
         
         # PATH= SYS_ARGS['config'] if 'config' in SYS_ARGS else os.sep.join([os.environ['HOME'],'.healthcareio','config.json'])
         
+        if os.path.exists(os.sep.join([PATH,'config.json'])) :
             e = analytics.engine(os.sep.join([PATH,'config.json'])) #--@TODO: make the configuration file globally accessible
             e.apply(type='claims',serialize=True)
             SYS_ARGS['engine'] = e
+            SYS_ARGS['config'] = json.loads(open(os.sep.join([PATH,'config.json'])).read())
+        else:
+            SYS_ARGS['config'] = {"owner":None,"store":None}
+
+        if 'args' not in SYS_ARGS['config'] :
+            SYS_ARGS['config']["args"] = {"batch":1,"resume":True,"folder":"/data"}
+        
+        me = pd.DataFrame(smart.top.read(name='healthcare-io.py')).args.unique().tolist()        
+        SYS_ARGS['me'] = me[0] #-- This key will identify the current process
 
         pointer = lambda : server.app.run(host='0.0.0.0',port=PORT,debug=DEBUG,threaded=False)
         pthread = Process(target=pointer,args=())

healthcareio/server/__init__.py

@@ -3,7 +3,78 @@ from healthcareio.params import SYS_ARGS
 import healthcareio.analytics
 import os
 import json
+import time
+import smart
+import transport
+import pandas as pd
+import numpy as np
+import x12
+
+from multiprocessing import Process
+from flask_socketio import SocketIO, emit, disconnect,send
+from healthcareio.server import proxy
+PATH = os.sep.join([os.environ['HOME'],'.healthcareio','config.json'])
 app = Flask(__name__)
+socket_ = SocketIO(app)
+
+def resume (files):
+    _args = SYS_ARGS['config']['store'].copy()
+    if 'mongo' in SYS_ARGS['config']['store']['type'] :
+        _args['type'] = 'mongo.MongoReader'
+        reader = transport.factory.instance(**_args)
+    _files = []
+    try:
+        pipeline = [{"$match":{"completed":{"$eq":True}}},{"$group":{"_id":"$name"}},{"$project":{"name":"$_id","_id":0}}]
+        _args = {"aggregate":"logs","cursor":{},"allowDiskUse":True,"pipeline":pipeline}
+        _files = reader.read(mongo = _args)
+        _files = [item['name'] for item in _files]
+    except Exception as e :
+        pass
+    print (["found ",len(files),"\tProcessed  ",len(_files)])
+    return list(set(files) - set(_files))
+        
+    
+
+def run ():
+    #
+    # let's get the files in the folder (perhaps recursively traverse them)
+    #
+    FILES = []
+    BATCH = int(SYS_ARGS['config']['args']['batch']) #-- number of processes (poorly named variable)
+
+    for root,_dir,f in os.walk(SYS_ARGS['config']['args']['folder']) :
+        if f :
+            FILES += [os.sep.join([root,name]) for name in f]
+    FILES = resume(FILES)
+    FILES = np.array_split(FILES,BATCH)
+    procs = []
+    for FILE_GROUP in FILES :
+        
+        FILE_GROUP = FILE_GROUP.tolist()
+        # logger.write({"process":index,"parse":SYS_ARGS['parse'],"file_count":len(row)})
+        # proc = Process(target=apply,args=(row,info['store'],_info,))
+        parser = x12.Parser(PATH) #os.sep.join([PATH,'config.json']))
+        parser.set.files(FILE_GROUP)
+        parser.start()
+        procs.append(parser)  
+    SYS_ARGS['procs']  = procs
+# @socket_.on('data',namespace='/stream')
+def push() :
+    _args = dict(SYS_ARGS['config']['store'].copy(),**{"type":"mongo.MongoReader"})
+    reader = transport.factory.instance(**_args)
+    pipeline = [{"$group":{"_id":"$parse","claims":{"$addToSet":"$name"}}},{"$project":{"_id":0,"type":"$_id","count":{"$size":"$claims"}}}]
+    _args = {"aggregate":"logs","cursor":{},"allowDiskUse":True,"pipeline":pipeline}
+    r = pd.DataFrame(reader.read(mongo=_args))
+    r = healthcareio.analytics.Apex.apply({"chart":{"type":"donut","axis":{"x":"count","y":"type"}},"data":r})
+    emit("update",r,json=True)
+    return r
+@socket_.on('connect')
+def client_connect(**r):
+    print ('Connection received')
+    print (r)
+    push()
+    pass
+    
 @app.route("/favicon.ico")
 def _icon():
     return send_from_directory(os.path.join([app.root_path, 'static','img','logo.svg']),
@@ -12,7 +83,7 @@ def _icon():
 def init():
     e = SYS_ARGS['engine']
     sections = {"remits":e.info['835'],"claims":e.info['837']}
-    _args = {"sections":sections}
+    _args = {"sections":sections,"store":SYS_ARGS["config"]["store"],"owner":SYS_ARGS['config']['owner'],"args":SYS_ARGS["config"]["args"]}
     return render_template("index.html",**_args)
 @app.route("/format/<id>/<index>",methods=['POST'])
 def _format(id,index):
@@ -21,43 +92,117 @@ def _format(id,index):
     key = '837' if id == 'claims' else '835'
     index = int(index)
     # p = e.info[key][index]
-    p = e.apply(type=id,index=index)
+    p = e.filter(type=id,index=index)
    
-    #
     r = []
-    for item in p[0]['pipeline'] :
+    for item in p['pipeline'] :
         _item= dict(item)       
-        del _item['sql']
-        del _item ['data']
-        
         _item = dict(_item,**healthcareio.analytics.Apex.apply(item))
+        del _item['data']
         if 'apex' in _item or 'html' in _item:
             r.append(_item)
         
-    r = {"id":p[0]['id'],"pipeline":r}    
+        
+    r = {"id":p['id'],"pipeline":r}  
     return json.dumps(r),200
+
 @app.route("/get/<id>/<index>",methods=['GET'])
 def get(id,index):
     e = SYS_ARGS['engine']
     key = '837' if id == 'claims' else '835'
     index = int(index)
     # p = e.info[key][index]
-    p = e.apply(type=id,index=index)
+    p = e.filter(type=id,index=index)
     r = {}
     for item in p[0]['pipeline'] :
         
         _item= [dict(item)]
         
-        r[item['label']] = item['data'].to_dict(orient='record')
+        # r[item['label']] = item['data'].to_dict(orient='record')
-        # del _item['sql']
+        r[item['label']] = item['data'].to_dict('record')
-        # del _item ['data']
-    #     print (item['label'])
-    #     _item['apex'] = healthcareio.analytics.Apex.apply(item)
-    #     if _item['apex']:
-    #         r.append(_item)
-        
-    # r = {"id":p[0]['id'],"pipeline":r}
     return json.dumps(r),200
+
+@app.route("/reset",methods=["POST"])
+def reset():
+    return "1",200
+@app.route("/data",methods=['GET'])
+def get_data ():
+    """
+    This function will return statistical data about the services i.e general statistics about what has/been processed
+    """
+    HEADER = {"Content-type":"application/json"}
+    _args = SYS_ARGS['config']
+    options = dict(proxy.get.files(_args),**proxy.get.processes(_args))
+    return json.dumps(options),HEADER
+@app.route("/log/<id>",methods=["POST","PUT","GET"])
+def log(id) :
+    HEADER = {"Content-Type":"application/json; charset=utf8"}
+    if id == 'params' and request.method in ['PUT', 'POST']:
+        info = request.json
+        _args = {"batch":info['batch'] if 'batch' in info else 1,"resume":True}
+        #
+        # We should update the configuration 
+        SYS_ARGS['config']['args'] = _args
+        PATH = os.sep.join([os.environ['HOME'],'.healthcareio','config.json'])
+        write = lambda content: (open(PATH,'w')).write(json.dumps(content))
+        proc = Process(target=write,args=(SYS_ARGS['config'],))
+        proc.start()
+        return "1",HEADER
+    pass
+@app.route("/io/<id>",methods=['POST'])
+def io_data(id):
+    if id == 'params' :
+        _args = request.json
+        #
+        # Expecting batch,folder as parameters
+        _args = request.json
+        _args['resume'] = True
+        print (_args)
+        #
+        # We should update the configuration 
+        SYS_ARGS['config']['args'] = _args
+        # PATH = os.sep.join([os.environ['HOME'],'.healthcareio','config.json'])
+        try:
+            write = lambda content: (open(PATH,'w')).write(json.dumps(content))
+            proc = Process(target=write,args=(SYS_ARGS['config'],))
+            proc.start()
+            # proc.join()
+            return "1",200
+        except Exception as e :
+            return "0",403
+            pass
+    elif id == 'stop' :
+        stop()
+        pass
+    elif id == 'run' :
+        # run()
+        _args = {"args":SYS_ARGS['config']['args'],"store":SYS_ARGS["config"]["store"]}
+        proxy.run(_args)
+        return "1",200
+        pass
+        
+@app.route("/export")
+def export_form():
+    _args = {"context":SYS_ARGS['context']}
+    return render_template("store.html",**_args)
+@app.route("/export",methods=['POST','PUT'])
+def apply_etl():
+    _info = request.json
+    m = {'s3':'s3.s3Writer','mongo':'mongo.MongoWriter'}
+    if _info :
+        dest_args = {'type':m[_info['type']],"args": _info['content'] }
+        src_args = SYS_ARGS['config']['store']
+        # print (_args)
+        # writer = transport.factory.instance(**_args)
+        proxy.publish(src_args,dest_args)
+        return "1",405
+        
+    else:
+        return "0",404
+@app.route("/update")
+def update():
+    pass
+    return "0",405
 @app.route("/reload",methods=['POST'])
 def reload():
     # e = SYS_ARGS['engine']
@@ -74,11 +219,20 @@ if __name__ == '__main__' :
     PORT = int(SYS_ARGS['port']) if 'port' in SYS_ARGS else 5500
     DEBUG= int(SYS_ARGS['debug']) if 'debug' in SYS_ARGS else 0
     SYS_ARGS['context'] = SYS_ARGS['context'] if 'context' in SYS_ARGS else ''
+
     #
     # 
     PATH= SYS_ARGS['config'] if 'config' in SYS_ARGS else os.sep.join([os.environ['HOME'],'.healthcareio','config.json'])
+    #
+    # Adjusting configuration with parameters (batch,folder,resume)
+    if 'args' not in SYS_ARGS['config'] :
+        SYS_ARGS['config']["args"] = {"batch":1,"resume":True,"folder":"/data"}
     
+    SYS_ARGS['procs'] = []
+    
+    
+    # SYS_ARGS['path'] = os.sep.join([os.environ['HOME'],'.healthcareio','config.json'])
     e = healthcareio.analytics.engine(PATH)
-    # e.apply(type='claims',serialize=True)
+    e.apply(type='claims',serialize=False)
     SYS_ARGS['engine'] = e
     app.run(host='0.0.0.0',port=PORT,debug=DEBUG,threaded=True)

healthcareio/server/index.py

@@ -12,8 +12,9 @@ def _icon():
 def init():
     e = SYS_ARGS['engine']
     sections = {"remits":e.info['835'],"claims":e.info['837']}
-    _args = {"sections":sections}
+    _args = {"sections":sections,"store":SYS_ARGS['config']['store']}
-    return render_template("index.html",**_args)
+    print (SYS_ARGS['config']['store'])
+    return render_template("setup.html",**_args)
 @app.route("/format/<id>/<index>",methods=['POST'])
 def _format(id,index):
     

healthcareio/server/proxy.py

@@ -0,0 +1,170 @@
+"""
+    This file serves as proxy to healthcare-io, it will be embedded into the API
+"""
+import os
+import transport
+import numpy as np
+import x12
+import pandas as pd
+import smart
+from analytics import Apex
+import time
+class get :
+    PROCS = []
+    PATH = os.sep.join([os.environ['HOME'],'.healthcareio','config.json'])
+    @staticmethod
+    def resume (files,args):
+        """
+        This function will determine the appropriate files to be processed by performing a simple complementary set operation against the logs
+        @TODO: Support data-stores other than mongodb
+        :param files   list of files within a folder
+        :param _args    configuration
+        """
+        _args = args['store'].copy()
+        if 'mongo' in _args['type'] :
+            _args['type'] = 'mongo.MongoReader'
+            reader = transport.factory.instance(**_args)
+        _files = []
+        try:
+            pipeline = [{"$match":{"completed":{"$eq":True}}},{"$group":{"_id":"$name"}},{"$project":{"name":"$_id","_id":0}}]
+            _args = {"aggregate":"logs","cursor":{},"allowDiskUse":True,"pipeline":pipeline}
+            _files = reader.read(mongo = _args)
+            _files = [item['name'] for item in _files]
+        except Exception as e :
+            pass
+        print (["found ",len(files),"\tProcessed  ",len(_files)])
+        return list(set(files) - set(_files))
+
+    @staticmethod
+    def processes(_args):
+        _info = pd.DataFrame(smart.top.read(name='healthcare-io.py'))[['name','cpu','mem']]
+        
+        if _info.shape[0] == 0 :
+            _info = pd.DataFrame({"name":["healthcare-io.py"],"cpu":[0],"mem":[0]})
+        # _info = pd.DataFrame(_info.groupby(['name']).sum())
+        # _info['name'] = ['healthcare-io.py']
+        m = {'cpu':'CPU','mem':'RAM','name':'name'}
+        _info.columns = [m[name] for name in _info.columns.tolist()]
+        _info.index = np.arange(_info.shape[0])
+
+        charts = []
+        for label in ['CPU','RAM'] :
+            value = _info[label].sum()
+            df = pd.DataFrame({"name":[label],label:[value]})
+            charts.append (
+                Apex.apply(
+                    {"data":df, "chart":{"type":"radial","axis":{"x":label,"y":"name"}}}
+                    )['apex']
+                )
+        #
+        # This will update the counts for the processes, upon subsequent requests so as to show the change
+        #     
+        N = 0
+        lprocs = []
+        for proc in get.PROCS :
+            if proc.is_alive() :
+                lprocs.append(proc)
+        N = len(lprocs)     
+        get.PROCS = lprocs
+        return {"process":{"chart":charts,"counts":N}}
+    @staticmethod
+    def files (_args):
+        _info = smart.folder.read(path='/data')
+        N = _info.files.tolist()[0]
+        if 'mongo' in _args['store']['type'] :
+            store_args = dict(_args['store'].copy(),**{"type":"mongo.MongoReader"})
+            # reader = transport.factory.instance(**_args)
+            
+            pipeline = [{"$group":{"_id":"$name","count":{"$sum":{"$cond":[{"$eq":["$completed",True]},1,0]}} }},{"$group":{"_id":None,"count":{"$sum":"$count"}}},{"$project":{"_id":0,"status":"completed","count":1}}]
+            query = {"mongo":{"aggregate":"logs","allowDiskUse":True,"cursor":{},"pipeline":pipeline}}
+            # _info = pd.DataFrame(reader.read(mongo={"aggregate":"logs","allowDiskUse":True,"cursor":{},"pipeline":pipeline}))
+            pipeline = [{"$group":{"_id":"$parse","claims":{"$addToSet":"$name"}}},{"$project":{"_id":0,"type":"$_id","count":{"$size":"$claims"}}}]
+            _query = {"mongo":{"aggregate":"logs","cursor":{},"allowDiskUse":True,"pipeline":pipeline}} #-- distribution claims/remits
+
+
+        else:
+            store_args = dict(_args['store'].copy(),**{"type":"disk.SQLiteReader"})
+            store_args['args']['table'] = 'logs'
+            query= {"sql":"select count(distinct json_extract(data,'$.name')) as count, 'completed' status from logs where json_extract(data,'$.completed') = true"}
+            _query={"sql":"select json_extract(data,'$.parse') as type,count(distinct json_extract(data,'$.name')) as count from logs group by type"} #-- distribution claim/remits
+        reader = transport.factory.instance(**store_args)
+        _info = pd.DataFrame(reader.read(**query))
+        if not _info.shape[0] :
+            _info = pd.DataFrame({"status":["completed"],"count":[0]})
+        _info['count'] = np.round( (_info['count'] * 100 )/N,2)
+        
+        charts = [Apex.apply({"data":_info,"chart":{"type":"radial","axis":{"y":"status","x":"count"}}})['apex']]
+        #
+        # Let us classify the files now i.e claims / remits
+        #
+        
+        
+        # pipeline = [{"$group":{"_id":"$parse","claims":{"$addToSet":"$name"}}},{"$project":{"_id":0,"type":"$_id","count":{"$size":"$claims"}}}]
+        # _args = {"aggregate":"logs","cursor":{},"allowDiskUse":True,"pipeline":pipeline}
+        # r = pd.DataFrame(reader.read(mongo=_args))
+        r = pd.DataFrame(reader.read(**_query)) #-- distribution claims/remits
+        r = Apex.apply({"chart":{"type":"donut","axis":{"x":"count","y":"type"}},"data":r})['apex']
+        r['chart']['height'] = '100%'
+        r['legend']['position'] = 'bottom'
+
+        charts += [r]
+
+        
+        return {"files":{"counts":N,"chart":charts}}
+
+        pass
+#
+# Process handling ....
+
+
+def run (_args) :
+    """
+    This function will run the jobs and insure as processes (as daemons).
+    :param _args    system configuration
+    """
+    FILES = []
+    BATCH = int(_args['args']['batch']) #-- number of processes (poorly named variable)
+
+    for root,_dir,f in os.walk(_args['args']['folder']) :
+        if f :
+            FILES += [os.sep.join([root,name]) for name in f]
+    FILES = get.resume(FILES,_args)
+    FILES = np.array_split(FILES,BATCH)
+    
+    for FILE_GROUP in FILES :
+        
+        FILE_GROUP = FILE_GROUP.tolist()
+        # logger.write({"process":index,"parse":_args['parse'],"file_count":len(row)})
+        # proc = Process(target=apply,args=(row,info['store'],_info,))
+        parser = x12.Parser(get.PATH) #os.sep.join([PATH,'config.json']))
+        parser.set.files(FILE_GROUP)   
+        parser.daemon = True
+        parser.start()
+        get.PROCS.append(parser)     
+        time.sleep(3)
+    #
+    # @TODO:consider submitting an update to clients via publish/subscribe framework
+    #
+    return get.PROCS
+def stop(_args):
+    for job in get.PROCS :
+        if job.is_alive() :
+            job.terminate()
+    get.PROCS = []
+    #
+    # @TODO: consider submitting an update to clients via publish/subscribe framework
+    pass
+def write(src_args,dest_args,files) :
+    #
+    # @TODO: Support for SQLite
+    pass
+def publish (src_args,dest_args,folder="/data"):
+    FILES = []
+    for root,_dir,f in os.walk(folder) :
+        if f :
+            FILES += [os.sep.join([root,name]) for name in f]
+    #
+    # @TODO: Add support for SQLite ....
+    
+    FILES = np.array_split(FILES,4)
+    

healthcareio/server/static/css/default.css

@@ -1,3 +1,4 @@
+
 .active {
     padding:4px;
     cursor:pointer;
@@ -6,3 +7,51 @@
 .active:hover{
     border-bottom:2px solid #ff6500; 
 }
+input[type=text]{
+    border:1px solid transparent;
+    background-color:#f3f3f3;
+    outline: 0px;
+    padding:8px;
+    font-weight:normal;
+    font-family:sans-serif;
+    color:black;
+}
+.active-button {
+    display:grid;
+    grid-template-columns: 32px auto;
+    gap:2px;
+    align-items:center;
+    border:2px solid #CAD5E0;
+    cursor:pointer;
+    
+}
+.active-button i {padding:4px;;}
+.active-button:hover { border-color:#ff6500}
+
+.system {display:grid; grid-template-columns: 45% 1px auto; gap:20px;  margin-left:5%; width:90%;}
+.system .status .item {display:grid; grid-template-columns: 75px 8px auto; gap:2px;}
+.input-form {display:grid; gap:2px;}
+.input-form .item {display:grid; grid-template-columns: 125px auto; gap:2px; align-items:center;}
+.input-form .item .label { font-weight:bold; padding-left:10px}
+.fa-cog {color:#4682B4}
+.fa-check {color:#00c6b3}
+.fa-times {color:maroon}
+
+.code {
+    margin:4px;
+    background:#000000 ;
+    padding:8px;
+    font-family: 'Courier New', Courier, monospace;
+    color:#d3d3d3;
+    font-size:12px;
+    line-height: 2;
+}
+
+.tabs {display:grid; grid-template-columns: repeat(3,1fr) auto; gap:0px; align-items:center; text-align: center;}
+.tab {border:1px solid transparent; border-bottom-color:#D3D3D3; font-weight:bold; padding:4px}
+
+.tabs .selected {border-color:#CAD5E0; border-bottom-color:transparent; }
+.system iframe {width:100%; height:100%; border:1px solid transparent;}
+.data-info {height:90%; padding:8px;}
+.fa-cloud {color:#4682B4}
+.fa-database{color:#cc8c91}

healthcareio/server/static/dialog.html

@@ -0,0 +1,21 @@
+<div class="dialog">
+    <div class="title-bar">
+        <div class="title bold" ></div>
+        <div class="active close" align="center"><i class="fas fa-times"></i></div>
+    </div>
+    <div class="message">
+        <div class="icon" align="center">
+            <i id="msg-icon"></i>
+        </div>
+        <div class="text"></div>
+    </div>
+
+    <div class="action">
+        <div class="active-button border-round" align="center">
+            <div align="center">
+                <i class="fas fa-check"></i>
+            </div>
+            <div class="bold">Ok</div>
+        </div>
+    </div>
+</div>

healthcareio/server/static/js/io/dialog.js

@@ -0,0 +1,75 @@
+/***
+ * This file will handle the dialog boxes as they and their associated configurations and function binding
+ */
+if (!dialog){
+    var dialog = {}
+}
+
+dialog.open = function(title,msg,pointer){
+    if (sessionStorage.dialog == null){
+
+    
+    var http = HttpClient.instance()
+        http.get(sessionStorage.io_context+'/static/dialog.html',function(x){
+            var html = x.responseText
+            jx.modal.show({html:html,id:'dialog'})
+            $('.dialog .title').text(title)            
+            $('.dialog .message .text').text(msg)
+            dialog.status.ask()
+            $('.dialog .action .active-button').on('click',pointer)
+            $('.dialog .title-bar .close').on('click',function(){dialog.close(0)})
+
+        })
+    }else{
+        var html = sessionStorage.dialog
+        jx.modal.show({html:html,id:'dialog'})
+        dialog.status.ask()
+        $('.dialog .action .active-button').on('click',pointer)
+        $('.dialog .title-bar .close').on('click',function(){dialog.close(0)})
+
+    }
+}
+dialog.bind = function(pointer){
+    if (pointer == null){
+        pointer = dialog.close
+    }
+    $('.dialog .action .active-button').off()
+    $('.dialog .action .active-button').on('click',pointer)
+}
+dialog.close = function(delay){
+    delay = (delay == null)?1750:delay
+    setTimeout(function(){
+        if ( $('.dialog').length > 0){
+            jx.modal.close()
+        }
+    },delay)
+}
+dialog.status = {}
+dialog.status.wait = function(){
+    $('.dialog .action .active-button').hide()
+}
+dialog.status.confirm = function(){
+    $('.dialog .action .active-button').show()
+}
+dialog.status.busy = function(){
+    $('.dialog .message #msg-icon').removeClass()
+    $('.dialog .message #msg-icon').addClass('fas fa-cog fa-4x fa-spin')
+    
+}
+dialog.status.fail = function(){
+    $('.dialog .message #msg-icon').removeClass()
+    $('.dialog .message #msg-icon').addClass('fas fa-times fa-4x')
+}
+dialog.status.ask = function(){
+    $('.dialog .message #msg-icon').removeClass()
+    $('.dialog .message #msg-icon').addClass('far fa-question-circle fa-4x')
+}
+dialog.status.warn = function(){
+    $('.dialog .message #msg-icon').removeClass()
+    $('.dialog .message #msg-icon').addClass('fas fa-exclamation-triangle fa-4x')
+}
+dialog.status.success = function(){
+    $('.dialog .message #msg-icon').removeClass()
+    $('.dialog .message #msg-icon').addClass('fas fa-check fa-4x')
+}
+

healthcareio/server/static/js/io/healthcare.js

@@ -0,0 +1,15 @@
+if (!healthcare) {
+    var healthcare = {io:{}}
+}
+healthcare.io = {'dialog':dialog,'confirmed':confirmed,'reset':reset,'update':update,'run':run,'publish':publish}
+healthcare.io.apply = function(){
+    var value = $('.input-form .item .procs').val()    
+    var folder= $('.input-form .item .folder').val()
+    $('.code .batch').html(value)
+    var http = HttpClient.instance()
+    http.setData({"batch":value,"resume":true,"folder":folder},"application/json")
+    http.post(sessionStorage.io_context+'/io/params',function(x){})
+}
+
+
+

healthcareio/server/static/js/io/io.js

@@ -0,0 +1,182 @@
+/**
+ * This file will depend on dialog.js (soft dependency). Some functions here will make calls to resources in dialog.js
+ */
+
+var reset = function(){
+    dialog.open('Healthcare/IO::Parser', 'Are you sure you would like to delete all data parsed? Click Ok to confirm',confirmed.reset)
+    
+}
+var update= function(){
+    dialog.open('Healthcare/IO::Parser','Update will change parsing configuration. Would you like to continue ?',confirmed.update)
+}
+var run = function(){
+    dialog.open('Healthcare/IO::Parser','Preparing parser, confirm to continue',confirmed.run)
+}
+var _queue = {socket:null}
+var confirmed = {}
+confirmed.run = function(){
+    dialog.status.busy()
+    dialog.status.wait()
+    $('.dialog .message .text').html('Initiating Parsing ...')
+    setTimeout(function(){
+        var http = HttpClient.instance()
+        http.post(sessionStorage.io_context+'/io/run',function(x){
+        //    dialog.handler = setInterval(function(){monitor.data()},750)
+        monitor.data()
+        //dialog.close()
+    
+        })
+    
+    },1000)
+}
+confirmed.reset = function(){
+    var uri = sessionStorage.io_context+'/reset'
+    var http= HttpClient.instance()
+    dialog.status.busy()
+    dialog.status.wait()
+    http.post(uri,function(x){
+       setTimeout(function(){
+        if (x.status == 200 && x.responseText == "1"){
+            dialog.status.success()
+            $('.dialog .message .text').html('Reset Healthcare/IO::Parser was successful!<br><div align="center">Dialog will be closing</div>')
+            dialog.close()
+        }else{
+            dialog.status.fail()
+            
+            
+        }
+
+       },2000)
+    })
+}
+
+confirmed.update = function(){
+    var uri = sessionStorage.io_context+'/update'
+    var email  = $('#email').val()
+    //
+    //-- validate the email
+    if (email.match(/^([^\s]+)@([^\s@]+)\.(org|com|edu|io)$/i)){
+        dialog.status.wait()
+        dialog.status.busy()
+        var http = HttpClient.instance()
+        http.setData({"email":email},"application/son")
+        setTimeout(function(){
+            http.post(uri,function(x){
+                if(x.status == 200 && x.responseText == "1"){
+                    dialog.status.success()
+                }else{
+                    
+                    dialog.status.fail()
+                    $('.dialog .message .text').html('Error code '+x.status)
+                    dialog.bind()
+                    dialog.status.confirm()
+                    $('.dialog .title-bar .title').html("Error found")
+            
+                }
+            })
+    
+        },1000)
+    }else{
+        dialog.status.fail()
+        dialog.bind()
+        $('.dialog .title-bar .title').text("Error found")
+        $('.dialog .message .text').html('Invvalid Email entered')
+        dialog.status.confirm()
+    }
+    
+}
+
+/**
+ * This namespace is designed to export data to either the cloud or to a database
+ */
+var publish={set:{}}
+publish.post = function(){
+    
+    if($('.jxmodal').length > 0){
+        jx.modal.close()
+    }
+    dialog.open('Export/ETL','Please wait')
+    dialog.status.busy()
+
+    var http = HttpClient.instance()
+    http.setData(JSON.parse(sessionStorage.export),"application/json")
+    http.post(sessionStorage.io_context+'/export',function(x){
+        if (x.status != 200){
+            setTimeout(function(){
+                $('.dialog .message .text').html('An error occurred with code '+x.status)
+                dialog.status.fail()
+                dialog.status.wait()
+    
+            },1500)
+            
+        }
+        //
+        // @TODO: Have progress be monitored for this bad boy i.e open the connection to socket and read in ...
+        //
+    })
+
+}
+publish.set.file = function(){
+    var file = $('#file')[0].files[0]
+    $('.file .name').html(file.name)
+    var button = $('.cloud input').prop('disabled',true)
+    var div = $('.cloud .file .fa-file-upload')[0]
+    $(div).empty()
+    $(div).addClass('fas fa-cog fa-spin')
+    var reader = new FileReader()
+    reader.readAsText(file)
+    
+    
+    reader.onload = function(){
+        _args = {"type":$('.cloud .id').html().trim(),"content":reader.result}
+        // _args = JSON.stringify(_args)
+        if (_args.content.match(/^\{.+/i) == null){
+            content = _args.content.split('\n')[1].split(',')
+            _args.content = {'bucket':'healthcareio','access_key':content[0].trim(),'secret_key':content[1].trim()}
+        }
+        sessionStorage.export = JSON.stringify(_args)
+    }
+    
+    reader.onloadend = function(){
+        setTimeout(function(){
+            var div = $('.cloud .file .fa-cog')[0]
+            $(div).empty()
+            $(div).addClass('fas fa-check')
+            $(div).removeClass('fa-spin')
+            // jx.modal.close()
+            
+            //setTimeout(jx.modal.close,1500)
+            publish.post()
+        },2000)    
+    }
+}
+publish.database = {}
+
+publish.database.init = function(id){
+    //
+    // we are expecting id in {mongo,couch,postgresql,mysql,sqlite}
+    // @TODO: Account for cloud service brokers like dropbox, box, one-drive and google-drive
+    sessionStorage.export = "{}"
+    p = {'id':id}
+    if (id.match(/(mongodb|postgresql|mysql|sqlite|couchdb)/i)){
+        var hide_id = '.store .cloud'
+        var show_id = '.store .database'
+    }else{
+        //
+        // @TODO: generate an error message
+        var show_id = '.store .cloud'
+        var hide_id = '.store .database'
+
+    }
+    var http = HttpClient.instance()
+    http.get(sessionStorage.io_context+'/export',function(x){
+        var html = x.responseText
+        jx.modal.show({'html':html,'id':'dialog'})
+        $(hide_id).hide(function(){
+            $(show_id).show()
+        })
+
+        $('.store .id').text(id)
+    })
+
+}

healthcareio/server/static/js/jx/rpc.js

@@ -13,13 +13,13 @@
 */
 if(!jx){
     var jx = {}
-}
+  }
-/**
+  /**
    * These are a few parsers that can come in handy:
    * urlparser:	This parser is intended to break down a url parameter string in key,value pairs
    */
   
-function urlparser(url){
+  function urlparser(url){
           if(url.toString().match(/\x3F/) != null){
               url = url.split('\x3F')[1]
               
@@ -41,15 +41,15 @@ function urlparser(url){
       }
   
       return r.data;
-}
+  }
   
-/**
+  /**
-* The following are corrections related to consistency in style & cohesion
+  * The following are corrections related to consistency in style & cohesion
-*/
+  */
-jx.ajax = {}
+  jx.ajax = {}
-jx.ajax.get = {} ;
+  jx.ajax.get = {} ;
-jx.ajax.debug = null;
+  jx.ajax.debug = null;
-jx.ajax.get.instance = function(){
+  jx.ajax.get.instance = function(){
       var factory = function(){
           this.obj = {}
           this.obj.headers = {}
@@ -61,8 +61,16 @@ jx.ajax.get.instance = function(){
                   this.obj.headers[key] = value;
               }
           }
-        this.setData = function(data){
+          this.setData = function(data,mimetype){
+              if(mimetype == null)
                   this.obj.data = data;
+              else {
+                  this.obj.headers['Content-Type'] = mimetype
+                  if(mimetype.match(/application\/json/i)){
+                      this.obj.data = JSON.stringify(data)
+                  }
+              }
+              
           }
           this.setAsync = function(flag){
               this.obj.async = (flag == true) ;
@@ -122,9 +130,10 @@ jx.ajax.get.instance = function(){
       }
       }//-- end of the factory method
       return new factory() ;
-}
+  }
+  
+  //
+  // backward compatibility
+  jx.ajax.getInstance = jx.ajax.get.instance ;
+  var HttpClient = jx.ajax.get ;
   
-//
-// backward compatibility
-jx.ajax.getInstance = jx.ajax.get.instance ;
-var HttpClient = jx.ajax.get ;

healthcareio/server/templates/header.html

@@ -0,0 +1,29 @@
+<meta charset="utf8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<meta http-equiv="Cache-Control" content="no-cache, no-store, must-revalidate" />
+<meta http-equiv="Pragma" content="no-cache" />
+<meta http-equiv="Expires" content="0" />
+<link rel="shortcut icon" href="{{context}}/static/img/logo.svg" type="image/icon type">
+<link rel="stylesheet" href="{{context}}/static/css/default.css" type="text/css">
+<script src="{{context}}/static/js/jx/rpc.js"></script>
+<script src="{{context}}/static/js/jx/dom.js"></script>
+<script src="{{context}}/static/js/jx/utils.js"></script>
+
+<script src="{{context}}/static/js/jquery.js"></script>
+
+<script src="https://cdn.jsdelivr.net/npm/apexcharts"></script>
+<link href="{{context}}/static/css/borders.css" type="text/css" rel="stylesheet">
+<link href="{{context}}/static/css/fa/css/all.css" type="text/css" rel="stylesheet">
+<script src="{{context}}/static/css/fa/js/all.js"></script>
+
+<div class="border-round border menu-bar">
+    <div class="menu">
+       Admin
+       <div class="menu-items border">
+           <div class="item">Setup</div>
+       </div>
+    </div>
+    <div class="menu">
+        Claims & Remits
+    </div>
+</div>

healthcareio/server/templates/index.html

@@ -8,6 +8,7 @@
 <script src="{{context}}/static/js/jx/rpc.js"></script>
 <script src="{{context}}/static/js/jx/dom.js"></script>
 <script src="{{context}}/static/js/jx/utils.js"></script>
+<script src="{{context}}/static/js/jx/ext/modal.js"></script>
 
 <script src="{{context}}/static/js/jquery.js"></script>
 
@@ -15,6 +16,9 @@
 <link href="{{context}}/static/css/borders.css" type="text/css" rel="stylesheet">
 <link href="{{context}}/static/css/fa/css/all.css" type="text/css" rel="stylesheet">
 <script src="{{context}}/static/css/fa/js/all.js"></script>
+<script src="{{context}}/static/js/io/dialog.js"></script>
+<script src="{{context}}/static/js/io/io.js"></script>
+<script src="{{context}}/static/js/io/healthcare.js"></script>
 <style>
     body {
         font-size:16px;
@@ -64,6 +68,7 @@
         scroll-behavior: smooth;
         gap:2px;
         padding:4px;
+        height:95%;
         
 
     }
@@ -81,12 +86,12 @@
         
 
     }
-    .dashboard .chart-pane .chart {
+    .dashboard .chart-pane .chart2 {
         max-height:99%;
         min-height:99%;
         height:99%;
     }
-    .dashboard .chart-pane .chart .graph {
+    .dashboard .chart-pane .chart2 .graph {
         
         
         max-height:100%;
@@ -95,7 +100,7 @@
 
 
     }
-    .dashboard .chart-pane .chart .graph .apexcharts-svg {
+    .dashboard .chart-pane .chart2 .graph .apexcharts-svg {
         /*border-radius:8px;*/
         background-color:#f3f3f3;
         max-height:100%;
@@ -161,7 +166,7 @@
     }
    
 
-    .gradient {  background-image: linear-gradient(to top,#F3F3F3, #D3D3D3, #F3F3F3);}
+    .gradient {  background-image: linear-gradient(to top,#F3F3F3, #FFFFFF, #FFFFFF);}
     .white {color:#ffffff}
     .scalar {
         display:grid;
@@ -172,23 +177,39 @@
         
         
         
+        
     }
     .shadow {
         box-shadow: 0px 4px 4px #d3d3d3;
     }
     .scalar-title { padding:8px; text-transform: capitalize; }
-    .scalar .value {font-size:32px; font-weight:bold; margin:4%;}
+    .scalar .value {font-size:32px; margin:4%;}
     .scalar .label {font-size:12px; text-transform:capitalize; text-overflow: ellipsis; display:grid; align-items: flex-end; text-align:center ;}
+    
+   
+    .monthly_patient_count, .taxonomy_code_distribution, .top_adjustment_codes, .adjustment_reasons {
+        grid-row:1 / span 3 ;
+        
+        
+    }
+
+
 </style>
 <script>
     sessionStorage.io_context  = "{{context}}"
     var plot = function(id,index){
+        $('.system').slideUp(function(){
+            $('.dashboard').slideDown()
+        })
+        
         var uri = ([sessionStorage.io_context,'format',id,index]).join('/')
         var httpclient = HttpClient.instance()
         //
         // @TODO: Let the user know something is going on .. spinner
         httpclient.post(uri,function(x){
+            
             var r = JSON.parse(x.responseText)
+            
             var pane = jx.dom.get.instance('DIV')
             var scalar_pane = jx.dom.get.instance('DIV')
             pane.id = r.id
@@ -202,9 +223,9 @@
             var p = jx.utils.patterns.visitor(r.pipeline,function(item){
                 var div = jx.dom.get.instance('DIV')
                 var frame = jx.dom.get.instance('DIV')
-                //div.className = 'chart border-round border'
+                //div.className = 'chart2 border-round border'
-                frame.className = 'chart border'
+                frame.className = 'chart2 border ' + item.label.toLowerCase().replace(/ /g,'_')
-                div.className = 'graph'
+                div.className = 'graph '
                 //frame.append(div)
                 //pane.append(frame)
                 
@@ -212,8 +233,11 @@
              if(item.apex != null){   item.apex.title = {text:item.label}
                     frame.append(div)
                     pane.append(frame)
+                    if (item.apex.colors ){
                         delete item.apex.colors
-                    item.apex.theme= {
+                    }
+                    
+                    /*item.apex.theme= {
                         mode: 'material', 
                         palette: 'palette6', 
                         monochrome: {
@@ -222,8 +246,11 @@
                             shadeTo: 'light',
                             shadeIntensity: 0.65
                         },
-                    }
+                    }*/
                     item.apex.chart.height = '100%'
+                    delete item.apex.chart.width
+                    
+                    
                     return new ApexCharts(div,item.apex)
                 }else{
                     //frame.className = ''
@@ -283,29 +310,56 @@
         }
     }
     $(document).ready(function(){
+        $('.dashabord').hide()
         $('.item-group').slideUp()
+        var index = 0;
+        jx.utils.patterns.visitor($('.menu .items'), function(_item){
+            var node = $(_item).children()[0]
+            $(node).attr('index',index)
+            node.onclick = function(){ toggle($(this).attr('index')) }
+            index += 1;
+            
+        })
+        var year = (new Date()).getFullYear()
+        $('.year').text(year)
     })
 </script>
 <title>Healthcare/IO Analytics</title>
 <body>
-    <div class="pane border">
+    <div class="pane">
         <div class="header border-bottom">
-            <div class="caption">Healthcare/IO</div>
+            <div class="caption">Healthcare/IO :: Parser</div>
-            <div class="small">Analytics Dashboard</div>
+            <div class="small">Dashboard</div>
         </div>
         
-        <div class="menu border-right">
+        <div class="menu">
+            
            <div>
+                <div class="items">
+                    <div class="bold active" style="margin:4px; height:28px; display:grid; gap:2px; grid-template-columns:auto 32px;"> 
+                        <div>Setup</div>
+                        <div align="center" class="glyph" >
+                            
+                            <i class="fas fa-angle-down"></i>
+                        </div>
+                    </div>
+                    <div class="item-group border border-round">
+                        <div class="item small active" onclick="setup.open()">Configure Parser</div>
+                        
+                        <div class="item small active" onclick="healthcare.io.reset()">Reset Parser</div>
+
+                    </div>
+                </div>
             {% for key in sections %}
                 <div class="items">
-                <div class="bold active" onclick="toggle({{loop.index -1}})" style="margin:4px; height:28px; display:grid; gap:2px; grid-template-columns:auto 32px;"> 
+                    <div class="bold active" style="margin:4px; height:28px; display:grid; gap:2px; grid-template-columns:auto 32px;"> 
                         <div>{{key|safe}}</div>
                         <div align="center" class="glyph" >
                             
                             <i class="fas fa-angle-down"></i>
                         </div>
                     </div>                    
-                <div class="item-group border">
+                    <div class="item-group border border-round">
                         
                         {% for item in sections[key] %}
                             <div class="item small active" onclick="plot('{{key}}',{{loop.index-1}})">{{item.id}}</div>
@@ -315,18 +369,21 @@
             {% endfor %}
            </div>
             
-            <div style="display:grid; align-items:flex-end">
+            <div style="display:none; align-items:flex-end">
-                <div class="logs border" style="height:250px"></div>
+                <div class="logs chart" style=" align-items:center; display:grid" align="center"></div>
 
             </div>
             
         </div>
-        <div class="dashboard"> 
+       <div>
+            <div class="dashboard" style="display:none"></div>
+            
+            {%include 'setup.html' %}
             
        </div>
             
     </div>
     <div class="footer small">
-        &copy; Vanderbilt University Medical Center
+        Healthcare/IO :: Parser &copy; <span class="year">    </span>
     </div>
 </body>

healthcareio/server/templates/setup.html

@@ -0,0 +1,391 @@
+<style>
+    .system {height:99%; overflow:hidden;}
+    .data-info .board{ height:300px; display:grid; grid-template-columns:auto 200px 200px; gap:20px; align-items:center}
+    /*.board { background-image: linear-gradient(to bottom, #ffffff,#ffffff,#f3f3f3,#d3d3d3d3)}*/
+    .number {font-size:48px; font-family:courier;padding:8px; ;}
+    .etl {display:grid; grid-template-columns: 250px auto; gap:2;}
+    .chart {box-shadow : 0px 1px 4px 2px #d3d3d3; width:200px; height:250px;
+        display:grid; align-items:center;
+        background-image: linear-gradient(to bottom,#f3f3f3,#ffffff);
+        overflow:hidden;
+        
+        
+    }
+    
+    .dialog { width:450px; min-height:200px; display:grid; grid-template-rows: 40px 80% auto; gap:4px}
+    .dialog .title-bar { border-top-left-radius: 8px; border-top-right-radius: 8px ; padding:4px; background-color:#f3f3f3; gap:2px; display:grid; grid-template-columns: auto 32px; align-items:center}
+    .dialog .action {display:grid; align-items: flex-end; padding-left:25%; padding-right:25%;}
+    .dialog .message {display:grid; align-items: center; grid-template-columns: 20% auto;}
+    .dialog .message .text {line-height:2; text-transform: capitalize;}
+    .fa-exclamation-triangle {color:orange}
+    .fa-question-circle{color:#009df7}
+</style>
+
+<script src="https://cdn.socket.io/socket.io-1.3.5.js"></script>
+<script src="{{context}}/static/js/io/dialog.js"></script>
+<script src="{{context}}/static/js/io/io.js"></script>
+<script src="{{context}}/static/js/io/healthcare.js"></script>
+
+<script>
+    var select = function(node){
+        var value = $($(node).children()[0]).attr('data-value')
+        
+        jx.utils.patterns.visitor($('.tab'),function(_item){
+            var button = $(_item).children()[0]
+            $(_item).removeClass('selected')
+            //alert([$(button).attr('data-value'),value])
+            if($(button).attr('data-value') == value){
+                $(node).addClass('selected')
+                $('.'+value).show()
+                
+            }else{
+                var m = '.'+ $(button).attr('data-value')
+                
+                    $(m).hide()
+                
+                
+            }
+        })
+        
+    }
+    var monitor = {}
+    monitor.listen = {handler:null}
+    
+    monitor.data = function(){
+        var http = HttpClient.instance()
+        http.get("/data",function(x){
+            var r = JSON.parse(x.responseText)
+            var keys = jx.utils.keys(r) //-- process,files
+            for (var i in keys){                
+                var prefix = keys[i]
+                if(prefix == 'process'){
+                    if(r[prefix].counts != 0){
+                        //
+                        // We should insure the listeners are enabled
+                        if(monitor.listen.handler == null){
+                            monitor.listen.handler = setInterval(
+                                function(){
+                                    console.log('running ...')
+                                    monitor.data()},5000)
+
+                        }
+                    }else{
+                        if (monitor.listen.handler != null){
+                            
+                            clearInterval(monitor.listen.handler)
+                        }
+                        dialog.close()
+                    }
+                }
+                monitor.render(prefix,r[prefix])
+            }            
+        })
+    }
+    monitor.render = function(prefix,r){
+        prefix = '.'+prefix
+        
+        var div = jx.dom.get.instance('DIV')
+        var label = jx.dom.get.instance('DIV')
+        div.align = 'center'
+        
+        div.innerHTML = r.counts
+        div.className = 'number'
+        label.innerHTML = prefix.replace(/\./,'')
+        label.style.textTransform = 'capitalize'
+        label.className = 'small bold border-top'
+        div.append(label)
+        
+
+            $(prefix + ' .board').empty()
+            $(prefix+' .board').append(div)
+        
+        var charts = jx.utils.patterns.visitor(r.chart,function(option){
+            
+            
+                var div = jx.dom.get.instance('div')
+            
+                div.className = 'chart'
+                div.align='center'
+                $(prefix+' .board').append(div)
+                
+                var chart = new ApexCharts($(div)[0],option)
+                //chart.render()
+                div.chart = chart
+    
+            return chart
+
+        })  
+        var observers = jx.utils.patterns.visitor(charts,function(_item){
+            var m = function(_chart){
+                this.chart = _chart ;
+                this.apply = function(caller){this.chart.render();                     
+                    caller.notify()
+                }
+            }
+            return new m(_item)
+        })
+        jx.utils.patterns.observer(observers,'apply')
+        //jx.utils.patterns.iterator(charts,'render')
+        
+         /*setTimeout(function(){
+             jx.utils.patterns.visitor(charts,function(_item){_item.render()})
+         },1000)  */ 
+    }
+    var setup = {}
+    setup.open = function(){
+        $('.dashboard').slideUp(
+            function(){
+                $('.setup').slideDown()
+            }
+        )
+        
+    }
+
+    $(document).ready(function(){
+        /*var shandler = new io();
+
+        if (shandler.disconnected ==false){
+            shandler.disconnect()
+        }
+        var socket = io.connect()
+        socket.on('connect',function(e){
+            socket.emit('connect',{name:'steve'})
+        })
+        socket.on('update',function(e){
+            console.log(e)
+            console.log()
+        })
+       var socket = io.connect('http://localhost:81',{cors:{AccessControlAllowOrigin:'*'}}) //('http://localhost:81/stream')
+        socket.on('procs',function(e){
+            
+        })
+        socket.on('data',function(e){
+            
+            $('.logs').empty()
+            var div = $('.logs')
+            
+            var option = e.apex
+            option.plotOptions.pie.size = 220 
+            option.plotOptions.pie = {dataLabels: {show:true,name:{show:true},value:{show:true}}}
+            
+            option.legend.show = false
+            console.log(option)
+            c = new ApexCharts(div[0],option)
+            c.render()
+            socket.emit("procs",{"name":"steve"})
+        })*/
+    
+        select($('.tab')[0])
+        monitor.data()
+        
+        $('.email').text($('#email').val())
+    })
+</script>
+
+    
+    <div class="system setup">
+        <div class="status">
+            
+            {% if not store.type %}
+               <div >
+                   <span class="caption bold border-bottom" style="padding-right:10">Current Configuration</span>
+                   <p></p>
+                   <div style="display:grid; align-items:center; grid-template-columns:32px auto;">
+                       <i class="fa fa-times" style="font-size:28; margin:4px;"></i> <span>System needs to be initialized !</span>
+                   </div>
+                   <p></p>
+                   <div class="active-button border-round" style="width:50%">
+                        <div class="icon">
+                            <i class="fas fa-cog"  style="font-size:28"></i>
+                        </div>
+                        <div class="bold" align="center">Initialize</div>
+                    </div>
+               </div>
+            {% else %}
+                <div class="border-right">
+                    
+                    <span class="caption bold border-bottom" style="padding-right:10">Current Configuration</span>                        
+                    <p>
+                    </p>
+                    <div class="item" style="display:grid; align-items:center">
+                        <div class="bold" style="text-transform: capitalize;">Owner </div><div class="bold">:</div>
+                        <div>
+                            
+                            <input type="text" id="email" value="{{owner}}">
+                        </div>
+                    </div>
+
+                    <div class="item">
+                        <div class="bold" style="text-transform: capitalize;">store</div><div class="bold">:</div>
+                        <div class="">{{store.type}}</div>
+                    </div>
+                    <p></p>
+                    <div class="active-button border-round" style="width:50%" onclick="healthcare.io.update()">
+                        <div class="icon">
+                            <I class="fas fa-download" style="font-size:28"></I>
+                        </div>
+                        <div class="bold" align="center">Update Config</div>
+                    </div>
+                    <p>
+                        <div class="code">
+                            #<br>
+                            healthcare-io.py --init <span class="email"></span> --store mongo
+                        </div>
+                    </p>
+               </div>
+               
+            {%endif%}
+            <p></p>
+            <br>
+            <div class="border-right">
+                <span class="caption bold border-bottom" style="padding-right:10">Manage Plan</span>
+                <p></p>
+                <div style="line-height: 2;">Insure your account is tied to a cloud service provider. 
+                    <br>We support <span class="bold">google-drive, dropbox, one-drive or box. </span>
+                </div>
+                <p>
+                    <div class="bold active-button border-round" style="width:50%" onclick="jx.modal.show({url:'https://healthcareio.the-phi.com/store/healthcareio/plans'})">
+                        <div>
+                            <img src="{{context}}/static/img/logo.svg" />
+                        </div>
+                        <div align="center">Open Plan Console</div>
+                    </div>
+                    </p>
+                
+            </div>
+            <br>
+            <div class="border-right" style="height:30%"></div>
+        </div>
+        <div class="_border-right"></div>
+        <div >
+            <span class="caption bold border-bottom" style="padding-right:10">
+                Manage Processes</span>
+            <p>
+                <div class="input-form" style="grid-template-columns: 30% auto;">
+                    <div class="item" style="grid-row:1; grid-column:1; ">
+                        <div class="label">Process #</div>
+                        <input type="text" class="procs batch"placeholder="#" style="width:64px; text-align:right" value="{{args.batch}}" onchange="healthcare.io.apply()"/>
+                    </div>
+                    <div class="item" style="grid-row:1; grid-column:2 ">
+                        <div class="label">Folder #</div>
+                        <input type="text" placeholder="Process counts" value="/data"/ class="data folder" disabled>
+                    </div>                                                         
+                    
+                </div>
+                <div style="display:grid; grid-template-columns:repeat(2,215px); gap:2px;">
+                    <div class="active-button border-round bold io-apply" style="margin-top:32; display:none" onclick="healthcare.io.apply()">
+                        <div class="icon"><i class="far fa-save" style="font-size:28; color:#4682B4"></i></div>
+                        <div align="center">Apply</div>
+                    </div>
+                    <div class="active-button border-round bold" style="margin-top:32" onclick="healthcare.io.stop()">
+                        <i class="far fa-stop-circle" style="font-size:28; color:maroon"></i>
+                        
+                        <div align="center">Stop</div>
+                    </div>
+                    <div class="active-button border-round bold" style="margin-top:32" onclick="healthcare.io.run()">
+                        <i class="fas fa-running" style="font-size:28; color:green"></i>
+                        <div align="center">Run</div>
+                    </div>
+
+                </div>                   
+               
+            </p>
+            <p>
+                <div class="code">
+                    <div class="bold"># The command that will be executed</div>
+                    <div>healthcare-io.py --parse --folder /data --batch <span class="batch">{{args.batch}}</span></div>
+                </div>
+            </p>
+            <p>
+                <div style="display:grid; grid-template-columns:auto 48px ; gap:2px">
+                    <div class="bold caption border-bottom">Process Monitoring</div>
+                    <div class="active" align="center" title="reload" onclick="monitor.data()"><i class="fas fa-sync"></i></div>
+                </div>
+                <div class="small">Powered by smart-top</div>
+                <p></p>
+                <div class="tabs">
+                    <div class="tab selected" onclick="select(this)">
+                        <div class="active" data-value="process" >
+                            <i class="far fa-clock" style="color:maroon"></i>
+                            <span>Process</span>
+                        </div>
+                    </div>
+                    <div class="tab" onclick="select(this)">
+                        <div class="active" data-value="files"><i class="fas fa-file-alt"></i> Files</div>
+                    </div>
+                    <div class="tab" onclick="select(this)">
+                        <div class="active" data-value="export"><i class="fas fa-upload"></i> Export</div>
+                    </div>
+                    
+                </div>
+                <div class="data-info">
+                    <div class="process ">
+                        
+                        <div class="board"></div>
+                       <div class="small" align="center">
+                            <div class="border-top bold" style="color:#4682B4;">Running Processes and resource usage</div>
+                       </div>
+                    </div>
+                    <div class="files">
+                        <div class="board"></div>
+                        <div class="small" align="center">
+                            <div class="border-top bold" style="color:#4682B4;">Summary of files found and processed</div>
+                       </div>
+
+                    </div>
+                    <div class="export">
+                        <p></p>                        
+                        <div class="etl">
+                            <div class="" >
+                                <div class="menu" style="position:absolute; width:200">
+                                    <div class="items ">
+                                        <div class="bold active" style="display:grid; grid-template-columns:80% auto;">
+                                            <span>
+                                                <i class="fas fa-cloud"></i>
+                                                Cloud</span>
+                                            <span class="glyph">
+                                                <i class="fas fa-angle-down"></i>
+                                            </span>
+                                        </div>
+                                        <div class="item-group border-round border small">
+                                            <div class="item" onclick="healthcare.io.publish.database.init('s3')">AWS S3</div>
+                                            <div class="item" onclick="healthcare.io.publish.database.init('bigquery')">Google Bigquery</div>
+                                            
+                                        </div>
+                                    </div>
+                                   <div class="items ">
+                                       <div class="bold active"style="display:grid; grid-template-columns:80% auto;">
+                                            <span>
+                                                <i class="fas fa-database"></i>
+                                                Database</span>
+                                            <span class="glyph">
+                                                <i class="fas fa-angle-down"></i>
+                                            </span>
+                                        </div>
+                                       <div class="item-group border-round border small">
+                                           <div class="bold">SQL</div>
+                                           
+                                                <div class="item" style="margin-left:15px; margin-right:32px" onclick="healthcare.io.publish.database.init('postgresql')">PostgreSQL</div>
+                                                <div class="item" style="margin-left:15px; margin-right:32px" onclick="healthcare.io.publish.database.init('mysql')">MySQL</div>
+                                                
+         
+                                           
+                                           <div class="bold">NoSQL</div>
+                                            <div class="item" style="margin-left:15px; margin-right:32px" onclick="healthcare.io.publish.database.init('mongodb')">Mongodb</div>
+                                            <div class="item" style="margin-left:15px; margin-right:32px" onclick="healthcare.io.publish.database.init('couchdb')">Couchdb</div>
+                                       </div>
+                                   </div>
+                                </div>
+                            </div>
+                            <div>
+                                <div class="active-button border-round" style="width:50%">
+                                    <div class="icon"><i class="fas fa-running" style="font-size:28"></i></div> <div class="bold" align="center">Start</div>
+                                </div>
+                            </div>
+                        </div>
+
+                    </div>
+                </div>
+            </p>
+        </div>
+    </div>

healthcareio/server/templates/store.html

@@ -0,0 +1,78 @@
+<link href="{{context}}/static/css/fa/css/all.css" type="text/css" rel="stylesheet">
+<link rel="stylesheet" href="{{context}}/static/css/default.css" type="text/css">
+<link href="{{context}}/static/css/borders.css" type="text/css" rel="stylesheet">
+<style>
+    .form , input{font-family:sans-serif; font-size:18px}
+    .form .small {font-size:14px; line-height:2}
+    .mongo, .couchdb{
+    
+        display:grid;
+        gap:2px;
+    }
+    .grid-full {display:grid; grid-template-columns: 100%;}
+    .grid-split-half {display:grid; grid-template-columns: 50% 50%; gap:2px;}
+    .store .title-bar {display:grid; align-items:center; grid-template-columns: auto 32px; padding:8px;}
+    .file {display:grid; align-items:center; grid-template-columns: 40% auto; gap:2px; font-family:sans-serif; padding:8px}
+    .file input {display:none}
+    .file label {padding:8px;}
+    
+    
+</style>
+<div class="store">
+    
+    <div class="title-bar">
+        <div class="caption">Export Module <span class="bold id"></span></div>
+        <div class="active" align="center" onclick="jx.modal.close()">
+            <i class="fas fa-times"></i>
+        </div>
+    </div>
+    <p></p>
+
+    <div class="cloud form border-round">
+        <div class="small border-round">
+            Please provide select access-key file or service account key file to perform ETL            
+            to <span class="bold id"></span>. <br>The files will be re-created in JSON format
+        </div>
+        <div class="file">
+            <label class="active-button border-round">
+                <span style="font-family:sans-serif">
+                    <i class="icon fas fa-file-upload" style="font-size:24px"></i>
+                </span>               
+                <div>Select key file</div> 
+                <input type="file" id="file" aria-label="File browser example" onchange="publish.set.file()">
+
+            </label>
+            <div class="name small black bold" style="padding-left:24px"></div>
+        </div>
+    </div>
+   <div class="database form border-round">
+       <div class="small border-round">
+        Tables / collections will be automatically inferred in the <span class="bold id"></span>
+       </div>
+        <div class="mongo">
+
+            <div class="grid-split-half">
+                <input type="text" class="host" placeholder="host:port"/>
+                <input type="text" class="host" placeholder="database"/>
+            </div>
+           
+            <div class="grid-split-half">
+                <input type="text" placeholder="user"/>
+            
+                <input type="text" placeholder="password"/>
+            </div>
+
+        </div>
+        <p></p>
+        <div>
+            <br>
+            <div class="active-button border-round" style="margin-left:30%; margin-right:30%">
+                <div class="icon">
+                    <i class="fas fa-check"></i>
+                </div>
+                <div class="bold" align="center">Export Now</div>
+            </div>
+        </div>
+    
+   </div>
+</div>

healthcareio/x12/__init__.py

@@ -49,8 +49,9 @@ class Formatters :
         """
         This function is designed to split an x12 row and 
         """
-        if row.startswith(prefix) is False:
         value = []
+        if row.startswith(prefix) is False:
+            
             
             for row_value in row.replace('~','').split(sep) :
                 
@@ -65,10 +66,12 @@ class Formatters :
                 else :
                     
                     value.append(row_value.replace('\n',''))
-            return [xchar.replace('\r','') for xchar in value] #row.replace('~','').split(sep)
+            value =  [xchar.replace('\r','') for xchar in value] #row.replace('~','').split(sep)
         else:
             
-            return [ [prefix]+ self.split(item,'>') for item in row.replace('~','').split(sep)[1:] ]
+            value =  [ [prefix]+ self.split(item,'>') for item in row.replace('~','').split(sep)[1:] ]
+
+        return value if type(value) == list and type(value[0]) != list else value[0]
     def get_config(self,config,row):
         """
         This function will return the meaningfull parts of the configuration for a given item
@@ -191,6 +194,9 @@ class Parser (Process):
         self.files = []
         self.set = void()
         self.set.files = self.set_files
+        self.emit = void()
+        self.emit.pre =  None
+        self.emit.post = None
     def set_files(self,files):
         self.files = files
     def get_map(self,row,config,version=None):
@@ -284,6 +290,7 @@ class Parser (Process):
                 try:
                     
                     tmp = self.get.value(row,_info)
+                    
                     # if 'P1080351470' in content[0] and 'PLB' in row:
                     #     print (_info)
                     #     print (row)
@@ -300,10 +307,10 @@ class Parser (Process):
                         else:
                             if label not in value:   
                                 value[label] = [tmp]
-                            elif len(list(tmp.keys())) == 1 :
+                            # elif len(list(tmp.keys())) == 1 :
-                                # print "\t",len(claim[label]),tmp
+                            #     # print "\t",len(claim[label]),tmp
-                                index = len(value[label]) -1 
+                            #     index = len(value[label]) -1 
-                                value[label][index] = dict(value[label][index],**tmp)
+                            #     value[label][index] = dict(value[label][index],**tmp)
                             else:
                                 value[label].append(tmp)                        
                         tmp['_index'] = len(value[label]) -1 
@@ -432,6 +439,9 @@ class Parser (Process):
         # self.finish(claims,logs,_code)
         return claims,logs,_code    
     def run(self):
+        if self.emit.pre :
+            self.emit.pre()
+
         for filename in self.files :
             content,logs,_code = self.read(filename)
             self.finish(content,logs,_code)
@@ -441,14 +451,22 @@ class Parser (Process):
         if args['type'] == 'mongo.MongoWriter' :
             args['args']['doc'] = 'claims' if _code == '837' else 'remits'
             _args['args']['doc'] = 'logs'
+        else:
+            args['args']['table'] = 'claims' if _code == '837' else 'remits'
+            _args['args']['table'] = 'logs'
+
         if content      :
             writer = transport.factory.instance(**args)
             writer.write(content)
             writer.close()
         if logs :
+            
             logger = transport.factory.instance(**_args)
             logger.write(logs)
+            
             logger.close()
+        if self.emit.post :
+            self.emit.post(content,logs)
 
 
 

setup.py

@@ -8,7 +8,7 @@ import sys
 def read(fname):
     return open(os.path.join(os.path.dirname(__file__), fname)).read() 
 args = {
-    "name":"healthcareio","version":"1.3.7",
+    "name":"healthcareio","version":"1.4.0",
     "author":"Vanderbilt University Medical Center",
     "author_email":"steve.l.nyemba@vumc.org",
     "license":"MIT",