docker user interface basics

healthcareio/Dockerfile

@@ -1,4 +1,8 @@
-FROM ubuntu:bionic-20200403
+#
+# Let us create an image for healthcareio
+#   The image will contain the {X12} Parser and the 
+# FROM ubuntu:bionic-20200403
+FROM ubuntu:focal
 RUN ["apt","update","--fix-missing"]
 RUN ["apt-get","upgrade","-y"]
 
@@ -6,9 +10,22 @@ RUN ["apt-get","-y","install","apt-utils"]
 
 RUN ["apt","update","--fix-missing"]
 RUN ["apt-get","upgrade","-y"]
-RUN ["apt-get","install","-y","sqlite3","sqlite3-pcre","libsqlite3-dev","python3-dev","python3","python3-pip","git","python3-virtualenv"]
+RUN ["apt-get","install","-y","mongo","sqlite3","sqlite3-pcre","libsqlite3-dev","python3-dev","python3","python3-pip","git","python3-virtualenv","wget"]
 #
 #
+RUN ["pip3","install","--upgrade","pip"]
+# RUN ["pip3","install","git+https://healthcare.the-phi.com/git/code/parser.git","botocore"]
 USER health-user
+#
+# This volume is where the data will be loaded from (otherwise it is assumed the user will have it in the container somehow)
+#
+VOLUME ["/data"]
+#
+# This is the port from which some degree of monitoring can/will happen
+EXPOSE 80
+# wget https://healthcareio.the-phi.com/git/code/parser.git/bootup.sh 
+COPY bootup.sh bootup.sh
+ENTRYPOINT ["bash","-C"]
+CMD ["bootup.sh"]
 # VOLUME ["/home/health-user/healthcare-io/","/home-healthuser/.healthcareio"]
 # RUN ["pip3","install","git+https://healthcareio.the-phi.com/git"]

healthcareio/server/__init__.py

@@ -3,7 +3,78 @@ from healthcareio.params import SYS_ARGS
 import healthcareio.analytics
 import os
 import json
+import time
+import smart
+import transport
+import pandas as pd
+import numpy as np
+import x12
+
+from multiprocessing import Process
+from flask_socketio import SocketIO, emit, disconnect,send
+from healthcareio.server import proxy
+PATH = os.sep.join([os.environ['HOME'],'.healthcareio','config.json'])
 app = Flask(__name__)
+socket_ = SocketIO(app)
+
+def resume (files):
+    _args = SYS_ARGS['config']['store'].copy()
+    if 'mongo' in SYS_ARGS['config']['store']['type'] :
+        _args['type'] = 'mongo.MongoReader'
+        reader = transport.factory.instance(**_args)
+    _files = []
+    try:
+        pipeline = [{"$match":{"completed":{"$eq":True}}},{"$group":{"_id":"$name"}},{"$project":{"name":"$_id","_id":0}}]
+        _args = {"aggregate":"logs","cursor":{},"allowDiskUse":True,"pipeline":pipeline}
+        _files = reader.read(mongo = _args)
+        _files = [item['name'] for item in _files]
+    except Exception as e :
+        pass
+    print (["found ",len(files),"\tProcessed  ",len(_files)])
+    return list(set(files) - set(_files))
+        
+    
+
+def run ():
+    #
+    # let's get the files in the folder (perhaps recursively traverse them)
+    #
+    FILES = []
+    BATCH = int(SYS_ARGS['config']['args']['batch']) #-- number of processes (poorly named variable)
+
+    for root,_dir,f in os.walk(SYS_ARGS['config']['args']['folder']) :
+        if f :
+            FILES += [os.sep.join([root,name]) for name in f]
+    FILES = resume(FILES)
+    FILES = np.array_split(FILES,BATCH)
+    procs = []
+    for FILE_GROUP in FILES :
+        
+        FILE_GROUP = FILE_GROUP.tolist()
+        # logger.write({"process":index,"parse":SYS_ARGS['parse'],"file_count":len(row)})
+        # proc = Process(target=apply,args=(row,info['store'],_info,))
+        parser = x12.Parser(PATH) #os.sep.join([PATH,'config.json']))
+        parser.set.files(FILE_GROUP)
+        parser.start()
+        procs.append(parser)  
+    SYS_ARGS['procs']  = procs
+# @socket_.on('data',namespace='/stream')
+def push() :
+    _args = dict(SYS_ARGS['config']['store'].copy(),**{"type":"mongo.MongoReader"})
+    reader = transport.factory.instance(**_args)
+    pipeline = [{"$group":{"_id":"$parse","claims":{"$addToSet":"$name"}}},{"$project":{"_id":0,"type":"$_id","count":{"$size":"$claims"}}}]
+    _args = {"aggregate":"logs","cursor":{},"allowDiskUse":True,"pipeline":pipeline}
+    r = pd.DataFrame(reader.read(mongo=_args))
+    r = healthcareio.analytics.Apex.apply({"chart":{"type":"donut","axis":{"x":"count","y":"type"}},"data":r})
+    emit("update",r,json=True)
+    return r
+@socket_.on('connect')
+def client_connect(**r):
+    print ('Connection received')
+    print (r)
+    push()
+    pass
+    
 @app.route("/favicon.ico")
 def _icon():
     return send_from_directory(os.path.join([app.root_path, 'static','img','logo.svg']),
@@ -12,7 +83,7 @@ def _icon():
 def init():
     e = SYS_ARGS['engine']
     sections = {"remits":e.info['835'],"claims":e.info['837']}
-    _args = {"sections":sections}
+    _args = {"sections":sections,"store":SYS_ARGS["config"]["store"],"owner":SYS_ARGS['config']['owner'],"args":SYS_ARGS["config"]["args"]}
     return render_template("index.html",**_args)
 @app.route("/format/<id>/<index>",methods=['POST'])
 def _format(id,index):
@@ -21,43 +92,117 @@ def _format(id,index):
     key = '837' if id == 'claims' else '835'
     index = int(index)
     # p = e.info[key][index]
-    p = e.apply(type=id,index=index)
+    p = e.filter(type=id,index=index)
    
-    #
     r = []
-    for item in p[0]['pipeline'] :
+    for item in p['pipeline'] :
         _item= dict(item)       
-        del _item['sql']
-        del _item ['data']
-        
         _item = dict(_item,**healthcareio.analytics.Apex.apply(item))
+        del _item['data']
         if 'apex' in _item or 'html' in _item:
             r.append(_item)
         
-    r = {"id":p[0]['id'],"pipeline":r}    
+        
+    r = {"id":p['id'],"pipeline":r}  
     return json.dumps(r),200
+
 @app.route("/get/<id>/<index>",methods=['GET'])
 def get(id,index):
     e = SYS_ARGS['engine']
     key = '837' if id == 'claims' else '835'
     index = int(index)
     # p = e.info[key][index]
-    p = e.apply(type=id,index=index)
+    p = e.filter(type=id,index=index)
     r = {}
     for item in p[0]['pipeline'] :
         
         _item= [dict(item)]
         
-        r[item['label']] = item['data'].to_dict(orient='record')
+        # r[item['label']] = item['data'].to_dict(orient='record')
-        # del _item['sql']
+        r[item['label']] = item['data'].to_dict('record')
-        # del _item ['data']
-    #     print (item['label'])
-    #     _item['apex'] = healthcareio.analytics.Apex.apply(item)
-    #     if _item['apex']:
-    #         r.append(_item)
-        
-    # r = {"id":p[0]['id'],"pipeline":r}
     return json.dumps(r),200
+
+@app.route("/reset",methods=["POST"])
+def reset():
+    return "1",200
+@app.route("/data",methods=['GET'])
+def get_data ():
+    """
+    This function will return statistical data about the services i.e general statistics about what has/been processed
+    """
+    HEADER = {"Content-type":"application/json"}
+    _args = SYS_ARGS['config']
+    options = dict(proxy.get.files(_args),**proxy.get.processes(_args))
+    return json.dumps(options),HEADER
+@app.route("/log/<id>",methods=["POST","PUT","GET"])
+def log(id) :
+    HEADER = {"Content-Type":"application/json; charset=utf8"}
+    if id == 'params' and request.method in ['PUT', 'POST']:
+        info = request.json
+        _args = {"batch":info['batch'] if 'batch' in info else 1,"resume":True}
+        #
+        # We should update the configuration 
+        SYS_ARGS['config']['args'] = _args
+        PATH = os.sep.join([os.environ['HOME'],'.healthcareio','config.json'])
+        write = lambda content: (open(PATH,'w')).write(json.dumps(content))
+        proc = Process(target=write,args=(SYS_ARGS['config'],))
+        proc.start()
+        return "1",HEADER
+    pass
+@app.route("/io/<id>",methods=['POST'])
+def io_data(id):
+    if id == 'params' :
+        _args = request.json
+        #
+        # Expecting batch,folder as parameters
+        _args = request.json
+        _args['resume'] = True
+        print (_args)
+        #
+        # We should update the configuration 
+        SYS_ARGS['config']['args'] = _args
+        # PATH = os.sep.join([os.environ['HOME'],'.healthcareio','config.json'])
+        try:
+            write = lambda content: (open(PATH,'w')).write(json.dumps(content))
+            proc = Process(target=write,args=(SYS_ARGS['config'],))
+            proc.start()
+            # proc.join()
+            return "1",200
+        except Exception as e :
+            return "0",403
+            pass
+    elif id == 'stop' :
+        stop()
+        pass
+    elif id == 'run' :
+        # run()
+        _args = {"args":SYS_ARGS['config']['args'],"store":SYS_ARGS["config"]["store"]}
+        proxy.run(_args)
+        return "1",200
+        pass
+        
+@app.route("/export")
+def export_form():
+    _args = {"context":SYS_ARGS['context']}
+    return render_template("store.html",**_args)
+@app.route("/export",methods=['POST','PUT'])
+def apply_etl():
+    _info = request.json
+    m = {'s3':'s3.s3Writer','mongo':'mongo.MongoWriter'}
+    if _info :
+        dest_args = {'type':m[_info['type']],"args": _info['content'] }
+        src_args = SYS_ARGS['config']['store']
+        # print (_args)
+        # writer = transport.factory.instance(**_args)
+        proxy.publish(src_args,dest_args)
+        return "1",405
+        
+    else:
+        return "0",404
+@app.route("/update")
+def update():
+    pass
+    return "0",405
 @app.route("/reload",methods=['POST'])
 def reload():
     # e = SYS_ARGS['engine']
@@ -74,11 +219,20 @@ if __name__ == '__main__' :
     PORT = int(SYS_ARGS['port']) if 'port' in SYS_ARGS else 5500
     DEBUG= int(SYS_ARGS['debug']) if 'debug' in SYS_ARGS else 0
     SYS_ARGS['context'] = SYS_ARGS['context'] if 'context' in SYS_ARGS else ''
+
     #
     # 
     PATH= SYS_ARGS['config'] if 'config' in SYS_ARGS else os.sep.join([os.environ['HOME'],'.healthcareio','config.json'])
+    #
+    # Adjusting configuration with parameters (batch,folder,resume)
+    if 'args' not in SYS_ARGS['config'] :
+        SYS_ARGS['config']["args"] = {"batch":1,"resume":True,"folder":"/data"}
     
+    SYS_ARGS['procs'] = []
+    
+    
+    # SYS_ARGS['path'] = os.sep.join([os.environ['HOME'],'.healthcareio','config.json'])
     e = healthcareio.analytics.engine(PATH)
-    # e.apply(type='claims',serialize=True)
+    e.apply(type='claims',serialize=False)
     SYS_ARGS['engine'] = e
     app.run(host='0.0.0.0',port=PORT,debug=DEBUG,threaded=True)

healthcareio/server/index.py

@@ -12,8 +12,9 @@ def _icon():
 def init():
     e = SYS_ARGS['engine']
     sections = {"remits":e.info['835'],"claims":e.info['837']}
-    _args = {"sections":sections}
+    _args = {"sections":sections,"store":SYS_ARGS['config']['store']}
-    return render_template("index.html",**_args)
+    print (SYS_ARGS['config']['store'])
+    return render_template("setup.html",**_args)
 @app.route("/format/<id>/<index>",methods=['POST'])
 def _format(id,index):
     

healthcareio/server/static/css/default.css

@@ -1,3 +1,4 @@
+
 .active {
     padding:4px;
     cursor:pointer;
@@ -6,3 +7,51 @@
 .active:hover{
     border-bottom:2px solid #ff6500; 
 }
+input[type=text]{
+    border:1px solid transparent;
+    background-color:#f3f3f3;
+    outline: 0px;
+    padding:8px;
+    font-weight:normal;
+    font-family:sans-serif;
+    color:black;
+}
+.active-button {
+    display:grid;
+    grid-template-columns: 32px auto;
+    gap:2px;
+    align-items:center;
+    border:2px solid #CAD5E0;
+    cursor:pointer;
+    
+}
+.active-button i {padding:4px;;}
+.active-button:hover { border-color:#ff6500}
+
+.system {display:grid; grid-template-columns: 45% 1px auto; gap:20px;  margin-left:5%; width:90%;}
+.system .status .item {display:grid; grid-template-columns: 75px 8px auto; gap:2px;}
+.input-form {display:grid; gap:2px;}
+.input-form .item {display:grid; grid-template-columns: 125px auto; gap:2px; align-items:center;}
+.input-form .item .label { font-weight:bold; padding-left:10px}
+.fa-cog {color:#4682B4}
+.fa-check {color:#00c6b3}
+.fa-times {color:maroon}
+
+.code {
+    margin:4px;
+    background:#000000 ;
+    padding:8px;
+    font-family: 'Courier New', Courier, monospace;
+    color:#d3d3d3;
+    font-size:12px;
+    line-height: 2;
+}
+
+.tabs {display:grid; grid-template-columns: repeat(3,1fr) auto; gap:0px; align-items:center; text-align: center;}
+.tab {border:1px solid transparent; border-bottom-color:#D3D3D3; font-weight:bold; padding:4px}
+
+.tabs .selected {border-color:#CAD5E0; border-bottom-color:transparent; }
+.system iframe {width:100%; height:100%; border:1px solid transparent;}
+.data-info {height:90%; padding:8px;}
+.fa-cloud {color:#4682B4}
+.fa-database{color:#cc8c91}

healthcareio/server/static/js/jx/rpc.js

@@ -61,8 +61,16 @@ jx.ajax.get.instance = function(){
                   this.obj.headers[key] = value;
               }
           }
-        this.setData = function(data){
+          this.setData = function(data,mimetype){
+              if(mimetype == null)
                   this.obj.data = data;
+              else {
+                  this.obj.headers['Content-Type'] = mimetype
+                  if(mimetype.match(/application\/json/i)){
+                      this.obj.data = JSON.stringify(data)
+                  }
+              }
+              
           }
           this.setAsync = function(flag){
               this.obj.async = (flag == true) ;
@@ -128,3 +136,4 @@ jx.ajax.get.instance = function(){
   // backward compatibility
   jx.ajax.getInstance = jx.ajax.get.instance ;
   var HttpClient = jx.ajax.get ;
+  

healthcareio/server/templates/index.html

@@ -8,6 +8,7 @@
 <script src="{{context}}/static/js/jx/rpc.js"></script>
 <script src="{{context}}/static/js/jx/dom.js"></script>
 <script src="{{context}}/static/js/jx/utils.js"></script>
+<script src="{{context}}/static/js/jx/ext/modal.js"></script>
 
 <script src="{{context}}/static/js/jquery.js"></script>
 
@@ -15,6 +16,9 @@
 <link href="{{context}}/static/css/borders.css" type="text/css" rel="stylesheet">
 <link href="{{context}}/static/css/fa/css/all.css" type="text/css" rel="stylesheet">
 <script src="{{context}}/static/css/fa/js/all.js"></script>
+<script src="{{context}}/static/js/io/dialog.js"></script>
+<script src="{{context}}/static/js/io/io.js"></script>
+<script src="{{context}}/static/js/io/healthcare.js"></script>
 <style>
     body {
         font-size:16px;
@@ -64,6 +68,7 @@
         scroll-behavior: smooth;
         gap:2px;
         padding:4px;
+        height:95%;
         
 
     }
@@ -81,12 +86,12 @@
         
 
     }
-    .dashboard .chart-pane .chart {
+    .dashboard .chart-pane .chart2 {
         max-height:99%;
         min-height:99%;
         height:99%;
     }
-    .dashboard .chart-pane .chart .graph {
+    .dashboard .chart-pane .chart2 .graph {
         
         
         max-height:100%;
@@ -95,7 +100,7 @@
 
 
     }
-    .dashboard .chart-pane .chart .graph .apexcharts-svg {
+    .dashboard .chart-pane .chart2 .graph .apexcharts-svg {
         /*border-radius:8px;*/
         background-color:#f3f3f3;
         max-height:100%;
@@ -161,7 +166,7 @@
     }
    
 
-    .gradient {  background-image: linear-gradient(to top,#F3F3F3, #D3D3D3, #F3F3F3);}
+    .gradient {  background-image: linear-gradient(to top,#F3F3F3, #FFFFFF, #FFFFFF);}
     .white {color:#ffffff}
     .scalar {
         display:grid;
@@ -172,23 +177,39 @@
         
         
         
+        
     }
     .shadow {
         box-shadow: 0px 4px 4px #d3d3d3;
     }
     .scalar-title { padding:8px; text-transform: capitalize; }
-    .scalar .value {font-size:32px; font-weight:bold; margin:4%;}
+    .scalar .value {font-size:32px; margin:4%;}
     .scalar .label {font-size:12px; text-transform:capitalize; text-overflow: ellipsis; display:grid; align-items: flex-end; text-align:center ;}
+    
+   
+    .monthly_patient_count, .taxonomy_code_distribution, .top_adjustment_codes, .adjustment_reasons {
+        grid-row:1 / span 3 ;
+        
+        
+    }
+
+
 </style>
 <script>
     sessionStorage.io_context  = "{{context}}"
     var plot = function(id,index){
+        $('.system').slideUp(function(){
+            $('.dashboard').slideDown()
+        })
+        
         var uri = ([sessionStorage.io_context,'format',id,index]).join('/')
         var httpclient = HttpClient.instance()
         //
         // @TODO: Let the user know something is going on .. spinner
         httpclient.post(uri,function(x){
+            
             var r = JSON.parse(x.responseText)
+            
             var pane = jx.dom.get.instance('DIV')
             var scalar_pane = jx.dom.get.instance('DIV')
             pane.id = r.id
@@ -202,8 +223,8 @@
             var p = jx.utils.patterns.visitor(r.pipeline,function(item){
                 var div = jx.dom.get.instance('DIV')
                 var frame = jx.dom.get.instance('DIV')
-                //div.className = 'chart border-round border'
+                //div.className = 'chart2 border-round border'
-                frame.className = 'chart border'
+                frame.className = 'chart2 border ' + item.label.toLowerCase().replace(/ /g,'_')
                 div.className = 'graph '
                 //frame.append(div)
                 //pane.append(frame)
@@ -212,8 +233,11 @@
              if(item.apex != null){   item.apex.title = {text:item.label}
                     frame.append(div)
                     pane.append(frame)
+                    if (item.apex.colors ){
                         delete item.apex.colors
-                    item.apex.theme= {
+                    }
+                    
+                    /*item.apex.theme= {
                         mode: 'material', 
                         palette: 'palette6', 
                         monochrome: {
@@ -222,8 +246,11 @@
                             shadeTo: 'light',
                             shadeIntensity: 0.65
                         },
-                    }
+                    }*/
                     item.apex.chart.height = '100%'
+                    delete item.apex.chart.width
+                    
+                    
                     return new ApexCharts(div,item.apex)
                 }else{
                     //frame.className = ''
@@ -283,29 +310,56 @@
         }
     }
     $(document).ready(function(){
+        $('.dashabord').hide()
         $('.item-group').slideUp()
+        var index = 0;
+        jx.utils.patterns.visitor($('.menu .items'), function(_item){
+            var node = $(_item).children()[0]
+            $(node).attr('index',index)
+            node.onclick = function(){ toggle($(this).attr('index')) }
+            index += 1;
+            
+        })
+        var year = (new Date()).getFullYear()
+        $('.year').text(year)
     })
 </script>
 <title>Healthcare/IO Analytics</title>
 <body>
-    <div class="pane border">
+    <div class="pane">
         <div class="header border-bottom">
-            <div class="caption">Healthcare/IO</div>
+            <div class="caption">Healthcare/IO :: Parser</div>
-            <div class="small">Analytics Dashboard</div>
+            <div class="small">Dashboard</div>
         </div>
         
-        <div class="menu border-right">
+        <div class="menu">
+            
            <div>
+                <div class="items">
+                    <div class="bold active" style="margin:4px; height:28px; display:grid; gap:2px; grid-template-columns:auto 32px;"> 
+                        <div>Setup</div>
+                        <div align="center" class="glyph" >
+                            
+                            <i class="fas fa-angle-down"></i>
+                        </div>
+                    </div>
+                    <div class="item-group border border-round">
+                        <div class="item small active" onclick="setup.open()">Configure Parser</div>
+                        
+                        <div class="item small active" onclick="healthcare.io.reset()">Reset Parser</div>
+
+                    </div>
+                </div>
             {% for key in sections %}
                 <div class="items">
-                <div class="bold active" onclick="toggle({{loop.index -1}})" style="margin:4px; height:28px; display:grid; gap:2px; grid-template-columns:auto 32px;"> 
+                    <div class="bold active" style="margin:4px; height:28px; display:grid; gap:2px; grid-template-columns:auto 32px;"> 
                         <div>{{key|safe}}</div>
                         <div align="center" class="glyph" >
                             
                             <i class="fas fa-angle-down"></i>
                         </div>
                     </div>                    
-                <div class="item-group border">
+                    <div class="item-group border border-round">
                         
                         {% for item in sections[key] %}
                             <div class="item small active" onclick="plot('{{key}}',{{loop.index-1}})">{{item.id}}</div>
@@ -315,18 +369,21 @@
             {% endfor %}
            </div>
             
-            <div style="display:grid; align-items:flex-end">
+            <div style="display:none; align-items:flex-end">
-                <div class="logs border" style="height:250px"></div>
+                <div class="logs chart" style=" align-items:center; display:grid" align="center"></div>
 
             </div>
             
         </div>
-        <div class="dashboard"> 
+       <div>
+            <div class="dashboard" style="display:none"></div>
+            
+            {%include 'setup.html' %}
             
        </div>
             
     </div>
     <div class="footer small">
-        &copy; Vanderbilt University Medical Center
+        Healthcare/IO :: Parser &copy; <span class="year">    </span>
     </div>
 </body>