From 8b4eb81564ce9bbb1c74c23990d179f4d69e2b39 Mon Sep 17 00:00:00 2001 From: Steve Nyemba Date: Mon, 2 Nov 2020 15:35:40 -0600 Subject: [PATCH 01/21] bug fix: payer information on 837 --- healthcareio/x12/__init__.py | 11 ++++++----- setup.py | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/healthcareio/x12/__init__.py b/healthcareio/x12/__init__.py index cd3a2fd..2e22d98 100644 --- a/healthcareio/x12/__init__.py +++ b/healthcareio/x12/__init__.py @@ -279,11 +279,12 @@ class Parser (Process): row = util.split(row.replace('\n','').replace('~','')) _info = util.get.config(self.config[_code][0],row) - + if _info : try: tmp = self.get.value(row,_info) + # if 'P1080351470' in content[0] and 'PLB' in row: # print (_info) # print (row) @@ -300,10 +301,10 @@ class Parser (Process): else: if label not in value: value[label] = [tmp] - elif len(list(tmp.keys())) == 1 : - # print "\t",len(claim[label]),tmp - index = len(value[label]) -1 - value[label][index] = dict(value[label][index],**tmp) + # elif len(list(tmp.keys())) == 1 : + # # print "\t",len(claim[label]),tmp + # index = len(value[label]) -1 + # value[label][index] = dict(value[label][index],**tmp) else: value[label].append(tmp) tmp['_index'] = len(value[label]) -1 diff --git a/setup.py b/setup.py index 61c69b0..0c9fb86 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ import sys def read(fname): return open(os.path.join(os.path.dirname(__file__), fname)).read() args = { - "name":"healthcareio","version":"1.3.7", + "name":"healthcareio","version":"1.4.0", "author":"Vanderbilt University Medical Center", "author_email":"steve.l.nyemba@vumc.org", "license":"MIT", From ed782b7e4044eff43bc37203c2d7e9fe4552552c Mon Sep 17 00:00:00 2001 From: Steve Nyemba Date: Fri, 11 Dec 2020 06:45:10 -0600 Subject: [PATCH 02/21] bug fixes with parser --- healthcareio/analytics.py | 134 ++++++++++++++++++++++++++++------ healthcareio/healthcare-io.py | 18 ++++- healthcareio/x12/__init__.py | 27 +++++-- 3 files changed, 150 insertions(+), 29 deletions(-) diff --git a/healthcareio/analytics.py b/healthcareio/analytics.py index 875a8cd..442ad60 100644 --- a/healthcareio/analytics.py +++ b/healthcareio/analytics.py @@ -11,7 +11,7 @@ import transport import matplotlib.pyplot as plt import re, base64 # from weasyprint import HTML, CSS -COLORS = ["#f79256","#7dcfb6","#fbd1a2","#00b2ca","#1d4e89","#4682B4","#c5c3c6","#4c5c68","#1985a1","#f72585","#7209b7","#3a0ca3","#4361ee","#4cc9f0","#ff595e","#ffca3a","#8ac926","#1982c4","#6a4c93"] +COLORS = ["#fbd1a2","#00b2ca","#1d4e89","#4682B4","#c5c3c6","#4c5c68","#1985a1","#f72585","#7209b7","#3a0ca3","#4361ee","#4cc9f0","#ff595e","#ffca3a","#8ac926","#1982c4","#6a4c93"] class stdev : def __init__(self) : self.values = [] @@ -149,11 +149,16 @@ class Apex : This class will format a data-frame to work with Apex charting engine """ @staticmethod - def apply(item): + def apply(item,theme={'mode':'light','palette':'palette6'}): pointer = item['chart']['type'] if hasattr(Apex,pointer) : pointer = getattr(Apex,pointer) + options = pointer(item) + if 'apex' in options and 'colors' in options['apex'] : + del options['apex']['colors'] + if 'apex' in options : + options['apex']['theme'] = theme options['responsive']= [ { 'breakpoint': 1, @@ -168,6 +173,18 @@ class Apex : print ("Oops") pass @staticmethod + def radial(item): + df = item['data'] + x = item['chart']['axis']['x'] + y = item['chart']['axis']['y'] + + labels = df[y].tolist() + values = [float(np.round(value,2)) for value in df[x].tolist()] + chart = {"type":"radialBar","height":200} + option = {"chart":chart,"series":values,"labels":labels,"plotOptions":{"radialBar":{"hollow":{"size":"70%"}}}} + return {'apex':option} + + @staticmethod def scatter(item): options = Apex.spline(item) options['apex']['chart']['type'] = 'scatter' @@ -175,7 +192,7 @@ class Apex : @staticmethod def scalar(item): _df = item['data'] - print (_df) + name = _df.columns.tolist()[0] value = _df[name].values.round(2)[0] html = '
:value
:label
' @@ -235,16 +252,17 @@ class Apex : @TODO: alias this with bar (!= column) """ df = item['data'] + N = df.shape[0] if df.shape[0] < 10 else 10 axis = item['chart']['axis'] y = axis['y'] if type(y) == list : y = y[0] axis['x'] = [axis['x']] if type(axis['x']) != list else axis['x'] - if not set(axis['x']) & set(df.columns.tolist()) : - print (set(axis['x']) & set(df.columns.tolist())) - print (axis['x']) - print (df.columns) + # if not set(axis['x']) & set(df.columns.tolist()) : + # print (set(axis['x']) & set(df.columns.tolist())) + # print (axis['x']) + # print (df.columns) # df.columns = axis['x'] series = [] _min=_max = 0 @@ -294,7 +312,6 @@ class Apex : values are x-axis """ df = item['data'] - if df.shape [0]> 1 : y_cols,x_cols = item['chart']['axis']['y'],item['chart']['axis']['x'] labels = df[y_cols].values.tolist() @@ -302,10 +319,11 @@ class Apex : values = df[x_cols].values.round(2).tolist() else: labels = [name.upper().replace('_',' ') for name in df.columns.tolist()] + df = df.astype(float) values = df.values.round(2).tolist()[0] if df.shape[1] > 1 else df.values.round(2).tolist() colors = COLORS[:len(values)] - options = {"series":values,"colors":colors,"labels":labels,"chart":{"type":"donut"},"plotOptions":{"pie":{"customScale":.8}},"legend":{"position":"right"}} + options = {"series":values,"colors":colors,"labels":labels,"dataLabels":{"enabled":True,"style":{"colors":["#000000"]},"dropShadow":{"enabled":False}},"chart":{"type":"donut","width":200},"plotOptions":{"pie":{"customScale":.9}},"legend":{"position":"right"}} return {"apex":options} pass @@ -329,43 +347,117 @@ class engine : _args['type'] = 'mongo.MongoReader' else: _args['type'] = 'disk.SQLiteReader' - self.reader = transport.factory.instance(**_args) + self.store_config = _args ; + + def filter (self,**args): + """ + type: claims or remits + filter optional identifier claims, procedures, taxonomy, ... + """ + + + _m = {'claim':'837','claims':'837','remits':'835','remit':'835'} + table = _m[ args['type']] + _analytics = self.info[table] + if 'index' in args : + index = int(args['index']) + _analytics = [_analytics[index]] + + _info = list(_analytics) #if 'filter' not in args else [item for item in analytics if args['filter'] == item['id']] + # conn = lite.connect(self.store_config['args']['path'],isolation_level=None) + # conn.create_aggregate("stdev",1,stdev) + DB_TYPE = 'mongo' if (type(self.reader) == transport.mongo.MongoReader) else 'sql' + if DB_TYPE == 'mongo' : + self.store_config['args']['doc'] = args['type'] + + self.reader = transport.factory.instance(**self.store_config) + r = [] + for row in _info : + pipeline = row['pipeline'] + + index = 0 + for item in pipeline: + if not item[DB_TYPE] : + continue + query = {DB_TYPE:item[DB_TYPE]} + + df = pd.DataFrame(self.reader.read(**query)) #item) + df = df.fillna('N/A') + # item['data'] = df + chart = item['chart'] + pipe = {"data":df,"chart":chart} + for key in list(item.keys()) : + if key not in ["chart","data","mongo","sql","couch"] : + pipe[key] = item[key] + + + + r.append(pipe) + self.reader.close() + return {"id":_info[0]['id'],'pipeline':r} + def apply (self,**args) : """ type: claims or remits filter optional identifier claims, procedures, taxonomy, ... """ + + _m = {'claim':'837','claims':'837','remits':'835','remit':'835'} # key = '837' if args['type'] == 'claims' else '835' table = _m[ args['type']] - analytics = self.info[table] + + _analytics = self.info[table] if 'index' in args : index = int(args['index']) - analytics = [analytics[index]] + _analytics = [_analytics[index]] - _info = list(analytics) if 'filter' not in args else [item for item in analytics if args['filter'] == item['id']] + _info = list(_analytics) if 'filter' not in args else [item for item in analytics if args['filter'] == item['id']] # conn = lite.connect(self.store_config['args']['path'],isolation_level=None) # conn.create_aggregate("stdev",1,stdev) - DB_TYPE = 'mongo' if (type(self.reader) == transport.mongo.MongoReader) else 'sql' + # + # @TODO: Find a better way to handle database variance + # + # DB_TYPE = 'mongo' if (type(self.reader) == transport.mongo.MongoReader) else 'sql' + + if 'mongo' in self.store_config['type'] : + DB_TYPE='mongo' + else: + DB_TYPE='sql' + self.store_config['args']['table'] = args['type'] + + self.reader = transport.factory.instance(**self.store_config) r = [] for row in _info : - - for item in row['pipeline'] : + pipeline = row['pipeline'] + index = 0 + for item in pipeline: # item['data'] = pd.read_sql(item['sql'],conn) - query = {DB_TYPE:item[DB_TYPE]} - item['data'] = self.reader.read(**item) + # query = {DB_TYPE:item[DB_TYPE]} + query = item[DB_TYPE] + if not query : + continue + if DB_TYPE == 'sql' : + query = {"sql":query} + + item['data'] = self.reader.read(**query) #item) if 'serialize' in args : - item['data'] = json.dumps(item['data'].to_dict(orient='record')) if type(item['data']) == pd.DataFrame else item['data'] + # item['data'] = json.dumps(item['data'].to_dict(orient='record')) if type(item['data']) == pd.DataFrame else item['data'] + item['data'] = json.dumps(item['data'].to_dict('record')) if type(item['data']) == pd.DataFrame else item['data'] else: item['data'] = (pd.DataFrame(item['data'])) - + pipeline[index] = item + index += 1 + # + # + row['pipeline']= pipeline # if 'info' in item: # item['info'] = item['info'].replace(":rows",str(item["data"].shape[0])) # conn.close() - + self.reader.close() return _info def _html(self,item) : diff --git a/healthcareio/healthcare-io.py b/healthcareio/healthcare-io.py index 037586c..be6012d 100644 --- a/healthcareio/healthcare-io.py +++ b/healthcareio/healthcare-io.py @@ -43,6 +43,8 @@ import numpy as np from multiprocessing import Process import time from healthcareio import x12 +import smart +import pandas as pd PATH = os.sep.join([os.environ['HOME'],'.healthcareio']) OUTPUT_FOLDER = os.sep.join([os.environ['HOME'],'healthcare-io']) @@ -337,10 +339,20 @@ if __name__ == '__main__' : # PATH= SYS_ARGS['config'] if 'config' in SYS_ARGS else os.sep.join([os.environ['HOME'],'.healthcareio','config.json']) - e = analytics.engine(os.sep.join([PATH,'config.json'])) #--@TODO: make the configuration file globally accessible - e.apply(type='claims',serialize=True) - SYS_ARGS['engine'] = e + if os.path.exists(os.sep.join([PATH,'config.json'])) : + e = analytics.engine(os.sep.join([PATH,'config.json'])) #--@TODO: make the configuration file globally accessible + e.apply(type='claims',serialize=True) + SYS_ARGS['engine'] = e + SYS_ARGS['config'] = json.loads(open(os.sep.join([PATH,'config.json'])).read()) + else: + SYS_ARGS['config'] = {"owner":None,"store":None} + + if 'args' not in SYS_ARGS['config'] : + SYS_ARGS['config']["args"] = {"batch":1,"resume":True,"folder":"/data"} + me = pd.DataFrame(smart.top.read(name='healthcare-io.py')).args.unique().tolist() + SYS_ARGS['me'] = me[0] #-- This key will identify the current process + pointer = lambda : server.app.run(host='0.0.0.0',port=PORT,debug=DEBUG,threaded=False) pthread = Process(target=pointer,args=()) pthread.start() diff --git a/healthcareio/x12/__init__.py b/healthcareio/x12/__init__.py index 2e22d98..66dc05a 100644 --- a/healthcareio/x12/__init__.py +++ b/healthcareio/x12/__init__.py @@ -49,8 +49,9 @@ class Formatters : """ This function is designed to split an x12 row and """ + value = [] if row.startswith(prefix) is False: - value = [] + for row_value in row.replace('~','').split(sep) : @@ -65,10 +66,12 @@ class Formatters : else : value.append(row_value.replace('\n','')) - return [xchar.replace('\r','') for xchar in value] #row.replace('~','').split(sep) + value = [xchar.replace('\r','') for xchar in value] #row.replace('~','').split(sep) else: - return [ [prefix]+ self.split(item,'>') for item in row.replace('~','').split(sep)[1:] ] + value = [ [prefix]+ self.split(item,'>') for item in row.replace('~','').split(sep)[1:] ] + + return value if type(value) == list and type(value[0]) != list else value[0] def get_config(self,config,row): """ This function will return the meaningfull parts of the configuration for a given item @@ -130,7 +133,7 @@ class Formatters : terms = value[1].split('>') return {'type':terms[0],'code':terms[1],"amount":float(value[2])} else: - + return {"code":value[2],"type":value[1],"amount":float(value[3])} def sv2(self,value): # @@ -191,6 +194,9 @@ class Parser (Process): self.files = [] self.set = void() self.set.files = self.set_files + self.emit = void() + self.emit.pre = None + self.emit.post = None def set_files(self,files): self.files = files def get_map(self,row,config,version=None): @@ -328,7 +334,7 @@ class Parser (Process): pass except Exception as e : - + print ('__',e.args) pass @@ -433,6 +439,9 @@ class Parser (Process): # self.finish(claims,logs,_code) return claims,logs,_code def run(self): + if self.emit.pre : + self.emit.pre() + for filename in self.files : content,logs,_code = self.read(filename) self.finish(content,logs,_code) @@ -442,14 +451,22 @@ class Parser (Process): if args['type'] == 'mongo.MongoWriter' : args['args']['doc'] = 'claims' if _code == '837' else 'remits' _args['args']['doc'] = 'logs' + else: + args['args']['table'] = 'claims' if _code == '837' else 'remits' + _args['args']['table'] = 'logs' + if content : writer = transport.factory.instance(**args) writer.write(content) writer.close() if logs : + logger = transport.factory.instance(**_args) logger.write(logs) + logger.close() + if self.emit.post : + self.emit.post(content,logs) From 2b533dc8fe054f6966871da11f9dad54cedd612c Mon Sep 17 00:00:00 2001 From: Steve Nyemba Date: Fri, 11 Dec 2020 06:55:34 -0600 Subject: [PATCH 03/21] docker user interface basics --- healthcareio/Dockerfile | 23 +- healthcareio/server/__init__.py | 196 +++++++++++++++-- healthcareio/server/index.py | 5 +- healthcareio/server/static/css/default.css | 49 +++++ healthcareio/server/static/js/jx/rpc.js | 241 +++++++++++---------- healthcareio/server/templates/index.html | 127 ++++++++--- 6 files changed, 464 insertions(+), 177 deletions(-) diff --git a/healthcareio/Dockerfile b/healthcareio/Dockerfile index 5cf8fd8..23464ef 100644 --- a/healthcareio/Dockerfile +++ b/healthcareio/Dockerfile @@ -1,4 +1,8 @@ -FROM ubuntu:bionic-20200403 +# +# Let us create an image for healthcareio +# The image will contain the {X12} Parser and the +# FROM ubuntu:bionic-20200403 +FROM ubuntu:focal RUN ["apt","update","--fix-missing"] RUN ["apt-get","upgrade","-y"] @@ -6,9 +10,22 @@ RUN ["apt-get","-y","install","apt-utils"] RUN ["apt","update","--fix-missing"] RUN ["apt-get","upgrade","-y"] -RUN ["apt-get","install","-y","sqlite3","sqlite3-pcre","libsqlite3-dev","python3-dev","python3","python3-pip","git","python3-virtualenv"] +RUN ["apt-get","install","-y","mongo","sqlite3","sqlite3-pcre","libsqlite3-dev","python3-dev","python3","python3-pip","git","python3-virtualenv","wget"] # # +RUN ["pip3","install","--upgrade","pip"] +# RUN ["pip3","install","git+https://healthcare.the-phi.com/git/code/parser.git","botocore"] USER health-user +# +# This volume is where the data will be loaded from (otherwise it is assumed the user will have it in the container somehow) +# +VOLUME ["/data"] +# +# This is the port from which some degree of monitoring can/will happen +EXPOSE 80 +# wget https://healthcareio.the-phi.com/git/code/parser.git/bootup.sh +COPY bootup.sh bootup.sh +ENTRYPOINT ["bash","-C"] +CMD ["bootup.sh"] # VOLUME ["/home/health-user/healthcare-io/","/home-healthuser/.healthcareio"] -# RUN ["pip3","install","git+https://healthcareio.the-phi.com/git"] \ No newline at end of file +# RUN ["pip3","install","git+https://healthcareio.the-phi.com/git"] diff --git a/healthcareio/server/__init__.py b/healthcareio/server/__init__.py index 3910930..d723bca 100644 --- a/healthcareio/server/__init__.py +++ b/healthcareio/server/__init__.py @@ -3,7 +3,78 @@ from healthcareio.params import SYS_ARGS import healthcareio.analytics import os import json +import time +import smart +import transport +import pandas as pd +import numpy as np +import x12 + +from multiprocessing import Process +from flask_socketio import SocketIO, emit, disconnect,send +from healthcareio.server import proxy +PATH = os.sep.join([os.environ['HOME'],'.healthcareio','config.json']) app = Flask(__name__) +socket_ = SocketIO(app) + +def resume (files): + _args = SYS_ARGS['config']['store'].copy() + if 'mongo' in SYS_ARGS['config']['store']['type'] : + _args['type'] = 'mongo.MongoReader' + reader = transport.factory.instance(**_args) + _files = [] + try: + pipeline = [{"$match":{"completed":{"$eq":True}}},{"$group":{"_id":"$name"}},{"$project":{"name":"$_id","_id":0}}] + _args = {"aggregate":"logs","cursor":{},"allowDiskUse":True,"pipeline":pipeline} + _files = reader.read(mongo = _args) + _files = [item['name'] for item in _files] + except Exception as e : + pass + print (["found ",len(files),"\tProcessed ",len(_files)]) + return list(set(files) - set(_files)) + + + +def run (): + # + # let's get the files in the folder (perhaps recursively traverse them) + # + FILES = [] + BATCH = int(SYS_ARGS['config']['args']['batch']) #-- number of processes (poorly named variable) + + for root,_dir,f in os.walk(SYS_ARGS['config']['args']['folder']) : + if f : + FILES += [os.sep.join([root,name]) for name in f] + FILES = resume(FILES) + FILES = np.array_split(FILES,BATCH) + procs = [] + for FILE_GROUP in FILES : + + FILE_GROUP = FILE_GROUP.tolist() + # logger.write({"process":index,"parse":SYS_ARGS['parse'],"file_count":len(row)}) + # proc = Process(target=apply,args=(row,info['store'],_info,)) + parser = x12.Parser(PATH) #os.sep.join([PATH,'config.json'])) + parser.set.files(FILE_GROUP) + parser.start() + procs.append(parser) + SYS_ARGS['procs'] = procs +# @socket_.on('data',namespace='/stream') +def push() : + _args = dict(SYS_ARGS['config']['store'].copy(),**{"type":"mongo.MongoReader"}) + reader = transport.factory.instance(**_args) + pipeline = [{"$group":{"_id":"$parse","claims":{"$addToSet":"$name"}}},{"$project":{"_id":0,"type":"$_id","count":{"$size":"$claims"}}}] + _args = {"aggregate":"logs","cursor":{},"allowDiskUse":True,"pipeline":pipeline} + r = pd.DataFrame(reader.read(mongo=_args)) + r = healthcareio.analytics.Apex.apply({"chart":{"type":"donut","axis":{"x":"count","y":"type"}},"data":r}) + emit("update",r,json=True) + return r +@socket_.on('connect') +def client_connect(**r): + print ('Connection received') + print (r) + push() + pass + @app.route("/favicon.ico") def _icon(): return send_from_directory(os.path.join([app.root_path, 'static','img','logo.svg']), @@ -12,7 +83,7 @@ def _icon(): def init(): e = SYS_ARGS['engine'] sections = {"remits":e.info['835'],"claims":e.info['837']} - _args = {"sections":sections} + _args = {"sections":sections,"store":SYS_ARGS["config"]["store"],"owner":SYS_ARGS['config']['owner'],"args":SYS_ARGS["config"]["args"]} return render_template("index.html",**_args) @app.route("/format//",methods=['POST']) def _format(id,index): @@ -21,43 +92,117 @@ def _format(id,index): key = '837' if id == 'claims' else '835' index = int(index) # p = e.info[key][index] - p = e.apply(type=id,index=index) - - # + p = e.filter(type=id,index=index) + r = [] - for item in p[0]['pipeline'] : - _item= dict(item) - del _item['sql'] - del _item ['data'] - + for item in p['pipeline'] : + _item= dict(item) _item = dict(_item,**healthcareio.analytics.Apex.apply(item)) + del _item['data'] if 'apex' in _item or 'html' in _item: r.append(_item) - r = {"id":p[0]['id'],"pipeline":r} + + r = {"id":p['id'],"pipeline":r} return json.dumps(r),200 + @app.route("/get//",methods=['GET']) def get(id,index): e = SYS_ARGS['engine'] key = '837' if id == 'claims' else '835' index = int(index) # p = e.info[key][index] - p = e.apply(type=id,index=index) + p = e.filter(type=id,index=index) r = {} for item in p[0]['pipeline'] : _item= [dict(item)] - r[item['label']] = item['data'].to_dict(orient='record') - # del _item['sql'] - # del _item ['data'] - # print (item['label']) - # _item['apex'] = healthcareio.analytics.Apex.apply(item) - # if _item['apex']: - # r.append(_item) - - # r = {"id":p[0]['id'],"pipeline":r} + # r[item['label']] = item['data'].to_dict(orient='record') + r[item['label']] = item['data'].to_dict('record') return json.dumps(r),200 + +@app.route("/reset",methods=["POST"]) +def reset(): + return "1",200 +@app.route("/data",methods=['GET']) +def get_data (): + """ + This function will return statistical data about the services i.e general statistics about what has/been processed + """ + HEADER = {"Content-type":"application/json"} + _args = SYS_ARGS['config'] + options = dict(proxy.get.files(_args),**proxy.get.processes(_args)) + return json.dumps(options),HEADER +@app.route("/log/",methods=["POST","PUT","GET"]) +def log(id) : + HEADER = {"Content-Type":"application/json; charset=utf8"} + if id == 'params' and request.method in ['PUT', 'POST']: + info = request.json + _args = {"batch":info['batch'] if 'batch' in info else 1,"resume":True} + # + # We should update the configuration + SYS_ARGS['config']['args'] = _args + PATH = os.sep.join([os.environ['HOME'],'.healthcareio','config.json']) + write = lambda content: (open(PATH,'w')).write(json.dumps(content)) + proc = Process(target=write,args=(SYS_ARGS['config'],)) + proc.start() + return "1",HEADER + pass +@app.route("/io/",methods=['POST']) +def io_data(id): + if id == 'params' : + _args = request.json + # + # Expecting batch,folder as parameters + _args = request.json + _args['resume'] = True + print (_args) + # + # We should update the configuration + SYS_ARGS['config']['args'] = _args + # PATH = os.sep.join([os.environ['HOME'],'.healthcareio','config.json']) + try: + write = lambda content: (open(PATH,'w')).write(json.dumps(content)) + proc = Process(target=write,args=(SYS_ARGS['config'],)) + proc.start() + # proc.join() + return "1",200 + except Exception as e : + return "0",403 + pass + elif id == 'stop' : + stop() + pass + elif id == 'run' : + # run() + _args = {"args":SYS_ARGS['config']['args'],"store":SYS_ARGS["config"]["store"]} + proxy.run(_args) + return "1",200 + pass + +@app.route("/export") +def export_form(): + _args = {"context":SYS_ARGS['context']} + return render_template("store.html",**_args) +@app.route("/export",methods=['POST','PUT']) +def apply_etl(): + _info = request.json + m = {'s3':'s3.s3Writer','mongo':'mongo.MongoWriter'} + if _info : + dest_args = {'type':m[_info['type']],"args": _info['content'] } + src_args = SYS_ARGS['config']['store'] + # print (_args) + # writer = transport.factory.instance(**_args) + proxy.publish(src_args,dest_args) + return "1",405 + + else: + return "0",404 +@app.route("/update") +def update(): + pass + return "0",405 @app.route("/reload",methods=['POST']) def reload(): # e = SYS_ARGS['engine'] @@ -74,11 +219,20 @@ if __name__ == '__main__' : PORT = int(SYS_ARGS['port']) if 'port' in SYS_ARGS else 5500 DEBUG= int(SYS_ARGS['debug']) if 'debug' in SYS_ARGS else 0 SYS_ARGS['context'] = SYS_ARGS['context'] if 'context' in SYS_ARGS else '' + # # PATH= SYS_ARGS['config'] if 'config' in SYS_ARGS else os.sep.join([os.environ['HOME'],'.healthcareio','config.json']) + # + # Adjusting configuration with parameters (batch,folder,resume) + if 'args' not in SYS_ARGS['config'] : + SYS_ARGS['config']["args"] = {"batch":1,"resume":True,"folder":"/data"} + SYS_ARGS['procs'] = [] + + + # SYS_ARGS['path'] = os.sep.join([os.environ['HOME'],'.healthcareio','config.json']) e = healthcareio.analytics.engine(PATH) - # e.apply(type='claims',serialize=True) + e.apply(type='claims',serialize=False) SYS_ARGS['engine'] = e app.run(host='0.0.0.0',port=PORT,debug=DEBUG,threaded=True) \ No newline at end of file diff --git a/healthcareio/server/index.py b/healthcareio/server/index.py index c61f130..99f3709 100644 --- a/healthcareio/server/index.py +++ b/healthcareio/server/index.py @@ -12,8 +12,9 @@ def _icon(): def init(): e = SYS_ARGS['engine'] sections = {"remits":e.info['835'],"claims":e.info['837']} - _args = {"sections":sections} - return render_template("index.html",**_args) + _args = {"sections":sections,"store":SYS_ARGS['config']['store']} + print (SYS_ARGS['config']['store']) + return render_template("setup.html",**_args) @app.route("/format//",methods=['POST']) def _format(id,index): diff --git a/healthcareio/server/static/css/default.css b/healthcareio/server/static/css/default.css index 0bded6c..5ea27b2 100644 --- a/healthcareio/server/static/css/default.css +++ b/healthcareio/server/static/css/default.css @@ -1,3 +1,4 @@ + .active { padding:4px; cursor:pointer; @@ -6,3 +7,51 @@ .active:hover{ border-bottom:2px solid #ff6500; } +input[type=text]{ + border:1px solid transparent; + background-color:#f3f3f3; + outline: 0px; + padding:8px; + font-weight:normal; + font-family:sans-serif; + color:black; +} +.active-button { + display:grid; + grid-template-columns: 32px auto; + gap:2px; + align-items:center; + border:2px solid #CAD5E0; + cursor:pointer; + +} +.active-button i {padding:4px;;} +.active-button:hover { border-color:#ff6500} + +.system {display:grid; grid-template-columns: 45% 1px auto; gap:20px; margin-left:5%; width:90%;} +.system .status .item {display:grid; grid-template-columns: 75px 8px auto; gap:2px;} +.input-form {display:grid; gap:2px;} +.input-form .item {display:grid; grid-template-columns: 125px auto; gap:2px; align-items:center;} +.input-form .item .label { font-weight:bold; padding-left:10px} +.fa-cog {color:#4682B4} +.fa-check {color:#00c6b3} +.fa-times {color:maroon} + +.code { + margin:4px; + background:#000000 ; + padding:8px; + font-family: 'Courier New', Courier, monospace; + color:#d3d3d3; + font-size:12px; + line-height: 2; +} + +.tabs {display:grid; grid-template-columns: repeat(3,1fr) auto; gap:0px; align-items:center; text-align: center;} +.tab {border:1px solid transparent; border-bottom-color:#D3D3D3; font-weight:bold; padding:4px} + +.tabs .selected {border-color:#CAD5E0; border-bottom-color:transparent; } +.system iframe {width:100%; height:100%; border:1px solid transparent;} +.data-info {height:90%; padding:8px;} +.fa-cloud {color:#4682B4} +.fa-database{color:#cc8c91} diff --git a/healthcareio/server/static/js/jx/rpc.js b/healthcareio/server/static/js/jx/rpc.js index 8dd08fe..22033f9 100755 --- a/healthcareio/server/static/js/jx/rpc.js +++ b/healthcareio/server/static/js/jx/rpc.js @@ -12,119 +12,128 @@ * Improve on how returned data is handled (if necessary). */ if(!jx){ - var jx = {} -} -/** - * These are a few parsers that can come in handy: - * urlparser: This parser is intended to break down a url parameter string in key,value pairs - */ - -function urlparser(url){ - if(url.toString().match(/\x3F/) != null){ - url = url.split('\x3F')[1] - - } - var p = url.split('&') ; - var r = {} ; - r.meta = [] ; - r.data = {} ; - var entry; - for(var i=0; i < p.length; i++){ - entry = p[i] ; - key = (entry.match('(.*)=') !=null)? entry.match('(.*)=')[1]:null ; - value = (entry.match('=(.*)$') != null)? entry.match('=(.*)$')[1]:null - if(key != null){ - key = key.replace('\x3F','') - r.meta.push(key) ; - r.data[key] = value ; - } - } - - return r.data; -} - -/** -* The following are corrections related to consistency in style & cohesion -*/ -jx.ajax = {} -jx.ajax.get = {} ; -jx.ajax.debug = null; -jx.ajax.get.instance = function(){ - var factory = function(){ - this.obj = {} - this.obj.headers = {} - this.obj.async = true; - this.setHeader = function(key,value){ - if(key.constructor != String && value == null){ - this.obj.headers = key ; - }else{ - this.obj.headers[key] = value; - } - } - this.setData = function(data){ - this.obj.data = data; - } - this.setAsync = function(flag){ - this.obj.async = (flag == true) ; - } - this.send = function(url,callback,method){ - - if(method == null){ - method = 'GET' - } - - p = jx.ajax.debug != null; - q = false; - if(p){ - q = jx.ajax.debug[url] != null; - } - - is_debuggable = p && q - - if(is_debuggable){ - x = {} ; - x.responseText = jx.ajax.debug [url] ; - callback(x) - }else{ - var http = new XMLHttpRequest() ; - http.onreadystatechange = function(){ - if(http.readyState == 4){ - - callback(http) - } - } - // - // In order to insure backward compatibility - // Previous versions allowed the user to set the variable on the wrapper (poor design) - if(this.async != null){ - this.setAsync(this.async) ; - } - http.open(method,url,this.obj.async) ; - for(key in this.obj.headers){ - value = this.obj.headers[key] ; - - http.setRequestHeader(key,value) - } - - http.send(this.obj.data) - } - - - } - this.put = function(url,callback){ - this.send(url,callback,'PUT') ; - } - this.get = function(url,callback){ - this.send(url,callback,'GET') ; - } - this.post = function(url,callback){ - this.send(url,callback,'POST') ; - } - }//-- end of the factory method - return new factory() ; -} - -// -// backward compatibility -jx.ajax.getInstance = jx.ajax.get.instance ; -var HttpClient = jx.ajax.get ; + var jx = {} + } + /** + * These are a few parsers that can come in handy: + * urlparser: This parser is intended to break down a url parameter string in key,value pairs + */ + + function urlparser(url){ + if(url.toString().match(/\x3F/) != null){ + url = url.split('\x3F')[1] + + } + var p = url.split('&') ; + var r = {} ; + r.meta = [] ; + r.data = {} ; + var entry; + for(var i=0; i < p.length; i++){ + entry = p[i] ; + key = (entry.match('(.*)=') !=null)? entry.match('(.*)=')[1]:null ; + value = (entry.match('=(.*)$') != null)? entry.match('=(.*)$')[1]:null + if(key != null){ + key = key.replace('\x3F','') + r.meta.push(key) ; + r.data[key] = value ; + } + } + + return r.data; + } + + /** + * The following are corrections related to consistency in style & cohesion + */ + jx.ajax = {} + jx.ajax.get = {} ; + jx.ajax.debug = null; + jx.ajax.get.instance = function(){ + var factory = function(){ + this.obj = {} + this.obj.headers = {} + this.obj.async = true; + this.setHeader = function(key,value){ + if(key.constructor != String && value == null){ + this.obj.headers = key ; + }else{ + this.obj.headers[key] = value; + } + } + this.setData = function(data,mimetype){ + if(mimetype == null) + this.obj.data = data; + else { + this.obj.headers['Content-Type'] = mimetype + if(mimetype.match(/application\/json/i)){ + this.obj.data = JSON.stringify(data) + } + } + + } + this.setAsync = function(flag){ + this.obj.async = (flag == true) ; + } + this.send = function(url,callback,method){ + + if(method == null){ + method = 'GET' + } + + p = jx.ajax.debug != null; + q = false; + if(p){ + q = jx.ajax.debug[url] != null; + } + + is_debuggable = p && q + + if(is_debuggable){ + x = {} ; + x.responseText = jx.ajax.debug [url] ; + callback(x) + }else{ + var http = new XMLHttpRequest() ; + http.onreadystatechange = function(){ + if(http.readyState == 4){ + + callback(http) + } + } + // + // In order to insure backward compatibility + // Previous versions allowed the user to set the variable on the wrapper (poor design) + if(this.async != null){ + this.setAsync(this.async) ; + } + http.open(method,url,this.obj.async) ; + for(key in this.obj.headers){ + value = this.obj.headers[key] ; + + http.setRequestHeader(key,value) + } + + http.send(this.obj.data) + } + + + } + this.put = function(url,callback){ + this.send(url,callback,'PUT') ; + } + this.get = function(url,callback){ + this.send(url,callback,'GET') ; + } + this.post = function(url,callback){ + this.send(url,callback,'POST') ; + } + }//-- end of the factory method + return new factory() ; + } + + // + // backward compatibility + jx.ajax.getInstance = jx.ajax.get.instance ; + var HttpClient = jx.ajax.get ; + \ No newline at end of file diff --git a/healthcareio/server/templates/index.html b/healthcareio/server/templates/index.html index 915cddc..768618e 100644 --- a/healthcareio/server/templates/index.html +++ b/healthcareio/server/templates/index.html @@ -8,6 +8,7 @@ + @@ -15,6 +16,9 @@ + + + Healthcare/IO Analytics -
+
-
Healthcare/IO
-
Analytics Dashboard
+
Healthcare/IO :: Parser
+
Dashboard
-