From 96c3dbe5a8ca831ec1b5cdb31a3adb006095577d Mon Sep 17 00:00:00 2001 From: Steve Nyemba Date: Sun, 24 Jan 2021 01:37:36 -0600 Subject: [PATCH] bug fix: including _id as unique identifier for a claim --- healthcareio/export/export.py | 7 ++++--- healthcareio/export/workers.py | 8 ++++++-- setup.py | 2 +- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/healthcareio/export/export.py b/healthcareio/export/export.py index 799b044..edae913 100644 --- a/healthcareio/export/export.py +++ b/healthcareio/export/export.py @@ -108,6 +108,7 @@ def meta(config) : def create (**_args) : skip = [] if 'skip' not in _args else _args['skip'] fields = ([_args['key']] if 'key' in _args else []) + _args['fields'] + fields = ['_id'] + fields table = _args['table'] sql = ['CREATE TABLE :table ',"(",",\n".join(["\t".join(["\t",name,"VARCHAR(125)"]) for name in fields]),")"] return " ".join(sql) @@ -145,7 +146,7 @@ def init (**_args) : _name = "".join(["$",field_name]) project[field_name] = {"$ifNull":[_name,""]} - project["_id"] = 0 + project["_id"] = 1 project = {"$project":project} r = [{"table":TABLE_NAME,"mongo":{"aggregate":TABLE_NAME,"pipeline":[project],"cursor":{},"allowDiskUse":True},"sql":create(table=TABLE_NAME,fields=_info['main'])}] @@ -154,12 +155,12 @@ def init (**_args) : # NOTE: Adding _index to the fields fields = _info['rel'][table] +["_index"] - project = {"_id":0,"claim_id":1,"_index":1} #dict.fromkeys(["_id","claim_id"]+fields,[ ".".join([table,field_name]) for field_name in fields]) + project = {"_id":1,"claim_id":1,"_index":1} #dict.fromkeys(["_id","claim_id"]+fields,[ ".".join([table,field_name]) for field_name in fields]) for field_name in fields : # project[field_name] = "$"+".".join([table,field_name]) _name = "$"+".".join([table,field_name]) project[field_name] = {"$ifNull":[_name,""]} #{"$cond":[{"$eq":[_name,None]},"",_name]} - project["_id"] = 0 + project["_id"] = 1 pipeline = [{"$unwind":"$"+table},{"$project":project}] r += [{"table":table,"mongo":{"aggregate":TABLE_NAME,"cursor":{},"pipeline":pipeline,"allowDiskUse":True},"sql":create(table=table,key='claim_id',fields=fields)}] diff --git a/healthcareio/export/workers.py b/healthcareio/export/workers.py index 3655c00..47dec96 100644 --- a/healthcareio/export/workers.py +++ b/healthcareio/export/workers.py @@ -183,8 +183,12 @@ class Writer(Worker): # print (['segment # ',index,len(rows)]) self.log(**{"context":self.name(),"segment":(index+1),"args":{"rows":len(rows),"table":table}}) if len(rows) : - - writer.write(list(rows)) + # + # @TODO: Upgrade to mongodb 4.0+ and remove the line below + # Upon upgrade use the operator "$toString" in export.init function + # + rows = [dict(item,**{"_id":str(item["_id"])}) for item in rows] + writer.write(rows) index += 1 # for _e in rows : # writer.write(_e) diff --git a/setup.py b/setup.py index 545b231..fc5d4ba 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ import sys def read(fname): return open(os.path.join(os.path.dirname(__file__), fname)).read() args = { - "name":"healthcareio","version":"1.5.6", + "name":"healthcareio","version":"1.5.8", "author":"Vanderbilt University Medical Center", "author_email":"steve.l.nyemba@vumc.org", "include_package_data":True,