data-transport/transport/disk.py

85 lines
2.3 KiB
Python

import os
from .__init__ import Reader,Writer
import json
class DiskReader(Reader) :
"""
This class is designed to read data from disk (location on hard drive)
@pre : isready() == True
"""
def __init__(self,**params):
"""
@param path absolute path of the file to be read
"""
Reader.__init__(self)
self.path = params['path'] ;
self.delimiter = params['delimiter'] if 'delimiter' in params else None
def isready(self):
return os.path.exists(self.path)
def read(self,size=-1):
"""
This function reads the rows from a designated location on disk
@param size number of rows to be read, -1 suggests all rows
"""
f = open(self.path,'rU')
i = 1
for row in f:
i += 1
if size == i:
break
if self.delimiter :
yield row.split(self.char)
yield row
f.close()
class DiskWriter(Writer):
"""
This function writes output to disk in a designated location. The function will write a text to a text file
- If a delimiter is provided it will use that to generate a xchar-delimited file
- If not then the object will be dumped as is
"""
def __init__(self,**params):
Writer.__init__(self)
self.cache['meta'] = {'cols':0,'rows':0,'delimiter':None}
if 'path' in params:
self.path = params['path']
else:
self.path = 'data-transport.log'
self.delimiter = params['delimiter'] if 'delimiter' in params else None
# if 'name' in params:
# self.name = params['name'];
# else:
# self.name = 'data-transport.log'
# if os.path.exists(self.path) == False:
# os.mkdir(self.path)
def meta(self):
return self.cache['meta']
def isready(self):
"""
This function determines if the class is ready for execution or not
i.e it determines if the preconditions of met prior execution
"""
return True
# p = self.path is not None and os.path.exists(self.path)
# q = self.name is not None
# return p and q
def format (self,row):
self.cache['meta']['cols'] += len(row) if isinstance(row,list) else len(row.keys())
self.cache['meta']['rows'] += 1
return (self.delimiter.join(row) if self.delimiter else json.dumps(row))+"\n"
def write(self,info):
"""
This function writes a record to a designated file
@param label <passed|broken|fixed|stats>
@param row row to be written
"""
f = open(self.path,'a')
f.write(self.format(info))
f.close()