history.py

Go to the documentation of this file.

#!/usr/bin/env python
"""

"""
import os, logging
from DybDbi.vrb import Vrb 
log = logging.getLogger(__name__)

class Gap(dict):pass

class Rec(dict):
    def __repr__(self):
        return "seqno %(seqno)4s %(ftimestart)s %(ftimeend)s  %(digest)s npay %(npay)4s %(msg)s " % self 
 
class History(dict):
    """
    Validity history dict keyed on timestart seconds

    After gap filling SEQNO ordering will not match time ordering, hence moved to 
    keying on timestart seconds : which should also be unique within an SS block
    (other tasks, simmasks would break that but would need to change blocking 
    in that case anyhow)
    """
    def __init__(self, kls, sitemask, subsite, wtask=0, task=-1 , paydict_=lambda rpt:{}, seqnocut=3867 , target_tablename=None, dupefirst=True ):
        """

        :param task: reading task
        :param wtask: writing task

        :param seqnocut: presentational only, SEQNO higher than this are highlighted
        :param target_tablename:
        :param dupefirst: duplicate the first entry to retain same SEQNO count, otherwise looses the first entry 


        Collects `Rec` instances into this dict keyed by TIMESTART seconds

        .. warn:: ensure the chunk does not have duplicate TIMESTARTs via sitemask, subsite, task choices

        """
        pass
        sqlcontext = "SITEMASK=%(sitemask)s and SUBSITE=%(subsite)s " % locals() 
        vrb = Vrb(kls, sqlcontext=sqlcontext, subsite=subsite, task=task )       ## ugly subsite repetition
        pass
        self.sitemask = sitemask
        self.subsite = subsite
        self.task = task
        self.wtask = wtask
        self.sqlcontext = sqlcontext 
        self.seqnocut = seqnocut
        self.target_tablename = target_tablename
        self.dupefirst = dupefirst
        pass
        self.kls = kls
        self.tn = kls.__name__[1:] 
        self.vrb = vrb
        self.gaps = []
        self._collect(paydict_)
        self.seqno2ik = {}        
        self._index()
        self._findgaps()


    def __repr__(self):
        return "%s %s nvrec %s gaps %s " % ( self.__class__.__name__, self.sqlcontext, len(self.vrb), len(self.gaps) )

    def _collect(self, paydict_):
        """
        Collect Rec instances into this dict keyed on timestart seconds

        :param paydict_: function of rpt instance that returns a dict
        """
        for seqno,vrec in self.vrb:
            cr = vrec.contextrange
            rpt = self.kls.Rpt().Clone()
            rpt.ctx( validityrec=vrec )
            dpay = paydict_(rpt)   # tuck away table specifics in the passed in paydict_ function
            npay = len(dpay) 
            pass 
            rec = Rec(seqno=seqno,vrec=vrec,timestart=cr.timestart,timeend=cr.timeend,npay=npay,payload=dpay,digest=rpt.digest)
            rec['ftimestart'] = rec['timestart'].AsString('s')
            rec['ftimeend']   = rec['timeend'].AsString('s')
            rec['msg'] = "***" if seqno > self.seqnocut else "   "
            pass
            key = cr.timestart.sec 
            self[key] = rec 
            pass
    def _index(self): 
        """
        Construct mapping from SEQNO to timestart ordered index
        """
        kets = sorted(self.keys()) 
        self.kets = kets
        for ik in range(0,len(kets)):
            ck = kets[ik]
            cv = self[ck]
            seqno = cv['seqno']
            self.seqno2ik[seqno] = ik
        pass 

    digests = property(lambda self:[self[t]['digest'] for t in sorted(self)], doc="payload digests in timestart seconds order")
    seqnos  = property(lambda self:[self[t]['seqno'] for t in sorted(self)] , doc="seqnos in timestart seconds order")


    def _findgaps(self):
        """
        Comparison of Rec with their priors, collecting any breaks 
        in chain::

            [ts-te][ts-te]

        """
        for ik in range(1,len(self.kets)):
            ck = self.kets[ik]
            pk = self.kets[ik-1] 
            cv = self[ck]
            pv = self[pk] 
            pass
            assert ck == cv['timestart'].sec
            assert pk == pv['timestart'].sec
            assert cv['npay'] == pv['npay'], (cv, pv)
            #assert cv['seqno'] > pv['seqno'], (cv, pv)    ## no longer the case after gap filling
            assert cv['timestart'].sec >= pv['timeend'].sec, (cv, pv)
            assert cv['timestart'].sec >= pv['timeend'].sec, (cv, pv)
            pass
            if cv['timestart'].sec != pv['timeend'].sec:
                gap = Gap(seqbef=pv['seqno'],seqaft=cv['seqno'],timestart=pv['timeend'], timeend=cv['timestart'] )             
                self.gaps.append(gap)
 
    def dump(self):
        """
        Dump validities, highlighting those with SEQNO greater than cut 
        """
        for i,t in enumerate(sorted(self)):
            rec = self[t]
            log.info("%4s %s %r" % (i,t,rec))

    def paydump(self, paykey='L2C2R8'):
        for i,t in enumerate(sorted(self)):
            rec = hist[t]
            pay = rec['payload']
            pav = pay.get(paykey,None)
            log.info("%4s %s %r %s" % (i,t,rec,pav ))
 
    def fillgaps(self):
        """
        Filling gaps via duplication of prior entry with timerange shifted
        """
        wseqs=[] 
        for gap in self.gaps:
            seqbef = gap['seqbef']
            wseq=self.writecopy( seqbef, gap['timestart'], gap['timeend'] , task=self.wtask )
            wseqs.append(wseq)
        log.info("wrote seqnos %r " % wseqs )

    def shunt(self, seqno):
        """
        :param seqno: if 0 all entries are shunted, otherwise only the specified SEQNO

        Shunting payload forward, '''pay it forward'''

        Originally the SEQNO were not aligned due to the
        treatment via chunks of (SITEMASK, SUBSITE).
        Effectively the deck was unshuffled into sorted manner.  

        To rectify this the source validity table (SEQNO, SITEMASK, SUBSITE)
        sequence is used to retrace the steps of the scraper and address multiple
        instances of this history class for each (SITEMASK,SUBSITE) chunk.
        This allowed the deck to be shuffled according to the source shuffle.    

        #. 1st rec untouched
        #. 2nd rec gets payload from 1st 
        #. 3rd rec gets payload from 2nd
        #. etc
        #. nth rec gets payload from (n-1)th
        #. last gets payload from penultimate

        """
        assert len(self.gaps) == 0, "must fill gaps before shunting "
        assert self.target_tablename , "must define target table when shunting "

        if seqno == 0:
            log.info("shunt into %s %s " %  ( len(self.kets),self.target_tablename) )
            iks = range(0,len(self.kets))
        else:
            iks = [self.seqno2ik[seqno]] 

        for ik in iks:
            ck = self.kets[ik]
            cv = self[ck]
            if ik == 0 and self.dupefirst:
                if seqno > 0:
                    assert cv['seqno'] == seqno 
                self.writecopy( cv['seqno'], cv['timestart'], cv['timeend'], task=self.wtask , target_tablename=self.target_tablename )
                continue            
            pk = self.kets[ik-1] 
            pv = self[pk] 
            pass
            assert ck == cv['timestart'].sec
            assert pk == pv['timestart'].sec
            assert cv['npay'] == pv['npay'], (cv, pv)
            assert cv['timestart'].sec >= pv['timeend'].sec, (cv, pv)
            assert cv['timestart'].sec >= pv['timeend'].sec, (cv, pv)
            assert cv['timestart'].sec == pv['timeend'].sec , "gaps need fixin 1st "
            pass 
            if seqno > 0:
                assert cv['seqno'] == seqno 
            self.writecopy( pv['seqno'], cv['timestart'], cv['timeend'], task=self.wtask , target_tablename=self.target_tablename )


    def writecopy(self, seqno, timestart=None, timeend=None, task=0, target_tablename=None ):
        """
        Copy validity rec and payload contents corresponding to seqno 
        into a new entry, optionally changing timestart and timeend        

        :param vrec: ValidityRec instance
        :param timestart: TimeStamp instance
        :param timeend: TimeStamp instance
        :param task:
        :param target_tablename:
        """ 
        from DybDbi import ContextRange, SimFlag, TimeStamp

        vrec = self.vrb(seqno)
        assert vrec.subsite == self.subsite
        assert vrec.contextrange.sitemask == self.sitemask
        #assert vrec.contextrange.simmask == self.simmask

        if not target_tablename:
            target_tablename = self.tn
        if not timestart:
            timestart = vrec.contextrange.timestart
        if not timeend:
            timeend = vrec.contextrange.timeend

        wrt = self.kls.Wrt().Clone()
        cr = ContextRange( self.sitemask,  SimFlag.kData , timestart, timeend )
        overlay = TimeStamp(0,0)
        wrt.ctx( contextrange=cr, subsite=vrec.subsite, task=task, versiondate=overlay, tablename=target_tablename )
        rpt = self.kls.Rpt().Clone()
        rpt.ctx( validityrec=vrec )
        n = len(rpt)
        for row in rpt:
            wrt.Write(row)
        wseqno = wrt.Close()
        return wseqno



if __name__ == '__main__':
    pass
    os.environ.setdefault('DBCONF','tmp_offline_db')
    logging.basicConfig(level=logging.INFO)
    from DybDbi import gDbi, GDcsAdWpHv
    gDbi.level = "WARNING"

    paydict = lambda rpt:dict([r.locationid,r.voltage] for r in rpt)
    #paydict = lambda rpt:{}
    hist = History( GDcsAdWpHv, sitemask=1, subsite=1 , paydict_=paydict )
    hist.findgaps()
    print hist
    if len(hist.gaps)>0:
       print "\n".join(map(repr,hist.gaps))

    for i,t in enumerate(sorted(hist)):
        rec = hist[t]
        log.info("%4s %s %r" % (i,t,rec))