vrb.py

Go to the documentation of this file.

#!/usr/bin/env python 
"""
For debugging::

   ipython vrb.py 

"""

import os, logging
from pprint import pformat
log = logging.getLogger(__name__)

def parse_cfg( val ):
    """
    Parse config like the below into dict:: 

        [mig20120820]
        src = GCalibPmtHighGainPariah -1 27877 6533
        tgt = GCalibPmtFineGain 1 1410 6533

    """
    bits = val.split(' ') ; assert len(bits) == 4, "unexpected fields in value %s %s " % (repr(bits), val)
    return dict(kln=bits[0],task=int(bits[1]),offseq=int(bits[2]),count=int(bits[3]))


class CFConf(dict):
    """
    Usage::

          from tools impoty CFConf
          def handler( val ):
              return dict()
          cfc = CFConf("offsets.cfg", "mig20120820", handler ) 

    The handler parses the config value and returns a dict  

    """
    def __init__(self, path, section , parse_cfg_ = lambda _:{} ):
        dict.__init__(self)
        from ConfigParser import ConfigParser
        cfp = ConfigParser()
        cfp.optionxform = str   ## avoid lowercasing keys, making the keys case sensitive
        path = os.path.expandvars(os.path.expanduser(path))
        cfp.read(path) 
        assert cfp.has_section(section), "section %s not present in config file %s " % ( section, path)
        for key, val in cfp.items(section):
            self[key] = parse_cfg_( val )

    def __repr__(self):
        return pformat(dict(self))



def pydigest_( rpt , first=0 ):
    """
    :param first: index of first column to include in digest
          
    The default `first=0` duplicates the C++ `rpt.digest`. 
    The python version can in addition restrict the comparison to ignore columns.
 
    The vals are strings like::
 
        67241993 20.937 0.203 6.333 0.94
        67241994 18.892 0.146 5.217 1.03

    Thus using a `first=1` allows digest content 
    comparisons between tables using sensorid and channelid keys 
 
    """
    from ROOT import TMD5   
    md5 = TMD5()
    for i,r in enumerate(rpt):
        vals = " ".join(r.values.split(" ")[first:])
        if i == 0:
            log.debug(vals) 
        md5.Update(vals, len(vals))
    md5.Final()
    return md5.AsString()


class CompareVrb(dict):
    def __init__(self, **kwa):
        dict.__init__(self, **kwa)

    def __call__(self):
        rpt = {}
        ldict = {}
        for tag, vrb in self.items():
            ldict[tag] = len(vrb)
            rpt[tag] = vrb.kls.Rpt().Clone()
        self.rpt = rpt

        lset = list(set(ldict.values()))   # uniquing by passing thru the set
        assert len(lset) == 1, ("all argument vrb required to have the same length", ldict )
        self.length = lset[0]

    def check_payload(self, first=0, check=None):
        """
        SEQNO by SEQNO comparisons of digests between tables

        :param first: column index to start the digest from, 1 to skip first column, 0 to include all
        :param check: function that accepts single arguement, the single seqno dict 

        C++ digests `rpt.digest` are constructed from all values in the payloads allowing 
        a quick way to check payload correspondence between multiple tables
        """
        for i in range(self.length):
            d = dict(index=i) 
            for tag,vrb in self.items():
                vrec = vrb.offseq_(i)
                self.rpt[tag].ctx( validityrec=vrec ) 
                d[tag] = dict(seqno=vrec.seqno,digest=self.rpt[tag].digest,pydigest=pydigest_(self.rpt[tag], first=first))
            tags = self.keys()
            for tag in tags[1:]:
                assert d[tags[0]]['pydigest'] == d[tag]['pydigest'], ("pydigest mismatch", d )
            if check:
                check(d)

    def check_validity(self, check=None ):
        """
        SEQNO by SEQNO comparisons of validity fields between tables
        """
        pass
        for i in range(self.length):
            v = dict(index=i)
            for tag,vrb in self.items():
                vrec = vrb.offseq_(i)
                for qwn in Vrec.qwns:
                    if qwn not in v:
                        v[qwn] = {}
                    qbit = qwn.split(".")
                    if len(qbit) == 2: 
                        g_ = lambda vrec:getattr(getattr(vrec,qbit[0]),qbit[1]) 
                    else:
                        g_ = lambda vrec:getattr(vrec,qwn)
                    v[qwn][tag] = g_(vrec) 
                pass
            if check:
                check(v)




class Vrec(object):
   qwns = "contextrange.timestart contextrange.timeend contextrange.sitemask contextrange.simmask subsite task aggregateno versiondate".split()
   def __init__(self, vrec ):
        self.vrec = vrec

   def vdict(self):
       v = {}
       for qwn in self.qwns:
           qbit = qwn.split(".")
           g_ = lambda vrec:getattr(getattr(vrec,qbit[0]),qbit[1]) if len(qbit) == 2 else lambda vrec:getattr(vrec,qwn)
           v[qwn] = g_(self.vrec) 
       return v


class Vrb(object):
    def __init__(self, kls, task=-1, offseq=0, sqlcontext="1=1", subsite=-1 ):
        """
        :param kls:
        :param task:
        :param offseq:  seqno offset, used by offseq_ for comparison between tables with shifted SEQNO 
        """
        self.vrb = kls.GetTableProxy().MakeValidityRecBuilder( sqlcontext, subsite, task )
        self.seqnos = map(lambda _:_.seqno, self.vrb)
        self.seqnos.sort()
        self.kls = kls
        self.offseq = offseq
        self.cursor = 0


    def __len__(self):
        return len(self.vrb)

    def __getitem__(self, rowNo ):
        """
        List access with square brackets queries by rowNo. 
        But the ordering appears to make no sense, so not very useful. 
        """
        return self.vrb.__getitem__(rowNo)

    def offseq_( self, count ):
        return self( self.offseq + count + 1 ) 

    def __call__(self, seqno):
        """
        Call access with round brackets queries by SEQNO

        :param seqno:
        """
        return self.vrb.GetValidityRecFromSeqNo(seqno)

    def purge(self):
        self.kls.GetTableProxy().GetCache().Purge()

    def __iter__(self):
        return self

    def next(self):
        """
        Adds iterability to the Vrb providing (seqno, vrec) tuples 
        """ 
        if self.cursor >= len(self.seqnos) or self.cursor < 0:
             raise StopIteration
        else:
             seqno = self.seqnos[self.cursor]
             vrec = self(seqno)
             assert seqno == vrec.seqno, ("seqno vrec.seqno mismatch", seqno, vrec.seqno)
             self.cursor += 1
             return ( seqno, vrec )

    def replace_insertdate( self, ts=None, seqnos=[], dbno=0 ):
        """
        The functionality of changeing INSERTDATES is not for general use. 
        It is intended for special purpose usage only, as INSERTDATEs are subsequently 
        fastforwarded to real times anyhow as part of the Standard Operation Procedures.
        """
        nrep = 0 
        if not ts:
            ts = TimeStamp()
        tbProxy = self.kls.GetTableProxy()
        dbProxy = tbProxy.GetDBProxy()
        for seqno,vrec in self:
            if len(seqnos) == 0 or seqno in seqnos: 
                print "%-4s %s %s " % ( seqno, vrec.insertdate, ts )
                dbProxy.ReplaceInsertDate( ts, seqno, dbno ) 
                nrep +=1
        return nrep



if __name__ == '__main__':
    os.environ.setdefault('DBCONF','tmp_offline_db')
    cfconf = 'test'
    cfc = CFConf("cfconf.cfg",cfconf, parse_cfg )

    import DybDbi
    DybDbi.gDbi.level = "WARNING"

    cv = CompareVrb()
    for k, d in cfc.items():
        kls = getattr(DybDbi, d['kln'] )
        vrb = Vrb(kls, d['task'], offseq=d['offseq'] )
        assert len(vrb) == d['count'], ("unexpected length", d, vrb ) 
        cv[k] = vrb
    cv()

    vrb = cv['tgt']
    vrec = vrb.offseq_(0)
    vrec_ = Vrec(vrec)
    print vrec_.vdict()