adlidsensor.py

Go to the documentation of this file.

"""
AD lid sensors scraping specialization 

Discussion from Wei:
 
#. we were discussing scrapping the average, its standard deviation, the minimum and the maximum within each hour. 
#. It seems average once per hour is sufficient. (Note: reactor flux will be available sparser than 1/hour). 

   ==============   ==============================================
    reference
   ==============   ==============================================
     :doc:`6673`      discussion 
     :doc:`6996`      for the current status given by David W. 
     :doc:`6983`      summarizes the lid sensor data so far. 
   ==============   ==============================================


"""
import os, logging
import random
from datetime import datetime, timedelta
from pprint import pformat

#from dcs import ADLS
from base import Regime, Scraper, Faker, DCS, SourceContext

log = logging.getLogger(__name__)
#log.addHandler(logging.FileHandler("adlidsensor.log"))

class AdLidSensor(Regime):
    """
    Regime frontend class with simple prescribed interface, 
    takes the cfg argument into this dict and no args in call 
    ... allowing the frontend to be entirely generic
    """
    srcs    = property( lambda self:AdLidSensorSource( DCS(self['source']) ))
    scraper = property( lambda self:AdLidSensorScraper(self.srcs, self.target, self ))
    faker   = property( lambda self:AdLidSensorFaker(self.srcs,self ))


class AdPhysLogical(dict):
    """
    Placeholder dict until physical to logical via DB lookup is worked out 
    """
    def __call__(self, physad ):
        assert physad in self 
        return self[physad] 
 

class AdLidSensorSource(list):
    """
    A list of SQLAlchemy dynamic classes 
    """
    def __init__(self, srcdb ):
        """Coordinates of source table/joins"""

        apl = AdPhysLogical(             ## place holder for testing, accurate as of Nov 30, 2011. 
                  AD1=("kDayaBay","kAD1"), 
                  AD2=("kDayaBay","kAD2"), 
                  AD3=("kLingAo" ,"kAD1"), 
                  AD4=("kFar"    ,"kAD1"), 
                  AD5=("kFar"    ,"kAD2"), 
                  AD6=("kFar"    ,"kAD3"), 
                  AD7=("kSAB"    ,"kAD1"), 
                  AD8=("kSAB"    ,"kAD2"), 
             )

        for physad in ("AD1","AD2","AD3","AD4","AD5","AD6","AD7","AD8",):
            ksite, ksubsite = apl( physad )
            dtn = SourceContext(table="%s_LidSensor" % physad , ksite=ksite , ksubsite=ksubsite ) 
            self.append( srcdb.kls(dtn) )


class AdLidSensorScraper(Scraper):
    """
    Specialization of generic scraper for AD temperature tables
    """
    #adls_matcher = ADLS()   

    def changed(self, sv  ):
        """
        returns changed decision to base class

        Caution DB/SQLAlchemy is providing decimal.Decimal
        values... unify types to float before comparison to avoid surprises
        """
        #pd = self._adlsdict( sv[0] )
        #ud = self._adlsdict( sv[-1] )
        #log.debug("prev %s " % pd )
        #log.debug("curr %s " % ud )
        log.debug("prev %s " % sv[0] )
        log.debug("curr %s " % sv[-1] )
        log.debug("source vector length %d" % len(sv));

        #if not hasattr(self, 'state'):          ## only on 1st call when no state
        #    kls = self.target.kls               ## the genDbi target  class
        #    keys = kls.SpecKeys().aslist()
        #    state = dict(zip(keys,map(lambda _:0, keys)))    ## state dict with all values 0
        #    self.state = state

        ## work of associating source to target attributes
        #for k in self.state:
            #sk = ..some_fn..( k )       ## source key from target key
            ## do running aggregates  min/max/avg
            #self.state[k] += sv[-1][sk]

        return False    ## True if sufficient changes to warrant non-age based propagation 

#        for adls in sorted(pd.keys()):
#            pv = float(pd[adls])    ## often decimal.Decimal types
#            uv = float(ud[adls])
#            df = abs(pv-uv) 
#            if float(df) > float(self.threshold):
#                chg = True
#            else:
#                chg = False
#            log.debug("pv %r uv %r df %r threshold %s change %s  " % (pv,uv,df, self.threshold, chg)) 
#            if chg:
#                return True
#        return False

    def propagate(self, sv ):
        """yields one or more target dicts ready for writing to target DB"""
        #dd = self._adlsdict(sv[-1])
        #log.debug("propagate %r " % dd)
        #yield self.state
        aggd=sv[-1].aggd
        assert aggd, "missing aggd - this should always be present"
        for k,v in sorted(aggd.items()):
            print k,v
        if aggd['NSamples']==0:
            kls = self.target.kls               ## the genDbi target  class
            keys = kls.SpecKeys().aslist()
            yield dict(zip(keys,map(lambda _:-1, keys))) ## state dict with all values -1
        else:    
            yield aggd

    def seed(self, sc):
        """
        Used for seeding target DB when testing into empty tables

        :param sc: source class, potentially different seeds will be needed 
                   for each source that feeds into a single target
        """
        kls = self.target.kls               ## the genDbi target  class
        keys = kls.SpecKeys().aslist()
        return dict(zip(keys,map(lambda _:0, keys)))    ## state dict with all values 0

    def _adlsdict(self, inst ):
        """converts source instance into target dicts"""
        dd = {}
        for k,v in inst.asdict.items():
            if k in 'id date_time'.split():
                continue
            #adls = self.adls_matcher(k)
            if adls and adls != "5":      ## SKIPPING Temp5 AS NO LONGER IN SPEC
                dd['Temp%s'%adls] = v
        return dd 


class AdLidSensorFaker(Faker):
    #adls_matcher = ADLS()   
    def fake(self, inst, id , dt=None ):
        """
        Invoked from base class, sets source instance attributes to form a fake 

        :param inst: source instance
        :param id: id to assign to the instance 
        """
        if dt==None:
            dt = datetime.now()
        for k,v in inst.asdict.items():
            if k == 'id':
                setattr( inst, k, id )
            elif k == 'date_time':
                setattr( inst, k, dt )
            else:
                # TODO switch on simulated data profiles
                #if self.profile == "modulo_ramp":
                #elif self.profile == "island_of_stability":
                #elif self.profile == "island_of_change":

                # TODO simulate actual fluid heights.  for now, simulate counts
                if k.startswith('Ultrasonic'):
                    setattr( inst, k, random.gauss(2000, 5) )
                elif k.startswith('Capacitance'):
                    setattr( inst, k, random.gauss(200, 10) )
                elif k.startswith('Temp_'):
                    setattr( inst, k, random.gauss(22.7, 0.1) )
                elif k.startswith('Tilt'):
                    setattr( inst, k, random.gauss(0, 0.1) )
                else:
                    setattr( inst, k, 0 );   

# source database variables
#  Ultrasonic_GdLS=0,
#  Ultrasonic_LS=0,
#  Capacitance_GdLS=0,
#  Capacitance_LS=0,
#  Capacitance_MO=0,
#  Tiltx_Sensor1=0,
#  Tilty_Sensor1=0,
#  Tiltx_Sensor2=0,
#  Tilty_Sensor2=0,
#  Tiltx_Sensor3=0,
#  Tilty_Sensor3=0,
#  Temp_GdLS=0,
#  Temp_LS=0,
#  Capacitance_Temp_GdLS=0,
#  Capacitance_Temp_LS=0,
#  Capacitance_Temp_MO=0,


if __name__ == '__main__':
    pass

    from base.parser import Parser
    cfg = Parser.config("adlidsensor_scraper")
    reg = cfg.regcls
    irg = reg(cfg)
    print irg.srcs    
    assert len(irg.srcs) > 0, "no source classes "