dbicnf.py

Go to the documentation of this file.

#!/usr/bin/env python
"""
An example using commandline parsing and pattern match against filenames, allowing smart
DBI writer scripts to be created that minimize code duplication. 

However make sure that arguments used are still captured into the 
repository either by creating one line scripts that invoke the
flexible scripts. Or arranging for flexible scripts to read driver files.

"""
import os, sys, re, argparse, logging, shlex
from pprint import pformat 
from datetime import datetime
log = logging.getLogger(__name__)

class TimeAction(argparse.Action):
    """Converts string date representations into datetimes """
    def __call__(self, parser, ns , values, option_string=None):
        log.debug('%r %r %r' % (ns, values, option_string))
        values = datetime.strptime( values , ns.timeformat )
        setattr(ns, self.dest, values)

class DbiCnf(dict):
    """
    DbiCnf is a dict holding parameters that are inputs to defining 
    the DBI writer and ingredients like contextrange etc..

    All outputs of this class such as ``timestart``, ``cr`` etc..
    are implemented as dynamically invoked properties, meaning 
    that the only important state held is in this dict 
    in the form of raw python types : str, int, datetime.

    This dict is composed with class defaults, ctor arguments, commandline 
    parsed results, path parameter regular expression parsed tokens, 
    interactive updating.

    Precedence in decreasing order:

    #. commandline arguments
    #. after ctor updates
    #. ctor keyword arguments
    #. basis defaults in ``DbiCnf.defaults``  

    Usage in writer scripts::

       from DybPython import DbiCnf
       cnf = DbiCnf()
       cnf()              ## performs the parse  

       from DybDbi import GCalibPmtSpec, CSV
       wrt = cnf.writer( GCalibPmtSpec ) 
 
       src = CSV( cnf.path )
       for r in src:
           instance = GCalibPmtSpec.Create( **r )
           wrt.Write( instance )

       if not cnf.dummy:
           assert wrt.close()

    Debugging/checking usage in ipython::

       from DybPython import DbiCnf
       cnf = DbiCnf(key=val,key2=val2)
       cnf['key3'] = 'val3'

       cnf()    ## performs command line parse
       cnf("All_AD1_Data.csv --task 20 --runtimestart 10 --dbconf tmp_offline_db:offline_db ")   ## test parsing gives desired params
       print cnf
       cnf['runtimestart'] = 10
       cnf.timestart            
       cnf['runtimestart'] = 1000    
       cnf.timestart                 ## will do timestart lookup for the changed run


    The simplest and recommended usage is to define a standard `.csv` file naming convention.
    For example when using the default context pattern::
   
       "^(?P<site>All|DayaBay|Far|LingAo|Mid|SAB)_(?P<subsite>AD1|AD2|AD3|AD4|All|IWS|OWS|RPC|Unknown)_(?P<simflag>MC|Data)\.csv"

    The tokens `site`, `subsite` and `simflag` are extracted from basenames such as the below by the pattern matching.

    #. SAB_AD1_Data.csv
    #. SAB_AD2_Data.csv
        
    """
    ## corresponds to an enum
    sites = "All|DayaBay|Far|LingAo|Mid|SAB" 
    subsites = "AD1|AD2|AD3|AD4|All|IWS|OWS|RPC|Unknown" 
    simflags = "Data|MC"

    def _defaults(self):
        return dict( 
             loglevel="INFO", 
             logpath=None,
           logformat='%(asctime)s %(name)s %(levelname)-8s %(message)s',
                 task=0, 
                 site=None,
              subsite=None,
              simflag=None,
            timeformat="%Y-%m-%d %H:%M:%S", 
             timestart=None, 
               timeend=None,
            runtimestart=None, 
              runtimeend=None,
                  ctxptn="^(?P<site>%s)_(?P<subsite>%s)_(?P<simflag>%s)\.csv" % ( self.sites, self.subsites, self.simflags ) ,
               nomatch=False,
               dbconf="tmp_offline_db",
                  dbno=0, 
               dummy=False,
          )
    defaults = property( _defaults )

    def argparser_(self):
        ap = argparse.ArgumentParser(description=__doc__, fromfile_prefix_chars='@', formatter_class=argparse.RawDescriptionHelpFormatter )

        po = ap.add_argument_group("positional")
        po.add_argument('path',                help='path to csv file, mandatory argument')

        op = ap.add_argument_group("operational")
        op.add_argument('-l','--loglevel',     help='logging level INFO,WARN,DEBUG... Default %(default)s ')
        op.add_argument(     "--logpath",      help="Path to write log file to. Default %(default)s ")
        op.add_argument(     "--logformat",    help="Used by logger. Default %(default)s ")
        op.add_argument('-n','--dummy',        help='Dummy run. Default %(default)s ', action="store_true" )
        
        ap.add_argument(     '--dbconf',       help='Section of ~/.my.cnf file. Default %(default)s when using runtimestart/runtimeend must be cascade such as tmp_offline_db:offline_db ')

        nu = ap.add_argument_group("not usually changed")
        nu.add_argument(     '--timeformat',   help='format for times. Default %(default)s ')
        nu.add_argument(     '--task',         help='set non zero for testing non-default algorithms. Default %(default)s ', type=int )
        nu.add_argument(     '--dbno',         help='DB number in cascade. Default %(default)s ', type=int )

        cx = ap.add_argument_group("context")
        cx.add_argument(     '--site',         help='string to be converted into enum integer with Site.FromString. Default %(default)s ', choices=self.sites.split("|"))
        cx.add_argument(     '--subsite',      help='string to be converted into enum integer with DetectorId.FromString. Default %(default)s ', choices=self.subsites.split("|"))
        cx.add_argument(     '--simflag',      help='string to be converted into enum integer with SimFlag.FromString. Default %(default)s ', choices=self.simflags.split("|"))
        cx.add_argument(     '--ctxptn',       help='Regular expression string to be matched against csv basenames. Often includes `site`, `subsite` and `simflag`. Default %(default)s ')
        cx.add_argument('-M','--nomatch',      help='Regular expression string is not forced to match csv basenames. Default %(default)s ', action="store_true" )

        ts = ap.add_mutually_exclusive_group()
        ts.add_argument(     '--timestart',    help='contextrange start time in UTC. Default %(default)s corresponds to TimeStamp.GetBOT  ', action=TimeAction )
        ts.add_argument(     '--runtimestart', help='contextrange start time in UTC corresponding to timestart for run number passed. Default %(default)s corresponds to use timestart ', type=int )

        te = ap.add_mutually_exclusive_group()
        te.add_argument(     '--timeend',      help='contextrange end time in UTC. Default %(default)s corresponds to TimeStamp.GetEOT ', action=TimeAction )
        te.add_argument(     '--runtimeend',   help='contextrange end time in UTC corresponding to timeend of run number passed. Default %(default)s corresponds to use timeend ', type=int )

        return ap 

    def __init__(self, *args, **kwa ):
        """
        :param kwa: ctor keyword arguments override class defaults ``DbiCnf.defaults`` updating into `self` 
        """
        dict.__init__(self)
        self.update( self.defaults )  
        self.update( kwa )            
 
    def __call__(self, args_=None ):
        """
        :param args_: default of None parses system arguments, define for interactive testing

        Performs parse steps:

        #. populates argparser defaults from `self`
        #. parses `args_` or sys.argv 
        #. parses args.path obtaining ``pathdict``
        #. updates `self` from args.path parsed tokens
        #. updates `self` raw initial parsed values 

        First positional argument is interpreted as a path and is pattern matched against 
        a regular expression. 
        """
        ap = self.argparser_()
        ap.set_defaults( **self )         

        if args_ and type(args_) == str:
            args_ = shlex.split(args_)     ## for interactive testing 

        args = ap.parse_args(args=args_)

        logging.basicConfig( level=getattr(logging,args.loglevel.upper()) )
        log.info( "initial args %r " %  args )
        
        pathdict = self.parse_path( args.path , args.ctxptn, args.nomatch )
 
        self.update( pathdict )   

        va = vars(args)           
        for k,v in va.items(): 
            if not v:continue   ## None does not trump pre-existers
            self[k] = v 

        if self['dbconf']:
            os.environ['DBCONF'] = self['dbconf'] 
            log.warn("DBCONF set to %s " % self['dbconf'] )

        self.irl = self._irunlookup()

        ## state for debug only 
        self['pathdict'] = pathdict        
        self.args = args      

    def __repr__(self):
        return self.__class__.__name__ + "\n" + pformat(dict(self))

    def _irunlookup(self):
        """
        **CAUTION** this takes the runtimestart/runtimeend directly from GDaqRunInfo lookup ...
        so they must be UTC ... if that is not the case, that needs to be fixed 
        """
        rl = filter(None, (self['runtimestart'], self['runtimeend']))
        if rl:
            from DybDbi import IRunLookup
            log.debug("_irunlookup %r " % rl )
            irl = IRunLookup(*map(int,rl))
            log.debug("IRunLookup %r gives ... \n %s " % ( rl, pformat(irl) ))  
        else:
            irl = {}
        return irl 

    def parse_path(self, path_ , ptn , nomatch ):
        """
        Extract context metadata from the path using the regular expression string 
        supplied. 

        :param path: path to .csv source file
        :param ptn: regular expression string that can contain tokens for any config parameters

        :rtype dict: dict of strings extracted from the path 
        """ 
        path = os.path.expandvars(os.path.expanduser(path_))  ## expand envvars or twiddles
        name = os.path.basename( path )
        ptn_ = re.compile(ptn)
        match = ptn_.match(  name )
        pathdict = {}
        if match:
            pathdict = match.groupdict()
        else:
            if nomatch:
                log.warning("did not match name %s with pattern %s " % ( name , ptn ) )
            else:
                log.fatal("failed to match name %s with pattern %s " % ( name , ptn ) )
                raise Exception
        log.debug( "pathdict %r " %  pathdict )
        return pathdict 


    # for consistent structure
    path  = property( lambda self:self['path'] ) 
    dbno  = property( lambda self:self['dbno'] ) 
    dummy = property( lambda self:self['dummy'] ) 

    runtimestart = property(lambda self:int(self['runtimestart']) if self['runtimestart'] else None)
    runtimeend   = property(lambda self:int(self['runtimeend']) if self['runtimeend'] else None)

    def _timestart(self):
        """
        """
        from DybDbi import TimeStamp
        rts = self.runtimestart
        if rts:
            if not rts in self.irl:
                self._irunlookup()
            timestart = self.irl[rts].vrec.contextrange.timestart
        else:
            timestart = self['timestart']
            timestart = TimeStamp.fromAssumedUTCDatetime( timestart ) if timestart else TimeStamp.GetBOT()
        return timestart
    timestart = property( _timestart, doc=_timestart.__doc__ )
         
    def _timeend(self):
        """
        """
        from DybDbi import TimeStamp
        rte = self.runtimeend 
        if rte:
            if not rte in self.irl:
                self._irunlookup()
            timeend = self.irl[rte].vrec.contextrange.timeend
        else:
            timeend   = self['timeend']
            timeend = TimeStamp.fromAssumedUTCDatetime( timeend ) if timeend else TimeStamp.GetEOT()
        return timeend  
    timeend = property( _timeend, doc=_timeend.__doc__ )
      
    # done longhand to avoid root hijacking --help
    def _site(self):
        """Convert string site into enum integer"""
        from DybDbi import Site
        return Site.FromString( self['site'] )
    site = property( _site )
    def _sitemask(self):
        """Convert string site into enum integer
           if multi-site masks are needed will have to revisit this
        """
        from DybDbi import Site
        return Site.FromString( self['site'] )
    sitemask = property( _sitemask, doc=_sitemask.__doc__ )  ## bit icky 

    def _subsite(self):
        """Convert string subsite/DetectorId into enum integer"""
        from DybDbi import DetectorId
        return DetectorId.FromString( self['subsite'] )
    subsite = property( _subsite , doc=_subsite.__doc__)

    def _simflag(self):
        """Convert string simflag into enum integer"""
        from DybDbi import SimFlag
        return SimFlag.FromString( self['simflag'] )
    simflag = property( _simflag, doc=_simflag.__doc__ )
    def _simmask(self):
        """Convert string simflag into enum integer
           (note the simflag is interpreted as the mask)
        """
        from DybDbi import SimFlag
        return SimFlag.FromString( self['simflag'] )
    simmask = property( _simmask, doc=_simmask.__doc__ )   

    def _cr(self):
        """
        Convert the strings into enum value, and datetimes into TimeStamps in
        order to create the ContextRange instance 

        :return: context range instance 
        """ 
        from DybDbi import ContextRange
        return ContextRange( self.sitemask, self.simmask, self.timestart, self.timeend )
    cr = property( _cr , doc=_cr.__doc__ )






    def writer(self, kls ):
        """
        Create a pre-configured DybDbi writer based on 
        arguments and source csv filename parsing and 
        creates the corresponding DB table if it does not exist.

        :param kls: DybDbi class, eg GCalibPmtHighGain

        """
        from DybDbi import TimeStamp

        assert kls.__name__[0] == 'G', "DybDbi classes must start with 'G' unlike %s " % kls.__name__
        name = kls.__name__[1:]
        assert self.dbno == 0 , "unexpected dbno %s " % self.dbno

        log.info("creating DB tables for %s in dbno %s  " % ( name , self.dbno )) 
        kls().CreateDatabaseTables( self.dbno , name )

        wrt = kls.Wrt() 
        wrt.ctx( 
             contextrange=self.cr, 
             dbno=self.dbno ,
             versiondate=TimeStamp(0,0),       
             subsite=self.subsite,
        )
        return wrt


    def logging_(self, args):
        """
        Hmm need some work ...
        """
        loglevel = getattr(logging,args.loglevel.upper())

        sh = logging.StreamHandler()
        sh.setLevel(loglevel)
        fmtr = logging.Formatter(args.logformat)
        sh.setFormatter(fmtr)
        log.addHandler(sh)

        if args.logpath:
            fh = logging.FileHandler(args.logpath,mode="w")
            fh.setFormatter(fmtr)
            fh.setLevel(loglevel)
            log.addHandler(fh)




if __name__ == '__main__':
    pass
    cnf = DbiCnf()      
    cnf()
    log.info( cnf )