/search.css" rel="stylesheet" type="text/css"/> /search.js">
| Classes | Job Modules | Data Objects | Services | Algorithms | Tools | Packages | Directories | Tracs |

In This Package:

configinfo.py
Go to the documentation of this file.
00001 #!/usr/bin/env python
00002 import logging
00003 from sqlalchemy.orm import class_mapper
00004 from sqlalchemy.sql import and_, or_, not_, select 
00005 from baseinfo import BaseInfo
00006 
00007 log = logging.getLogger(__name__)
00008 
00009 class ConfigInfo(BaseInfo):
00010     """
00011     Gathers from DAQ tables and propagates new configs to the non-DBI offline tables ``DaqRunConfig``::
00012 
00013         mysql> describe DaqRunConfig ;
00014         +-----------------+--------------+------+-----+---------+-------+
00015         | Field           | Type         | Null | Key | Default | Extra |
00016         +-----------------+--------------+------+-----+---------+-------+
00017         | schemaVersion   | mediumint(9) | NO   | PRI | 0       |       | 
00018         | dataVersion     | mediumint(9) | NO   | PRI | 0       |       | 
00019         | creationVersion | mediumint(9) | YES  |     | NULL    |       | 
00020         | className       | varchar(64)  | NO   | PRI |         |       | 
00021         | objectId        | varchar(64)  | NO   | PRI |         |       | 
00022         | name            | varchar(128) | NO   | PRI |         |       | 
00023         | parentPosition  | mediumint(9) | NO   | PRI | 0       |       | 
00024         | intValue        | bigint(20)   | YES  |     | NULL    |       | 
00025         | floatValue      | float        | YES  |     | NULL    |       | 
00026         | stringValue     | varchar(64)  | YES  |     | NULL    |       | 
00027         +-----------------+--------------+------+-----+---------+-------+
00028         10 rows in set (0.01 sec)
00029 
00030     Source DAQ tables:
00031 
00032         ==============     ======================================
00033          table               notes
00034         ==============     ======================================
00035          OKSOBJECT
00036          OKSDATA 
00037          OKSDATAVAL
00038          OKSDATAREL
00039         ==============     ======================================
00040 
00041     """ 
00042     _configClassList = ['Partition', 'Segment', 'ROSApplication', 'ROSConfiguration', 'Detector', 'Crate', 'LTB%', 'FEE%'] 
00043 
00044     def __init__(self,  source, target, cfg={} ):
00045         BaseInfo.__init__(self, source, target , cfg  )
00046 
00047         self._schemaVersion = None
00048         self._dataVersion = None
00049         self._baseVersion = None
00050 
00051         self.msgs = []
00052 
00053     def _set_versions(self, sdv ):
00054         assert len(sdv) == 3, sdv
00055         self._schemaVersion, self._dataVersion, self._baseVersion = sdv 
00056     def _get_versions(self):
00057         return  self._schemaVersion, self._dataVersion, self._baseVersion
00058     versions = property( _get_versions, _set_versions )
00059 
00060     def _is_offline(self):
00061         """
00062         Query non-DBI offline table ``DaqRunConfig`` to see if the version set (schemaVersion,dataVersion) 
00063         has already been collected offline
00064         """
00065         off = self.target()
00066         kdrc  = self.target.kls_("DaqRunConfig")
00067         n = off.query(kdrc).filter(and_(kdrc.schemaVersion == self._schemaVersion,kdrc.dataVersion == self._dataVersion)).count()
00068         return n>0 
00069     is_offline = property( _is_offline , doc=_is_offline.__doc__ )
00070 
00071     def is_object_offline(self, creationVersion, className, objectId  ):
00072         """
00073         Query non-DBI offline table ``DaqRunConfig`` to see if object corresponding to arguments  
00074         has already been collected
00075         """
00076         off = self.target()
00077         kdrc  = self.target.kls_("DaqRunConfig")
00078         obc = and_(
00079                   kdrc.schemaVersion == self._schemaVersion,
00080                   kdrc.creationVersion == creationVersion,
00081                   kdrc.className == className,
00082                   kdrc.objectId == objectId,
00083                  )
00084         n = off.query(kdrc).filter(obc).count()
00085         return n>0 
00086 
00087     def _okselect( self ):
00088         """
00089         Based on ``check_newConfig``
00090 
00091         Selects config entries matching version arguments and corresponding 
00092         to config classes of interest from DAQ tables OKSOBJECT, OKSDATA
00093         """
00094         t1 = self.source.table_("OKSOBJECT")
00095         t2 = self.source.table_("OKSDATA")
00096 
00097         likes = [t1.c.CLASSNAME.like(_) for _ in  self._configClassList ] 
00098         lkc = or_( *likes )
00099         dvc = or_( 
00100                  t1.c.DATAVERSION == self._dataVersion , 
00101                  and_(
00102                       t1.c.DATAVERSION == self._baseVersion , 
00103                       t1.c.CREATIONVERSION <= self._dataVersion 
00104                      )
00105                  )
00106         top =  and_( 
00107               t1.c.SCHEMAVERSION == self._schemaVersion ,  
00108               t1.c.SCHEMAVERSION == t2.c.SCHEMAVERSION , 
00109               t1.c.DATAVERSION == t2.c.VERSION , 
00110               dvc , 
00111               lkc ,
00112               )
00113         s = select([t1.c.CLASSNAME,t1.c.ID,t1.c.CREATIONVERSION], top )
00114         result = self.source.engine.execute(s)
00115         return result 
00116 
00117     okselect = property(_okselect, doc=_okselect.__doc__ )
00118 
00119 
00120     def qconfig( self, kls , creationVersion , className , objectId ):
00121         """ 
00122         Can be used for both OKSDATAVAL and OKSDATAREL as they share these attributes
00123         """
00124         plk = and_(
00125                     kls.SCHEMAVERSION == self._schemaVersion,
00126                     kls.CREATIONVERSION == creationVersion,
00127                     kls.CLASSNAME == className,
00128                     kls.OBJECTID == objectId,
00129                   )
00130         daq = self.source()
00131         return daq.query(kls).filter(plk)
00132 
00133     def objects( self, creationVersion , className , objectId ):
00134         """
00135         :param creationVersion:
00136         :param className:
00137         :param objectId:
00138 
00139         Finds entries from ``OKSDATAVAL`` and ``OKSDATAREL`` corresponding to 
00140         the arguments.
00141 
00142         The ``ODR`` entries are splayed out into two ``ODV`` entries with 
00143         some name changes:: 
00144  
00145                 mysql> describe OKSDATAVAL ;
00146                 +-----------------+--------------+------+-----+---------+-------+
00147                 | Field           | Type         | Null | Key | Default | Extra |
00148                 +-----------------+--------------+------+-----+---------+-------+
00149                 | SCHEMAVERSION   | mediumint(9) | NO   | PRI | 0       |       | 0
00150                 | DATAVERSION     | mediumint(9) | NO   | PRI | 0       |       | 1
00151                 | CREATIONVERSION | mediumint(9) | NO   |     | 0       |       | 2
00152                 | CLASSNAME       | varchar(64)  | NO   | PRI |         |       | 3
00153                 | OBJECTID        | varchar(64)  | NO   | PRI |         |       | 4
00154                 | NAME            | varchar(128) | NO   | PRI |         |       | 5
00155                 | PARENTPOSITION  | mediumint(9) | NO   | PRI | 0       |       | 6
00156                 | INTEGERVALUE    | bigint(20)   | YES  |     | NULL    |       | 7
00157                 | NUMBERVALUE     | double       | YES  |     | NULL    |       | 8
00158                 | STRINGVALUE     | text         | YES  |     | NULL    |       | 9
00159                 +-----------------+--------------+------+-----+---------+-------+
00160                 10 rows in set (0.01 sec)
00161 
00162                 mysql> describe OKSDATAREL ;
00163                 +-----------------+--------------+------+-----+---------+-------+
00164                 | Field           | Type         | Null | Key | Default | Extra |
00165                 +-----------------+--------------+------+-----+---------+-------+
00166                 | SCHEMAVERSION   | mediumint(9) | NO   | PRI | 0       |       | 0  
00167                 | DATAVERSION     | mediumint(9) | NO   | PRI | 0       |       | 1
00168                 | CREATIONVERSION | mediumint(9) | NO   |     | 0       |       | 2
00169                 | CLASSNAME       | varchar(64)  | NO   | PRI |         |       | 3
00170                 | OBJECTID        | varchar(64)  | NO   | PRI |         |       | 4
00171                 | NAME            | varchar(128) | NO   | PRI |         |       | 5
00172                 | PARENTPOSITION  | mediumint(9) | NO   | PRI | 0       |       | 6
00173                 | VALUECLASSNAME  | varchar(64)  | NO   |     |         |       | 7
00174                 | VALUEOBJECTID   | varchar(64)  | NO   |     |         |       | 8
00175                 | VALUEVERSION    | mediumint(9) | YES  |     | NULL    |       | 9
00176                 +-----------------+--------------+------+-----+---------+-------+
00177                 10 rows in set (0.00 sec)
00178 
00179         Target table is similar to ``OKSDATAVAL``::
00180 
00181                 mysql> describe DaqRunConfig ;
00182                 +-----------------+--------------+------+-----+---------+-------+
00183                 | Field           | Type         | Null | Key | Default | Extra |
00184                 +-----------------+--------------+------+-----+---------+-------+
00185                 | schemaVersion   | mediumint(9) | NO   | PRI | 0       |       | 
00186                 | dataVersion     | mediumint(9) | NO   | PRI | 0       |       | 
00187                 | creationVersion | mediumint(9) | YES  |     | NULL    |       | 
00188                 | className       | varchar(64)  | NO   | PRI |         |       | 
00189                 | objectId        | varchar(64)  | NO   | PRI |         |       | 
00190                 | name            | varchar(128) | NO   | PRI |         |       | 
00191                 | parentPosition  | mediumint(9) | NO   | PRI | 0       |       | 
00192                 | intValue        | bigint(20)   | YES  |     | NULL    |       | 
00193                 | floatValue      | float        | YES  |     | NULL    |       | 
00194                 | stringValue     | varchar(64)  | YES  |     | NULL    |       | 
00195                 +-----------------+--------------+------+-----+---------+-------+
00196         
00197         Duplicates the legacy splaying:: 
00198 
00199               #print row
00200               childClassName, childObjectId = (row[7], row[8])
00201               # change to OKSDATAVAL format
00202               rowval = (row[0], row[1], row[2], row[3], row[4], 'childClassName_' + row[5], row[6], None, None, childClassName)
00203               records.append(rowval)
00204               rowval = (row[0], row[1], row[2], row[3], row[4], 'childObjectId_' + row[5], row[6], None, None, childObjectId)
00205               records.append(rowval)
00206 
00207 
00208         """
00209         asdict = lambda i:dict([(p.key,getattr(i,p.key)) for p in class_mapper(i.__class__).iterate_properties])  ## mapped instance as clean dict, without SA instrumentation 
00210 
00211         kodv = self.source.kls_("OKSDATAVAL")
00212         qodv = self.qconfig( kodv, creationVersion, className, objectId )
00213         odvs = map(asdict, qodv.all())
00214 
00215         kodr = self.source.kls_("OKSDATAREL")
00216         qodr = self.qconfig( kodr, creationVersion, className, objectId )
00217 
00218         for odr in qodr.all():
00219             a,b = asdict(odr), asdict(odr)    
00220 
00221             a['NAME'] = 'childClassName_%s' % odr.NAME
00222             a['VALUECLASSNAME'] = None
00223             a['VALUEOBJECTID'] = None
00224             a['VALUEVERSION'] =  odr.VALUECLASSNAME
00225             odvs.append(a) 
00226 
00227             b['NAME'] = 'childObjectId_%s' % odr.NAME
00228             b['VALUECLASSNAME'] = None
00229             b['VALUEOBJECTID'] = None
00230             b['VALUEVERSION'] =  odr.VALUEOBJECTID
00231             odvs.append(b) 
00232             
00233         return odvs
00234 
00235     @classmethod
00236     def tst_insert(cls, odvs):
00237         """
00238         Create tmp_test.DaqRunConfig by creating DB and  `~/.my.cnf` section then creating empty DB and table:: 
00239 
00240             mysql> create database tmp_test ;
00241             mysql> create table if not exists tmp_test.DaqRunConfig like tmp_offline_db.DaqRunConfig ;
00242 
00243         Custom mapping creates ``DaqRunConfig`` ``kls`` with attributes that correspond 
00244         to the default attributes(==columns) of the basis ``OKSDATAVAL``.
00245         Allows ``ODV`` dicts to be consumed by the ``kls`` ctor.
00246           
00247         """
00248         mapping = dict(
00249                     schemaVersion="SCHEMAVERSION",
00250                     dataVersion="DATAVERSION",
00251                     creationVersion="CREATIONVERSION",
00252                     className="CLASSNAME",
00253                     objectId="OBJECTID",
00254                     name="NAME",
00255                     parentPosition="PARENTPOSITION",
00256                     intValue="INTEGERVALUE",
00257                     floatValue="NUMBERVALUE",
00258                     stringValue="STRINGVALUE",
00259                    )
00260         rapping = dict((v,k) for (k,v) in mapping.items())
00261         kls  = tstm.kls_("DaqRunConfig", mapping=rapping )    ## once only 
00262 
00263         tst = tstm()
00264         for odv in odvs:
00265             drc = kls()    ## generated kwa ctor is a feature of declarative base that is not provided by classical mapping
00266             for k,v in odv.items():
00267                 setattr( drc, k , v )
00268             tst.add(drc)
00269         tst.commit()
00270 
00271 
00272     def __repr__(self):
00273         return "%s(schemaVersion=%s,dataVersion=%s,baseVersion=%s)" % ( self.__class__.__name__ , self._schemaVersion, self._dataVersion, self._baseVersion)
00274 
00275     def __call__(self):
00276         """
00277         Early returns if:
00278 
00279         #. if this version combination is already offline
00280  
00281         """ 
00282         if self.is_offline:
00283             msg = " %s if offline already " % self 
00284             log.warn(msg)
00285             self.msgs.append(msg)    
00286             if self.cfg.get('test',False):
00287                 log.warn("proceeding due to test option")
00288             else: 
00289                 return None  
00290 
00291         result = self.okselect
00292         log.info("okselect rowcount %s " % result.rowcount ) 
00293 
00294         if result.rowcount == 0:
00295             msg = "no config rows from okselect "
00296             log.warn(msg)
00297             self.msgs.append(msg)    
00298             return None  
00299 
00300         ## NASTY DOUBLY NESTED QUERYING
00301         for className, objectId, creationVersion in result:
00302             ioo = self.is_object_offline( creationVersion , className , objectId )
00303             odvs = self.objects( creationVersion=creationVersion, className=className, objectId=objectId )
00304             for odv in odvs:
00305                 print odv
00306             ConfigInfo.tst_insert( odvs ) 
00307 
00308 
00309 
00310 if __name__ == '__main__':
00311 
00312     logging.basicConfig(level=logging.INFO)
00313 
00314     from NonDbi import MetaDB
00315     daqm = MetaDB("tmp_daqdb")
00316     offm = MetaDB("tmp_offline_db")    
00317     tstm = MetaDB("tmp_test")
00318 
00319     cfg = dict(test=True)
00320     ci = ConfigInfo( source=daqm, target=offm , cfg=cfg )
00321 
00322     dataVersion = 1  
00323     schemaVersion = 11 
00324     baseVersion   = 1 
00325 
00326     ci.versions = ( schemaVersion, dataVersion, baseVersion )
00327     ci()
00328 
00329 
00330     #odvs = ConfigInfo.objects( schemaVersion=11 , creationVersion=1, className='FEEDACThreshold', classId='PedestalThreshold_0_7' )
00331     #for odv in odvs:
00332     #    print odv  
00333 
00334 
| Classes | Job Modules | Data Objects | Services | Algorithms | Tools | Packages | Directories | Tracs |

Generated on Fri May 16 2014 09:50:03 for Scraper by doxygen 1.7.4