/search.css" rel="stylesheet" type="text/css"/> /search.js">
00001 #!/usr/bin/env python 00002 import logging 00003 from sqlalchemy.orm import class_mapper 00004 from sqlalchemy.sql import and_, or_, not_, select 00005 from baseinfo import BaseInfo 00006 00007 log = logging.getLogger(__name__) 00008 00009 class ConfigInfo(BaseInfo): 00010 """ 00011 Gathers from DAQ tables and propagates new configs to the non-DBI offline tables ``DaqRunConfig``:: 00012 00013 mysql> describe DaqRunConfig ; 00014 +-----------------+--------------+------+-----+---------+-------+ 00015 | Field | Type | Null | Key | Default | Extra | 00016 +-----------------+--------------+------+-----+---------+-------+ 00017 | schemaVersion | mediumint(9) | NO | PRI | 0 | | 00018 | dataVersion | mediumint(9) | NO | PRI | 0 | | 00019 | creationVersion | mediumint(9) | YES | | NULL | | 00020 | className | varchar(64) | NO | PRI | | | 00021 | objectId | varchar(64) | NO | PRI | | | 00022 | name | varchar(128) | NO | PRI | | | 00023 | parentPosition | mediumint(9) | NO | PRI | 0 | | 00024 | intValue | bigint(20) | YES | | NULL | | 00025 | floatValue | float | YES | | NULL | | 00026 | stringValue | varchar(64) | YES | | NULL | | 00027 +-----------------+--------------+------+-----+---------+-------+ 00028 10 rows in set (0.01 sec) 00029 00030 Source DAQ tables: 00031 00032 ============== ====================================== 00033 table notes 00034 ============== ====================================== 00035 OKSOBJECT 00036 OKSDATA 00037 OKSDATAVAL 00038 OKSDATAREL 00039 ============== ====================================== 00040 00041 """ 00042 _configClassList = ['Partition', 'Segment', 'ROSApplication', 'ROSConfiguration', 'Detector', 'Crate', 'LTB%', 'FEE%'] 00043 00044 def __init__(self, source, target, cfg={} ): 00045 BaseInfo.__init__(self, source, target , cfg ) 00046 00047 self._schemaVersion = None 00048 self._dataVersion = None 00049 self._baseVersion = None 00050 00051 self.msgs = [] 00052 00053 def _set_versions(self, sdv ): 00054 assert len(sdv) == 3, sdv 00055 self._schemaVersion, self._dataVersion, self._baseVersion = sdv 00056 def _get_versions(self): 00057 return self._schemaVersion, self._dataVersion, self._baseVersion 00058 versions = property( _get_versions, _set_versions ) 00059 00060 def _is_offline(self): 00061 """ 00062 Query non-DBI offline table ``DaqRunConfig`` to see if the version set (schemaVersion,dataVersion) 00063 has already been collected offline 00064 """ 00065 off = self.target() 00066 kdrc = self.target.kls_("DaqRunConfig") 00067 n = off.query(kdrc).filter(and_(kdrc.schemaVersion == self._schemaVersion,kdrc.dataVersion == self._dataVersion)).count() 00068 return n>0 00069 is_offline = property( _is_offline , doc=_is_offline.__doc__ ) 00070 00071 def is_object_offline(self, creationVersion, className, objectId ): 00072 """ 00073 Query non-DBI offline table ``DaqRunConfig`` to see if object corresponding to arguments 00074 has already been collected 00075 """ 00076 off = self.target() 00077 kdrc = self.target.kls_("DaqRunConfig") 00078 obc = and_( 00079 kdrc.schemaVersion == self._schemaVersion, 00080 kdrc.creationVersion == creationVersion, 00081 kdrc.className == className, 00082 kdrc.objectId == objectId, 00083 ) 00084 n = off.query(kdrc).filter(obc).count() 00085 return n>0 00086 00087 def _okselect( self ): 00088 """ 00089 Based on ``check_newConfig`` 00090 00091 Selects config entries matching version arguments and corresponding 00092 to config classes of interest from DAQ tables OKSOBJECT, OKSDATA 00093 """ 00094 t1 = self.source.table_("OKSOBJECT") 00095 t2 = self.source.table_("OKSDATA") 00096 00097 likes = [t1.c.CLASSNAME.like(_) for _ in self._configClassList ] 00098 lkc = or_( *likes ) 00099 dvc = or_( 00100 t1.c.DATAVERSION == self._dataVersion , 00101 and_( 00102 t1.c.DATAVERSION == self._baseVersion , 00103 t1.c.CREATIONVERSION <= self._dataVersion 00104 ) 00105 ) 00106 top = and_( 00107 t1.c.SCHEMAVERSION == self._schemaVersion , 00108 t1.c.SCHEMAVERSION == t2.c.SCHEMAVERSION , 00109 t1.c.DATAVERSION == t2.c.VERSION , 00110 dvc , 00111 lkc , 00112 ) 00113 s = select([t1.c.CLASSNAME,t1.c.ID,t1.c.CREATIONVERSION], top ) 00114 result = self.source.engine.execute(s) 00115 return result 00116 00117 okselect = property(_okselect, doc=_okselect.__doc__ ) 00118 00119 00120 def qconfig( self, kls , creationVersion , className , objectId ): 00121 """ 00122 Can be used for both OKSDATAVAL and OKSDATAREL as they share these attributes 00123 """ 00124 plk = and_( 00125 kls.SCHEMAVERSION == self._schemaVersion, 00126 kls.CREATIONVERSION == creationVersion, 00127 kls.CLASSNAME == className, 00128 kls.OBJECTID == objectId, 00129 ) 00130 daq = self.source() 00131 return daq.query(kls).filter(plk) 00132 00133 def objects( self, creationVersion , className , objectId ): 00134 """ 00135 :param creationVersion: 00136 :param className: 00137 :param objectId: 00138 00139 Finds entries from ``OKSDATAVAL`` and ``OKSDATAREL`` corresponding to 00140 the arguments. 00141 00142 The ``ODR`` entries are splayed out into two ``ODV`` entries with 00143 some name changes:: 00144 00145 mysql> describe OKSDATAVAL ; 00146 +-----------------+--------------+------+-----+---------+-------+ 00147 | Field | Type | Null | Key | Default | Extra | 00148 +-----------------+--------------+------+-----+---------+-------+ 00149 | SCHEMAVERSION | mediumint(9) | NO | PRI | 0 | | 0 00150 | DATAVERSION | mediumint(9) | NO | PRI | 0 | | 1 00151 | CREATIONVERSION | mediumint(9) | NO | | 0 | | 2 00152 | CLASSNAME | varchar(64) | NO | PRI | | | 3 00153 | OBJECTID | varchar(64) | NO | PRI | | | 4 00154 | NAME | varchar(128) | NO | PRI | | | 5 00155 | PARENTPOSITION | mediumint(9) | NO | PRI | 0 | | 6 00156 | INTEGERVALUE | bigint(20) | YES | | NULL | | 7 00157 | NUMBERVALUE | double | YES | | NULL | | 8 00158 | STRINGVALUE | text | YES | | NULL | | 9 00159 +-----------------+--------------+------+-----+---------+-------+ 00160 10 rows in set (0.01 sec) 00161 00162 mysql> describe OKSDATAREL ; 00163 +-----------------+--------------+------+-----+---------+-------+ 00164 | Field | Type | Null | Key | Default | Extra | 00165 +-----------------+--------------+------+-----+---------+-------+ 00166 | SCHEMAVERSION | mediumint(9) | NO | PRI | 0 | | 0 00167 | DATAVERSION | mediumint(9) | NO | PRI | 0 | | 1 00168 | CREATIONVERSION | mediumint(9) | NO | | 0 | | 2 00169 | CLASSNAME | varchar(64) | NO | PRI | | | 3 00170 | OBJECTID | varchar(64) | NO | PRI | | | 4 00171 | NAME | varchar(128) | NO | PRI | | | 5 00172 | PARENTPOSITION | mediumint(9) | NO | PRI | 0 | | 6 00173 | VALUECLASSNAME | varchar(64) | NO | | | | 7 00174 | VALUEOBJECTID | varchar(64) | NO | | | | 8 00175 | VALUEVERSION | mediumint(9) | YES | | NULL | | 9 00176 +-----------------+--------------+------+-----+---------+-------+ 00177 10 rows in set (0.00 sec) 00178 00179 Target table is similar to ``OKSDATAVAL``:: 00180 00181 mysql> describe DaqRunConfig ; 00182 +-----------------+--------------+------+-----+---------+-------+ 00183 | Field | Type | Null | Key | Default | Extra | 00184 +-----------------+--------------+------+-----+---------+-------+ 00185 | schemaVersion | mediumint(9) | NO | PRI | 0 | | 00186 | dataVersion | mediumint(9) | NO | PRI | 0 | | 00187 | creationVersion | mediumint(9) | YES | | NULL | | 00188 | className | varchar(64) | NO | PRI | | | 00189 | objectId | varchar(64) | NO | PRI | | | 00190 | name | varchar(128) | NO | PRI | | | 00191 | parentPosition | mediumint(9) | NO | PRI | 0 | | 00192 | intValue | bigint(20) | YES | | NULL | | 00193 | floatValue | float | YES | | NULL | | 00194 | stringValue | varchar(64) | YES | | NULL | | 00195 +-----------------+--------------+------+-----+---------+-------+ 00196 00197 Duplicates the legacy splaying:: 00198 00199 #print row 00200 childClassName, childObjectId = (row[7], row[8]) 00201 # change to OKSDATAVAL format 00202 rowval = (row[0], row[1], row[2], row[3], row[4], 'childClassName_' + row[5], row[6], None, None, childClassName) 00203 records.append(rowval) 00204 rowval = (row[0], row[1], row[2], row[3], row[4], 'childObjectId_' + row[5], row[6], None, None, childObjectId) 00205 records.append(rowval) 00206 00207 00208 """ 00209 asdict = lambda i:dict([(p.key,getattr(i,p.key)) for p in class_mapper(i.__class__).iterate_properties]) ## mapped instance as clean dict, without SA instrumentation 00210 00211 kodv = self.source.kls_("OKSDATAVAL") 00212 qodv = self.qconfig( kodv, creationVersion, className, objectId ) 00213 odvs = map(asdict, qodv.all()) 00214 00215 kodr = self.source.kls_("OKSDATAREL") 00216 qodr = self.qconfig( kodr, creationVersion, className, objectId ) 00217 00218 for odr in qodr.all(): 00219 a,b = asdict(odr), asdict(odr) 00220 00221 a['NAME'] = 'childClassName_%s' % odr.NAME 00222 a['VALUECLASSNAME'] = None 00223 a['VALUEOBJECTID'] = None 00224 a['VALUEVERSION'] = odr.VALUECLASSNAME 00225 odvs.append(a) 00226 00227 b['NAME'] = 'childObjectId_%s' % odr.NAME 00228 b['VALUECLASSNAME'] = None 00229 b['VALUEOBJECTID'] = None 00230 b['VALUEVERSION'] = odr.VALUEOBJECTID 00231 odvs.append(b) 00232 00233 return odvs 00234 00235 @classmethod 00236 def tst_insert(cls, odvs): 00237 """ 00238 Create tmp_test.DaqRunConfig by creating DB and `~/.my.cnf` section then creating empty DB and table:: 00239 00240 mysql> create database tmp_test ; 00241 mysql> create table if not exists tmp_test.DaqRunConfig like tmp_offline_db.DaqRunConfig ; 00242 00243 Custom mapping creates ``DaqRunConfig`` ``kls`` with attributes that correspond 00244 to the default attributes(==columns) of the basis ``OKSDATAVAL``. 00245 Allows ``ODV`` dicts to be consumed by the ``kls`` ctor. 00246 00247 """ 00248 mapping = dict( 00249 schemaVersion="SCHEMAVERSION", 00250 dataVersion="DATAVERSION", 00251 creationVersion="CREATIONVERSION", 00252 className="CLASSNAME", 00253 objectId="OBJECTID", 00254 name="NAME", 00255 parentPosition="PARENTPOSITION", 00256 intValue="INTEGERVALUE", 00257 floatValue="NUMBERVALUE", 00258 stringValue="STRINGVALUE", 00259 ) 00260 rapping = dict((v,k) for (k,v) in mapping.items()) 00261 kls = tstm.kls_("DaqRunConfig", mapping=rapping ) ## once only 00262 00263 tst = tstm() 00264 for odv in odvs: 00265 drc = kls() ## generated kwa ctor is a feature of declarative base that is not provided by classical mapping 00266 for k,v in odv.items(): 00267 setattr( drc, k , v ) 00268 tst.add(drc) 00269 tst.commit() 00270 00271 00272 def __repr__(self): 00273 return "%s(schemaVersion=%s,dataVersion=%s,baseVersion=%s)" % ( self.__class__.__name__ , self._schemaVersion, self._dataVersion, self._baseVersion) 00274 00275 def __call__(self): 00276 """ 00277 Early returns if: 00278 00279 #. if this version combination is already offline 00280 00281 """ 00282 if self.is_offline: 00283 msg = " %s if offline already " % self 00284 log.warn(msg) 00285 self.msgs.append(msg) 00286 if self.cfg.get('test',False): 00287 log.warn("proceeding due to test option") 00288 else: 00289 return None 00290 00291 result = self.okselect 00292 log.info("okselect rowcount %s " % result.rowcount ) 00293 00294 if result.rowcount == 0: 00295 msg = "no config rows from okselect " 00296 log.warn(msg) 00297 self.msgs.append(msg) 00298 return None 00299 00300 ## NASTY DOUBLY NESTED QUERYING 00301 for className, objectId, creationVersion in result: 00302 ioo = self.is_object_offline( creationVersion , className , objectId ) 00303 odvs = self.objects( creationVersion=creationVersion, className=className, objectId=objectId ) 00304 for odv in odvs: 00305 print odv 00306 ConfigInfo.tst_insert( odvs ) 00307 00308 00309 00310 if __name__ == '__main__': 00311 00312 logging.basicConfig(level=logging.INFO) 00313 00314 from NonDbi import MetaDB 00315 daqm = MetaDB("tmp_daqdb") 00316 offm = MetaDB("tmp_offline_db") 00317 tstm = MetaDB("tmp_test") 00318 00319 cfg = dict(test=True) 00320 ci = ConfigInfo( source=daqm, target=offm , cfg=cfg ) 00321 00322 dataVersion = 1 00323 schemaVersion = 11 00324 baseVersion = 1 00325 00326 ci.versions = ( schemaVersion, dataVersion, baseVersion ) 00327 ci() 00328 00329 00330 #odvs = ConfigInfo.objects( schemaVersion=11 , creationVersion=1, className='FEEDACThreshold', classId='PedestalThreshold_0_7' ) 00331 #for odv in odvs: 00332 # print odv 00333 00334