/search.css" rel="stylesheet" type="text/css"/> /search.js">
| Classes | Job Modules | Data Objects | Services | Algorithms | Tools | Packages | Directories | Tracs |

In This Package:

SiteConfig.py
Go to the documentation of this file.
00001 # SiteConfig
00002 #
00003 # Contains the site configuration for the current cluster/computer
00004 #
00005 # Created by: dandwyer@caltech.edu 2010/08/09
00006 
00007 import ProcessManager.Status as Status
00008 from ProcessManager.FileDescription import FileDescription
00009 
00010 class CatalogProxy:
00011     """Class used to provide DybPython.Catalog-like functionality"""
00012     def __init__(self):
00013         """Constructor"""
00014         self.runs = {}
00015         return
00016     
00017 class SiteConfig:
00018     """ Configuration information for a given computer site """
00019     def __init__(self):
00020         """Constructor"""
00021         import os
00022         self.nuwaDir=None
00023         self.cmtConfig=None
00024         self.outputDir=None
00025         self.jobLogDir=None
00026         self.statsOffsetDir=None
00027         self.figuresOffsetDir=None
00028         self.xmlOffsetDir=None
00029         self.statsRootDir=None
00030         self.figuresRootDir=None
00031         self.xmlRootDir=None
00032         self.runsXmlDir=None
00033         self.batchExec=None
00034         self.batchWrapper="$PROCESSMANAGERROOT/scripts/batchNuWa.sh"
00035         self.batchLogDir=self.jobLogDir
00036         self.batchLogDir=None
00037         self.stateRootDir=None
00038         self.nuwaDataDir=None
00039         self.catalog=None
00040         self.catalogPaths=None
00041         return
00042 
00043     def jobId(self, jobName=None, runNumber=None, seqNumber=None):
00044         """ Return a string to identify the job """
00045         jobId = ""
00046         if jobName:
00047             jobId += jobName
00048         else:
00049             jobId += "noname"
00050         if runNumber:
00051             jobId += "_run%07d" % runNumber
00052         if seqNumber:
00053             jobId += "_seq%04d" % seqNumber
00054         return jobId
00055         
00056     def batchCommand(self, batchArgs, jobName, runNumber, seqNumber):
00057         """Construct a batch command"""
00058         print "SiteConfig: batchCommand is undefined for this site"
00059         return ""
00060 
00061     def runSepDir(self, runNumber):
00062         """Make an appropriate structure for run subdirectories"""
00063         # group runs into subdirectories of 1000/100
00064         runK = runNumber / 1000
00065         runH = runNumber / 100
00066         return "runs_%04d000/runs_%05d00" % (runK, runH)
00067 
00068     def runSubDir(self, runNumber):
00069         """Return run subpath + directory"""
00070         return self.runSepDir(runNumber)+"/run_%07d" % runNumber
00071 
00072     def statsRelativeDir(self, runNumber):
00073         """Construct the relative path of the root files from the index"""
00074         return self.runSubDir(runNumber)+"/root"
00075 
00076     def statsDirectory(self, runNumber):
00077         """Construct the absolute path of the root files from the index"""
00078         return self.statsRootDir+"/"+self.statsRelativeDir(runNumber)
00079 
00080     def statsSeqFile(self, jobName, runNumber, seqNumber):
00081         """Construct the stats ROOT file path and name"""
00082         filename = "%s.root" % self.jobId(jobName, runNumber, seqNumber)
00083         return self.statsDirectory(runNumber)+"/"+filename
00084 
00085     def statsInstallPath(self, jobName, runNumber):
00086         """Construct the relative summed stats ROOT file path and name for index lookup"""
00087         filename = "%s_total.root" % self.jobId(jobName, runNumber)
00088         return self.statsOffsetDir+'/'+self.statsRelativeDir(runNumber)+"/"+filename
00089 
00090     def statsSumFile(self, jobName, runNumber):
00091         """Construct the summed stats ROOT file path and name"""
00092         filename = "%s_total.root" % self.jobId(jobName, runNumber)
00093         return self.statsDirectory(runNumber)+"/"+filename
00094 
00095     def statsLockFile(self, jobName, runNumber):
00096         """Construct the stats lock file path and name"""
00097         filename = "%s_total.lock" % self.jobId(jobName, runNumber)
00098         return self.statsDirectory(runNumber)+"/"+filename
00099 
00100     def runLockFile(self, runNumber):
00101         """Construct the run summary lock file path and name"""
00102         filename = "run_%07d.lock" % runNumber
00103         return self.xmlRootDir+"/"+self.runSubDir(runNumber)+"/"+filename
00104 
00105     def figuresDirectory(self, runNumber):
00106         """Construct the figures directory for the given run"""
00107         return self.figuresRootDir+"/"+self.runSubDir(runNumber)
00108 
00109     def figuresPathReplace(self, runNumber):
00110         """Construct the figures directory for the given run"""
00111         runPath = "run_%07d" % runNumber
00112         return runPath+","+self.runSubDir(runNumber)
00113 
00114     def xmlRelativeDir(self, runNumber):
00115         """Construct the xml directory for the given run"""
00116         return self.xmlOffsetDir+"/"+self.runSubDir(runNumber)
00117 
00118     def xmlDirectory(self, runNumber):
00119         """Construct the xml directory for the given run"""
00120         return self.outputDir+'/'+self.xmlRelativeDir(runNumber)
00121 
00122     def xmlIndexFile(self, runNumber):
00123         """Construct the run index file path and name"""
00124         filename = "run_%07d.xml" % runNumber
00125         return self.xmlDirectory(runNumber)+"/"+filename
00126 
00127     def runIndexDir(self):
00128         """Construct the index of all runs file path"""
00129         return self.runsXmlDir
00130 
00131     def runIndexFile(self):
00132         """Construct the index of all runs file path and name"""
00133         return self.runIndexDir() + "/runs.xml"
00134 
00135     def runIndexLockFile(self):
00136         """Construct the index of all runs lock file path and name"""
00137         return self.runsXmlDir + "/runs.lock"
00138 
00139     def logDirectory(self, runNumber):
00140         """Construct the path for the nuwa log output directory"""
00141         return self.jobLogDir+"/"+self.runSepDir(runNumber)
00142 
00143     def logOutFile(self, jobName, runNumber, seqNumber=None):
00144         """Construct the path for the nuwa log output file"""
00145         import os
00146         filename = "%s_%d.out" % (self.jobId(jobName, runNumber, seqNumber),
00147                                   os.getpid() )
00148         return self.logDirectory(runNumber)+"/"+filename
00149 
00150     def logErrFile(self, jobName, runNumber, seqNumber=None):
00151         """Construct the path for the log error file"""
00152         import os
00153         filename = "%s_%d.err" % (self.jobId(jobName, runNumber, seqNumber),
00154                                   os.getpid() )
00155         return self.logDirectory(runNumber)+"/"+filename
00156 
00157     def batchLogDirectory(self, runNumber):
00158         """Construct the directory for batch log files"""
00159         return self.batchLogDir+"/"+self.runSepDir(runNumber)
00160 
00161     def batchLogOutFile(self, jobName, runNumber, seqNumber=None):
00162         """Construct the path for the batch log output file"""
00163         import os
00164         jobId = self.jobId(jobName, runNumber, seqNumber)
00165         filename = "batch_%s_%d.out" % (self.jobId(jobName, runNumber,
00166                                                    seqNumber),
00167                                         os.getpid() )
00168         return self.batchLogDirectory(runNumber)+"/"+filename
00169 
00170     def batchLogErrFile(self, jobName, runNumber, seqNumber=None):
00171         """Construct the path for the batch log error file"""
00172         import os
00173         filename = "batch_%s_%d.err" % (self.jobId(jobName, runNumber,
00174                                                    seqNumber),
00175                                         os.getpid() )
00176         return self.batchLogDirectory(runNumber)+"/"+filename
00177 
00178     def batchJobFile(self, jobName, runNumber, seqNumber=None):
00179         """Construct the path for the batch log error file"""
00180         import os
00181         filename = "batch_%s_%d.sh" % (self.jobId(jobName, runNumber,
00182                                                    seqNumber),
00183                                         os.getpid() )
00184         return self.batchLogDirectory(runNumber)+"/"+filename
00185 
00186     def stateFile(self, jobType, jobName, runNumber, seqNumber):
00187         """Construct the path and filename for logging state of this job"""
00188         filename = "%s_%s.state" % (self.jobId(jobName, runNumber,seqNumber),
00189                                     jobType)
00190         return self.stateRootDir+"/"+self.runSubDir(runNumber)+"/"+filename
00191 
00192     def nuwaDataDirectory(self, fileName, runNumber, seqNumber):
00193         """Construct the path for NuWa raw data files (.root)"""
00194         fileOnly = fileName.split('/')[-1]
00195         category = fileOnly.split('.')[1]
00196         return self.nuwaDataDir+"/"+category+"/daq/"+self.runSubDir(runNumber)
00197 
00198     def nuwaDataFile(self, fileName, runNumber, seqNumber):
00199         """Construct the path and filename for NuWa raw data files (.root)"""
00200         fileOnly = fileName.split('/')[-1]
00201         if fileOnly.endswith(".data"):
00202             fileOnly = fileOnly[:-5]+".root"
00203         if not fileOnly.startswith("daq."):            
00204             fileOnly = "daq."+fileOnly.partition('.')[2]
00205         return self.nuwaDataDirectory(fileName, runNumber, seqNumber)+"/"+fileOnly
00206 
00207     def reconDirectory(self, fileName, runNumber, seqNumber):
00208         """Construct the path for NuWa raw data files (.root)"""
00209         fileOnly = fileName.split('/')[-1]
00210         category = fileOnly.split('.')[1]
00211         return self.nuwaDataDir+"/"+category+"/recon/"+self.runSubDir(runNumber)
00212 
00213     def reconFile(self, jobName, fileName, runNumber, seqNumber):
00214         """Construct the path and filename for NuWa raw data files (.root)"""
00215         fileOnly = fileName.split('/')[-1]
00216         fileTokens = fileOnly.split('.')
00217         fileTokens[0] = 'recon'
00218         fileTokens[1] = jobName
00219         fileTokens[-1] = 'root'
00220         reconFileName = '.'.join(fileTokens)
00221         return self.reconDirectory(fileName, runNumber, seqNumber)+"/"+reconFileName
00222 
00223     def filesInRun(self, runNumber):
00224         """Return the files (with full local path) given this run number"""
00225         if not self.catalog:
00226             self.loadCatalog()
00227         fileList = []
00228         try:
00229             fileList = self.catalog.runs[runNumber]
00230         except KeyError:
00231            print "SiteConfig: No files in run ", runNumber
00232         return fileList
00233            
00234     def loadCatalog(self):
00235         """Load file Catalog entries"""
00236         if self.catalogPaths:
00237             # Mimic catalog by loading files from directories
00238             self.catalog = CatalogProxy()
00239             import os
00240             for path in self.catalogPaths:
00241                 if not os.path.isdir(path):
00242                     print "SiteConfig: Skipping invalid catalog path: ", path
00243                     continue
00244                 print "SiteConfig: Loading files from path: ", path
00245                 entries = os.listdir(path)
00246                 for entry in entries:
00247                     if len(entry)<6:
00248                         continue
00249                     if entry[-5:]!='.root' and entry[-5:]!='.data':
00250                         continue
00251                     fileDesc = FileDescription(entry)
00252                     if not fileDesc.isValid:
00253                         continue
00254                     if not self.catalog.runs.has_key(fileDesc.runNumber):
00255                         # Add new run
00256                         self.catalog.runs[fileDesc.runNumber] = []
00257                     # Add file to run catalog
00258                     self.catalog.runs[fileDesc.runNumber].append( path+'/'+entry )
00259                     self.catalog.runs[fileDesc.runNumber].sort()
00260         else:    
00261             # Use standard catalog
00262             import DybPython.Catalog as Catalog
00263             self.catalog = Catalog
00264         return    
00265                 
00266 class SiteConfigLocal(SiteConfig):
00267     """ Configuration information for a local install """
00268     def __init__(self):
00269         """Constructor"""
00270         SiteConfig.__init__(self)
00271         self.nuwaDir="$SITEROOT"
00272         self.cmtConfig="$CMTCONFIG"
00273         self.outputDir="."
00274         self.jobLogDir=self.outputDir+"/logs"
00275         self.statsOffsetDir="diagnostics"
00276         self.figuresOffsetDir="diagnostics"
00277         self.xmlOffsetDir="diagnostics"
00278         self.statsRootDir=self.outputDir+'/'+self.statsOffsetDir
00279         self.figuresRootDir=self.outputDir+'/'+self.figuresOffsetDir
00280         self.xmlRootDir=self.outputDir+'/'+self.xmlOffsetDir
00281         self.runsXmlDir=self.outputDir
00282         self.batchExec=None
00283         self.batchLogDir=self.jobLogDir
00284         self.stateRootDir=self.outputDir+"/state"
00285         self.nuwaDataDir=self.outputDir
00286         
00287     def batchCommand(self, batchArgs, jobName, runNumber, seqNumber):
00288         """Construct a batch command"""
00289         import os
00290         pid = os.getpid()
00291         batchCommand = "%(wrapper)s %(nuwaDir)s %(cmtConfig)s " % {
00292             'wrapper':self.batchWrapper,
00293             'nuwaDir':self.nuwaDir,
00294             'cmtConfig':self.cmtConfig
00295             }
00296         batchCommand += "'runProcess.py %(args)s " % {
00297             'args': ' '.join(batchArgs) }
00298         batchCommand += "> %(outlog)s 2> %(errlog)s &'" % {
00299             'outlog':self.batchLogOutFile(jobName, runNumber, seqNumber),
00300             'errlog':self.batchLogErrFile(jobName, runNumber, seqNumber) }
00301         return batchCommand
00302 
00303 class SiteConfigPdsf(SiteConfig):
00304     """ Configuration information for a given site """
00305     def __init__(self):
00306         """Constructor"""
00307         SiteConfig.__init__(self)
00308         #self.nuwaDir="/common/dayabay/releases/NuWa/trunk-opt/NuWa-trunk"
00309         self.nuwaDir="/eliza7/dayabay/scratch/dandwyer/NuWa-trunk-opt"
00310         self.cmtConfig="$CMTCONFIG"
00311         self.outputDir="/project/projectdirs/dayabay/www/dybprod"
00312         self.jobLogDir=self.outputDir+"/logs"
00313         self.statsOffsetDir="diagnostics"
00314         self.figuresOffsetDir="diagnostics"
00315         self.xmlOffsetDir="diagnostics"
00316         self.statsRootDir=self.outputDir+'/'+self.statsOffsetDir
00317         self.figuresRootDir=self.outputDir+'/'+self.figuresOffsetDir
00318         self.xmlRootDir=self.outputDir+'/'+self.xmlOffsetDir
00319         self.runsXmlDir=self.outputDir
00320         self.batchExec="qsub"
00321         self.batchLogDir=self.jobLogDir
00322         self.stateRootDir=self.outputDir+"/state"
00323         self.nuwaDataDir="/eliza16/dayabay/nuwaData/exp"
00324         
00325     def batchCommand(self, batchArgs, jobName, runNumber, seqNumber):
00326         """Construct a batch command"""
00327         import os
00328         pid = os.getpid()
00329         jobId = "%s_%d" % (jobName[:2],runNumber)
00330         if seqNumber:
00331             jobId += "_%d" % seqNumber
00332         batchCommand = "%(exec)s -o %(outlog)s -e %(errlog)s -N %(jobId)s " % {
00333             'exec':self.batchExec,
00334             'outlog':self.batchLogOutFile(jobName, runNumber, seqNumber),
00335             'errlog':self.batchLogErrFile(jobName, runNumber, seqNumber),
00336             'jobId':jobId}
00337         batchCommand += "%(wrapper)s %(nuwaDir)s %(cmtConfig)s " % {
00338             'wrapper':self.batchWrapper,
00339             'nuwaDir':self.nuwaDir,
00340             'cmtConfig':self.cmtConfig
00341             }
00342         batchCommand += "'runProcess.py %(args)s'" % {
00343             'args': ' '.join(batchArgs) }  
00344         return batchCommand
00345 
00346 class SiteConfigPdsfTest(SiteConfig):
00347     """ Configuration information for a given site """
00348     def __init__(self):
00349         """Constructor"""
00350         SiteConfig.__init__(self)
00351         self.nuwaDir="$SITEROOT"
00352         self.cmtConfig="$CMTCONFIG"
00353         self.outputDir="/eliza16/dayabay/users/dandwyer/dybprodTest"
00354         self.jobLogDir=self.outputDir+"/logs"
00355         self.statsOffsetDir="diagnostics"
00356         self.figuresOffsetDir="diagnostics"
00357         self.xmlOffsetDir="diagnostics"
00358         self.statsRootDir=self.outputDir+'/'+self.statsOffsetDir
00359         self.figuresRootDir=self.outputDir+'/'+self.figuresOffsetDir
00360         self.xmlRootDir=self.outputDir+'/'+self.xmlOffsetDir
00361         self.runsXmlDir=self.outputDir
00362         self.batchExec="qsub"
00363         self.batchLogDir=self.jobLogDir
00364         self.stateRootDir=self.outputDir+"/state"
00365         self.nuwaDataDir=self.outputDir
00366         
00367     def batchCommand(self, batchArgs, jobName, runNumber, seqNumber):
00368         """Construct a batch command"""
00369         import os
00370         pid = os.getpid()
00371         jobId = "%s_%d" % (jobName[:2],runNumber)
00372         if seqNumber:
00373             jobId += "_%d" % seqNumber
00374         batchCommand = "%(exec)s -o %(outlog)s -e %(errlog)s -N %(jobId)s " % {
00375             'exec':self.batchExec,
00376             'outlog':self.batchLogOutFile(jobName, runNumber, seqNumber),
00377             'errlog':self.batchLogErrFile(jobName, runNumber, seqNumber),
00378             'jobId':jobId}
00379         batchCommand += "%(wrapper)s %(nuwaDir)s %(cmtConfig)s " % {
00380             'wrapper':self.batchWrapper,
00381             'nuwaDir':self.nuwaDir,
00382             'cmtConfig':self.cmtConfig
00383             }
00384         batchCommand += "'runProcess.py %(args)s'" % {
00385             'args': ' '.join(batchArgs) }  
00386         return batchCommand
00387 
00388 class SiteConfigPdsfSim(SiteConfig):
00389     """ Configuration information for a given site """
00390     def __init__(self):
00391         """Constructor"""
00392         SiteConfig.__init__(self)
00393         #self.nuwaDir="/common/dayabay/releases/NuWa/trunk-opt/NuWa-trunk"
00394         self.nuwaDir="$SITEROOT"
00395         self.cmtConfig="$CMTCONFIG"
00396         self.outputDir="/project/projectdirs/dayabay/www/dybprodSim"
00397         self.jobLogDir=self.outputDir+"/logs"
00398         self.statsOffsetDir="diagnostics"
00399         self.figuresOffsetDir="diagnostics"
00400         self.xmlOffsetDir="diagnostics"
00401         self.statsRootDir=self.outputDir+'/'+self.statsOffsetDir
00402         self.figuresRootDir=self.outputDir+'/'+self.figuresOffsetDir
00403         self.xmlRootDir=self.outputDir+'/'+self.xmlOffsetDir
00404         self.runsXmlDir=self.outputDir
00405         self.batchExec="qsub"
00406         self.batchLogDir=self.jobLogDir
00407         self.stateRootDir=self.outputDir+"/state"
00408         self.nuwaDataDir="/eliza16/dayabay/nuwaData/sim"
00409         
00410     def batchCommand(self, batchArgs, jobName, runNumber, seqNumber):
00411         """Construct a batch command"""
00412         import os
00413         pid = os.getpid()
00414         jobId = "%s_%d" % (jobName[:2],runNumber)
00415         if seqNumber:
00416             jobId += "_%d" % seqNumber
00417         batchCommand = "%(exec)s -o %(outlog)s -e %(errlog)s -N %(jobId)s " % {
00418             'exec':self.batchExec,
00419             'outlog':self.batchLogOutFile(jobName, runNumber, seqNumber),
00420             'errlog':self.batchLogErrFile(jobName, runNumber, seqNumber),
00421             'jobId':jobId}
00422         batchCommand += "%(wrapper)s %(nuwaDir)s %(cmtConfig)s " % {
00423             'wrapper':self.batchWrapper,
00424             'nuwaDir':self.nuwaDir,
00425             'cmtConfig':self.cmtConfig
00426             }
00427         batchCommand += "'runProcess.py %(args)s'" % {
00428             'args': ' '.join(batchArgs) }  
00429         return batchCommand
00430 
00431 class SiteConfigIhep(SiteConfig):
00432     """ Configuration information for a given site """
00433     def __init__(self):
00434         """Constructor"""
00435         SiteConfig.__init__(self)
00436         self.nuwaDir="/afs/ihep.ac.cn/soft/dayabay/NuWa-64/opt/NuWa-trunk/"
00437         self.cmtConfig="x86_64-slc5-gcc41-opt"
00438         self.outputDir="/dybfs/rec/odm"
00439         #self.outputDir="/publicfs/dyb/data/rec/odm"
00440         self.jobLogDir=self.outputDir+"/logs"
00441         self.statsOffsetDir="diagnostics"
00442         self.figuresOffsetDir="diagnostics"
00443         self.xmlOffsetDir="diagnostics"
00444         self.statsRootDir=self.outputDir+'/'+self.statsOffsetDir
00445         self.figuresRootDir=self.outputDir+'/'+self.figuresOffsetDir
00446         self.xmlRootDir=self.outputDir+'/'+self.xmlOffsetDir
00447         self.runsXmlDir=self.outputDir
00448         self.batchExec="qsub"
00449         self.batchLogDir=self.jobLogDir
00450         self.stateRootDir=self.outputDir+"/state"
00451         self.nuwaDataDir="/publicfs/dyb/data/mc/EH1/odm/rec"
00452         
00453     def batchCommand(self, batchArgs, jobName, runNumber, seqNumber):
00454         """Construct a batch command"""
00455         import os
00456         pid = os.getpid()
00457         jobId = "%s_%d" % (jobName[:2],runNumber)
00458         if seqNumber:
00459             jobId += "_%d" % seqNumber
00460         batchCommand = "%(exec)s -q dyb64q -o %(outlog)s -e %(errlog)s -N %(jobId)s " % {
00461             'exec':self.batchExec,
00462             'outlog':self.batchLogOutFile(jobName, runNumber, seqNumber),
00463             'errlog':self.batchLogErrFile(jobName, runNumber, seqNumber),
00464             'jobId':jobId}
00465         nuwaCommand = "runProcess.py %(args)s\n" % {
00466             'args': ' '.join(batchArgs) }  
00467         jobDir = self.batchLogDirectory(runNumber)
00468         if not os.path.exists(jobDir):
00469             os.mkdir(jobdir)
00470         jobFile = self.batchJobFile(jobName, runNumber, seqNumber)
00471         job = open(jobFile,"w")
00472         job.write("#!/bin/bash\n")
00473         job.write("source /afs/ihep.ac.cn/soft/dayabay/NuWa-64/env/NuWa-trunk/nuwa-opt.bash\n")
00474         job.write(nuwaCommand+"\n")
00475         job.close()
00476         batchCommand += jobFile
00477         return batchCommand
00478 
00479 siteConfigs = {"local":SiteConfigLocal(),
00480                "pdsf":SiteConfigPdsf(),
00481                "pdsfSim":SiteConfigPdsfSim(),
00482                "pdsfTest":SiteConfigPdsfTest(),
00483                "ihep":SiteConfigIhep()
00484                }
| Classes | Job Modules | Data Objects | Services | Algorithms | Tools | Packages | Directories | Tracs |

Generated on Fri May 16 2014 09:54:04 for ProcessManager by doxygen 1.7.4