SiteConfig.py

Go to the documentation of this file.

# SiteConfig
#
# Contains the site configuration for the current cluster/computer
#
# Created by: dandwyer@caltech.edu 2010/08/09

import ProcessManager.Status as Status
from ProcessManager.FileDescription import FileDescription

class CatalogProxy:
    """Class used to provide DybPython.Catalog-like functionality"""
    def __init__(self):
        """Constructor"""
        self.runs = {}
        return
    
class SiteConfig:
    """ Configuration information for a given computer site """
    def __init__(self):
        """Constructor"""
        import os
        self.nuwaDir=None
        self.cmtConfig=None
        self.outputDir=None
        self.jobLogDir=None
        self.statsOffsetDir=None
        self.figuresOffsetDir=None
        self.xmlOffsetDir=None
        self.statsRootDir=None
        self.figuresRootDir=None
        self.xmlRootDir=None
        self.runsXmlDir=None
        self.batchExec=None
        self.batchWrapper="$PROCESSMANAGERROOT/scripts/batchNuWa.sh"
        self.batchLogDir=self.jobLogDir
        self.batchLogDir=None
        self.stateRootDir=None
        self.nuwaDataDir=None
        self.catalog=None
        self.catalogPaths=None
        return

    def jobId(self, jobName=None, runNumber=None, seqNumber=None):
        """ Return a string to identify the job """
        jobId = ""
        if jobName:
            jobId += jobName
        else:
            jobId += "noname"
        if runNumber:
            jobId += "_run%07d" % runNumber
        if seqNumber:
            jobId += "_seq%04d" % seqNumber
        return jobId
        
    def batchCommand(self, batchArgs, jobName, runNumber, seqNumber):
        """Construct a batch command"""
        print "SiteConfig: batchCommand is undefined for this site"
        return ""

    def runSepDir(self, runNumber):
        """Make an appropriate structure for run subdirectories"""
        # group runs into subdirectories of 1000/100
        runK = runNumber / 1000
        runH = runNumber / 100
        return "runs_%04d000/runs_%05d00" % (runK, runH)

    def runSubDir(self, runNumber):
        """Return run subpath + directory"""
        return self.runSepDir(runNumber)+"/run_%07d" % runNumber

    def statsRelativeDir(self, runNumber):
        """Construct the relative path of the root files from the index"""
        return self.runSubDir(runNumber)+"/root"

    def statsDirectory(self, runNumber):
        """Construct the absolute path of the root files from the index"""
        return self.statsRootDir+"/"+self.statsRelativeDir(runNumber)

    def statsSeqFile(self, jobName, runNumber, seqNumber):
        """Construct the stats ROOT file path and name"""
        filename = "%s.root" % self.jobId(jobName, runNumber, seqNumber)
        return self.statsDirectory(runNumber)+"/"+filename

    def statsInstallPath(self, jobName, runNumber):
        """Construct the relative summed stats ROOT file path and name for index lookup"""
        filename = "%s_total.root" % self.jobId(jobName, runNumber)
        return self.statsOffsetDir+'/'+self.statsRelativeDir(runNumber)+"/"+filename

    def statsSumFile(self, jobName, runNumber):
        """Construct the summed stats ROOT file path and name"""
        filename = "%s_total.root" % self.jobId(jobName, runNumber)
        return self.statsDirectory(runNumber)+"/"+filename

    def statsLockFile(self, jobName, runNumber):
        """Construct the stats lock file path and name"""
        filename = "%s_total.lock" % self.jobId(jobName, runNumber)
        return self.statsDirectory(runNumber)+"/"+filename

    def runLockFile(self, runNumber):
        """Construct the run summary lock file path and name"""
        filename = "run_%07d.lock" % runNumber
        return self.xmlRootDir+"/"+self.runSubDir(runNumber)+"/"+filename

    def figuresDirectory(self, runNumber):
        """Construct the figures directory for the given run"""
        return self.figuresRootDir+"/"+self.runSubDir(runNumber)

    def figuresPathReplace(self, runNumber):
        """Construct the figures directory for the given run"""
        runPath = "run_%07d" % runNumber
        return runPath+","+self.runSubDir(runNumber)

    def xmlRelativeDir(self, runNumber):
        """Construct the xml directory for the given run"""
        return self.xmlOffsetDir+"/"+self.runSubDir(runNumber)

    def xmlDirectory(self, runNumber):
        """Construct the xml directory for the given run"""
        return self.outputDir+'/'+self.xmlRelativeDir(runNumber)

    def xmlIndexFile(self, runNumber):
        """Construct the run index file path and name"""
        filename = "run_%07d.xml" % runNumber
        return self.xmlDirectory(runNumber)+"/"+filename

    def runIndexDir(self):
        """Construct the index of all runs file path"""
        return self.runsXmlDir

    def runIndexFile(self):
        """Construct the index of all runs file path and name"""
        return self.runIndexDir() + "/runs.xml"

    def runIndexLockFile(self):
        """Construct the index of all runs lock file path and name"""
        return self.runsXmlDir + "/runs.lock"

    def logDirectory(self, runNumber):
        """Construct the path for the nuwa log output directory"""
        return self.jobLogDir+"/"+self.runSepDir(runNumber)

    def logOutFile(self, jobName, runNumber, seqNumber=None):
        """Construct the path for the nuwa log output file"""
        import os
        filename = "%s_%d.out" % (self.jobId(jobName, runNumber, seqNumber),
                                  os.getpid() )
        return self.logDirectory(runNumber)+"/"+filename

    def logErrFile(self, jobName, runNumber, seqNumber=None):
        """Construct the path for the log error file"""
        import os
        filename = "%s_%d.err" % (self.jobId(jobName, runNumber, seqNumber),
                                  os.getpid() )
        return self.logDirectory(runNumber)+"/"+filename

    def batchLogDirectory(self, runNumber):
        """Construct the directory for batch log files"""
        return self.batchLogDir+"/"+self.runSepDir(runNumber)

    def batchLogOutFile(self, jobName, runNumber, seqNumber=None):
        """Construct the path for the batch log output file"""
        import os
        jobId = self.jobId(jobName, runNumber, seqNumber)
        filename = "batch_%s_%d.out" % (self.jobId(jobName, runNumber,
                                                   seqNumber),
                                        os.getpid() )
        return self.batchLogDirectory(runNumber)+"/"+filename

    def batchLogErrFile(self, jobName, runNumber, seqNumber=None):
        """Construct the path for the batch log error file"""
        import os
        filename = "batch_%s_%d.err" % (self.jobId(jobName, runNumber,
                                                   seqNumber),
                                        os.getpid() )
        return self.batchLogDirectory(runNumber)+"/"+filename

    def batchJobFile(self, jobName, runNumber, seqNumber=None):
        """Construct the path for the batch log error file"""
        import os
        filename = "batch_%s_%d.sh" % (self.jobId(jobName, runNumber,
                                                   seqNumber),
                                        os.getpid() )
        return self.batchLogDirectory(runNumber)+"/"+filename

    def stateFile(self, jobType, jobName, runNumber, seqNumber):
        """Construct the path and filename for logging state of this job"""
        filename = "%s_%s.state" % (self.jobId(jobName, runNumber,seqNumber),
                                    jobType)
        return self.stateRootDir+"/"+self.runSubDir(runNumber)+"/"+filename

    def nuwaDataDirectory(self, fileName, runNumber, seqNumber):
        """Construct the path for NuWa raw data files (.root)"""
        fileOnly = fileName.split('/')[-1]
        category = fileOnly.split('.')[1]
        return self.nuwaDataDir+"/"+category+"/daq/"+self.runSubDir(runNumber)

    def nuwaDataFile(self, fileName, runNumber, seqNumber):
        """Construct the path and filename for NuWa raw data files (.root)"""
        fileOnly = fileName.split('/')[-1]
        if fileOnly.endswith(".data"):
            fileOnly = fileOnly[:-5]+".root"
        if not fileOnly.startswith("daq."):            
            fileOnly = "daq."+fileOnly.partition('.')[2]
        return self.nuwaDataDirectory(fileName, runNumber, seqNumber)+"/"+fileOnly

    def reconDirectory(self, fileName, runNumber, seqNumber):
        """Construct the path for NuWa raw data files (.root)"""
        fileOnly = fileName.split('/')[-1]
        category = fileOnly.split('.')[1]
        return self.nuwaDataDir+"/"+category+"/recon/"+self.runSubDir(runNumber)

    def reconFile(self, jobName, fileName, runNumber, seqNumber):
        """Construct the path and filename for NuWa raw data files (.root)"""
        fileOnly = fileName.split('/')[-1]
        fileTokens = fileOnly.split('.')
        fileTokens[0] = 'recon'
        fileTokens[1] = jobName
        fileTokens[-1] = 'root'
        reconFileName = '.'.join(fileTokens)
        return self.reconDirectory(fileName, runNumber, seqNumber)+"/"+reconFileName

    def filesInRun(self, runNumber):
        """Return the files (with full local path) given this run number"""
        if not self.catalog:
            self.loadCatalog()
        fileList = []
        try:
            fileList = self.catalog.runs[runNumber]
        except KeyError:
           print "SiteConfig: No files in run ", runNumber
        return fileList
           
    def loadCatalog(self):
        """Load file Catalog entries"""
        if self.catalogPaths:
            # Mimic catalog by loading files from directories
            self.catalog = CatalogProxy()
            import os
            for path in self.catalogPaths:
                if not os.path.isdir(path):
                    print "SiteConfig: Skipping invalid catalog path: ", path
                    continue
                print "SiteConfig: Loading files from path: ", path
                entries = os.listdir(path)
                for entry in entries:
                    if len(entry)<6:
                        continue
                    if entry[-5:]!='.root' and entry[-5:]!='.data':
                        continue
                    fileDesc = FileDescription(entry)
                    if not fileDesc.isValid:
                        continue
                    if not self.catalog.runs.has_key(fileDesc.runNumber):
                        # Add new run
                        self.catalog.runs[fileDesc.runNumber] = []
                    # Add file to run catalog
                    self.catalog.runs[fileDesc.runNumber].append( path+'/'+entry )
                    self.catalog.runs[fileDesc.runNumber].sort()
        else:    
            # Use standard catalog
            import DybPython.Catalog as Catalog
            self.catalog = Catalog
        return    
                
class SiteConfigLocal(SiteConfig):
    """ Configuration information for a local install """
    def __init__(self):
        """Constructor"""
        SiteConfig.__init__(self)
        self.nuwaDir="$SITEROOT"
        self.cmtConfig="$CMTCONFIG"
        self.outputDir="."
        self.jobLogDir=self.outputDir+"/logs"
        self.statsOffsetDir="diagnostics"
        self.figuresOffsetDir="diagnostics"
        self.xmlOffsetDir="diagnostics"
        self.statsRootDir=self.outputDir+'/'+self.statsOffsetDir
        self.figuresRootDir=self.outputDir+'/'+self.figuresOffsetDir
        self.xmlRootDir=self.outputDir+'/'+self.xmlOffsetDir
        self.runsXmlDir=self.outputDir
        self.batchExec=None
        self.batchLogDir=self.jobLogDir
        self.stateRootDir=self.outputDir+"/state"
        self.nuwaDataDir=self.outputDir
        
    def batchCommand(self, batchArgs, jobName, runNumber, seqNumber):
        """Construct a batch command"""
        import os
        pid = os.getpid()
        batchCommand = "%(wrapper)s %(nuwaDir)s %(cmtConfig)s " % {
            'wrapper':self.batchWrapper,
            'nuwaDir':self.nuwaDir,
            'cmtConfig':self.cmtConfig
            }
        batchCommand += "'runProcess.py %(args)s " % {
            'args': ' '.join(batchArgs) }
        batchCommand += "> %(outlog)s 2> %(errlog)s &'" % {
            'outlog':self.batchLogOutFile(jobName, runNumber, seqNumber),
            'errlog':self.batchLogErrFile(jobName, runNumber, seqNumber) }
        return batchCommand

class SiteConfigPdsf(SiteConfig):
    """ Configuration information for a given site """
    def __init__(self):
        """Constructor"""
        SiteConfig.__init__(self)
        #self.nuwaDir="/common/dayabay/releases/NuWa/trunk-opt/NuWa-trunk"
        self.nuwaDir="/eliza7/dayabay/scratch/dandwyer/NuWa-trunk-opt"
        self.cmtConfig="$CMTCONFIG"
        self.outputDir="/project/projectdirs/dayabay/www/dybprod"
        self.jobLogDir=self.outputDir+"/logs"
        self.statsOffsetDir="diagnostics"
        self.figuresOffsetDir="diagnostics"
        self.xmlOffsetDir="diagnostics"
        self.statsRootDir=self.outputDir+'/'+self.statsOffsetDir
        self.figuresRootDir=self.outputDir+'/'+self.figuresOffsetDir
        self.xmlRootDir=self.outputDir+'/'+self.xmlOffsetDir
        self.runsXmlDir=self.outputDir
        self.batchExec="qsub"
        self.batchLogDir=self.jobLogDir
        self.stateRootDir=self.outputDir+"/state"
        self.nuwaDataDir="/eliza16/dayabay/nuwaData/exp"
        
    def batchCommand(self, batchArgs, jobName, runNumber, seqNumber):
        """Construct a batch command"""
        import os
        pid = os.getpid()
        jobId = "%s_%d" % (jobName[:2],runNumber)
        if seqNumber:
            jobId += "_%d" % seqNumber
        batchCommand = "%(exec)s -o %(outlog)s -e %(errlog)s -N %(jobId)s " % {
            'exec':self.batchExec,
            'outlog':self.batchLogOutFile(jobName, runNumber, seqNumber),
            'errlog':self.batchLogErrFile(jobName, runNumber, seqNumber),
            'jobId':jobId}
        batchCommand += "%(wrapper)s %(nuwaDir)s %(cmtConfig)s " % {
            'wrapper':self.batchWrapper,
            'nuwaDir':self.nuwaDir,
            'cmtConfig':self.cmtConfig
            }
        batchCommand += "'runProcess.py %(args)s'" % {
            'args': ' '.join(batchArgs) }  
        return batchCommand

class SiteConfigPdsfTest(SiteConfig):
    """ Configuration information for a given site """
    def __init__(self):
        """Constructor"""
        SiteConfig.__init__(self)
        self.nuwaDir="$SITEROOT"
        self.cmtConfig="$CMTCONFIG"
        self.outputDir="/eliza16/dayabay/users/dandwyer/dybprodTest"
        self.jobLogDir=self.outputDir+"/logs"
        self.statsOffsetDir="diagnostics"
        self.figuresOffsetDir="diagnostics"
        self.xmlOffsetDir="diagnostics"
        self.statsRootDir=self.outputDir+'/'+self.statsOffsetDir
        self.figuresRootDir=self.outputDir+'/'+self.figuresOffsetDir
        self.xmlRootDir=self.outputDir+'/'+self.xmlOffsetDir
        self.runsXmlDir=self.outputDir
        self.batchExec="qsub"
        self.batchLogDir=self.jobLogDir
        self.stateRootDir=self.outputDir+"/state"
        self.nuwaDataDir=self.outputDir
        
    def batchCommand(self, batchArgs, jobName, runNumber, seqNumber):
        """Construct a batch command"""
        import os
        pid = os.getpid()
        jobId = "%s_%d" % (jobName[:2],runNumber)
        if seqNumber:
            jobId += "_%d" % seqNumber
        batchCommand = "%(exec)s -o %(outlog)s -e %(errlog)s -N %(jobId)s " % {
            'exec':self.batchExec,
            'outlog':self.batchLogOutFile(jobName, runNumber, seqNumber),
            'errlog':self.batchLogErrFile(jobName, runNumber, seqNumber),
            'jobId':jobId}
        batchCommand += "%(wrapper)s %(nuwaDir)s %(cmtConfig)s " % {
            'wrapper':self.batchWrapper,
            'nuwaDir':self.nuwaDir,
            'cmtConfig':self.cmtConfig
            }
        batchCommand += "'runProcess.py %(args)s'" % {
            'args': ' '.join(batchArgs) }  
        return batchCommand

class SiteConfigPdsfSim(SiteConfig):
    """ Configuration information for a given site """
    def __init__(self):
        """Constructor"""
        SiteConfig.__init__(self)
        #self.nuwaDir="/common/dayabay/releases/NuWa/trunk-opt/NuWa-trunk"
        self.nuwaDir="$SITEROOT"
        self.cmtConfig="$CMTCONFIG"
        self.outputDir="/project/projectdirs/dayabay/www/dybprodSim"
        self.jobLogDir=self.outputDir+"/logs"
        self.statsOffsetDir="diagnostics"
        self.figuresOffsetDir="diagnostics"
        self.xmlOffsetDir="diagnostics"
        self.statsRootDir=self.outputDir+'/'+self.statsOffsetDir
        self.figuresRootDir=self.outputDir+'/'+self.figuresOffsetDir
        self.xmlRootDir=self.outputDir+'/'+self.xmlOffsetDir
        self.runsXmlDir=self.outputDir
        self.batchExec="qsub"
        self.batchLogDir=self.jobLogDir
        self.stateRootDir=self.outputDir+"/state"
        self.nuwaDataDir="/eliza16/dayabay/nuwaData/sim"
        
    def batchCommand(self, batchArgs, jobName, runNumber, seqNumber):
        """Construct a batch command"""
        import os
        pid = os.getpid()
        jobId = "%s_%d" % (jobName[:2],runNumber)
        if seqNumber:
            jobId += "_%d" % seqNumber
        batchCommand = "%(exec)s -o %(outlog)s -e %(errlog)s -N %(jobId)s " % {
            'exec':self.batchExec,
            'outlog':self.batchLogOutFile(jobName, runNumber, seqNumber),
            'errlog':self.batchLogErrFile(jobName, runNumber, seqNumber),
            'jobId':jobId}
        batchCommand += "%(wrapper)s %(nuwaDir)s %(cmtConfig)s " % {
            'wrapper':self.batchWrapper,
            'nuwaDir':self.nuwaDir,
            'cmtConfig':self.cmtConfig
            }
        batchCommand += "'runProcess.py %(args)s'" % {
            'args': ' '.join(batchArgs) }  
        return batchCommand

class SiteConfigIhep(SiteConfig):
    """ Configuration information for a given site """
    def __init__(self):
        """Constructor"""
        SiteConfig.__init__(self)
        self.nuwaDir="/afs/ihep.ac.cn/soft/dayabay/NuWa-64/opt/NuWa-trunk/"
        self.cmtConfig="x86_64-slc5-gcc41-opt"
        self.outputDir="/dybfs/rec/odm"
        #self.outputDir="/publicfs/dyb/data/rec/odm"
        self.jobLogDir=self.outputDir+"/logs"
        self.statsOffsetDir="diagnostics"
        self.figuresOffsetDir="diagnostics"
        self.xmlOffsetDir="diagnostics"
        self.statsRootDir=self.outputDir+'/'+self.statsOffsetDir
        self.figuresRootDir=self.outputDir+'/'+self.figuresOffsetDir
        self.xmlRootDir=self.outputDir+'/'+self.xmlOffsetDir
        self.runsXmlDir=self.outputDir
        self.batchExec="qsub"
        self.batchLogDir=self.jobLogDir
        self.stateRootDir=self.outputDir+"/state"
        self.nuwaDataDir="/publicfs/dyb/data/mc/EH1/odm/rec"
        
    def batchCommand(self, batchArgs, jobName, runNumber, seqNumber):
        """Construct a batch command"""
        import os
        pid = os.getpid()
        jobId = "%s_%d" % (jobName[:2],runNumber)
        if seqNumber:
            jobId += "_%d" % seqNumber
        batchCommand = "%(exec)s -q dyb64q -o %(outlog)s -e %(errlog)s -N %(jobId)s " % {
            'exec':self.batchExec,
            'outlog':self.batchLogOutFile(jobName, runNumber, seqNumber),
            'errlog':self.batchLogErrFile(jobName, runNumber, seqNumber),
            'jobId':jobId}
        nuwaCommand = "runProcess.py %(args)s\n" % {
            'args': ' '.join(batchArgs) }  
        jobDir = self.batchLogDirectory(runNumber)
        if not os.path.exists(jobDir):
            os.mkdir(jobdir)
        jobFile = self.batchJobFile(jobName, runNumber, seqNumber)
        job = open(jobFile,"w")
        job.write("#!/bin/bash\n")
        job.write("source /afs/ihep.ac.cn/soft/dayabay/NuWa-64/env/NuWa-trunk/nuwa-opt.bash\n")
        job.write(nuwaCommand+"\n")
        job.close()
        batchCommand += jobFile
        return batchCommand

siteConfigs = {"local":SiteConfigLocal(),
               "pdsf":SiteConfigPdsf(),
               "pdsfSim":SiteConfigPdsfSim(),
               "pdsfTest":SiteConfigPdsfTest(),
               "ihep":SiteConfigIhep()
               }