/search.css" rel="stylesheet" type="text/css"/> /search.js">
| Classes | Job Modules | Data Objects | Services | Algorithms | Tools | Packages | Directories | Tracs |

In This Package:

Public Member Functions | Public Attributes | Static Public Attributes
DybPython::dbsrv::PartitionMgr Class Reference

List of all members.

Public Member Functions

def __init__
def survey
def assign_parts
def dir
def is_last
def qminmax
def qrange
def offset
def min
def max
def where
def int_dirname
def has_partitionset
def archived_chunks
def available_chunks

Public Attributes

 basedir
 and_where
 plast
 key
 kpo
 size
 parts

Static Public Attributes

tuple int_ptn = re.compile("^\d*$")

Detailed Description

Definition at line 1379 of file dbsrv.py.


Constructor & Destructor Documentation

def DybPython::dbsrv::PartitionMgr::__init__ (   self,
  opts,
  basedir = None 
)
#. for example 1000,0,3 for three chunks of 1000 starting at 1 

Definition at line 1381 of file dbsrv.py.

01382                                           :
01383         """
01384         #. for example 1000,0,3 for three chunks of 1000 starting at 1 
01385         """
01386         self.basedir = basedir  
01387         pass
01388         size = int(opts.partitionsize)  
01389         pass
01390         if opts.partitionrange is None:
01391             parts = None
01392         else:
01393             a, n = map(int,opts.partitionrange.split(","))  
01394             parts = range(a,n)
01395         pass  
01396         key, kpo = opts.partitionkey.split(",")
01397         and_where = opts.where  
01398         pass
01399         self.and_where = and_where
01400 
01401         if opts.partitionlast is None:
01402             plast = None
01403         else:             
01404             plast = int(opts.partitionlast) 
01405         pass
01406         self.plast = plast
01407         pass
01408         self.key = key       # key string eg SEQNO
01409         self.kpo = int(kpo)  # key position in csv
01410         pass
01411         self.size = size
01412         self.parts = parts
01413 


Member Function Documentation

def DybPython::dbsrv::PartitionMgr::survey (   self,
  db 
)
Need to identify the last partition after which not 
all of the tables will have full partitions.

For sensible partitioning should restrict to tables 
whose SEQNO stride along roughly together. 

Definition at line 1414 of file dbsrv.py.

01415                          :
01416         """
01417         Need to identify the last partition after which not 
01418         all of the tables will have full partitions.
01419 
01420         For sensible partitioning should restrict to tables 
01421         whose SEQNO stride along roughly together. 
01422 
01423         """
01424         size = self.size
01425         log.info("PartitionMgr survey size %s " % size  )
01426         svy = {}
01427         mlp = 0
01428         
01429         for table in db.ptables():
01430             mm = db(self.qminmax(table,where="TRUE"))[0]
01431             last = mm['max']/size
01432             svy[table] = dict(mm, last=last )
01433         pass
01434         svy['COMMON'] = dict(
01435             min=max(map(lambda _:_['min'],svy.values())),  
01436             max=min(map(lambda _:_['max'],svy.values())),
01437             last=min(map(lambda _:_['last'],svy.values()))
01438           )  
01439 
01440         log.info("\n"+pformat(svy)) 
01441         autoplast = svy['COMMON']['last']
01442         log.info("auto survey determines last partition index as %s " % autoplast )
01443         if self.plast is None:
01444             self.plast = autoplast  
01445         else:
01446             if self.plast != autoplast:
01447                 log.warn("`--partitionlast` option gives %s BUT auto-survey gives %s " % (self.plast, autoplast))  
01448             pass
01449         pass
01450         if self.parts is None:
01451             parts = range(0,self.plast+1)  # plast is the 0-based index of the last partition, hence the need for  +1
01452             log.info("auto definition of partition range %s " % repr(parts))
01453             self.parts = parts 
01454         else:
01455             log.info("using partition range from options %s " % repr(self.parts))
01456         return svy
 
def DybPython::dbsrv::PartitionMgr::assign_parts (   self,
  chunks 
)
:param chunks:

Definition at line 1457 of file dbsrv.py.

01458                                     :
01459         """
01460         :param chunks:
01461         """
01462         parts = sorted(chunks)
01463         plast = parts[-1] + 1
01464         parts += [plast]
01465 
01466         self.parts = parts
01467         self.plast = plast
01468         log.info("assign_parts %s plast %s " % ( self.parts, self.plast ))

def DybPython::dbsrv::PartitionMgr::dir (   self,
  p 
)

Definition at line 1469 of file dbsrv.py.

01469                     : 
01470         """
01471         """ 
01472         if self.is_last(p):
01473             leaf = "last"
01474         else:
01475             leaf = p
01476         pass
01477         return os.path.join(self.basedir,str(self.size),str(leaf)) 
01478 
def DybPython::dbsrv::PartitionMgr::is_last (   self,
  p 
)

Definition at line 1479 of file dbsrv.py.

01480                         :
01481         return p == self.plast

def DybPython::dbsrv::PartitionMgr::qminmax (   self,
  table,
  where = "1=1" 
)

Definition at line 1482 of file dbsrv.py.

01483                                          :
01484         key = self.key
01485         return "select max(%(key)s) as max, min(%(key)s) as min, count(*) as count from %(table)s where %(where)s " % locals()

def DybPython::dbsrv::PartitionMgr::qrange (   self,
  table,
  where = "1=1",
  keycount = True 
)

Definition at line 1486 of file dbsrv.py.

01487                                                        :
01488         key = self.key
01489         if keycount:
01490             keycount = ", count(distinct(%(key)s)) as keycount " % locals()
01491         else:
01492             keycount = ""
01493         pass
01494         return "select max(%(key)s) as max, min(%(key)s) as min, count(*) as count  %(keycount)s from %(table)s where %(where)s " % locals()

def DybPython::dbsrv::PartitionMgr::offset (   self,
  p 
)

Definition at line 1495 of file dbsrv.py.

01495                        : 
01496         return self.size*p 
def DybPython::dbsrv::PartitionMgr::min (   self,
  p 
)

Definition at line 1497 of file dbsrv.py.

01497                     : 
01498         return self.size*p + 1
def DybPython::dbsrv::PartitionMgr::max (   self,
  p 
)

Definition at line 1499 of file dbsrv.py.

01499                     : 
01500         return self.size*(p+1)
def DybPython::dbsrv::PartitionMgr::where (   self,
  p 
)

Definition at line 1501 of file dbsrv.py.

01502                       :
01503         n = len(self.parts) 
01504         size = self.size 
01505         pmin = self.min(p) 
01506         pmax = self.max(p) 
01507         pkey = self.key
01508         p1 = p + 1
01509         if self.and_where is None:
01510             and_where = ""
01511         else:
01512             and_where = " and %s " % self.and_where 
01513 
01514         return "/* [%(p)s] %(size)s-partition %(p1)-3s/%(n)s  */ %(pkey)s >= %(pmin)s and %(pkey)s <= %(pmax)s %(and_where)s " % locals()

def DybPython::dbsrv::PartitionMgr::int_dirname (   self,
  dir 
)
:param dir:
:return: list of integers corresponding to subfolder of *dir* with integer names, in ascending order 

Definition at line 1515 of file dbsrv.py.

01516                               :
01517         """
01518         :param dir:
01519         :return: list of integers corresponding to subfolder of *dir* with integer names, in ascending order 
01520         """ 
01521         return sorted(map(int,filter(lambda _:self.int_ptn.match(_), os.listdir(dir))))

def DybPython::dbsrv::PartitionMgr::has_partitionset (   self)
Checks if there is a partitionset folder for the configured partition size.
ie looks for the 10000 folder, in default case.

Definition at line 1522 of file dbsrv.py.

01523                               :
01524         """
01525         Checks if there is a partitionset folder for the configured partition size.
01526         ie looks for the 10000 folder, in default case.
01527         """
01528         psizes = self.int_dirname(self.basedir)
01529         if not self.size in psizes:
01530             msg = "partition dump at %s does not have partitions of the configured size %s : %s " % (self.basedir, self.size, psizes)
01531             log.warn(msg)
01532             return False 
01533         return True 

def DybPython::dbsrv::PartitionMgr::archived_chunks (   self)
:return: list of integers in ascending order corresponding to partition archives found

Definition at line 1534 of file dbsrv.py.

01535                              :
01536         """
01537         :return: list of integers in ascending order corresponding to partition archives found
01538         """
01539         zdir  = os.path.join(self.basedir, "archive", str(self.size))
01540         ptn = re.compile("^(?P<size>\d*)_(?P<part>\d*).tar.gz.dna$")
01541         chunks = []
01542         for path in os.listdir(zdir):
01543             name = os.path.basename(path)
01544             match = ptn.match(name)
01545             if match:
01546                 d = match.groupdict()
01547                 chunks.append(int(d['part']))
01548             pass 
01549         return sorted(chunks)

def DybPython::dbsrv::PartitionMgr::available_chunks (   self)
:return: list of intergers in ascending order corresponding to the partition folders found

Looks for the partition folder, named according to the partition size eg "basedir/10000" 
and determines the chunks by looking at the integer named partition subfolders "basedir/10000/0" 

Definition at line 1550 of file dbsrv.py.

01550                               : 
01551         """
01552         :return: list of intergers in ascending order corresponding to the partition folders found
01553 
01554         Looks for the partition folder, named according to the partition size eg "basedir/10000" 
01555         and determines the chunks by looking at the integer named partition subfolders "basedir/10000/0" 
01556         """
01557         if not self.has_partitionset():
01558             log.warn("no partitionset for partition size %s" % self.size)
01559             return []  
01560         pfold = os.path.join( self.basedir, str(self.size) ) 
01561         chunks = self.int_dirname(pfold)
01562         log.info("partition folder %s for configured size %s has chunks %s " % (pfold, self.size, repr(chunks)) )
01563         return chunks
01564 
01565 

Member Data Documentation

tuple DybPython::dbsrv::PartitionMgr::int_ptn = re.compile("^\d*$") [static]

Definition at line 1380 of file dbsrv.py.

Definition at line 1383 of file dbsrv.py.

Definition at line 1383 of file dbsrv.py.

Definition at line 1383 of file dbsrv.py.

Definition at line 1383 of file dbsrv.py.

Definition at line 1383 of file dbsrv.py.

Definition at line 1383 of file dbsrv.py.

Definition at line 1383 of file dbsrv.py.


The documentation for this class was generated from the following file:
| Classes | Job Modules | Data Objects | Services | Algorithms | Tools | Packages | Directories | Tracs |

Generated on Fri May 16 2014 09:55:41 for DybPython by doxygen 1.7.4