/search.css" rel="stylesheet" type="text/css"/> /search.js">
| Classes | Job Modules | Data Objects | Services | Algorithms | Tools | Packages | Directories | Tracs |

In This Package:

Public Member Functions | Static Public Attributes
DybPython::vlut::scan::Scan Class Reference
Inheritance diagram for DybPython::vlut::scan::Scan:
Inheritance graph
[legend]
Collaboration diagram for DybPython::vlut::scan::Scan:
Collaboration graph
[legend]

List of all members.

Public Member Functions

def compare
def minmax
def compare_
def get_or_create
def make_scan
def load
def txtload
def save
def txtsave

Static Public Attributes

tuple compare = classmethod( compare )
tuple minmax = classmethod( minmax )
tuple compare_ = classmethod( compare_ )
tuple get_or_create = classmethod( get_or_create )
tuple make_scan = classmethod( make_scan )
tuple load = classmethod(load)
tuple txtload = classmethod(txtload)

Detailed Description

Definition at line 73 of file scan.py.


Member Function Documentation

def DybPython::vlut::scan::Scan::compare (   cls,
  vs 
)

Definition at line 74 of file scan.py.

00075                             :
00076 
00077         path = vs.path 
00078         log.info("Scan.compare vs %s creating %s  " % ( vs, path ))
00079 
00080         cf = vs.compare
00081 
00082         assert cf in ('vary','dbconf',)        
00083         assert len(vs[cf]) == 2 
00084 
00085         xv = vs.varycopy(0)
00086         x  = xv.make_scan() 
00087         
00088         yv = vs.varycopy(1)
00089         y = yv.make_scan() 
00090 
00091         sc = cls.compare_( x , y ) 
00092         log.info("Scan.compare saving comparison scan %s to %s " % ( sc['_stat'], path) ) 
00093         sc.save( path )
00094         return sc

def DybPython::vlut::scan::Scan::minmax (   cls,
  d,
  pfx,
  val 
)

Definition at line 97 of file scan.py.

00098                                    :
00099         mx, mi = pfx+'max',pfx+'min'
00100         if not d.has_key(mx) or val > d[mx]:d[mx] = val 
        if not d.has_key(mi) or val < d[mi]:d[mi] = val 
def DybPython::vlut::scan::Scan::compare_ (   cls,
  x,
  y,
  diffkeys = ['dig',
  t,
  n,
  i2vfn = lambda _:_['t'] 
)
Compare two dict-of-list-of-dict, checking same shape for the
non leaf parts. The leaf dict is compared kv by kv and differences
result in the differing values being incorportated into a tuple of
differers in the newly dict-of-list-of-dict



:param diffkeys: list of keys to regard when detecting dict differences
         default ignores collision and ncollision differences 

Changes:

#. moved from diffing on SEQNO with `rvs` to digest diffing with `dig` allowing comparison of non-SEQNO aligned 
 

Definition at line 104 of file scan.py.

00104                                                                          :_['t'] ):
00105         """
00106         Compare two dict-of-list-of-dict, checking same shape for the
00107         non leaf parts. The leaf dict is compared kv by kv and differences
00108         result in the differing values being incorportated into a tuple of
00109         differers in the newly dict-of-list-of-dict
00110 
00111 
00112 
00113         :param diffkeys: list of keys to regard when detecting dict differences
00114                          default ignores collision and ncollision differences 
00115 
00116         Changes:
00117 
00118         #. moved from diffing on SEQNO with `rvs` to digest diffing with `dig` allowing comparison of non-SEQNO aligned 
00119  
00120         """
00121         assert type(x) == cls and type(y) == cls
00122         xk = sorted(x.keys())
00123         yk = sorted(y.keys())
00124 
00125         if xk != yk:
00126             log.info( "xk-yk: %s " % set(xk).difference(set(yk)) )
00127             log.info( "yk-xk: %s " % set(yk).difference(set(xk)) )
00128             for ix,xkk in enumerate(xk):
00129                 print "x",ix,xkk
00130             for iy,ykk in enumerate(yk):
00131                 print "y",iy,ykk
00132 
00133         assert xk == yk, (xk,yk)
00134         ndif = 0 
00135         cf = {}
00136 
00137         mm = dict( dif={} , all={} )
00138         for k in xk:             ## insertdate
00139             lxk = len(x[k])
00140             lyk = len(y[k])
00141 
00142             if lxk != lyk:
00143                print "x[k]", k
00144                for ix,dx in enumerate(x[k]):print ix,dx 
00145                print "y[k]", k
00146                for iy,dy in enumerate(y[k]):print iy,dy 
00147 
00148             assert lxk == lyk, ( lxk,lyk )
00149             cf[k] = []
00150             for i in range(lxk):
00151                 dx = x[k][i]
00152                 dy = y[k][i]   
00153 
00154                 ivx = i2vfn(dx)   
00155                 ivy = i2vfn(dy)   
00156                 assert ivx == ivy , ("differing index to time conversion for x and y ", ivx,ivy )
00157 
00158                 cls.minmax( mm['all'] , 'i' , ivx )
00159                 cls.minmax( mm['all'] , 'k' , k )
00160 
00161                 dxk = sorted(dx.keys())
00162                 assert dxk == sorted(dy.keys()) 
00163                 ddif = 0
00164                 dcf = {}
00165                 for kk in dxk:
00166                     if dx[kk] != dy[kk]:
00167                         dcf[kk] = (dx[kk],dy[kk])
00168                         if len(diffkeys) == 0 or kk in diffkeys:   
00169                             ddif += 1
00170                             cls.minmax( mm['dif'] , 'i' , ivx )
00171                             cls.minmax( mm['dif'] , 'k' , k )
00172                             log.debug("count difference in %s %s %s  " % ( kk, dcf[kk], diffkeys )) 
00173                         else:
00174                             log.debug("ignore difference in %s %s %s " % ( kk, dcf[kk], diffkeys )) 
00175                     else:
00176                         dcf[kk] = dx[kk]
00177                         log.debug("equality in %s %s %s " % ( kk, dx[kk], dy[kk] ))
00178                 if ddif > 0: 
00179                     ndif += 1
00180                     log.warn( "dcf %s" % dcf )
00181 
00182                 cf[k].append( dcf )
00183         sc = cls(cf)
00184         sc['_stat'] = dict(ndif=ndif, minmax=mm )
00185         log.warn("Scan.compare_ _stat %s " % sc['_stat'] )
00186         return sc
Handles both leaf and comparison VlutSpec 
action is effected by options `_recompare` and `_rescan` and whether
a prior scan exists

Definition at line 189 of file scan.py.

00190                                  :
00191         """
00192         Handles both leaf and comparison VlutSpec 
00193         action is effected by options `_recompare` and `_rescan` and whether
00194         a prior scan exists
00195 
00196         """
00197         vs = _vs.copy( category="cache" )
00198         path = vs.path
00199         exists = os.path.exists( path )
00200         if vs.compare:
00201             if not exists or vs.recompare: 
00202                 log.info("Scan.get_or_create making comparison scan %s " % path )
00203                 scan = cls.compare( vs ) 
00204             else:
00205                 log.info("Scan.get_or_create loading comparison scan %s " % path )
00206                 scan = Scan.load( path )
00207         else:
00208             if not exists or vs.rescan:
00209                 log.info("Scan.get_or_create making normal scan %s " % path )
00210                 cls.make_scan( vs )
00211                 scan = Scan.load( path )
00212             else:
00213                 log.info("Scan.get_or_create loading normal scan from %s " % path )
00214                 scan = Scan.load( path )
        return scan
def DybPython::vlut::scan::Scan::make_scan (   cls,
  vs 
)
Invokes `dbifullscan` for each `dbconf` in a separate process allowing 
the full DBI stack to be booted in each process (working around DBI non-resetability).
The scans are persisted at locations controlled by the `vs` instance.

:param vs: VlutSpec instance specifiying dbconf/tn/ctx/opts

Definition at line 217 of file scan.py.

00218                             :
00219         """
00220         Invokes `dbifullscan` for each `dbconf` in a separate process allowing 
00221         the full DBI stack to be booted in each process (working around DBI non-resetability).
00222         The scans are persisted at locations controlled by the `vs` instance.
00223 
00224         :param vs: VlutSpec instance specifiying dbconf/tn/ctx/opts
00225         """ 
00226         assert vs['category'] == 'cache', (vs, vs['category'] )
00227         if type(vs['dbconf']) == list:
00228             dbconfs = vs['dbconf']
00229         else:
00230             dbconfs = [vs['dbconf']]  
00231         
00232         for dbconf in dbconfs:
00233             v = vs.spawn( dbconf=dbconf )
00234             p = Process( target=dbifullscan , args=(v,) )
00235             p.start()
00236             p.join()    # blocks to serialize
00237             assert p.exitcode == 0, "dbiscan: subprocess retuned non-zero exitcode %d " % p.exitcode
        pass
def DybPython::vlut::scan::perdict::load (   cls,
  path 
) [inherited]

Definition at line 58 of file scan.py.

00059                        :
00060         with open(path,"rb") as fp:
00061             obj = pickle.load(fp)
        return cls(obj)
def DybPython::vlut::scan::perdict::txtload (   cls,
  path 
) [inherited]

Definition at line 66 of file scan.py.

00067                           :
00068         with open(path,"w") as fp:
00069             obj = eval(fp.read())   ## sensitive to form of datetime import 
        return cls(obj)
def DybPython::vlut::scan::perdict::save (   self,
  path 
) [inherited]

Definition at line 55 of file scan.py.

00056                         :
00057         with open(path,"wb") as fp:
            pickle.dump( dict(self) , fp )
def DybPython::vlut::scan::perdict::txtsave (   self,
  path 
) [inherited]

Definition at line 63 of file scan.py.

00064                            :
00065         with open(path,"w") as fp:
            fp.write( pformat(dict(self)) )

Member Data Documentation

tuple DybPython::vlut::scan::Scan::compare = classmethod( compare ) [static]

Definition at line 95 of file scan.py.

tuple DybPython::vlut::scan::Scan::minmax = classmethod( minmax ) [static]

Definition at line 101 of file scan.py.

tuple DybPython::vlut::scan::Scan::compare_ = classmethod( compare_ ) [static]

Definition at line 187 of file scan.py.

Definition at line 215 of file scan.py.

tuple DybPython::vlut::scan::Scan::make_scan = classmethod( make_scan ) [static]

Definition at line 238 of file scan.py.

tuple DybPython::vlut::scan::perdict::load = classmethod(load) [static, inherited]

Definition at line 62 of file scan.py.

tuple DybPython::vlut::scan::perdict::txtload = classmethod(txtload) [static, inherited]

Definition at line 70 of file scan.py.


The documentation for this class was generated from the following file:
| Classes | Job Modules | Data Objects | Services | Algorithms | Tools | Packages | Directories | Tracs |

Generated on Fri May 16 2014 09:55:41 for DybPython by doxygen 1.7.4