diff.py

Go to the documentation of this file.

#!/usr/bin/env python
"""

Example, validation of partitioned backups vs direct dump::

    [blyth@belle7 DybPython]$ RANGE=0,35 ./diff.py /var/dbbackup/dbsrv/belle7.nuu.edu.tw/channelquality_db_dybdb2_direct/10000 /var/dbbackup/dbsrv/dybdb2.ihep.ac.cn/channelquality_db_dybdb2/10000
    2013-06-26 15:23:42,558 __main__ INFO     rng ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12', '13', '14', '15', '16', '17', '18', '19', '20', '21', '22', '23', '24', '25', '26', '27', '28', '29', '30', '31', '32', '33', '34'] 
    2013-06-26 15:23:43,188 __main__ INFO     diff -r --brief /var/dbbackup/dbsrv/belle7.nuu.edu.tw/channelquality_db_dybdb2_direct/10000/0 /var/dbbackup/dbsrv/dybdb2.ihep.ac.cn/channelquality_db_dybdb2/10000/0 => 0 
    2013-06-26 15:23:44,708 __main__ INFO     diff -r --brief /var/dbbackup/dbsrv/belle7.nuu.edu.tw/channelquality_db_dybdb2_direct/10000/1 /var/dbbackup/dbsrv/dybdb2.ihep.ac.cn/channelquality_db_dybdb2/10000/1 => 0 
    2013-06-26 15:24:14,393 __main__ INFO     diff -r --brief /var/dbbackup/dbsrv/belle7.nuu.edu.tw/channelquality_db_dybdb2_direct/10000/2 /var/dbbackup/dbsrv/dybdb2.ihep.ac.cn/channelquality_db_dybdb2/10000/2 => 0 
    2013-06-26 15:24:40,702 __main__ INFO     diff -r --brief /var/dbbackup/dbsrv/belle7.nuu.edu.tw/channelquality_db_dybdb2_direct/10000/3 /var/dbbackup/dbsrv/dybdb2.ihep.ac.cn/channelquality_db_dybdb2/10000/3 => 0 
    2013-06-26 15:25:07,644 __main__ INFO     diff -r --brief /var/dbbackup/dbsrv/belle7.nuu.edu.tw/channelquality_db_dybdb2_direct/10000/4 /var/dbbackup/dbsrv/dybdb2.ihep.ac.cn/channelquality_db_dybdb2/10000/4 => 0 
    2013-06-26 15:25:36,576 __main__ INFO     diff -r --brief /var/dbbackup/dbsrv/belle7.nuu.edu.tw/channelquality_db_dybdb2_direct/10000/5 /var/dbbackup/dbsrv/dybdb2.ihep.ac.cn/channelquality_db_dybdb2/10000/5 => 0 
    2013-06-26 15:26:04,959 __main__ INFO     diff -r --brief /var/dbbackup/dbsrv/belle7.nuu.edu.tw/channelquality_db_dybdb2_direct/10000/6 /var/dbbackup/dbsrv/dybdb2.ihep.ac.cn/channelquality_db_dybdb2/10000/6 => 0 
    2013-06-26 15:26:30,391 __main__ INFO     diff -r --brief /var/dbbackup/dbsrv/belle7.nuu.edu.tw/channelquality_db_dybdb2_direct/10000/7 /var/dbbackup/dbsrv/dybdb2.ihep.ac.cn/channelquality_db_dybdb2/10000/7 => 0 
    2013-06-26 15:26:59,597 __main__ INFO     diff -r --brief /var/dbbackup/dbsrv/belle7.nuu.edu.tw/channelquality_db_dybdb2_direct/10000/8 /var/dbbackup/dbsrv/dybdb2.ihep.ac.cn/channelquality_db_dybdb2/10000/8 => 0 
    2013-06-26 15:27:26,654 __main__ INFO     diff -r --brief /var/dbbackup/dbsrv/belle7.nuu.edu.tw/channelquality_db_dybdb2_direct/10000/9 /var/dbbackup/dbsrv/dybdb2.ihep.ac.cn/channelquality_db_dybdb2/10000/9 => 0 
    2013-06-26 15:27:59,946 __main__ INFO     diff -r --brief /var/dbbackup/dbsrv/belle7.nuu.edu.tw/channelquality_db_dybdb2_direct/10000/10 /var/dbbackup/dbsrv/dybdb2.ihep.ac.cn/channelquality_db_dybdb2/10000/10 => 0 
    2013-06-26 15:28:30,351 __main__ INFO     diff -r --brief /var/dbbackup/dbsrv/belle7.nuu.edu.tw/channelquality_db_dybdb2_direct/10000/11 /var/dbbackup/dbsrv/dybdb2.ihep.ac.cn/channelquality_db_dybdb2/10000/11 => 0 
    2013-06-26 15:28:59,000 __main__ INFO     diff -r --brief /var/dbbackup/dbsrv/belle7.nuu.edu.tw/channelquality_db_dybdb2_direct/10000/12 /var/dbbackup/dbsrv/dybdb2.ihep.ac.cn/channelquality_db_dybdb2/10000/12 => 0 
    2013-06-26 15:29:28,941 __main__ INFO     diff -r --brief /var/dbbackup/dbsrv/belle7.nuu.edu.tw/channelquality_db_dybdb2_direct/10000/13 /var/dbbackup/dbsrv/dybdb2.ihep.ac.cn/channelquality_db_dybdb2/10000/13 => 0 
    2013-06-26 15:29:57,775 __main__ INFO     diff -r --brief /var/dbbackup/dbsrv/belle7.nuu.edu.tw/channelquality_db_dybdb2_direct/10000/14 /var/dbbackup/dbsrv/dybdb2.ihep.ac.cn/channelquality_db_dybdb2/10000/14 => 0 
    2013-06-26 15:30:27,899 __main__ INFO     diff -r --brief /var/dbbackup/dbsrv/belle7.nuu.edu.tw/channelquality_db_dybdb2_direct/10000/15 /var/dbbackup/dbsrv/dybdb2.ihep.ac.cn/channelquality_db_dybdb2/10000/15 => 0 
    2013-06-26 15:30:57,652 __main__ INFO     diff -r --brief /var/dbbackup/dbsrv/belle7.nuu.edu.tw/channelquality_db_dybdb2_direct/10000/16 /var/dbbackup/dbsrv/dybdb2.ihep.ac.cn/channelquality_db_dybdb2/10000/16 => 0 
    2013-06-26 15:31:26,704 __main__ INFO     diff -r --brief /var/dbbackup/dbsrv/belle7.nuu.edu.tw/channelquality_db_dybdb2_direct/10000/17 /var/dbbackup/dbsrv/dybdb2.ihep.ac.cn/channelquality_db_dybdb2/10000/17 => 0 
    2013-06-26 15:31:57,606 __main__ INFO     diff -r --brief /var/dbbackup/dbsrv/belle7.nuu.edu.tw/channelquality_db_dybdb2_direct/10000/18 /var/dbbackup/dbsrv/dybdb2.ihep.ac.cn/channelquality_db_dybdb2/10000/18 => 0 
    2013-06-26 15:32:25,482 __main__ INFO     diff -r --brief /var/dbbackup/dbsrv/belle7.nuu.edu.tw/channelquality_db_dybdb2_direct/10000/19 /var/dbbackup/dbsrv/dybdb2.ihep.ac.cn/channelquality_db_dybdb2/10000/19 => 0 
    2013-06-26 15:32:52,782 __main__ INFO     diff -r --brief /var/dbbackup/dbsrv/belle7.nuu.edu.tw/channelquality_db_dybdb2_direct/10000/20 /var/dbbackup/dbsrv/dybdb2.ihep.ac.cn/channelquality_db_dybdb2/10000/20 => 0 
    2013-06-26 15:33:20,600 __main__ INFO     diff -r --brief /var/dbbackup/dbsrv/belle7.nuu.edu.tw/channelquality_db_dybdb2_direct/10000/21 /var/dbbackup/dbsrv/dybdb2.ihep.ac.cn/channelquality_db_dybdb2/10000/21 => 0 
    2013-06-26 15:33:47,635 __main__ INFO     diff -r --brief /var/dbbackup/dbsrv/belle7.nuu.edu.tw/channelquality_db_dybdb2_direct/10000/22 /var/dbbackup/dbsrv/dybdb2.ihep.ac.cn/channelquality_db_dybdb2/10000/22 => 0 
    2013-06-26 15:34:16,491 __main__ INFO     diff -r --brief /var/dbbackup/dbsrv/belle7.nuu.edu.tw/channelquality_db_dybdb2_direct/10000/23 /var/dbbackup/dbsrv/dybdb2.ihep.ac.cn/channelquality_db_dybdb2/10000/23 => 0 
    2013-06-26 15:34:45,463 __main__ INFO     diff -r --brief /var/dbbackup/dbsrv/belle7.nuu.edu.tw/channelquality_db_dybdb2_direct/10000/24 /var/dbbackup/dbsrv/dybdb2.ihep.ac.cn/channelquality_db_dybdb2/10000/24 => 0 
    2013-06-26 15:35:13,254 __main__ INFO     diff -r --brief /var/dbbackup/dbsrv/belle7.nuu.edu.tw/channelquality_db_dybdb2_direct/10000/25 /var/dbbackup/dbsrv/dybdb2.ihep.ac.cn/channelquality_db_dybdb2/10000/25 => 0 
    2013-06-26 15:35:40,851 __main__ INFO     diff -r --brief /var/dbbackup/dbsrv/belle7.nuu.edu.tw/channelquality_db_dybdb2_direct/10000/26 /var/dbbackup/dbsrv/dybdb2.ihep.ac.cn/channelquality_db_dybdb2/10000/26 => 0 
    2013-06-26 15:36:10,699 __main__ INFO     diff -r --brief /var/dbbackup/dbsrv/belle7.nuu.edu.tw/channelquality_db_dybdb2_direct/10000/27 /var/dbbackup/dbsrv/dybdb2.ihep.ac.cn/channelquality_db_dybdb2/10000/27 => 0 
    2013-06-26 15:36:37,114 __main__ INFO     diff -r --brief /var/dbbackup/dbsrv/belle7.nuu.edu.tw/channelquality_db_dybdb2_direct/10000/28 /var/dbbackup/dbsrv/dybdb2.ihep.ac.cn/channelquality_db_dybdb2/10000/28 => 0 
    2013-06-26 15:37:05,515 __main__ INFO     diff -r --brief /var/dbbackup/dbsrv/belle7.nuu.edu.tw/channelquality_db_dybdb2_direct/10000/29 /var/dbbackup/dbsrv/dybdb2.ihep.ac.cn/channelquality_db_dybdb2/10000/29 => 0 
    2013-06-26 15:37:33,092 __main__ INFO     diff -r --brief /var/dbbackup/dbsrv/belle7.nuu.edu.tw/channelquality_db_dybdb2_direct/10000/30 /var/dbbackup/dbsrv/dybdb2.ihep.ac.cn/channelquality_db_dybdb2/10000/30 => 0 
    2013-06-26 15:38:03,650 __main__ INFO     diff -r --brief /var/dbbackup/dbsrv/belle7.nuu.edu.tw/channelquality_db_dybdb2_direct/10000/31 /var/dbbackup/dbsrv/dybdb2.ihep.ac.cn/channelquality_db_dybdb2/10000/31 => 0 
    [blyth@belle7 DybPython]$ 
    [blyth@belle7 DybPython]$ diff -r --brief /var/dbbackup/dbsrv/belle7.nuu.edu.tw/channelquality_db_dybdb2_direct/10000/_ /var/dbbackup/dbsrv/dybdb2.ihep.ac.cn/channelquality_db_dybdb2/10000/_
    [blyth@belle7 DybPython]$ echo $?
    0
    [blyth@belle7 DybPython]$ diff -r --brief /var/dbbackup/dbsrv/belle7.nuu.edu.tw/channelquality_db_dybdb2_direct/10000/last /var/dbbackup/dbsrv/dybdb2.ihep.ac.cn/channelquality_db_dybdb2/10000/last
    [blyth@belle7 DybPython]$ echo $?
    0


    

"""
import os, sys, logging
log = logging.getLogger(__name__)

def int_(s):
    try:
        x = int(s)
    except ValueError:
        x = s
    return x 

isorted_ = lambda _:sorted(_,key=int_)

def do(cmd): 
    p = os.popen(cmd)
    ret = p.read()
    st = p.close()
    if st is None:
        rc = 0
    else: 
        rc = os.WEXITSTATUS(st)  
    return ret, rc

class Dir(list):
    def __init__(self, base):
        self[:]=isorted_(os.listdir(base))   # subdir names relative to base
        self.base = base   
    def path(self, e):
        return os.path.join(self.base, e )


class Diff(object):
    def __init__(self, a, b ):
        cmd = "diff -r --brief %(a)s %(b)s" % locals()
        ret, rc = do(cmd)
        log.info("%s => %s " % ( cmd, "\n".join(filter(len,[str(rc), ret])) )) 


if __name__ == '__main__':
    logging.basicConfig(level=logging.INFO, format="%(asctime)s %(name)s %(levelname)-8s %(message)s")
    dirs = sys.argv[1:]
    assert len(dirs) == 2

    a = Dir(dirs[0])
    b = Dir(dirs[1])
    
    sa, sb = set(a), set(b)
    si, su = sa & sb, sa | sb   # intersection, union

    rng = map(str,range(*map(int,os.environ.get('RANGE',"0,1000").split(","))))

    log.info("rng %s " % repr(rng))

    for x in isorted_(su):
        if not x in rng:continue
        if x in si:
            pa, pb = a.path(x), b.path(x)
            Diff(pa, pb)
        else:
            log.warn("skip %s that is not in both" % x )