Other aop/bitten_debug

Table Of Contents

Previous topic

DBSVN : dybaux SVN pre-commit hook

Next topic

MySQL DB Repair

This Page

Daya Bay Links

Content Skeleton

Bitten Debugging

Warning

DO NOT access the live trac.db always extract from backup tarball on another node

Extract trac.db from backup tarball

Extract dybsvn/db/trac.db from the altbackup tarball on C:

[blyth@cms01 ~]$ tar zxf /data/var/scm/alt.backup/dayabay/tracs/dybsvn/2013/04/11/104702/dybsvn.tar.gz dybsvn/db/trac.db
tar: dybsvn/db/trac.db: Wrote only 9216 of 10240 bytes

Arghh, its big, extract onto a disk with ~7GB of space:

[blyth@cms01 env]$ cd /data/env/tmp
[blyth@cms01 tmp]$ time tar zxf /data/var/scm/alt.backup/dayabay/tracs/dybsvn/2013/04/11/104702/dybsvn.tar.gz dybsvn/db/trac.db

real    6m19.715s
user    1m11.404s
sys     1m4.570s

[blyth@cms01 tmp]$ du -hs dybsvn/db/trac.db
6.7G    dybsvn/db/trac.db

Examining bitten tables

[blyth@cms01 tmp]$ sqlite3 dybsvn/db/trac.db
SQLite version 3.1.2
Enter ".help" for instructions
sqlite> .tables
attachment          bitten_report       node_change         ticket
auth_cookie         bitten_report_item  permission          ticket_change
bitten_build        bitten_rule         report              ticket_custom
bitten_config       bitten_slave        revision            version
bitten_error        bitten_step         session             wiki
bitten_log          component           session_attribute
bitten_log_message  enum                system
bitten_platform     milestone           tags
sqlite>

sqlite> .tables bitten%
bitten_build        bitten_log          bitten_report       bitten_slave
bitten_config       bitten_log_message  bitten_report_item  bitten_step
bitten_error        bitten_platform     bitten_rule
sqlite>

table counts

sqlite> select count(*) from bitten_build ;
14433
sqlite> select count(*) from bitten_config ;
5
sqlite> select count(*) from bitten_error ;
8630
sqlite> select count(*) from bitten_log ;
336829
sqlite> select count(*) from bitten_log_message ;
46789033
sqlite> select count(*) from bitten_platform ;
16
sqlite> select count(*) from bitten_report ;
111263
sqlite> select count(*) from bitten_report_item ;
22410788
sqlite> select count(*) from bitten_rule ;
16
sqlite> select count(*) from bitten_slave ;
179125
sqlite> select count(*) from bitten_step ;
338484
sqlite>

bitten_build

Last 30 builds:

sqlite> .headers ON
sqlite> .mode column
sqlite> .width 10 15 10 15 5 25 15 15 5
sqlite>  select * from bitten_build order by id desc limit 30 ;
id          config           rev         rev_time         platf  slave                      started          stopped          statu
----------  ---------------  ----------  ---------------  -----  -------------------------  ---------------  ---------------  -----
20499       opt.dybinst      20242       1365636603       28                                0                0                P
20498       opt.dybinst      20242       1365636603       34     pdyb-02                    1365637872       0                I
20497       opt.dybinst      20242       1365636603       30                                0                0                P
20496       opt.dybinst      20242       1365636603       33     farm4.dyb.local            1365638017       1365646096       F
20495       opt.dybinst      20242       1365636603       36     daya0004.rcf.bnl.gov       1365637929       1365643916       F
20494       dybinst          20242       1365636603       27                                0                0                P
20493       dybinst          20242       1365636603       35     pdyb-03                    1365637904       0                I
20492       dybinst          20242       1365636603       31                                0                0                P
20491       dybinst          20242       1365636603       32     farm2.dyb.local            1365637844       0                I
20490       dybinst          20242       1365636603       37     daya0001.rcf.bnl.gov       1365637818       1365643733       F
20489       dybinst          20242       1365636603       14                                0                0                P
20488       dybinst          20242       1365636603       15     belle7.nuu.edu.tw          1365638035       0                I
20487       opt.dybinst      20225       1365547583       28                                0                0                P
20486       opt.dybinst      20225       1365547583       34     pdyb-02                    1365549055       1365562435       F
20485       opt.dybinst      20225       1365547583       30                                0                0                P
20484       opt.dybinst      20225       1365547583       33     farm4.dyb.local            1365549060       1365557099       F
20483       opt.dybinst      20225       1365547583       36     daya0001.rcf.bnl.gov       1365549042       1365553685       F
20482       dybinst          20225       1365547583       27                                0                0                P
20481       dybinst          20225       1365547583       35     pdyb-03                    1365552651       1365576366       F
20480       dybinst          20225       1365547583       31                                0                0                P
20479       dybinst          20225       1365547583       32     farm2.dyb.local            1365548972       1365563836       F
20478       dybinst          20225       1365547583       37     daya0001.rcf.bnl.gov       1365577881       1365584661       F
20477       dybinst          20225       1365547583       14                                0                0                P
20476       dybinst          20225       1365547583       15     belle7.nuu.edu.tw          1365577969       1365589200       F
20475       opt.dybinst      20216       1365525489       28                                0                0                P
20474       opt.dybinst      20216       1365525489       34     pdyb-02                    1365526891       1365541177       F
20473       opt.dybinst      20216       1365525489       30                                0                0                P
20472       opt.dybinst      20216       1365525489       33     farm4.dyb.local            1365526962       1365535201       F
20471       opt.dybinst      20216       1365525489       36     daya0004.rcf.bnl.gov       1365526903       1365533113       F
20470       dybinst          20216       1365525489       27                                0                0                P
sqlite>

Last 40 on farm4.dyb.local:

sqlite> select id, rev, datetime(rev_time,'unixepoch') as rev_time,datetime(started,'unixepoch') as started,datetime(stopped,'unixepoch') as stopped, status from bitten_build where slave = 'farm4.dyb.local' order by id desc limit 40 ;
id          rev         rev_time              started               stopped               status
----------  ----------  --------------------  --------------------  --------------------  -------------------------
20496       20242       2013-04-10 23:30:03   2013-04-10 23:53:37   2013-04-11 02:08:16   F
20484       20225       2013-04-09 22:46:23   2013-04-09 23:11:00   2013-04-10 01:24:59   F
20472       20216       2013-04-09 16:38:09   2013-04-09 17:02:42   2013-04-09 19:20:01   F
20460       20193       2013-04-05 20:53:44   2013-04-05 21:17:39   2013-04-05 23:29:17   S
20448       20181       2013-04-04 15:48:18   2013-04-04 16:10:48   2013-04-04 18:23:48   S
20436       20180       2013-04-04 01:44:20   2013-04-04 02:07:46   2013-04-04 04:54:13   S
20424       20176       2013-04-03 20:01:07   2013-04-03 20:25:50   2013-04-03 22:41:50   S
20412       20164       2013-04-01 19:15:47   2013-04-01 20:56:05   2013-04-01 23:10:30   S
20400       20163       2013-04-01 18:16:26   2013-04-01 18:40:21   2013-04-01 20:55:25   S
20388       20160       2013-04-01 05:39:14   2013-04-01 06:01:11   2013-04-01 08:53:16   S
20376       20159       2013-04-01 02:26:11   2013-04-01 02:47:01   2013-04-01 05:05:27   S
20364       20156       2013-03-29 22:03:47   2013-03-29 22:44:18   2013-03-30 00:58:23   S
20352       20154       2013-03-29 20:05:25   2013-03-29 20:26:31   2013-03-29 22:43:38   S
20340       20147       2013-03-28 04:59:30   2013-03-28 05:23:23   2013-03-28 07:36:29   S
20328       20145       2013-03-27 18:07:13   2013-03-27 18:40:08   2013-03-27 20:56:27   S
20316       20143       2013-03-27 16:05:23   2013-03-27 16:25:27   2013-03-27 18:39:29   S
20304       20137       2013-03-27 03:19:32   2013-03-27 05:55:59   2013-03-27 06:54:00   S
20292       20137       2013-03-27 03:19:32   2013-03-27 03:39:46   2013-03-27 05:55:20   S
20280       20121       2013-03-25 04:37:48   2013-03-25 05:02:27   2013-03-25 07:18:09   S
20268       20093       2013-03-20 23:10:20   2013-03-20 23:31:29   2013-03-21 01:46:23   S
20256       20084       2013-03-20 16:15:42   2013-03-20 16:38:06   2013-03-20 18:50:20   S
20244       20072       2013-03-19 10:37:07   2013-03-19 11:25:05   2013-03-19 13:38:24   S
20232       20070       2013-03-19 01:25:52   2013-03-19 09:52:44   2013-03-19 11:24:26   S
20220       20069       2013-03-18 14:40:35   2013-03-18 15:01:02   2013-03-18 17:12:13   S
20208       20057       2013-03-18 07:26:02   2013-03-18 07:50:55   2013-03-18 09:59:59   S
20196       20054       2013-03-18 02:58:06   2013-03-18 06:43:01   2013-03-18 07:40:15   S
20184       20054       2013-03-18 02:58:06   2013-03-18 05:44:14   2013-03-18 06:42:18   S
20172       20054       2013-03-18 02:58:06   2013-03-18 03:29:00   2013-03-18 05:43:29   S
20160       19972       2013-03-11 20:03:17   2013-03-11 20:35:43   2013-03-11 22:48:47   S
20148       19951       2013-03-07 15:53:19   2013-03-07 16:23:57   2013-03-07 18:35:55   S
20136       19929       2013-03-05 18:57:20   2013-03-05 19:31:06   2013-03-05 21:41:52   S
20124       19906       2013-03-02 13:32:11   2013-03-02 14:02:51   2013-03-02 16:16:55   S
20112       19903       2013-03-01 21:54:31   2013-03-01 22:28:10   2013-03-02 00:41:01   S
20100       19901       2013-03-01 17:39:47   2013-03-01 18:13:08   2013-03-01 20:27:24   S
20088       19891       2013-03-01 07:21:58   2013-03-01 10:09:32   2013-03-01 11:06:08   S
20076       19891       2013-03-01 07:21:58   2013-03-01 07:53:24   2013-03-01 10:08:55   S
20064       19860       2013-02-27 06:56:28   2013-02-27 07:29:26   2013-02-27 10:12:27   S
20052       19856       2013-02-26 22:28:51   2013-02-26 22:59:59   2013-02-27 01:16:54   S
20040       19850       2013-02-26 17:37:31   2013-02-26 18:12:05   2013-02-26 20:59:12   S
20028       19835       2013-02-22 17:57:28   2013-02-22 18:29:28   2013-02-22 20:46:47   F
sqlite>

More digestable in terms of lag until build starts and duration minutes:

sqlite> .width 8 8 20 7 7 7
sqlite> select id, rev, datetime(rev_time,'unixepoch') as rev_time, ( started - rev_time )/60 as lagmin, (stopped - started)/60 as durmin, status from bitten_build where slave = 'farm4.dyb.local' order by id desc limit 40 ;
id        rev       rev_time              lagmin   durmin   status
--------  --------  --------------------  -------  -------  -------
20496     20242     2013-04-10 23:30:03   23       134      F
20484     20225     2013-04-09 22:46:23   24       133      F
20472     20216     2013-04-09 16:38:09   24       137      F
20460     20193     2013-04-05 20:53:44   23       131      S
20448     20181     2013-04-04 15:48:18   22       133      S
20436     20180     2013-04-04 01:44:20   23       166      S
20424     20176     2013-04-03 20:01:07   24       136      S
20412     20164     2013-04-01 19:15:47   100      134      S
20400     20163     2013-04-01 18:16:26   23       135      S
20388     20160     2013-04-01 05:39:14   21       172      S
20376     20159     2013-04-01 02:26:11   20       138      S
20364     20156     2013-03-29 22:03:47   40       134      S
20352     20154     2013-03-29 20:05:25   21       137      S
20340     20147     2013-03-28 04:59:30   23       133      S
20328     20145     2013-03-27 18:07:13   32       136      S
20316     20143     2013-03-27 16:05:23   20       134      S
20304     20137     2013-03-27 03:19:32   156      58       S
20292     20137     2013-03-27 03:19:32   20       135      S
20280     20121     2013-03-25 04:37:48   24       135      S
20268     20093     2013-03-20 23:10:20   21       134      S
20256     20084     2013-03-20 16:15:42   22       132      S
20244     20072     2013-03-19 10:37:07   47       133      S
20232     20070     2013-03-19 01:25:52   506      91       S
20220     20069     2013-03-18 14:40:35   20       131      S
20208     20057     2013-03-18 07:26:02   24       129      S
20196     20054     2013-03-18 02:58:06   224      57       S
20184     20054     2013-03-18 02:58:06   166      58       S
20172     20054     2013-03-18 02:58:06   30       134      S
20160     19972     2013-03-11 20:03:17   32       133      S
20148     19951     2013-03-07 15:53:19   30       131      S
20136     19929     2013-03-05 18:57:20   33       130      S
20124     19906     2013-03-02 13:32:11   30       134      S
20112     19903     2013-03-01 21:54:31   33       132      S
20100     19901     2013-03-01 17:39:47   33       134      S
20088     19891     2013-03-01 07:21:58   167      56       S
20076     19891     2013-03-01 07:21:58   31       135      S
20064     19860     2013-02-27 06:56:28   32       163      S
20052     19856     2013-02-26 22:28:51   31       136      S
20040     19850     2013-02-26 17:37:31   34       167      S
20028     19835     2013-02-22 17:57:28   32       137      F
sqlite>
sqlite> select id, rev, datetime(rev_time,'unixepoch') as rev_time, ( started - rev_time )/60 as lagmin, (stopped - started)/60 as durmin, status, config, slave  from bitten_build where status != 'P' and config = 'opt.dybinst'  order by id desc limit 40 ;
id        rev       rev_time              lagmin   durmin   status   config           slave
--------  --------  --------------------  -------  -------  -------  ---------------  ---------------
20498     20242     2013-04-10 23:30:03   21       -227606  I        opt.dybinst      pdyb-02
20496     20242     2013-04-10 23:30:03   23       134      F        opt.dybinst      farm4.dyb.local
20495     20242     2013-04-10 23:30:03   22       99       F        opt.dybinst      daya0004.rcf.bn
20486     20225     2013-04-09 22:46:23   24       223      F        opt.dybinst      pdyb-02
20484     20225     2013-04-09 22:46:23   24       133      F        opt.dybinst      farm4.dyb.local
20483     20225     2013-04-09 22:46:23   24       77       F        opt.dybinst      daya0001.rcf.bn
20474     20216     2013-04-09 16:38:09   23       238      F        opt.dybinst      pdyb-02
20472     20216     2013-04-09 16:38:09   24       137      F        opt.dybinst      farm4.dyb.local
20471     20216     2013-04-09 16:38:09   23       103      F        opt.dybinst      daya0004.rcf.bn
20462     20193     2013-04-05 20:53:44   24       206      S        opt.dybinst      pdyb-02
20460     20193     2013-04-05 20:53:44   23       131      S        opt.dybinst      farm4.dyb.local
20459     20193     2013-04-05 20:53:44   508      76       S        opt.dybinst      daya0001.rcf.bn
20450     20181     2013-04-04 15:48:18   21       214      S        opt.dybinst      pdyb-02
20448     20181     2013-04-04 15:48:18   22       133      S        opt.dybinst      farm4.dyb.local
20447     20181     2013-04-04 15:48:18   21       72       S        opt.dybinst      daya0001.rcf.bn
20438     20180     2013-04-04 01:44:20   23       243      S        opt.dybinst      pdyb-02
20436     20180     2013-04-04 01:44:20   23       166      S        opt.dybinst      farm4.dyb.local
20435     20180     2013-04-04 01:44:20   22       93       S        opt.dybinst      daya0001.rcf.bn
20426     20176     2013-04-03 20:01:07   22       222      S        opt.dybinst      pdyb-02
20424     20176     2013-04-03 20:01:07   24       136      S        opt.dybinst      farm4.dyb.local
20423     20176     2013-04-03 20:01:07   24       79       S        opt.dybinst      daya0001.rcf.bn
20414     20164     2013-04-01 19:15:47   171      216      S        opt.dybinst      pdyb-02
20412     20164     2013-04-01 19:15:47   100      134      S        opt.dybinst      farm4.dyb.local
20411     20164     2013-04-01 19:15:47   66       100      S        opt.dybinst      daya0004.rcf.bn
20402     20163     2013-04-01 18:16:26   23       206      S        opt.dybinst      pdyb-02
20400     20163     2013-04-01 18:16:26   23       135      S        opt.dybinst      farm4.dyb.local
20399     20163     2013-04-01 18:16:26   24       100      S        opt.dybinst      daya0004.rcf.bn
20390     20160     2013-04-01 05:39:14   24       259      S        opt.dybinst      pdyb-02
20388     20160     2013-04-01 05:39:14   21       172      S        opt.dybinst      farm4.dyb.local
20387     20160     2013-04-01 05:39:14   24       123      S        opt.dybinst      daya0004.rcf.bn
20378     20159     2013-04-01 02:26:11   502      217      S        opt.dybinst      pdyb-02
20376     20159     2013-04-01 02:26:11   20       138      S        opt.dybinst      farm4.dyb.local
20375     20159     2013-04-01 02:26:11   24       101      S        opt.dybinst      daya0004.rcf.bn
20366     20156     2013-03-29 22:03:47   106      204      S        opt.dybinst      pdyb-02
20364     20156     2013-03-29 22:03:47   40       134      S        opt.dybinst      farm4.dyb.local
20363     20156     2013-03-29 22:03:47   22       99       S        opt.dybinst      daya0004.rcf.bn
20354     20154     2013-03-29 20:05:25   21       202      S        opt.dybinst      pdyb-02
20352     20154     2013-03-29 20:05:25   21       137      S        opt.dybinst      farm4.dyb.local
20351     20154     2013-03-29 20:05:25   22       103      S        opt.dybinst      daya0004.rcf.bn
20342     20147     2013-03-28 04:59:30   21       225      S        opt.dybinst      pdyb-02