Megatest

Changes On Branch bbeec077ff5bc2a2
Login
☰Home Timeline Files Branches Tags Tickets Wiki Login
Hide Diff Patch Unified Diff

Changes In Branch refactor-sqlite3-calls Through [bbeec077ff] Excluding Merge-Ins

This is equivalent to a diff from a1236d6bf9 to bbeec077ff

2014-10-08
22:46
Merged in fixes from v1.60 branch check-in: 7636717b91 user: matt tags: refactor-sqlite3-calls
12:11
Fixed sync bug (I think) check-in: 18f70c2641 user: mrwellan tags: v1.60
03:55
updated stats check-in: bbeec077ff user: matt tags: refactor-sqlite3-calls
03:41
Replace sqlite3:for-each-row with sqlite3:first-result where it makes sense check-in: 92efe0fe78 user: matt tags: refactor-sqlite3-calls
2014-10-07
21:13
Streamlined syncback and added more control for syncing to and from megatest.db and the new databases check-in: a1236d6bf9 user: matt tags: v1.60, v1.6002_beta4
17:11
Added syncback check-in: 08847d5384 user: mrwellan tags: v1.60

Modified db.scm from [c2976edd1b] to [6079da4d93].

︙︙
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618

  ;; (db:delay-if-busy)
  (sqlite3:execute db "UPDATE tests SET state=?,status=?,event_time=strftime('%s','now') WHERE run_id=? AND testname=? AND item_path=?;" 
 		   state status run-id test-name item-path))

;; NEW BEHAVIOR: Count tests running in only one run!
;;
(define (db:get-count-tests-running dbstruct run-id)
  (let ((res 0))
    (sqlite3:for-each-row
     (lambda (count)
       (set! res count))
     (db:get-db dbstruct run-id)
     ;; WARNING BUG EDIT ME - merged from v1.55 - not sure what is right here ...
     ;; "SELECT count(id) FROM tests WHERE state in ('RUNNING','LAUNCHED','REMOTEHOSTSTART') AND run_id NOT IN (SELECT id FROM runs WHERE state='deleted') AND NOT (uname = 'n/a' AND item_path = '');")
     "SELECT count(id) FROM tests WHERE state in ('RUNNING','LAUNCHED','REMOTEHOSTSTART') AND run_id=?;" 
     run-id) ;; NOT IN (SELECT id FROM runs WHERE state='deleted');")
    res))

;; NEW BEHAVIOR: Count tests running in only one run!
;;
(define (db:get-count-tests-actually-running dbstruct run-id)
  (let ((res 0))
    (sqlite3:for-each-row
     (lambda (count)
       (set! res count))
     (db:get-db dbstruct run-id)
     ;; WARNING BUG EDIT ME - merged from v1.55 - not sure what is right here ...
     ;; "SELECT count(id) FROM tests WHERE state in ('RUNNING','LAUNCHED','REMOTEHOSTSTART') AND run_id NOT IN (SELECT id FROM runs WHERE state='deleted') AND NOT (uname = 'n/a' AND item_path = '');")
     "SELECT count(id) FROM tests WHERE state in ('RUNNING','REMOTEHOSTSTART') AND run_id=?;" 
     run-id) ;; NOT IN (SELECT id FROM runs WHERE state='deleted');")
    res))

;; NEW BEHAVIOR: Look only at single run with run-id
;; 
;; (define (db:get-running-stats dbstruct run-id)
(define (db:get-count-tests-running-for-run-id dbstruct run-id)
  (let ((res 0))
    (sqlite3:for-each-row
     (lambda (count)
       (set! res count))  ;; select * from tests where run_id=1 and uname = 'n/a' and item_path='';
     (db:get-db dbstruct run-id)
     "SELECT count(id) FROM tests WHERE state in ('RUNNING','LAUNCHED','REMOTEHOSTSTART') AND run_id=? AND NOT (uname = 'n/a' AND item_path = '');" run-id)
    res))

 ;; override states to count with list of strings.
;;
(define (db:get-count-tests-running-for-run-id-blah db run-id states)
  (let ((res 0)
	(sqrystr (conc "SELECT count(id) FROM tests WHERE state in ('"
		       (if states








<
|
<
<
|
|
|
|
|
<




<
|
<
<
|
|
|
|
|
<





<
|
<
<
|
|
<








1569
1570
1571
1572
1573
1574
1575

1576


1577
1578
1579
1580
1581

1582
1583
1584
1585

1586


1587
1588
1589
1590
1591

1592
1593
1594
1595
1596

1597


1598
1599

1600
1601
1602
1603
1604
1605
1606

  ;; (db:delay-if-busy)
  (sqlite3:execute db "UPDATE tests SET state=?,status=?,event_time=strftime('%s','now') WHERE run_id=? AND testname=? AND item_path=?;" 
 		   state status run-id test-name item-path))

;; NEW BEHAVIOR: Count tests running in only one run!
;;
(define (db:get-count-tests-running dbstruct run-id)

  (sqlite3:first-result 


   (db:get-db dbstruct run-id)
   ;; WARNING BUG EDIT ME - merged from v1.55 - not sure what is right here ...
   ;; "SELECT count(id) FROM tests WHERE state in ('RUNNING','LAUNCHED','REMOTEHOSTSTART') AND run_id NOT IN (SELECT id FROM runs WHERE state='deleted') AND NOT (uname = 'n/a' AND item_path = '');")
   "SELECT count(id) FROM tests WHERE state in ('RUNNING','LAUNCHED','REMOTEHOSTSTART') AND run_id=?;" 
   run-id))


;; NEW BEHAVIOR: Count tests running in only one run!
;;
(define (db:get-count-tests-actually-running dbstruct run-id)

  (sqlite3:first-result


   (db:get-db dbstruct run-id)
   ;; WARNING BUG EDIT ME - merged from v1.55 - not sure what is right here ...
   ;; "SELECT count(id) FROM tests WHERE state in ('RUNNING','LAUNCHED','REMOTEHOSTSTART') AND run_id NOT IN (SELECT id FROM runs WHERE state='deleted') AND NOT (uname = 'n/a' AND item_path = '');")
   "SELECT count(id) FROM tests WHERE state in ('RUNNING','REMOTEHOSTSTART') AND run_id=?;" 
   run-id)) ;; NOT IN (SELECT id FROM runs WHERE state='deleted');")


;; NEW BEHAVIOR: Look only at single run with run-id
;; 
;; (define (db:get-running-stats dbstruct run-id)
(define (db:get-count-tests-running-for-run-id dbstruct run-id)

  (sqlite3:first-result


   (db:get-db dbstruct run-id)
   "SELECT count(id) FROM tests WHERE state in ('RUNNING','LAUNCHED','REMOTEHOSTSTART') AND run_id=? AND NOT (uname = 'n/a' AND item_path = '');" run-id))


 ;; override states to count with list of strings.
;;
(define (db:get-count-tests-running-for-run-id-blah db run-id states)
  (let ((res 0)
	(sqrystr (conc "SELECT count(id) FROM tests WHERE state in ('"
		       (if states
︙︙
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651

1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676

1677
1678
1679
1680
1681
1682
1683
1684
1685

     ;; "SELECT count(id) FROM tests WHERE state in ('RUNNING','LAUNCHED','REMOTEHOSTSTART') AND run_id=? AND NOT (uname = 'n/a' AND item_path = '');" run-id)
    res))
   
    
(define (db:get-count-tests-running-in-jobgroup dbstruct run-id jobgroup)
  (if (not jobgroup)
      0 ;; 
      (let ((res        0)
	    (testnames '()))
	;; get the testnames
	(sqlite3:for-each-row
	 (lambda (testname)
	   (set! testnames (cons testname testnames)))
	 (db:get-db dbstruct #f)
	 "SELECT testname FROM test_meta WHERE jobgroup=?"
	 jobgroup)
	;; get the jobcount NB// EXTEND THIS TO OPPERATE OVER ALL RUNS?
	(if (not (null? testnames))
	    (sqlite3:for-each-row
	     (lambda (count)
	       (set! res count))
	     (db:get-db dbstruct run-id)
	     (conc "SELECT count(id) FROM tests WHERE state in ('RUNNING','LAUNCHED','REMOTEHOSTSTART') AND testname in ('"
		   (string-intersperse testnames "','")
		   "');")))

             ;; DEBUG FIXME - need to merge this v.155 query correctly   
             ;; AND testname in (SELECT testname FROM test_meta WHERE jobgroup=?)
             ;; AND NOT (uname = 'n/a' AND item_path = '');"
	res)))

;; done with run when:
;;   0 tests in LAUNCHED, NOT_STARTED, REMOTEHOSTSTART, RUNNING
(define (db:estimated-tests-remaining dbstruct run-id)
  (let ((res 0))
    (sqlite3:for-each-row
     (lambda (count)
       (set! res count))
     (db:get-db dbstruct run-id) ;; NB// KILLREQ means the jobs is still probably running
     "SELECT count(id) FROM tests WHERE state in ('LAUNCHED','NOT_STARTED','REMOTEHOSTSTART','RUNNING','KILLREQ');")
    res))

;; map run-id, testname item-path to test-id
(define (db:get-test-id dbstruct run-id testname item-path)
  (let* ((db (db:get-db dbstruct run-id))
	 (res #f))
    (sqlite3:for-each-row
     (lambda (id) ;;  run-id testname state status event-time host cpuload diskfree uname rundir item-path run_duration final_logf comment )
       (set! res id)) ;; (vector id run-id testname state status event-time host cpuload diskfree uname rundir item-path run_duration final_logf comment )))
     (db:get-db dbstruct run-id)
     "SELECT id FROM tests WHERE testname=? AND item_path=?;"

     testname item-path)
    res))

(define db:test-record-fields '("id"           "run_id"        "testname"  "state"      "status"      "event_time"
				"host"         "cpuload"       "diskfree"  "uname"      "rundir"   "item_path"
                                "run_duration" "final_logf" "comment"   "shortdir"))

;; fields *must* be a non-empty list
;;








<
|









|
<
<



|
>



<




<
|
<
<
|
|
<



|
|
<
<
<


>
|
<








1615
1616
1617
1618
1619
1620
1621

1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632


1633
1634
1635
1636
1637
1638
1639
1640

1641
1642
1643
1644

1645


1646
1647

1648
1649
1650
1651
1652



1653
1654
1655
1656

1657
1658
1659
1660
1661
1662
1663

     ;; "SELECT count(id) FROM tests WHERE state in ('RUNNING','LAUNCHED','REMOTEHOSTSTART') AND run_id=? AND NOT (uname = 'n/a' AND item_path = '');" run-id)
    res))
   
    
(define (db:get-count-tests-running-in-jobgroup dbstruct run-id jobgroup)
  (if (not jobgroup)
      0 ;; 

      (let ((testnames '()))
	;; get the testnames
	(sqlite3:for-each-row
	 (lambda (testname)
	   (set! testnames (cons testname testnames)))
	 (db:get-db dbstruct #f)
	 "SELECT testname FROM test_meta WHERE jobgroup=?"
	 jobgroup)
	;; get the jobcount NB// EXTEND THIS TO OPPERATE OVER ALL RUNS?
	(if (not (null? testnames))
	    (sqlite3:first-result


	     (db:get-db dbstruct run-id)
	     (conc "SELECT count(id) FROM tests WHERE state in ('RUNNING','LAUNCHED','REMOTEHOSTSTART') AND testname in ('"
		   (string-intersperse testnames "','")
		   "');"))
	    0))))
             ;; DEBUG FIXME - need to merge this v.155 query correctly   
             ;; AND testname in (SELECT testname FROM test_meta WHERE jobgroup=?)
             ;; AND NOT (uname = 'n/a' AND item_path = '');"


;; done with run when:
;;   0 tests in LAUNCHED, NOT_STARTED, REMOTEHOSTSTART, RUNNING
(define (db:estimated-tests-remaining dbstruct run-id)

  (sqlite3:first-result


   (db:get-db dbstruct run-id) ;; NB// KILLREQ means the jobs is still probably running
   "SELECT count(id) FROM tests WHERE state in ('LAUNCHED','NOT_STARTED','REMOTEHOSTSTART','RUNNING','KILLREQ');"))


;; map run-id, testname item-path to test-id
(define (db:get-test-id dbstruct run-id testname item-path)
  (let* ((db (db:get-db dbstruct run-id)))
    (db:first-result-default



     (db:get-db dbstruct run-id)
     "SELECT id FROM tests WHERE testname=? AND item_path=?;"
     #f ;; the default
     testname item-path)))


(define db:test-record-fields '("id"           "run_id"        "testname"  "state"      "status"      "event_time"
				"host"         "cpuload"       "diskfree"  "uname"      "rundir"   "item_path"
                                "run_duration" "final_logf" "comment"   "shortdir"))

;; fields *must* be a non-empty list
;;
︙︙
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822

1823
1824
1825
1826
1827
1828
1829
1830
1831

       (set! res (apply vector a b)))
     (db:get-db dbstruct run-id)
     (conc "SELECT " db:test-record-qry-selector " FROM tests WHERE testname=? AND item_path=?;")
     test-name item-path)
    res))

(define (db:test-get-rundir-from-test-id dbstruct run-id test-id)
  ;; (db:delay-if-busy)
  (let ((db (db:get-db dbstruct run-id))
	(res #f))
    (sqlite3:for-each-row
     (lambda (tpath)
       (set! res tpath))
     (db:get-db dbstruct run-id)
     "SELECT rundir FROM tests WHERE id=?;"

     test-id)
    res))

;;======================================================================
;; S T E P S
;;======================================================================

(define (db:teststep-set-status! dbstruct run-id test-id teststep-name state-in status-in comment logfile)
  (let ((db (db:get-db dbstruct run-id)))








|
<
<
<
<
<
|
|
>
|
<








1786
1787
1788
1789
1790
1791
1792
1793





1794
1795
1796
1797

1798
1799
1800
1801
1802
1803
1804

       (set! res (apply vector a b)))
     (db:get-db dbstruct run-id)
     (conc "SELECT " db:test-record-qry-selector " FROM tests WHERE testname=? AND item_path=?;")
     test-name item-path)
    res))

(define (db:test-get-rundir-from-test-id dbstruct run-id test-id)
  (db:first-result-default





   (db:get-db dbstruct run-id)
   "SELECT rundir FROM tests WHERE id=?;"
   #f ;; default result
   test-id))


;;======================================================================
;; S T E P S
;;======================================================================

(define (db:teststep-set-status! dbstruct run-id test-id teststep-name state-in status-in comment logfile)
  (let ((db (db:get-db dbstruct run-id)))
︙︙
2410
2411
2412
2413
2414
2415
2416


















2417
2418
2419
2420
2421
2422
2423

		 (set! result (append (if (null? tests) (list waitontest-name) tests) result)))
	     ;; if the test is not found then clearly the waiton is not met...
	     ;; (if (not ever-seen)(set! result (cons waitontest-name result)))))
	     (if (not ever-seen)
		 (set! result (append (if (null? tests)(list waitontest-name) tests) result)))))
	 waitons)
	(delete-duplicates result))))



















;;======================================================================
;; Extract ods file from the db
;;======================================================================

;; NOT REWRITTEN YET!!!!!









>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>








2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414

		 (set! result (append (if (null? tests) (list waitontest-name) tests) result)))
	     ;; if the test is not found then clearly the waiton is not met...
	     ;; (if (not ever-seen)(set! result (cons waitontest-name result)))))
	     (if (not ever-seen)
		 (set! result (append (if (null? tests)(list waitontest-name) tests) result)))))
	 waitons)
	(delete-duplicates result))))

;;======================================================================
;; SQLITE3 HELPERS
;;======================================================================

;; convert to -inline
(define (db:first-result-default db stmt default . params)
  (handle-exceptions
   exn
   (let ((err-status ((condition-property-accessor 'sqlite3 'status #f) exn)))
     ;; check for (exn sqlite3) ((condition-property-accessor 'exn 'message) exn)
     (if (eq? err-status 'done)
	 default
	 (begin
	   (debug:print 0 "ERROR:  query " stmt " failed " ((condition-property-accessor 'exn 'message) exn))
	   (print-call-chain)
	   default)))
   (apply sqlite3:first-result db stmt params)))

;;======================================================================
;; Extract ods file from the db
;;======================================================================

;; NOT REWRITTEN YET!!!!!
︙︙

Added tests/stats.txt version [2a209bca81]. 





























































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77

DB Stats:  a1236d6bf92ec5cb8955f490761b21b0d3eea9d3
========
Cmd                                     Count   TotTime   Avg       
get-count-tests-running-for-run-id         1035    237.0                0.23
get-count-tests-running-in-jobgroup         884    119.0                0.13
get-count-tests-running                     884    169.0                0.19
get-prereqs-not-met                         884    732.0                0.83
get-test-info-by-id                         673    122.0                0.18
get-keys                                    476      1.0                0.00
get-test-id                                 356     42.0                0.12
testmeta-get-record                         203     24.0                0.12
roll-up-pass-fail-counts                    159     39.0                0.25
register-test                               140     30.0                0.21
test-set-rundir-shortdir                    128     98.0                0.77
test-set-status-state                        94     45.0                0.48
find-and-mark-incomplete                     32      0.0                0.00
state-status-msg                             25      4.0                0.16
delete-tests-in-state                        12      4.0                0.33
get-tests-for-run-mindata                     8      0.0                0.00
get-all-run-ids                               5      2.0                0.40
get-run-info                                  4      0.0                0.00
register-run                                  4      5.0                1.25
set-tests-state-status                        4     15.0                3.75
get-tests-for-run                             4     15.0                3.75

# After converting first three functions above to sqlite3:first-result
DB Stats
========
Cmd                                     Count   TotTime   Avg       
get-count-tests-running-for-run-id         1138    179.0                0.16
get-count-tests-running-in-jobgroup         987     91.0                0.09
get-count-tests-running                     987    171.0                0.17
get-prereqs-not-met                         987    892.0                0.90
get-test-info-by-id                         672     95.0                0.14
get-keys                                    476      0.0                0.00
get-test-id                                 355     41.0                0.12
testmeta-get-record                         203     15.0                0.07
roll-up-pass-fail-counts                    159     30.0                0.19
register-test                               140     22.0                0.16
test-set-rundir-shortdir                    128    855.0                6.68
test-set-status-state                        94     20.0                0.21
find-and-mark-incomplete                     36      1.0                0.03
state-status-msg                             24      5.0                0.21
delete-tests-in-state                        12      2.0                0.17
get-tests-for-run-mindata                     9      0.0                0.00
get-all-run-ids                               5      1.0                0.20
register-run                                  4      1.0                0.25
get-tests-for-run                             4     11.0                2.75
get-run-info                                  4      0.0                0.00
set-tests-state-status                        4     17.0                4.25

DB Stats another run, converted one or two non-relevant functions to sqlite3:first-result
========
Cmd                                     Count   TotTime   Avg       
get-count-tests-running-for-run-id          987    157.0                0.16
get-count-tests-running-in-jobgroup         836     79.0                0.09
get-count-tests-running                     836    121.0                0.14
get-prereqs-not-met                         836    513.0                0.61
get-test-info-by-id                         673     85.0                0.13
get-keys                                    476      0.0                0.00
get-test-id                                 356     32.0                0.09
testmeta-get-record                         203     19.0                0.09
roll-up-pass-fail-counts                    159     27.0                0.17
register-test                               140     23.0                0.16
test-set-rundir-shortdir                    128     35.0                0.27
test-set-status-state                        94     20.0                0.21
find-and-mark-incomplete                     40      0.0                0.00
state-status-msg                             25      5.0                0.20
delete-tests-in-state                        12      1.0                0.08
get-tests-for-run-mindata                    10      0.0                0.00
get-all-run-ids                               5      0.0                0.00
set-tests-state-status                        4     15.0                3.75
register-run                                  4      2.0                0.50
get-run-info                                  4      1.0                0.25
get-tests-for-run                             4     12.0                3.00

This page was generated in about 0.14s by Fossil 2.24 [452f8e1ffc] 2024-03-26 15:36:00