Megatest

Check-in [ebea00e4bb]
Login
☰Home Timeline Files Branches Tags Tickets Wiki Login
Overview
Comment:Merged rollup-runs branch into trunk
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: ebea00e4bbe36fd26a8b37983f5f85cff8590ef7
User & Date: mrwellan on 2011-08-24 12:50:12
Other Links: manifest | tags
Context
2011-08-24
13:37
Cleaned up the test steps display a bit more check-in: 55d1298d58 user: mrwellan tags: trunk
12:50
Merged rollup-runs branch into trunk check-in: ebea00e4bb user: mrwellan tags: trunk
12:33
Tweaked test steps reporting in dashboard check-in: fcb61c8998 user: matt tags: trunk
07:27
Removing junk file Closed-Leaf check-in: d14b97e156 user: mrwellan tags: rollup-runs
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Modified configf.scm from [2d522c454c] to [ed3f434b69].

︙
28
29
30
31
32
33
34
35

36
37

38
39

40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60

61
62

63
64
65
66
67
68
69
70
71
72
73
74
75













76
77


78
79
80
81
82
83
84

28
29
30
31
32
33
34

35
36

37
38

39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59

60
61
62
63













64
65
66
67
68
69
70
71
72
73
74
75
76


77
78
79
80
81
82
83
84
85








-
+

-
+

-
+




















-
+


+
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-
+
+









(define (config:assoc-safe-add alist key val)
  (let ((newalist (filter (lambda (x)(not (equal? key (car x)))) alist)))
    (append newalist (list (list key val)))))

;; read a config file, returns two level hierarchial hash-table,
;; adds to ht if given (must be #f otherwise)
(define (read-config path . ht)
(define (read-config path ht allow-system)
  (if (not (file-exists? path))
      (if (null? ht)(make-hash-table) (car ht))
      (if (not ht)(make-hash-table) ht)
      (let ((inp        (open-input-file path))
	    (res        (if (null? ht)(make-hash-table)(car ht)))
	    (res        (if (not ht)(make-hash-table) ht))
	    (include-rx (regexp "^\\[include\\s+(.*)\\]\\s*$"))
	    (section-rx (regexp "^\\[(.*)\\]\\s*$"))
	    (blank-l-rx (regexp "^\\s*$"))
	    (key-sys-pr (regexp "^(\\S+)\\s+\\[system\\s+(\\S+.*)\\]\\s*$"))
	    (key-val-pr (regexp "^(\\S+)\\s+(.*)$"))
	    (comment-rx (regexp "^\\s*#.*"))
	    (cont-ln-rx (regexp "^(\\s+)(\\S+.*)$")))
	(let loop ((inl               (read-line inp))
		   (curr-section-name "default")
		   (var-flag #f);; turn on for key-var-pr and cont-ln-rx, turn off elsewhere
		   (lead     #f))
	  (if (eof-object? inl) 
	      (begin
		(close-input-port inp)
		res)
	      (regex-case 
	       inl 
	       (comment-rx _                  (loop (read-line inp) curr-section-name #f #f))
	       (blank-l-rx _                  (loop (read-line inp) curr-section-name #f #f))
	       (include-rx ( x include-file ) (begin
						(read-config include-file res)
						(read-config include-file res allow-system)
						(loop (read-line inp) curr-section-name #f #f)))
	       (section-rx ( x section-name ) (loop (read-line inp) section-name #f #f))
	       (key-sys-pr ( x key cmd      ) (if allow-system
	       (key-sys-pr ( x key cmd      ) (let ((alist (hash-table-ref/default res curr-section-name '()))
						    (val   (let* ((cmdres  (cmd-run->list cmd))
								  (status  (cadr cmdres))
								  (res     (car  cmdres)))
							     (if (not (eq? status 0))
								 (begin
								   (debug:print 0 "ERROR: problem with " inl ", return code " status)
								   (exit 1)))
							     (if (null? res)
								 ""
								 (string-intersperse res " ")))))
						(hash-table-set! res curr-section-name 
								 (config:assoc-safe-add alist key val))
						  (let ((alist (hash-table-ref/default res curr-section-name '()))
							(val   (let* ((cmdres  (cmd-run->list cmd))
								      (status  (cadr cmdres))
								      (res     (car  cmdres)))
								 (if (not (eq? status 0))
								     (begin
								       (debug:print 0 "ERROR: problem with " inl ", return code " status)
								       (exit 1)))
								 (if (null? res)
								     ""
								     (string-intersperse res " ")))))
						    (hash-table-set! res curr-section-name 
								     (config:assoc-safe-add alist key val))
								 ;; (append alist (list (list key val))))
						(loop (read-line inp) curr-section-name #f #f)))
						    (loop (read-line inp) curr-section-name #f #f))
						  (loop (read-line inp) curr-section-name #f #f)))
	       (key-val-pr ( x key val      ) (let ((alist (hash-table-ref/default res curr-section-name '())))
						(hash-table-set! res curr-section-name 
								 (config:assoc-safe-add alist key val))
						(loop (read-line inp) curr-section-name key #f)))
	       ;; if a continued line
	       (cont-ln-rx ( x whsp val     ) (let ((alist (hash-table-ref/default res curr-section-name '())))
						(if var-flag             ;; if set to a string then we have a continued var
︙
100
101
102
103
104
105
106
107

108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123

124
125
126
127

101
102
103
104
105
106
107

108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123

124
125
126
127
128








-
+















-
+





  
(define (find-and-read-config fname)
  (let* ((curr-dir   (current-directory))
         (configinfo (find-config fname))
	 (toppath    (car configinfo))
	 (configfile (cadr configinfo)))
    (if toppath (change-directory toppath)) 
    (let ((configdat  (if configfile (read-config configfile) #f))) ;; (make-hash-table))))
    (let ((configdat  (if configfile (read-config configfile #f #t) #f))) ;; (make-hash-table))))
      (if toppath (change-directory curr-dir))
      (list configdat toppath configfile fname))))

(define (config-lookup cfgdat section var)
  (let ((sectdat (hash-table-ref/default cfgdat section '())))
    (if (null? sectdat)
	#f
	(let ((match (assoc var sectdat)))
	  (if match
	      (cadr match)
	      #f))
	)))

(define (setup)
  (let* ((configf (find-config))
	 (config  (if configf (read-config configf) #f)))
	 (config  (if configf (read-config configf #f #t) #f)))
    (if config
	(setenv "RUN_AREA_HOME" (pathname-directory configf)))
    config))

Modified db.scm from [04b9f6fc1e] to [aed40706ae].

︙
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74

60
61
62
63
64
65
66

67
68
69
70
71
72
73








-








                     final_logf TEXT DEFAULT 'logs/final.log',
                     logdat     BLOB, 
                     run_duration INTEGER DEFAULT 0,
                     comment    TEXT DEFAULT '',
                     event_time TIMESTAMP,
                     fail_count INTEGER DEFAULT 0,
                     pass_count INTEGER DEFAULT 0,
                     tags       TEXT DEFAULT '',
                     CONSTRAINT testsconstraint UNIQUE (run_id, testname, item_path)
          );")
	  (sqlite3:execute db "CREATE INDEX tests_index ON tests (run_id, testname);")
	  (sqlite3:execute db "CREATE VIEW runs_tests AS SELECT * FROM runs INNER JOIN tests ON runs.id=tests.run_id;")
	  (sqlite3:execute db "CREATE TABLE test_steps 
                              (id INTEGER PRIMARY KEY,
                               test_id INTEGER,
︙
107
108
109
110
111
112
113

114

115
116
117
118
119
120
121
122
123
124












125
126
127
128
129
130
131

106
107
108
109
110
111
112
113

114
115
116
117
118
119
120
121
122
123

124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142








+
-
+









-
+
+
+
+
+
+
+
+
+
+
+
+








	  (print "Adding megatest-version to metadata")
	  (sqlite3:execute db (db:set-var db "MEGATEST_VERSION" megatest-version))))
     ;;      (if (< mver 1.18)
     ;; 	 (begin
     ;; 	   (print "Adding tags column to tests table")
     ;; 	   (sqlite3:execute db "ALTER TABLE tests ADD COLUMN tags TEXT DEFAULT '';")))
     (if (< mver 1.20)
	 (begin
	 (sqlite3:execute db "CREATE TABLE test_meta (id INTEGER PRIMARY KEY,
	   (sqlite3:execute db "CREATE TABLE test_meta (id INTEGER PRIMARY KEY,
                                     testname    TEXT DEFAULT '',
                                     author      TEXT DEFAULT '',
                                     owner       TEXT DEFAULT '',
                                     description TEXT DEFAULT '',
                                     reviewed    TIMESTAMP,
                                     iterated    TEXT DEFAULT '',
                                     avg_runtime REAL,
                                     avg_disk    REAL,
                                     tags        TEXT DEFAULT '',
                                CONSTRAINT test_meta_contstraint UNIQUE (id,testname));"))
                                CONSTRAINT test_meta_contstraint UNIQUE (id,testname));")
	   (for-each 
	    (lambda (stmt)
	      (sqlite3:execute db stmt))
	    (list 
	     "ALTER TABLE tests ADD COLUMN expected_value REAL;" ;; DO NOT Add a default, we want it to be NULL
	     "ALTER TABLE tests ADD COLUMN value REAL;"
	     "ALTER TABLE tests ADD COLUMN tol REAL;"
	     "ALTER TABLE tests ADD COLUMN tol_perc REAL;"
	     "ALTER TABLE tests ADD COLUMN first_err TEXT;"
	     "ALTER TABLE tests ADD COLUMN first_warn TEXT;"
	     ))))
     (if (< mver megatest-version)
	 (db:set-var db "MEGATEST_VERSION" megatest-version)))))

;;======================================================================
;; meta get and set vars
;;======================================================================
︙
197
198
199
200
201
202
203





















204
205
206
207
208
209
210

208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242








+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+








	       "")
	   (if (and (> (length count) 1)
		    (number? (cadr count)))
	       (conc " OFFSET " (cadr count))
	       ""))
     runpatt)
    (vector header res)))

;; replace header and keystr with a call to runs:get-std-run-fields
;; keypatt: '(("key1" "patt1")("key2" "patt2")...)
(define (db:get-runs db keys keypatts runpatt)
  (let* ((res      '())
	 (remfields (list "id" "runname" "state" "status" "owner" "event_time"))
	 (header    (append (map key:get-fieldname keys)
			    remfields))
	 (keystr    (conc (keys->keystr keys) ","
			  (string-intersperse remfields ","))))
    (sqlite3:for-each-row
     (lambda (a . x) ;; turn all the fields returned into a vector and add to the list
       (set! res (cons (apply vector a x) res)))
     db
     (conc "SELECT " keystr " FROM runs WHERE runname LIKE ? "
	   (map (lambda (keypatt)
		  (conc "AND " (car keypatt) " LIKE " (cadr keypatt) " "))
		keypatts)
	   "ORDER BY event_time DESC;")
     runpatt)
    (vector header res)))

;; use this one for db-get-run-info
(define-inline (db:get-row    vec)(vector-ref vec 1))

;; use (get-value-by-header (db:get-header runinfo)(db:get-row runinfo))
(define (db:get-run-info db run-id)
  (let* ((res      #f)
︙
309
310
311
312
313
314
315
316
317


318
319
320
321
322
323
324

341
342
343
344
345
346
347


348
349
350
351
352
353
354
355
356








-
-
+
+








;; done with run when:
;;   0 tests in LAUNCHED, NOT_STARTED, REMOTEHOSTSTART, RUNNING
(define (db:estimated-tests-remaining db run-id)
  (let ((res 0))
    (sqlite3:for-each-row
     (lambda (count)
       (set! res count))
     db
     "SELECT count(id) FROM tests WHERE state in ('LAUNCHED','NOT_STARTED','REMOTEHOSTSTART','RUNNING') AND run_id=?;" run-id)
     db ;; NB// KILLREQ means the jobs is still probably running
     "SELECT count(id) FROM tests WHERE state in ('LAUNCHED','NOT_STARTED','REMOTEHOSTSTART','RUNNING','KILLREQ') AND run_id=?;" run-id)
    res))

;; NB// Sync this with runs:get-test-info
(define (db:get-test-info db run-id testname item-path)
  (let ((res #f))
    (sqlite3:for-each-row
     (lambda (id run-id testname state status event-time host cpuload diskfree uname rundir item-path run_duration final_logf comment)
︙
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454


















455
456
457
458
459
460
461

462
463
464
465
466
467
468


















469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493








-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+








     (lambda (id test-id stepname state status event-time)
       (set! res (cons (vector id test-id stepname state status event-time) res)))
     db
     "SELECT id,test_id,stepname,state,status,event_time FROM test_steps WHERE test_id=? ORDER BY id ASC;" ;; event_time DESC,id ASC;
     test-id)
    (reverse res)))

;; check that *all* the prereqs are "COMPLETED"
(define (db-get-prereqs-met db run-id waiton)
  (let ((res          #f)
	(not-complete 0)
	(tests        (db-get-tests-for-run db run-id)))
    (for-each
     (lambda (test-name)
       (for-each 
	(lambda (test)
	  (if (equal? (db:test-get-testname test) test-name)
	      (begin
		(set! res #t)
		(if (not (equal? (db:test-get-state test) "COMPLETED"))
		    (set! not-complete (+ 1 not-complete))))))
	tests))
     waiton)
    (and (or (null? waiton) res)
	 (eq? not-complete 0))))
;; ;; check that *all* the prereqs are "COMPLETED"
;; (define (db-get-prereqs-met db run-id waiton)
;;   (let ((res          #f)
;; 	(not-complete 0)
;; 	(tests        (db-get-tests-for-run db run-id)))
;;     (for-each
;;      (lambda (test-name)
;;        (for-each 
;; 	(lambda (test)
;; 	  (if (equal? (db:test-get-testname test) test-name)
;; 	      (begin
;; 		(set! res #t)
;; 		(if (not (equal? (db:test-get-state test) "COMPLETED"))
;; 		    (set! not-complete (+ 1 not-complete))))))
;; 	tests))
;;      waiton)
;;     (and (or (null? waiton) res)
;; 	 (eq? not-complete 0))))

;; USE: (lset-difference string=? '("a" "b" "c") '("d" "c" "e" "a"))
;;
;; Return a list of prereqs that were NOT met
;;  Tests (and all items) in waiton list must be "COMPLETED" and "PASS"
(define (db-get-prereqs-not-met db run-id waiton)
  (if (null? waiton)
︙
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498

504
505
506
507
508
509
510



















511








-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-


					(if (not (and (equal? (db:test-get-state test) "COMPLETED")
						      (member (db:test-get-status test) '("PASS" "WARN" "CHECK"))))
					    (set! result (cons waitontest-name result))))))
				tests)
		      (if (not ever-seen)(set! result (cons waitontest-name result)))))
		  waiton)
	(delete-duplicates result))))
;;  
;;  	     ;; subtract from the waiton list the "COMPLETED" tests
;;  	     ;;(completed-tests (filter (lambda (x)
;;  	     ;;   			(equal? (db:test-get-state x) "COMPLETED"))
;;  	     ;;   		      tests))
;;  	     (completed-tests (let ((non-completed (make-hash-table)))
;;  				(for-each (lambda (x)
;;  					    ;; could add check for PASS here
;;  					    (if (not (and (equal? (db:test-get-state x) "COMPLETED")
;;  							  (equal? (db:test-get-status x) "PASS")))
;;  						(hash-table-set! non-completed (db:test-get-testname x) x)))
;;  					    ;; (debug:print 0 "Completed: " (db:test-get-testname x))))
;;  					  tests)
;;  				(filter (lambda (x)
;;  					  (not (hash-table-ref/default non-completed (db:test-get-testname x) #f)))
;;  					tests)))
;;  	     (pre-dep-names   (map db:test-get-testname completed-tests))
;;  	     (result          (lset-difference string=? waiton pre-dep-names)))
;;  	(print "pre-dep-names: " pre-dep-names " waiton: " waiton " result: " result)

Modified launch.scm from [aadffc4da1] to [f0e35ea73c].

︙
143
144
145
146
147
148
149
150

151
152
153
154
155
156
157

143
144
145
146
147
148
149

150
151
152
153
154
155
156
157








-
+








      (set! fullcmd (append launcher (car hosts)(list remote-megatest "-execute" cmdparms))))
     (launcher
      (set! fullcmd (append launcher (list remote-megatest "-execute" cmdparms))))
     (else
      (set! fullcmd (list remote-megatest "-execute" cmdparms))))
    (if (args:get-arg "-xterm")(set! fullcmd (append fullcmd (list "-xterm"))))
    (debug:print 1 "Launching megatest for test " test-name " in " work-area" ...")
    (test-set-status! db run-id test-name "LAUNCHED" "n/a" itemdat) ;; (if launch-results launch-results "FAILED"))
    (test-set-status! db run-id test-name "LAUNCHED" "n/a" itemdat #f #f) ;; (if launch-results launch-results "FAILED"))
    ;; set 
    ;; set pre-launch-env-vars before launching, keep the vars in prevvals and put the envionment back when done
    (let* ((commonprevvals (alist->env-vars
			    (hash-table-ref/default *configdat* "env-override" '())))
	   (testprevvals   (alist->env-vars
			    (hash-table-ref/default test-conf "pre-launch-env-overrides" '())))
	   (miscprevvals   (alist->env-vars ;; consolidate this code with the code in megatest.scm for "-execute"
︙

Modified megatest-version.scm from [ebeb01abfb] to [9a855b99f9]. 

1
2
3


1
2

3



-
+

;; Always use two digit decimal
;; 1.01, 1.02...1.10,1.11 ... 1.99,2.00..
(define megatest-version 1.20)
(define megatest-version 1.21)

Modified megatest.scm from [8f3220a6de] to [3218809c9e].

︙
35
36
37
38
39
40
41








42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61

62
63
64
65
66
67
68

35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77








+
+
+
+
+
+
+
+




















+








  -m comment              : insert a comment for this test

Run data
  :runname                : required, name for this particular test run
  :state                  : required if updating step state; e.g. start, end, completed
  :status                 : required if updating step status; e.g. pass, fail, n/a

Values and record errors and warnings
  -set-values             : update or set values in the megatest db 
  :value                  : value measured
  :expected_value         : value expected
  :tol                    : tolerance |value-expect| <= tol
  :first_err              : record an error message
  :first_warn             : record a warning message

Queries
  -list-runs patt         : list runs matching pattern \"patt\", % is the wildcard
  -testpatt patt          : in list-runs show only these tests, % is the wildcard
  -itempatt patt          : in list-runs show only tests with items that match patt
  -showkeys               : show the keys used in this megatest setup

Misc 
  -force                  : override some checks
  -xterm                  : start an xterm instead of launching the test
  -remove-runs            : remove the data for a run, requires all fields be specified
                            and :runname ,-testpatt and -itempatt
                            and -testpatt
  -keepgoing              : continue running until no jobs are \"LAUNCHED\" or
                            \"NOT_STARTED\"
  -rerun FAIL,WARN...     : re-run if called on a test that previously ran (nullified
                            if -keepgoing is also specified)
  -rebuild-db             : bring the database schema up to date
  -rollup N               : fill run (set by :runname)  with latest test(s) from
                            past N days, requires keys
  -rename-run <runb>      : rename run (set by :runname) to <runb>, requires keys
  -update-meta            : update the tests metadata for all tests

Helpers
  -runstep stepname  ...  : take remaining params as comand and execute as stepname
                            log will be in stepname.log. Best to put command in quotes
  -logpro file            : with -exec apply logpro file to stepname.log, creates
                            stepname.html and sets log to same
                            If using make use stepname_logpro.log as your target
︙
92
93
94
95
96
97
98







99
100
101
102
103
104
105

106
107
108
109
110
111
112
113

114
115
116
117
118
119
120

101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138








+
+
+
+
+
+
+







+








+








			"-runstep"
			"-logpro"
			"-m"
			"-rerun"
			"-days"
			"-rename-run"
			"-to"
			;; values and messages
			":first_err"
			":first_warn"
			":value"
			":expected_value"
			":tol"
			;; misc
			"-debug" ;; for *verbosity* > 2
			) 
		 (list  "-h"
		        "-force"
		        "-xterm"
		        "-showkeys"
		        "-test-status"
			"-set-values"
			"-summarize-items"
		        "-gui"
			"-runall"    ;; run all tests
			"-remove-runs"
			"-keepgoing"
			"-usequeue"
			"-rebuild-db"
			"-rollup"
			"-update-meta"
			"-v" ;; verbose 2, more than normal (normal is 1)
			"-q" ;; quiet 0, errors/warnings only
		       )
		 args:arg-hash
		 0))

(if (args:get-arg "-h")
︙
356
357
358
359
360
361
362
363

364
365
366
367
368
369
370

374
375
376
377
378
379
380

381
382
383
384
385
386
387
388








-
+








	    (set-run-config-vars db run-id)
            ;; environment overrides are done *before* the remaining critical envars.
            (alist->env-vars env-ovrd)
	    (set-megatest-env-vars db run-id)
	    (set-item-env-vars itemdat)
            (save-environment-as-files "megatest")
	    (test-set-meta-info db run-id test-name itemdat)
	    (test-set-status! db run-id test-name "REMOTEHOSTSTART" "n/a" itemdat (args:get-arg "-m"))
	    (test-set-status! db run-id test-name "REMOTEHOSTSTART" "n/a" itemdat (args:get-arg "-m") #f)
	    (if (args:get-arg "-xterm")
		(set! fullrunscript "xterm")
                (if (not (file-execute-access? fullrunscript))
                    (system (conc "chmod ug+x " fullrunscript))))
	    ;; We are about to actually kick off the test
	    ;; so this is a good place to remove the records for 
	    ;; any previous runs
︙
444
445
446
447
448
449
450
451

452
453
454
455
456
457
458

462
463
464
465
466
467
468

469
470
471
472
473
474
475
476








-
+








						     ;;  (test-set-status! db run-id test-name "KILLED"  "FAIL"
						     ;;       	     itemdat (args:get-arg "-m"))
						     ;;  (sqlite3:finalize! db)
						     ;;  (exit 1)))))
						     (begin
						       (debug:print 0 "WARNING: Request received to kill job but problem with process, attempting to kill manager process")
						       (test-set-status! db run-id test-name "KILLED"  "FAIL"
									 itemdat (args:get-arg "-m"))
									 itemdat (args:get-arg "-m") #f)
						       (sqlite3:finalize! db)
						       (exit 1))))
					       ;;     (thread-terminate! job-thread)))
					       (set! kill-tries (+ 1 kill-tries))
					       (mutex-unlock! m)))
					 ;; (handle-exceptions
					       ;;  exn
︙
488
489
490
491
492
493
494
495

496
497
498
499
500
501
502

506
507
508
509
510
511
512

513
514
515
516
517
518
519
520








-
+








		      (test-set-status! db run-id test-name
					(if kill-job? "KILLED" "COMPLETED")
					(if (vector-ref exit-info 1) ;; look at the exit-status
					    (if (and (not kill-job?) 
						     (eq? (vector-ref exit-info 2) 0))
						"PASS"
						"FAIL")
					    "FAIL") itemdat (args:get-arg "-m"))))
					    "FAIL") itemdat (args:get-arg "-m") #f)))
		;; for automated creation of the rollup html file this is a good place...
		(if (not (equal? item-path ""))
		   (tests:summarize-items db run-id test-name #f)) ;; don't force - just update if no
		)
	      (mutex-unlock! m)
	      ;; (exec-results (cmd-run->list fullrunscript)) ;;  (list ">" (conc test-name "-run.log"))))
	      ;; (success      exec-results)) ;; (eq? (cadr exec-results) 0)))
︙
536
537
538
539
540
541
542

543
544
545
546
547
548
549

554
555
556
557
558
559
560
561
562
563
564
565
566
567
568








+








		(exit 6)))
	  (sqlite3:finalize! db)
	  (set! *didsomething* #t))))

(if (or (args:get-arg "-setlog")       ;; since setting up is so costly lets piggyback on -test-status
	(args:get-arg "-set-toplog")
	(args:get-arg "-test-status")
	(args:get-arg "-set-values")
	(args:get-arg "-runstep")
	(args:get-arg "-summarize-items"))
    (if (not (getenv "MT_CMDINFO"))
	(begin
	  (debug:print 0 "ERROR: MT_CMDINFO env var not set, commands -test-status, -runstep and -setlog must be called *inside* a megatest environment!")
	  (exit 5))
	(let* ((startingdir (current-directory))
︙
614
615
616
617
618
619
620
621


622
623

624
625
626
627
628
629
630
631
632


















633
634
635
636
637
638
639

633
634
635
636
637
638
639

640
641
642
643
644









645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669








-
+
+


+
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+








		    (teststep-set-status! db run-id test-name stepname "end" exitstat itemdat (args:get-arg "-m"))
		    (sqlite3:finalize! db)
		    (if (not (eq? exitstat 0))
			(exit 254)) ;; (exit exitstat) doesn't work?!?
		  ;; open the db
		  ;; mark the end of the test
		  )))
	  (if (args:get-arg "-test-status")
	  (if (or (args:get-arg "-test-status")
		  (args:get-arg "-set-values"))
	      (let ((newstatus (cond
				((number? status)       (if (equal? status 0) "PASS" "FAIL"))
				((and (string? status)
				((string->number status)(if (equal? (string->number status) 0) "PASS" "FAIL"))
				(else status))))
		(test-set-status! db run-id test-name state newstatus itemdat (args:get-arg "-m")))
	      (if (and state status)
		  (if (not (args:get-arg "-setlog"))
		      (begin
			(debug:print 0 "ERROR: You must specify :state and :status with every call to -test-status\n" help)
			(sqlite3:finalize! db)
			(exit 6)))))
				      (string->number status))(if (equal? (string->number status) 0) "PASS" "FAIL"))
				(else status)))
		    ;; transfer relevant keys into a hash to be passed to test-set-status!
		    ;; could use an assoc list I guess. 
		    (otherdata (let ((res (make-hash-table)))
				 (for-each (lambda (key)
					     (if (args:get-arg key)
						 (hash-table-set! res key (args:get-arg key))))
					   (list ":value" ":tol" ":expected_value" ":first_err" ":first_warn"))
				 res)))
		(if (and (args:get-arg "-test-status")
			 (or (not state)
			     (not status)))
		    (begin
		      (debug:print 0 "ERROR: You must specify :state and :status with every call to -test-status\n" help)
		      (sqlite3:finalize! db)
		      (exit 6)))
		(test-set-status! db run-id test-name state newstatus itemdat (args:get-arg "-m") otherdata)))
	  (sqlite3:finalize! db)
	  (set! *didsomething* #t))))

(if (args:get-arg "-showkeys")
    (let ((db #f)
	  (keys #f))
      (if (not (setup-for-run))
︙
663
664
665
666
667
668
669
















670
671
672
673
674
675
676

693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722








+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+








	    (debug:print 0 "Failed to setup, exiting") 
	    (exit 1)))
      ;; now can find our db
      (set! db (open-db))
      (patch-db db)
      (sqlite3:finalize! db)
      (set! *didsomething* #t)))

;;======================================================================
;; Update the tests meta data from the testconfig files
;;

(if (args:get-arg "-update-meta")
    (begin
      (if (not (setup-for-run))
	  (begin
	    (debug:print 0 "Failed to setup, exiting") 
	    (exit 1)))
      ;; now can find our db
      (set! db (open-db))
      (runs:update-all-test_meta db)
      (sqlite3:finalize! db)
      (set! *didsomething* #t)))

(if (not *didsomething*)
    (debug:print 0 help))

(if (not (eq? *globalexitstatus* 0))
    (if (or (args:get-arg "-runtests")(args:get-arg "-runall"))
        (begin
︙

Modified runconfig.scm from [1140c67c42] to [4dece87b4e]. 

1
2
3
4
5
6
7
8
9

10
11
12
13
14
15
16

1
2
3
4
5
6
7
8

9
10
11
12
13
14
15
16









-
+








;;======================================================================
;; read a config file, loading only the section pertinent
;; to this run field1val/field2val/field3val ...
;;======================================================================
(define (setup-env-defaults db fname run-id . already-seen)
  (let* ((keys    (get-keys db))
	 (keyvals (get-key-vals db run-id))
	 (thekey  (string-intersperse (map (lambda (x)(if x x "-na-")) keyvals) "/"))
	 (confdat (read-config fname))
	 (confdat (read-config fname #f #f))
	 (whatfound (make-hash-table))
	 (sections (list "default" thekey)))
    (debug:print 4 "Using key=\"" thekey "\"")
    (for-each 
     (lambda (section)
       (let ((section-dat (hash-table-ref/default confdat section #f)))
	 (if section-dat
︙

Modified runs.scm from [a71e17f8a8] to [0a86a73c3e].

︙
71
72
73
74
75
76
77
78

79
80
81
82
83
84

85
86
87
88


89
90
91
92
93
94
95
96
97
98



































99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120


121
122


123
124
125
126
127
128
129

71
72
73
74
75
76
77

78
79
80
81
82
83

84
85
86
87

88
89
90
91
92
93
94





95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149


150
151
152

153
154
155
156
157
158
159
160
161








-
+





-
+



-
+
+





-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+




















-
-
+
+

-
+
+








     (lambda (a . r)
       (set! res (cons (list->vector (cons a r)) res)))
     db 
     (conc "SELECT " keystr " FROM runs WHERE runname like ? " key-patt ";")
     runnamepatt)
    (vector header res)))

(define (register-test db run-id test-name item-path tags)
(define (register-test db run-id test-name item-path)
  (let ((item-paths (if (equal? item-path "")
			(list item-path)
			(list item-path ""))))
    (for-each 
     (lambda (pth)
       (sqlite3:execute db "INSERT OR IGNORE INTO tests (run_id,testname,event_time,item_path,state,status,tags) VALUES (?,?,strftime('%s','now'),?,'NOT_STARTED','n/a',?);" 
       (sqlite3:execute db "INSERT OR IGNORE INTO tests (run_id,testname,event_time,item_path,state,status) VALUES (?,?,strftime('%s','now'),?,'NOT_STARTED','n/a');" 
			run-id 
			test-name
			pth 
			(conc "," (string-intersperse tags ",") ",")))
			;; (conc "," (string-intersperse tags ",") ",")
			))
     item-paths )))

;;  (define db (open-db))
;;  (test-set-status! db 2 "runfirst" "COMPLETED" "PASS" "summer")

(define (test-set-status! db run-id test-name state status itemdat-or-path . comment)
  (let ((item-path (if (string? itemdat-or-path) itemdat-or-path (item-list->path itemdat-or-path))))
    (sqlite3:execute db "UPDATE tests SET state=?,status=?,event_time=strftime('%s','now') WHERE run_id=? AND testname=? AND item_path=?;" 
		     state status run-id test-name item-path)
    (if (and (not (equal? item-path "")) ;; need to update the top test record if PASS or FAIL and this is a subtest
(define (test-set-status! db run-id test-name state status itemdat-or-path comment dat)
  (let ((item-path (if (string? itemdat-or-path) itemdat-or-path (item-list->path itemdat-or-path)))
	(otherdat  (if dat dat (make-hash-table))))
    ;; update the primary record IF state AND status are defined
    (if (and state status)
	(sqlite3:execute db "UPDATE tests SET state=?,status=?,event_time=strftime('%s','now') WHERE run_id=? AND testname=? AND item_path=?;" 
			 state status run-id test-name item-path))
    ;; add metadata (need to do this way to avoid SQL injection issues)
    ;; :value
    (let ((val (hash-table-ref/default otherdat ":value" #f)))
      (if val
	  (sqlite3:execute db "UPDATE tests SET value=? WHERE run_id=? AND testname=? AND item_path=?;" val run-id test-name item-path)))
    ;; :expected_value
    (let ((val (hash-table-ref/default otherdat ":expected_value" #f)))
      (if val
	  (sqlite3:execute db "UPDATE tests SET expected_value=? WHERE run_id=? AND testname=? AND item_path=?;" val run-id test-name item-path)))
    ;; :tol
    (let ((val (hash-table-ref/default otherdat ":tol" #f)))
      (if val
	  (sqlite3:execute db "UPDATE tests SET tol=? WHERE run_id=? AND testname=? AND item_path=?;" val run-id test-name item-path)))
    ;; :first_err
    (let ((val (hash-table-ref/default otherdat ":first_err" #f)))
      (if val
	  (sqlite3:execute db "UPDATE tests SET first_err=? WHERE run_id=? AND testname=? AND item_path=?;" val run-id test-name item-path)))
    ;; :first_warn
    (let ((val (hash-table-ref/default otherdat ":first_warn" #f)))
      (if val
	  (sqlite3:execute db "UPDATE tests SET first_warn=? WHERE run_id=? AND testname=? AND item_path=?;" val run-id test-name item-path)))
    ;; :tol_perc
    (let ((val (hash-table-ref/default otherdat ":tol_perc" #f)))
      (if val
	  (sqlite3:execute db "UPDATE tests SET tol_perc=? WHERE run_id=? AND testname=? AND item_path=?;" val run-id test-name item-path)))

    ;; need to update the top test record if PASS or FAIL and this is a subtest
    (if (and (not (equal? item-path ""))
	     (or (equal? status "PASS")
		 (equal? status "WARN")
		 (equal? status "FAIL")))
	(begin
	  (sqlite3:execute 
	   db
	   "UPDATE tests 
             SET fail_count=(SELECT count(id) FROM tests WHERE run_id=? AND testname=? AND item_path != '' AND status='FAIL'),
                 pass_count=(SELECT count(id) FROM tests WHERE run_id=? AND testname=? AND item_path != '' AND (status='PASS' OR status='WARN'))
             WHERE run_id=? AND testname=? AND item_path='';"
	   run-id test-name run-id test-name run-id test-name)
	  (sqlite3:execute
	   db
	   "UPDATE tests
             SET state=CASE WHEN (SELECT count(id) FROM tests WHERE run_id=? AND testname=? AND item_path != '' AND state in ('RUNNING','NOT_STARTED')) > 0 THEN 
                          'RUNNING'
                       ELSE 'COMPLETED' END,
                status=CASE WHEN fail_count > 0 THEN 'FAIL' WHEN pass_count > 0 AND fail_count=0 THEN 'PASS' ELSE 'UNKNOWN' END
             WHERE run_id=? AND testname=? AND item_path='';"
	   run-id test-name run-id test-name)))
    (if (and (not (null? comment))
	     (car comment))
    (if (and (string? comment)
	     (string-match (regexp "\\S+") comment))
	(sqlite3:execute db "UPDATE tests SET comment=? WHERE run_id=? AND testname=? AND item_path=?;"
			 (car comment) run-id test-name item-path))))
			 (car comment) run-id test-name item-path))
    ))

(define (test-set-log! db run-id test-name itemdat logf) 
  (let ((item-path (item-list->path itemdat)))
    (sqlite3:execute db "UPDATE tests SET final_logf=? WHERE run_id=? AND testname=? AND item_path=?;" 
		     logf run-id test-name item-path)))

(define (test-set-toplog! db run-id test-name logf)
︙
398
399
400
401
402
403
404
405

406
407
408



409


410
411
412
413
414
415
416

430
431
432
433
434
435
436

437
438
439
440
441
442
443

444
445
446
447
448
449
450
451
452








-
+



+
+
+
-
+
+








  (setenv "MT_TEST_NAME" test-name) ;; 
  (setenv "MT_RUNNAME"   (args:get-arg ":runname"))
  (set-megatest-env-vars db run-id) ;; these may be needed by the launching process
  (change-directory *toppath*)
  (let* ((test-path    (conc *toppath* "/tests/" test-name))
	 (test-configf (conc test-path "/testconfig"))
	 (testexists   (and (file-exists? test-configf)(file-read-access? test-configf)))
	 (test-conf    (if testexists (read-config test-configf) (make-hash-table)))
	 (test-conf    (if testexists (read-config test-configf #f #t) (make-hash-table)))
	 (waiton       (let ((w (config-lookup test-conf "requirements" "waiton")))
			 (if (string? w)(string-split w)'())))
	 (tags         (let ((t (config-lookup test-conf "setup" "tags")))
			 ;; we want our tags to be separated by commas and fully delimited by commas
			 ;; so that queries with "like" can tie to the commas at either end of each tag
			 ;; while also allowing the end user to freely use spaces and commas to separate tags
			 (if (string? t)(string-split t ",") '()))))
			 (if (string? t)(string-substitute (regexp "[,\\s]+") "," (conc "," t ",") #t)
			     '()))))
    (if (not testexists)
	(begin
	  (debug:print 0 "ERROR: Can't find config file " test-configf)
	  (exit 2))
	;; put top vars into convenient variables and open the db
	(let* (;; db is always at *toppath*/db/megatest.db
	       (items       (hash-table-ref/default test-conf "items" '()))
︙
426
427
428
429
430
431
432
433
434
435
436
437

438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454

462
463
464
465
466
467
468





469










470
471
472
473
474
475
476








-
-
-
-
-
+
-
-
-
-
-
-
-
-
-
-








	      (begin
		(print "items: ")(pp (item-assoc->item-list items))
		(print "itestable: ")(pp (item-table->item-list itemstable))))
	  (if (args:get-arg "-m")
	      (db:set-comment-for-run db run-id (args:get-arg "-m")))

	  ;; Here is where the test_meta table is best updated
	  (let ((currrecord (db:testmeta-get-record db test-name)))
	    (if (not currrecord)
		(begin
		  (set! currrecord (make-vector 10 #f))
		  (db:testmeta-add-record db test-name)))
	  (runs:update-test_meta db test-name test-conf)
	    (for-each 
	     (lambda (key)
	       (let* ((idx (cadr key))
		      (fld (car  key))
		      (val (config-lookup test-conf "test_meta" fld)))
		 (if (and val (not (equal? (vector-ref currrecord idx) val)))
		     (begin
		       (print "Updating " test-name " " fld " to " val)
		       (db:testmeta-update-field db test-name fld val)))))
	     '(("author" 2)("owner" 3)("description" 4)("reviewed" 5)("tags" 9))))

	  ;; braindead work-around for poorly specified allitems list BUG!!! FIXME
	  (if (null? allitems)(set! allitems '(())))
	  (let loop ((itemdat (car allitems))
		     (tal     (cdr allitems)))
	    ;; (lambda (itemdat) ;;; ((ripeness "overripe") (temperature "cool") (season "summer"))
	    ;; Handle lists of items
︙
465
466
467
468
469
470
471
472

473
474
475
476
477
478
479

487
488
489
490
491
492
493

494
495
496
497
498
499
500
501








-
+








	      (if (runs:can-run-more-tests db)
		  (begin
		    (let loop2 ((ts (db:get-test-info db run-id test-name item-path)) ;; #f)
				(ct 0))
		      (if (and (not ts)
			       (< ct 10))
			  (begin
			    (register-test db run-id test-name item-path tags)
			    (register-test db run-id test-name item-path)
			    (db:test-set-comment db run-id test-name item-path "")
			    (loop2 (db:get-test-info db run-id test-name item-path)
				   (+ ct 1)))
			  (if ts
			      (set! testdat ts)
			      (begin
				(debug:print 0 "WARNING: Couldn't register test " test-name " with item path " item-path ", skipping")
︙
680
681
682
683
684
685
686



































687
688
689

690
691
692
693
694

702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745

746
747
748
749
750
751








+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+


-
+






	    (let* ((keys       (db-get-keys db))
		   (keynames   (map key:get-fieldname keys))
		   (keyvallst  (keys->vallist keys #t)))
	      (proc db keys keynames keyvallst)))
	(sqlite3:finalize! db)
	(set! *didsomething* #t))))

;;======================================================================
;; Rollup runs
;;======================================================================

;; Update the test_meta table for this test
(define (runs:update-test_meta db test-name test-conf)
  (let ((currrecord (db:testmeta-get-record db test-name)))
    (if (not currrecord)
	(begin
	  (set! currrecord (make-vector 10 #f))
	  (db:testmeta-add-record db test-name)))
    (for-each 
     (lambda (key)
       (let* ((idx (cadr key))
	      (fld (car  key))
	      (val (config-lookup test-conf "test_meta" fld)))
	 (if (and val (not (equal? (vector-ref currrecord idx) val)))
	     (begin
	       (print "Updating " test-name " " fld " to " val)
	       (db:testmeta-update-field db test-name fld val)))))
     '(("author" 2)("owner" 3)("description" 4)("reviewed" 5)("tags" 9)))))

;; Update test_meta for all tests
(define (runs:update-all-test_meta db)
  (let ((test-names (get-all-legal-tests)))
    (for-each 
     (lambda (test-name)
       (let* ((test-path    (conc *toppath* "/tests/" test-name))
	      (test-configf (conc test-path "/testconfig"))
	      (testexists   (and (file-exists? test-configf)(file-read-access? test-configf)))
	      ;; read configs with tricks turned off (i.e. no system)
	      (test-conf    (if testexists (read-config test-configf #f #f)(make-hash-table))))
	 (runs:update-test_meta db test-name test-conf)))
     test-names)))
	 
(define (runs:rollup-run db keys keynames keyvallst n)
  (let* ((new-run-id   (register-run db keys))
	 (similar-runs (db:get-similar-runs db keys))
	 (similar-runs (db:get-runs db keys))
	 (tests-n-days (db:get-tests-n-days db similar-runs)))
    (for-each 
     (lambda (test-id)
       (db:rollup-test db run-id test-id))
     tests-n-days)))

Modified tests/tests.scm from [8fcf733fb4] to [197f496658].

︙
8
9
10
11
12
13
14
15
16


17
18

19
20
21
22
23
24
25

8
9
10
11
12
13
14


15
16
17

18
19
20
21
22
23
24
25








-
-
+
+

-
+








(include "../process.scm")
(include "../launch.scm")
(include "../items.scm")
(include "../runs.scm")
(include "../megatest-version.scm")

(define conffile #f)
(test "Read a config" #t (hash-table? (read-config "test.config")))
(test "Read a config that doesn't exist" #t (hash-table? (read-config "nada.config")))
(test "Read a config" #t (hash-table? (read-config "test.config" #f #f)))
(test "Read a config that doesn't exist" #t (hash-table? (read-config "nada.config" #f #f)))

(set! conffile (read-config "test.config"))
(set! conffile (read-config "test.config" #f #f))
(test "Get available diskspace" #t (number? (get-df "./")))
(test "Get best dir" #t (let ((bestdir (get-best-disk conffile)))
			      (or (equal? "./"   bestdir)
				  (equal? "/tmp" bestdir))))
(test "Multiline variable" 4 (length (string-split (config-lookup conffile "metadata" "description") "\n")))

;; db
︙

Modified tests/tests/runfirst/main.sh from [f575e06d2a] to [b033ad6b4d]. 

1
2
3
4
5
6
7
8
9
10
11


1
2
3
4
5
6
7
8
9
10

11











-
+

#!/bin/bash

# megatest -step wasting_time :state start :status n/a -m "This is a test step comment"
# sleep 20
# megatest -step wasting_time :state end :status $?

touch ../I_was_here

$MT_MEGATEST -runstep wasting_time -logpro wasting_time.logpro "sleep 8;echo all done eh?" -m "This is a test step comment"

$MT_MEGATEST -test-status :state COMPLETED :status $? -m "This is a test level comment" -set-toplog the_top_log.html
$MT_MEGATEST -test-status :state COMPLETED :status $? -m "This is a test level comment" -set-toplog the_top_log.html :value 1e6 :expected_value 1.1e6 :tol 100e3

Modified tests/tests/singletest/main.sh from [c23e537bd9] to [e63ffb76fa]. 

1
2
3
4
5
6
7
8
9


1
2
3
4
5
6
7
8

9









-
+

#!/bin/bash

# megatest -step wasting_time :state start :status n/a -m "This is a test step comment"
# sleep 20
# megatest -step wasting_time :state end :status $?

$MT_MEGATEST -runstep wasting_time -logpro wasting_time.logpro "sleep 5;echo all done eh?" -m "This is a test step comment"

$MT_MEGATEST -test-status :state COMPLETED :status $? -m "This is a test level comment" -set-toplog the_top_log.html
$MT_MEGATEST -test-status :state COMPLETED :status $? -m "This is a test level comment" -set-toplog the_top_log.html :first_err "This is the first error"

Modified tests/tests/singletest2/main.sh from [c23e537bd9] to [54d3e4ef49]. 

1
2
3
4
5
6
7
8
9


1
2
3
4
5
6
7
8

9









-
+

#!/bin/bash

# megatest -step wasting_time :state start :status n/a -m "This is a test step comment"
# sleep 20
# megatest -step wasting_time :state end :status $?

$MT_MEGATEST -runstep wasting_time -logpro wasting_time.logpro "sleep 5;echo all done eh?" -m "This is a test step comment"

$MT_MEGATEST -test-status :state COMPLETED :status $? -m "This is a test level comment" -set-toplog the_top_log.html
$MT_MEGATEST -test-status :state COMPLETED :status $? -m "This is a test level comment" -set-toplog the_top_log.html :first_warn "This is the first warning"

This page was generated in about 0.14s by Fossil 2.24 [452f8e1ffc] 2024-03-26 15:36:00