Megatest

Check-in [a7e3370085]
Login
☰Home Timeline Files Branches Tags Tickets Wiki Login
Overview
Comment:Merged recent changes from v1.55 and fixed few related bugs
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | v1.60
Files: files | file ages | folders
SHA1: a7e337008529c3be02559542b948d47b799ffd62
User & Date: matt on 2014-07-22 22:00:01
Other Links: branch diff | manifest | tags
Context
2014-07-23
00:33
Changed some configs for fdktestqa check-in: 68f95981bf user: matt tags: v1.60
2014-07-22
22:00
Merged recent changes from v1.55 and fixed few related bugs check-in: a7e3370085 user: matt tags: v1.60
12:06
Fixed dashboard crash on sort choices check-in: 4ba94f5735 user: mrwellan tags: v1.55, v1.5524
2014-07-15
00:06
Merged recent changes from v1.55 check-in: c2c7cdb91e user: matt tags: v1.60
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Modified dashboard.scm from [d5cde3a2ed] to [be49d754f0].

︙
134
135
136
137
138
139
140
141


142
143
144
145
146
147
148

134
135
136
137
138
139
140

141
142
143
144
145
146
147
148
149








-
+
+








(define *state-ignore-hash*  (make-hash-table))

(define *tests-sort-options* (vector (vector "Sort +a" 'testname   "ASC")
				     (vector "Sort -a" 'testname   "DESC")
				     (vector "Sort +t" 'event_time "ASC")
				     (vector "Sort -t" 'event_time "DESC")
				     (vector "Sort +s" 'statestatus "ASC")
				     (vector "Sort -s" 'statestatus "DESC")))
				     (vector "Sort -s" 'statestatus "DESC")
				     (vector "Sort +a" 'testname   "ASC")))

(define *tests-sort-type-index* '(("+testname" 0)
				  ("-testname" 1)
				  ("+event_time" 2)
				  ("-event_time" 3)
				  ("+statestatus" 4)
				  ("-statestatus" 5)))
︙
475
476
477
478
479
480
481
482

483
484
485
486
487
488
489

476
477
478
479
480
481
482

483
484
485
486
487
488
489
490








-
+








			   (teststatus (db:test-get-status   test))
			   (teststate  (db:test-get-state    test))
			   ;;(teststart  (db:test-get-event_time test))
			   (runtime    (db:test-get-run_duration test))
			   (buttontxt  (cond
					((equal? teststate "COMPLETED") teststatus)
					((and (equal? teststate "NOT_STARTED")
					      (member teststatus '("ZERO_ITEMS" "BLOCKED" "PREQ_FAIL" "PREQ_DISCARDED" "KEEP_TRYING" "TEN_STRIKES")))
					      (member teststatus '("ZERO_ITEMS" "BLOCKED" "PREQ_FAIL" "PREQ_DISCARDED" "TIMED_OUT" "KEEP_TRYING" "TEN_STRIKES")))
					 teststatus)
					(else
					 teststate)))
			   (button     (vector-ref columndat rown))
			   (color      (car (gutils:get-color-for-state-status teststate teststatus)))
			   (curr-color (vector-ref buttondat 1)) ;; (iup:attribute button "BGCOLOR"))
			   (curr-title (vector-ref buttondat 2))) ;; (iup:attribute button "TITLE")))
︙

Added docs/megatest-state-status.dot version [45d0ee8608]. 

















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47

+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

digraph megatest_state_status {
  ranksep=0.05
  // rankdir=LR

node [shape=box,style=filled];

// subgraph cluster_notstarted {
//   label="Not started";

  "NOT_STARTED FAILS" [
  label = "{ NOT_STARTED/FAILS |{ NO_ITEMS |<here> FAIL_PREREQ |<here> FAIL_TIMEOUT }}";
  shape= "record";
  ]

"NOT_STARTED n/a" -> "LAUNCHED n/a" [label=" launch"];
"NOT_STARTED WAIT" -> "LAUNCHED n/a"

  "NOT_STARTED n/a";
  "NOT_STARTED WAIT" [
  label = "{NOT_STARTED WAIT|{ NO_SLOTS | <here> WAIT_PREREQ}}";
  shape = "record";
]

// struct3 [shape=record,label="hello\nworld |{ b |{c|<here> d|e}| f}| g | h"];

  "NOT_STARTED n/a" -> "NOT_STARTED FAILS";
  "NOT_STARTED n/a" -> "NOT_STARTED WAIT";

  "RUNNING" [
     shape="record";
     label="{RUNNING|{n/a|<here> PASS |<here> FAIL}}";
  ]

  "COMPLETED" [
      shape="record";
      label = "{COMPLETED|{PASS | <here> FAIL |<here> CHECK|<here> SKIP}}";
  ]


"RUNNING" -> "COMPLETED";
"RUNNING" -> "INCOMPLETE" [label="test dead for > 24hrs"];


"LAUNCHED n/a" -> "REMOTEHOSTSTART n/a" -> "RUNNING";

}









Modified launch.scm
from [45d25922cf]
to [d2834f7a04].




︙


362
363
364
365
366
367
368
369

370
371
372

373
374
375
376
377
378
379

362
363
364
365
366
367
368

369
370
371

372
373
374
375
376
377
378
379








-
+


-
+








							     (if p-id
								 (begin
								   (debug:print 0 "Killing " (cadr parts) "; kill -9  " p-id)
								   ;; (process-signal pid signal/kill))))) ;; 
								   (system (conc "kill -9 " p-id))))))
							 (car processes)))
						      (system (conc "kill -9 -" pid))
						      (tests:test-set-status! test-id "KILLED"  "FAIL" (args:get-arg "-m") #f)))
						      (tests:test-set-status! test-id "KILLED"  "KILLED" (args:get-arg "-m") #f)))
						   (begin
						     (debug:print 0 "WARNING: Request received to kill job but problem with process, attempting to kill manager process")
;;						     (tests:test-set-status! run-id test-id "KILLED"  "FAIL"
						     (tests:test-set-status! test-id "KILLED"  "KILLED" (args:get-arg "-m") #f)
						     (tests:test-set-status! run-id test-id "KILLED"  "FAIL" (args:get-arg "-m") #f)
						     (exit 1) ;; IS THIS NECESSARY OR WISE???
						     )))
					     (set! kill-tries (+ 1 kill-tries))
					     (mutex-unlock! m)))
				       (if keep-going
					   (begin




︙







Modified lock-queue.scm
from [5c82c36b12]
to [0c7d16446b].




︙


19
20
21
22
23
24
25
26

27
28
29
30
31
32
33







34
35
36



37
38
39
40
41
42
43
44



45
46
47
48

49
50
51
52
53
54
55

19
20
21
22
23
24
25

26
27
28
29
30
31
32
33
34
35
36
37
38
39
40



41
42
43
44
45
46
47
48



49
50
51
52
53
54

55
56
57
58
59
60
61
62








-
+







+
+
+
+
+
+
+
-
-
-
+
+
+





-
-
-
+
+
+



-
+








(declare (uses common))

;;======================================================================
;; attempt to prevent overlapping updates of rollup files by queueing
;; update requests in an sqlite db
;;======================================================================

(define (lock-queue:open-db fname)
(define (lock-queue:open-db fname #!key (count 10))
  (let* ((actualfname (conc fname ".lockdb"))
	 (dbexists (file-exists? actualfname))
	 (db       (sqlite3:open-database actualfname))
	 (handler  (make-busy-timeout 136000)))
    (if dbexists
	db
	(begin
	  (handle-exceptions
	   exn
	   (begin
	     (thread-sleep! 10)
	     (if (> count 0)
		 (lock-queue:open-db fname count: (- count 1))
		 db))
	  (sqlite3:execute 
	   db
	   "CREATE TABLE IF NOT EXISTS queue (
	   (sqlite3:execute 
	    db
	    "CREATE TABLE IF NOT EXISTS queue (
  	      id         INTEGER PRIMARY KEY,
              test_id    INTEGER,
              start_time INTEGER,
              state      TEXT,
              CONSTRAINT queue_constraint UNIQUE (test_id));")
	  (sqlite3:execute
	   db
	   "CREATE TABLE IF NOT EXISTS runlocks (
	   (sqlite3:execute
	    db
	    "CREATE TABLE IF NOT EXISTS runlocks (
              id         INTEGER PRIMARY KEY,
              test_id    INTEGER,
              run_lock   TEXT,
              CONSTRAINT runlock_constraint UNIQUE (run_lock));")))
              CONSTRAINT runlock_constraint UNIQUE (run_lock));"))))
    (sqlite3:set-busy-handler! db handler)
    db))

(define (lock-queue:set-state db test-id newstate #!key (remtries 10))
  (handle-exceptions
   exn
   (if (> remtries 0)




︙


81
82
83
84
85
86
87
88

89
90
91
92
93
94




95

96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114

115







116
117


118
119
120









121
122
123
124
125
126
127

128
129







130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150





















151
152
153
154

88
89
90
91
92
93
94

95
96
97
98
99
100
101
102
103
104
105

106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124

125
126
127
128
129
130
131
132
133


134
135
136


137
138
139
140
141
142
143
144
145
146
147
148
149
150
151

152
153
154
155
156
157
158
159
160
161





















162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186








-
+






+
+
+
+
-
+


















-
+

+
+
+
+
+
+
+
-
-
+
+

-
-
+
+
+
+
+
+
+
+
+






-
+


+
+
+
+
+
+
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+





	;; Actually this should not be needed as mystart cannot be simultaneously less than and test-id same as 
	(if (not (equal? tid test-id)) 
	    (set! res tid)))
      db
      "SELECT test_id FROM queue WHERE start_time > ?;" mystart)
     res)))

(define (lock-queue:get-lock db test-id)
(define (lock-queue:get-lock db test-id #!key (count 10))
  (let ((res       #f)
	(lckqry    (sqlite3:prepare db "SELECT test_id,run_lock FROM runlocks WHERE run_lock='locked';"))
	(mklckqry  (sqlite3:prepare db "INSERT INTO runlocks (test_id,run_lock) VALUES (?,'locked');")))
    (let ((result 
	   (handle-exceptions
	    exn
	    (begin
	      (thread-sleep! 10)
	      (if (> count 0)
		  (lock-queue:get-lock db test-id count: (- count 1)))
	    #f
	      #f)
	    (sqlite3:with-transaction
	     db
	     (lambda ()
	       (sqlite3:for-each-row (lambda (tid lockstate)
				       (set! res (list tid lockstate)))
				     lckqry)
	       (if res
		   (if (equal? (car res) test-id)
		       #t ;; already have the lock
		       #f)
		   (begin
		     (sqlite3:execute mklckqry test-id)
		     ;; if no error handled then return #t for got the lock
		     #t)))))))
      (sqlite3:finalize! lckqry)
      (sqlite3:finalize! mklckqry)
      result)))

(define (lock-queue:release-lock fname test-id)
(define (lock-queue:release-lock fname test-id #!key (count 10))
  (let ((db (lock-queue:open-db fname)))
    (handle-exceptions
     exn
     (begin
       (thread-sleep! 10)
       (if (> count 0)
	   (lock-queue:release-lock fname test-id count: (- count 1))
	   #f))
    (sqlite3:execute db "DELETE FROM runlocks WHERE test_id=?;" test-id)
    (sqlite3:finalize! db)))
     (sqlite3:execute db "DELETE FROM runlocks WHERE test_id=?;" test-id)
     (sqlite3:finalize! db))))

(define (lock-queue:steal-lock db test-id)
  (sqlite3:execute db "DELETE FROM runlocks WHERE run_lock='locked';")
(define (lock-queue:steal-lock db test-id #!key (count 10))
  (handle-exceptions
   exn
   (begin
     (thread-sleep! 10)
     (if (> count 0)
	 (lock-queue:steal-lock db test-id count: (- count 1))
	 #f))
   (sqlite3:execute db "DELETE FROM runlocks WHERE run_lock='locked';"))
  (lock-queue:get-lock db test-it))

;; returns #f if ok to skip the task
;; returns #t if ok to proceed with task
;; otherwise waits
;;
(define (lock-queue:wait-turn fname test-id)
(define (lock-queue:wait-turn fname test-id #!key (count 10))
  (let ((db      (lock-queue:open-db fname))
	(mystart (current-seconds)))
    (handle-exceptions
     exn
     (begin
       (thread-sleep! 10)
       (if (> count 0)
	   (lock-queue:wait-turn fname test-id count: (- count 1))
	   #f))
    (sqlite3:execute
     db
     "INSERT OR REPLACE INTO queue (test_id,start_time,state) VALUES (?,?,'waiting');"
     test-id mystart)
    (thread-sleep! 1) ;; give other tests a chance to register
    (let ((result 
	   (let loop ((younger-waiting (lock-queue:any-younger? db mystart test-id)))
	     (if younger-waiting
		 (begin
		   ;; no need for us to wait. mark in the lock queue db as skipping
		   (lock-queue:set-state db test-id "skipping")
		   #f) ;; let the calling process know that nothing needs to be done
		 (if (lock-queue:get-lock db test-id)
		     #t
		     (if (> (- (current-seconds) mystart) 36000) ;; waited too long, steal the lock
			 (lock-queue:steal-lock db test-id)
			 (begin
			   (thread-sleep! 1)
			   (loop (lock-queue:any-younger? db mystart test-id)))))))))
      (sqlite3:finalize! db)
      result)))
     (sqlite3:execute
      db
      "INSERT OR REPLACE INTO queue (test_id,start_time,state) VALUES (?,?,'waiting');"
      test-id mystart)
     (thread-sleep! 1) ;; give other tests a chance to register
     (let ((result 
	    (let loop ((younger-waiting (lock-queue:any-younger? db mystart test-id)))
	      (if younger-waiting
		  (begin
		    ;; no need for us to wait. mark in the lock queue db as skipping
		    (lock-queue:set-state db test-id "skipping")
		    #f) ;; let the calling process know that nothing needs to be done
		  (if (lock-queue:get-lock db test-id)
		      #t
		      (if (> (- (current-seconds) mystart) 36000) ;; waited too long, steal the lock
			  (lock-queue:steal-lock db test-id)
			  (begin
			    (thread-sleep! 1)
			    (loop (lock-queue:any-younger? db mystart test-id)))))))))
       (sqlite3:finalize! db)
       result))))
	  
            
;; (use trace)
;; (trace lock-queue:get-lock lock-queue:release-lock lock-queue:wait-turn lock-queue:any-younger? lock-queue:set-state)








Modified runs.scm
from [d806e46083]
to [4d9e155bfe].




︙


389
390
391
392
393
394
395

396


397
398
399
400
401

402
403
404
405
406
407
408

389
390
391
392
393
394
395
396

397
398
399
400
401
402
403
404
405
406
407
408
409
410
411








+
-
+
+





+








(define runs:nothing-left-in-queue-count 0)

(define (runs:expand-items hed tal reg reruns regfull newtal jobgroup max-concurrent-jobs run-id waitons item-path testmode test-record can-run-more items runname tconfig reglen test-registry test-records itemmap)
  (let* ((loop-list       (list hed tal reg reruns))
	 (prereqs-not-met (rmt:get-prereqs-not-met run-id waitons item-path testmode itemmap: itemmap))
	 ;; (prereqs-not-met (mt:lazy-get-prereqs-not-met run-id waitons item-path mode: testmode itemmap: itemmap))
	 (fails           (runs:calc-fails prereqs-not-met))
	 (prereq-fails    (runs:calc-prereq-fail prereqs-not-met))
	 (non-completed   (runs:calc-not-completed prereqs-not-met)))
	 (non-completed   (runs:calc-not-completed prereqs-not-met))
	 (runnables       (runs:calc-runnable prereqs-not-met)))
    (debug:print-info 4 "START OF INNER COND #2 "
		      "\n can-run-more:    " can-run-more
		      "\n testname:        " hed
		      "\n prereqs-not-met: " (runs:pretty-string prereqs-not-met)
		      "\n non-completed:   " (runs:pretty-string non-completed) 
		      "\n prereq-fails:    " (runs:pretty-string prereq-fails)
		      "\n fails:           " (runs:pretty-string fails)
		      "\n testmode:        " testmode
		      "\n (member 'toplevel testmode): " (member 'toplevel testmode)
		      "\n (null? non-completed):    " (null? non-completed)
		      "\n reruns:          " reruns
		      "\n items:           " items
		      "\n can-run-more:    " can-run-more)




︙


451
452
453
454
455
456
457

458
459
460
461
462
463
464

454
455
456
457
458
459
460
461
462
463
464
465
466
467
468








+








		(tests:testqueue-set-items! test-record items-list)
		(list hed tal reg reruns))
	      (begin
		(debug:print 0 "ERROR: The proc from reading the items table did not yield a list - please report this")
		(exit 1))))))

     ((and (null? fails)
	   (null? prereq-fails)
	   (not (null? non-completed)))
      (let* ((allinqueue (map (lambda (x)(if (string? x) x (db:test-get-testname x)))
        		      (append newtal reruns)))
	     ;; prereqstrs is a list of test names as strings that are prereqs for hed
             (prereqstrs (delete-duplicates (map (lambda (x)(if (string? x) x (db:test-get-testname x)))
						 prereqs-not-met)))
	     ;; a prereq that is not found in allinqueue will be put in the notinqueue list




︙


491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552

553
554
555
556
557
558
559
560
561
562
563
564

565
566
567
568
569



570

571
572
573
574


575

576
577
578
579
580
581
582
583
584
585
586
587
588

589
590
591
592
593
594
595

495
496
497
498
499
500
501






















































502
503
504
505
506
507
508
509
510
511
512
513
514

515
516
517
518
519
520
521
522
523

524
525
526
527
528
529
530

531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552








-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-

+











-
+





+
+
+
-
+




+
+
-
+













+








		  #f
		  (list (runs:queue-next-hed trimmed-tal trimmed-reg reglen regfull)
			(runs:queue-next-tal trimmed-tal trimmed-reg reglen regfull)
			(runs:queue-next-reg trimmed-tal trimmed-reg reglen regfull)
			reruns)))
	      (list (car newtal)(append (cdr newtal) reg) '() reruns))))

     ;; (debug:print-info 1 "allinqueue: " allinqueue)
     ;; (debug:print-info 1 "prereqstrs: " prereqstrs)
     ;; (debug:print-info 1 "notinqueue: " notinqueue)
     ;; (debug:print-info 1 "tal:        " tal)
     ;; (debug:print-info 1 "newtal:     " newtal)
     ;; (debug:print-info 1 "reg:        " reg)

;; == ==       ;; num-retries code was here
;; == ==       ;; we use this opportunity to move contents of reg to tal
;; == ==       ;; but also lets check that the prerequisites are all in the newtal or reruns lists
;; == == 
;; == ==       (let* ((allinqueue (map (lambda (x)(if (string? x) x (db:test-get-testname x)))
;; == ==         		      (append newtal reruns)))
;; == == 	     ;; prereqstrs is a list of test names as strings that are prereqs for hed
;; == ==              (prereqstrs (map (lambda (x)(if (string? x) x (db:test-get-testname x)))
;; == ==         		      prereqs-not-met))
;; == == 	     ;; a prereq that is not found in allinqueue will be put in the notinqueue list
;; == == 	     ;; 
;; == ==              (notinqueue (filter (lambda (x)
;; == ==         			   (not (member x allinqueue)))
;; == ==         			 prereqstrs)))
;; == ==         (if (not (null? notinqueue))
;; == ==             (if (runs:can-keep-running? hed 5) ;; try five times
;; == ==         	(begin
;; == == 		  (debug:print-info 4 "increment cant-run-tests for " hed)
;; == ==         	  (runs:inc-cant-run-tests hed)
;; == ==         	  (list (car newtal)(append (cdr newtal) reg) '() reruns))
;; == ==         	(begin
;; == == 		  
;; == == 		  (if (runs:lownoise (conc "no fails prereq, null notinqueue " hed) 30)
;; == == 		      (begin
;; == == 			(debug:print 1 "WARNING: test " hed " has no failed prerequisites but does have prerequistes that are NOT in the queue: " (string-intersperse notinqueue ", "))
;; == == 			(debug:print-info 4 "allinqueue: " allinqueue)
;; == == 			(debug:print-info 4 "prereqstrs: " prereqstrs)
;; == == 			(debug:print-info 4 "notinqueue: " notinqueue)))
;; == == 		  (if (and (null? tal)(null? reg))
;; == == 		      (list (car newtal)(append (cdr newtal) reg) '() reruns)
;; == == 		      (list (runs:queue-next-hed tal reg reglen regfull)
;; == == 			    (runs:queue-next-tal tal reg reglen regfull)
;; == == 			    (runs:queue-next-reg tal reg reglen regfull)
;; == == 			    reruns))))
;; == == 	    ;; have prereqs in queue, keep going.
;; == == 	    (begin
;; == == 	      (if (runs:lownoise (conc "no fails prereq " hed) 30)
;; == == 		  (debug:print-info 1 "no fails in prerequisites for " hed ", waiting on tests; "
;; == == 				    (string-intersperse (map (lambda (x)
;; == == 							       (if (string? x)
;; == == 								   x
;; == == 								   (runs:make-full-test-name (db:test-get-testname x)
;; == == 											     (db:test-get-item-path x))))
;; == == 							     non-completed) ", ")
;; == == 				    ". Delaying launch of " hed "."))
;; == == 	      (list (car newtal)(append (cdr newtal) reg) '() reruns))))) ;; an issue with prereqs not yet met?

     ((and (null? fails)
	   (null? prereq-fails)
	   (null? non-completed))
      (if  (runs:can-keep-running? hed 5)
	  (begin
	    (runs:inc-cant-run-tests hed)
	    (debug:print-info 1 "no fails in prerequisites for " hed " but also none running, keeping " hed " for now. Try count: " (hash-table-ref/default *seen-cant-run-tests* hed 0))
	    ;; num-retries code was here
	    ;; we use this opportunity to move contents of reg to tal
	    (list (car newtal)(append (cdr newtal) reg) '() reruns)) ;; an issue with prereqs not yet met?
	  (begin
	    (debug:print-info 1 "no fails in prerequisites for " hed " but nothing seen running in a while, dropping test " hed " from the run queue")
	    (let ((test-id (rmt:get-test-id run-id hed "")))
	      (mt:test-set-state-status-by-id run-id test-id "DEQUEDED" "TIMED_OUT" "Nothing seen running in a while."))
	      (mt:test-set-state-status-by-id test-id "NOT_STARTED" "TIMED_OUT" "Nothing seen running in a while."))
	    (list (runs:queue-next-hed tal reg reglen regfull)
		  (runs:queue-next-tal tal reg reglen regfull)
		  (runs:queue-next-reg tal reg reglen regfull)
		  reruns))))

     ((and 
       (or (not (null? fails))
	   (not (null? prereq-fails)))
     ((and (not (null? fails))(member 'normal testmode))
       (member 'normal testmode))
      (debug:print-info 1 "test "  hed " (mode=" testmode ") has failed prerequisite(s); "
			(string-intersperse (map (lambda (t)(conc (db:test-get-testname t) ":" (db:test-get-state t)"/"(db:test-get-status t))) fails) ", ")
			", removing it from to-do list")
      (let ((test-id (rmt:get-test-id run-id hed "")))
	(if (not (null? prereq-fails))
	    (mt:test-set-state-status-by-id run-id test-id "NOT_STARTED" "PREQ_DISCARDED" "Failed to run due to prior failed prerequisites")
	(mt:test-set-state-status-by-id run-id test-id "NOT_STARTED" "PREQ_FAIL" "Failed to run due to failed prerequisites"))
	    (mt:test-set-state-status-by-id run-id test-id "NOT_STARTED" "PREQ_FAIL"      "Failed to run due to failed prerequisites")))
      (if (or (not (null? reg))(not (null? tal)))
	  (begin
	    (hash-table-set! test-registry hed 'CANNOTRUN)
	    (list (runs:queue-next-hed tal reg reglen regfull)
		  (runs:queue-next-tal tal reg reglen regfull)
		  (runs:queue-next-reg tal reg reglen regfull)
		  (cons hed reruns)))
	  #f)) ;; #f flags do not loop

     ((and (not (null? fails))(member 'toplevel testmode))
      (if (or (not (null? reg))(not (null? tal)))
	   (list (car newtal)(append (cdr newtal) reg) '() reruns)
	  #f)) 
     ((null? runnables) #f) ;; if we get here and non-completed is null the it's all over.
     (else
      (debug:print 0 "WARNING: FAILS or incomplete tests maybe preventing completion of this run. Watch for issues with test " hed ", continuing for now")
      ;; (list (runs:queue-next-hed tal reg reglen regfull)
      ;;   	(runs:queue-next-tal tal reg reglen regfull)
      ;;   	(runs:queue-next-reg tal reg reglen regfull)
      ;;   	reruns)
      (list (car newtal)(cdr newtal) reg reruns)))))




︙


836
837
838
839
840
841
842
843

844
845
846
847
848
849
850

793
794
795
796
797
798
799

800
801
802
803
804
805
806
807








-
+








	    (if (not (vector? t))
		t
		(let ((state  (db:test-get-state t))
		      (status (db:test-get-status t)))
		  (case (string->symbol state)
		    ((COMPLETED) #f)
		    ((NOT_STARTED)
		     (if (member status '("TEN_STRIKES" "BLOCKED"))
		     (if (member status '("TEN_STRIKES" "BLOCKED" "PREQ_FAIL" "ZERO_ITEMS" "PREQ_DISCARDED" "TIMED_OUT" ))
			 #f
			 t))
		    ((DELETED) #f)
		    (else t)))))
	  tests))

;; every time though the loop increment the test/itempatt val.




︙


1087
1088
1089
1090
1091
1092
1093















1094
1095
1096
1097
1098
1099
1100









1101
1102
1103
1104
1105
1106
1107

1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088








+
+
+
+
+
+
+
+
+
+
+
+
+
+
+







+
+
+
+
+
+
+
+
+








(define (runs:calc-fails prereqs-not-met)
  (filter (lambda (test)
	    (and (vector? test) ;; not (string? test))
		 (equal? (db:test-get-state test) "COMPLETED")
		 (not (member (db:test-get-status test)
			      '("PASS" "WARN" "CHECK" "WAIVED" "SKIP")))))
	  prereqs-not-met))

(define (runs:calc-prereq-fail prereqs-not-met)
  (filter (lambda (test)
	    (and (vector? test) ;; not (string? test))
		 (equal? (db:test-get-state test) "NOT_STARTED")
		 (not (member (db:test-get-status test)
			      '("n/a" "KEEP_TRYING")))))
	  prereqs-not-met))

(define (runs:calc-not-completed prereqs-not-met)
  (filter
   (lambda (t)
     (or (not (vector? t))
	 (not (equal? "COMPLETED" (db:test-get-state t)))))
   prereqs-not-met))

(define (runs:calc-not-completed prereqs-not-met)
  (filter
   (lambda (t)
     (or (not (vector? t))
	 (not (equal? "COMPLETED" (db:test-get-state t)))))
   prereqs-not-met))

(define (runs:calc-runnable prereqs-not-met)
  (filter 
   (lambda (t)
     (or (not (vector? t))
	 (and (equal? "NOT_STARTED" (db:test-get-state t))
	      (member (db:test-get-status t)
			      '("n/a" "KEEP_TRYING")))))
   prereqs-not-met))

(define (runs:pretty-string lst)
  (map (lambda (t)
	 (if (not (vector? t))
	     (conc t)
	     (conc (db:test-get-testname t) ":" (db:test-get-state t) "/" (db:test-get-status t))))
       lst))




︙


1190
1191
1192
1193
1194
1195
1196
1197

1198
1199
1200
1201
1202
1203

1204
1205
1206
1207
1208
1209
1210

1171
1172
1173
1174
1175
1176
1177

1178
1179
1180
1181
1182
1183

1184
1185
1186
1187
1188
1189
1190
1191








-
+





-
+








      (case (if force ;; (args:get-arg "-force")
		'NOT_STARTED
		(if testdat
		    (string->symbol (test:get-state testdat))
		    'failed-to-insert))
	((failed-to-insert)
	 (debug:print 0 "ERROR: Failed to insert the record into the db"))
	((NOT_STARTED COMPLETED DELETED)
	((NOT_STARTED COMPLETED DELETED INCOMPLETE)
	 (let ((runflag #f))
	   (cond
	    ;; -force, run no matter what
	    (force (set! runflag #t))
	    ;; NOT_STARTED, run no matter what
	    ((member (test:get-state testdat) '("DELETED" "NOT_STARTED"))(set! runflag #t))
	    ((member (test:get-state testdat) '("DELETED" "NOT_STARTED" "INCOMPLETE"))(set! runflag #t))
	    ;; not -rerun and PASS, WARN or CHECK, do no run
	    ((and (or (not rerun)
		      keepgoing)
		  ;; Require to force re-run for COMPLETED or *anything* + PASS,WARN or CHECK
		  (or (member (test:get-status testdat) '("PASS" "WARN" "CHECK" "SKIP" "WAIVED"))
		      (member (test:get-state  testdat) '("COMPLETED")))) 
	     (debug:print-info 2 "running test " test-name "/" item-path " suppressed as it is " (test:get-state testdat) " and " (test:get-status testdat))




︙


1479
1480
1481
1482
1483
1484
1485
1486

1487
1488
1489
1490
1491
1492
1493

1460
1461
1462
1463
1464
1465
1466

1467
1468
1469
1470
1471
1472
1473
1474








-
+









(define (runs:remove-test-directory db test remove-data-only)
  (let* ((run-dir       (db:test-get-rundir test))    ;; run dir is from the link tree
	 (real-dir      (if (file-exists? run-dir)
			    (resolve-pathname run-dir)
			    #f)))
    (if (not remove-data-only)
	(mt:test-set-state-status-by-id (db:test-get-run-id test)(db:test-get-id test) "REMOVING" "LOCKED" #f))
	(mt:test-set-state-status-by-id (db:test-get-run_id test)(db:test-get-id test) "REMOVING" "LOCKED" #f))
    (debug:print-info 1 "Attempting to remove " (if real-dir (conc " dir " real-dir " and ") "") " link " run-dir)
    (if (and real-dir 
	     (> (string-length real-dir) 5)
	     (file-exists? real-dir)) ;; bad heuristic but should prevent /tmp /home etc.
	(begin ;; let* ((realpath (resolve-pathname run-dir)))
	  (debug:print-info 1 "Recursively removing " real-dir)
	  (if (file-exists? real-dir)




︙







Modified tests/Makefile
from [e2f7a8e3c3]
to [f7e02bc2a7].




︙


81
82
83
84
85
86
87

88
89
90
91
92
93
94

81
82
83
84
85
86
87
88
89
90
91
92
93
94
95








+








test6: fullprep
	cd fullrun;$(MEGATEST) -preclean -runtests runfirst -testpatt %/1 -reqtarg ubuntu/nfs/none :runname $(RUNNAME)_itempatt -v
	cd fullrun;$(MEGATEST) -preclean -runtests runfirst -testpatt %blahha% -reqtarg ubuntu/nfs/none :runname $(RUNNAME)_itempatt -debug 10
	cd fullrun;$(MEGATEST) -rollup :runname newrun -target ubuntu/nfs/none -debug 10

test7: 
	@echo Only a/c testname c should remain. If there is a run a/b/c then there is a cache issue.
	cd simplerun;$(DASHBOARD) &
	(cd simplerun; \
	 $(MEGATEST) -server - -daemonize; \
         $(MEGATEST) -remove-runs -target %/% :runname % -testpatt %; \
         $(MEGATEST) -preclean -runtests %  -target a/b :runname c; sleep 5; \
	 $(MEGATEST) -remove-runs -target a/c :runname c; \
	 $(MEGATEST) -preclean -runtests %  -target a/c :runname c; \
	 $(MEGATEST) -remove-runs -target a/b :runname c -testpatt % ; \




︙


108
109
110
111
112
113
114

115
116
117
118
119
120
121

109
110
111
112
113
114
115
116
117
118
119
120
121
122
123








+









# Some simple checks for bootstrapping and run loop logic 

test9 : minsetup test9a test9b test9c test9d test9e

test9a :
	@echo Run super-simple mintest e, no waitons.
	cd mintest;$(DASHBOARD)&
	cd mintest;$(MEGATEST) -preclean -runtests e -target $(VER) :runname `date +%H.%M.%S` -debug $(DEBUG)

test9b :
	@echo Run simple mintest d with one waiton c
	cd mintest;$(MEGATEST) -preclean -runtests d -target $(VER) :runname `date +%H.%M.%S` -debug $(DEBUG)

test9c :




︙







Modified tests/fdktestqa/fdk.config
from [3481fe6c37]
to [6c0fedec41].



1
2
3
4
5
6

7
8
9
10
11
12

1
2
3
4
5
6
7
8
9
10
11
12
13







+







[fields]
SYSTEM TEXT
RELEASE TEXT

[setup]
# Adjust max_concurrent_jobs to limit how much you load your machines
# max_concurrent_jobs 150
max_concurrent_jobs 500

# This is your link path, you can move it but it is generally better to keep it stable
linktree #{shell readlink -f #{getenv PWD}/../simplelinks}

[include testqa/configs/megatest.abc.config]








Modified tests/fullrun/megatest.config
from [9e6357a013]
to [232c6802d7].




︙


107
108
109
110
111
112
113

114
115
116
117
118
119
120

107
108
109
110
111
112
113
114
115
116
117
118
119
120
121








+








# The empty var should have a definition with null string
EMPTY_VAR

WRAPPEDVAR This var should have the work blah thrice: \
blah \
blah

MAX_ALLOWED_LOAD 200
# XTERM   [system xterm]
# RUNDEAD [system exit 56]

[server]

# If the server can't be started on this port it will try the next port until
# it succeeds




︙







Modified utils/loadrunner
from [9ef382563c]
to [18bd8142d3].




︙


12
13
14
15
16
17
18
19
20
21
22




23
24
25
26
27
28
29

12
13
14
15
16
17
18




19
20
21
22
23
24
25
26
27
28
29








-
-
-
-
+
+
+
+








#
function launchjob () {
  # Can't always trust $PWD
  CURRWD=`pwd`
  if [[ $TARGETHOST_LOGF == "" ]]; then
      TARGETHOST_LOGF=NBFAKE-`date +%GWW%V.%u_%T`
  fi
  echo "#======================================================================"
  echo "# NBFAKE Running command:"
  echo "#     \"$*\""
  echo "#======================================================================"
  # echo "#======================================================================"
  # echo "# NBFAKE Running command:"
  # echo "#     \"$*\""
  # echo "#======================================================================"
  
  if [[ $TARGETHOST == ""  ]]; then
    sh -c "cd $CURRWD;export DISPLAY=$DISPLAY; export PATH=$PATH; nohup $* > $TARGETHOST_LOGF 2>&1 &"
  else
    ssh -n -f $TARGETHOST "sh -c \"cd $CURRWD;export DISPLAY=$DISPLAY; export PATH=$PATH; nohup $* > $TARGETHOST_LOGF 2>&1 &\""
  fi
}




︙


43
44
45
46
47
48
49

50
51
52
53
54
55
56
57

58
59
60
61
62



63
64
65
66
67
68

69
70
71
72

73
74
75
76

43
44
45
46
47
48
49
50
51
52
53
54
55
56
57

58
59
60



61
62
63
64
65
66
67
68

69
70
71
72

73
74
75
76
77








+







-
+


-
-
-
+
+
+





-
+



-
+





    numcpu=2
fi

# NB// max_load is in units of percent.
#
lperc=$(echo "100 * $load / $numcpu"|bc)
lperc2=$(echo "100 * $load2 / $numcpu"|bc)
let lperc2adj="$lperc2 + 0.5"
if [[ "x$MAX_ALLOWED_LOAD" == "x" ]]; then
  max_load=100
else
  max_load=$MAX_ALLOWED_LOAD
fi

lfile=/tmp/loadrunner-$USER.lockfile
lockfile -r 5 -l 60 $lfile
lockfile -r 5 -l 60 $lfile &> /dev/null

if [[  $lperc -lt $max_load ]];then
  if [[ $lperc -le $lperc2 ]];then
    echo "Load acceptable: lperc=$lperc %, max_load=$max_load %, load=$load, numcpu=$numcpu, MAX_ALLOWED_LOAD=$MAX_ALLOWED_LOAD % and $lperc2 < $lperc"
    echo "Starting command: \"$@\""
  if [[ $lperc -le $lperc2adj ]];then
    # echo "Load acceptable: lperc=$lperc %, max_load=$max_load %, load=$load, numcpu=$numcpu, MAX_ALLOWED_LOAD=$MAX_ALLOWED_LOAD % and $lperc2 < $lperc"
    # echo "Starting command: \"$@\""
    launchjob "$@"
    # we sleep ten seconds here to keep the lock a little longer and give time for
    # the uptime to show a response
    sleep 10
  else
   echo "$LOADRUNNER $@" | at now + 2 minutes 2> /dev/null
   echo "$LOADRUNNER $@" | at now + 2 minutes &> /dev/null
  fi
else
  # echo "Load too high: lperc=$lperc, max_load=$max_load, waiting two minutes before trying to run command: \"$@\""
  echo "$LOADRUNNER $@" | at now + 2 minutes 2> /dev/null
  echo "$LOADRUNNER $@" | at now + 2 minutes &> /dev/null
fi

sleep $(get_delay_time 10)
rm -f $lfile













This page was generated in about
0.14s by
Fossil 2.24 [452f8e1ffc] 2024-03-26 15:36:00