Changes In Branch fix-chained-waiton Through [750fe28e23] Excluding Merge-Ins
This is equivalent to a diff from 1f31d511c0 to 750fe28e23
2016-07-25
| ||
20:43 | catch up workspace Closed-Leaf check-in: c028f8bd3a user: bb tags: fix-chained-waiton | |
2016-06-21
| ||
16:42 | Merged final redir-logs into v1.61 check-in: bba0809d25 user: mrwellan tags: v1.61 | |
04:45 | Merged v1.61 into fix-chained-waiton check-in: 750fe28e23 user: matt tags: fix-chained-waiton | |
04:12 | Merged v1.61 into trunk check-in: 6f4408fd20 user: matt tags: trunk | |
04:06 | Merging first phase of redir-logs into v1.61 check-in: 1f31d511c0 user: matt tags: v1.61 | |
03:57 | Merged filters-fix into redir-logs check-in: a3957fea2d user: matt tags: redir-logs | |
2016-05-18
| ||
11:21 | Forced cleanup db on changing versions check-in: c2ba631f76 user: ritikaag tags: v1.61 | |
2016-05-16
| ||
09:05 | Speculative change to track tests that were waitons of a stuck or non-runnable test check-in: 0b57bca235 user: mrwellan tags: fix-chained-waiton | |
Modified runs.scm from [2ce0f7497c] to [d689aacbb6].
︙ | ︙ | |||
182 183 184 185 186 187 188 189 190 191 192 193 194 195 | (define (runs:run-tests target runname test-patts user flags #!key (run-count 1)) ;; test-names (let* ((keys (keys:config-get-fields *configdat*)) (keyvals (keys:target->keyval keys target)) (run-id (rmt:register-run keyvals runname "new" "n/a" user)) ;; test-name))) ;; (deferred '()) ;; delay running these since they have a waiton clause (runconfigf (conc *toppath* "/runconfigs.config")) (test-records (make-hash-table)) ;; need to process runconfigs before generating these lists (all-tests-registry #f) ;; (tests:get-all)) ;; (tests:get-valid-tests (make-hash-table) test-search-path)) ;; all valid tests to check waiton names (all-test-names #f) ;; (hash-table-keys all-tests-registry)) (test-names #f) ;; Generated by a call to (tests:filter-test-names all-test-names test-patts)) (required-tests #f) ;; Put fully qualified test/testpath names in this list to be done (task-key (conc (hash-table->alist flags) " " (get-host-name) " " (current-process-id))) (tdbdat (tasks:open-db)) | > | 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 | (define (runs:run-tests target runname test-patts user flags #!key (run-count 1)) ;; test-names (let* ((keys (keys:config-get-fields *configdat*)) (keyvals (keys:target->keyval keys target)) (run-id (rmt:register-run keyvals runname "new" "n/a" user)) ;; test-name))) ;; (deferred '()) ;; delay running these since they have a waiton clause (runconfigf (conc *toppath* "/runconfigs.config")) (test-records (make-hash-table)) (test-deps (make-hash-table)) ;; need to process runconfigs before generating these lists (all-tests-registry #f) ;; (tests:get-all)) ;; (tests:get-valid-tests (make-hash-table) test-search-path)) ;; all valid tests to check waiton names (all-test-names #f) ;; (hash-table-keys all-tests-registry)) (test-names #f) ;; Generated by a call to (tests:filter-test-names all-test-names test-patts)) (required-tests #f) ;; Put fully qualified test/testpath names in this list to be done (task-key (conc (hash-table->alist flags) " " (get-host-name) " " (current-process-id))) (tdbdat (tasks:open-db)) |
︙ | ︙ | |||
241 242 243 244 245 246 247 | (if (not test-patts) ;; first time in - adjust testpatt (set! test-patts (common:args-get-testpatt runconf))) ;; Now generate all the tests lists (set! all-tests-registry (tests:get-all)) ;; hash of testname => path-to-test (set! all-test-names (hash-table-keys all-tests-registry)) (set! test-names (tests:filter-test-names all-test-names test-patts)) | | | 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 | (if (not test-patts) ;; first time in - adjust testpatt (set! test-patts (common:args-get-testpatt runconf))) ;; Now generate all the tests lists (set! all-tests-registry (tests:get-all)) ;; hash of testname => path-to-test (set! all-test-names (hash-table-keys all-tests-registry)) (set! test-names (tests:filter-test-names all-test-names test-patts)) ;; I think seeding required-tests with all test-names makes sense but lack analysis to back that up. ;; NEW STRATEGY HERE: ;; 1. fill required tests with test-patts ;; 2. scan testconfigs and if waitons, itemwait, itempatt calc prior test test-patt ;; 3. repeat until all deps propagated |
︙ | ︙ | |||
331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 | (if (and waiton (not (member waiton test-names))) (let* ((waiton-record (hash-table-ref/default test-records waiton #f)) (waiton-tconfig (if waiton-record (vector-ref waiton-record 1) #f)) (waiton-itemized (and waiton-tconfig (or (hash-table-ref/default waiton-tconfig "items" #f) (hash-table-ref/default waiton-tconfig "itemstable" #f)))) (itemmaps (tests:get-itemmaps config)) ;; (configf:lookup config "requirements" "itemmap")) (new-test-patts (tests:extend-test-patts test-patts hed waiton itemmaps))) (debug:print-info 0 #f "Test " waiton " has " (if waiton-record "a" "no") " waiton-record and" (if waiton-itemized " " " no ") "items") ;; need to account for test-patt here, if I am test "a", selected with a test-patt of "hed/b%" ;; and we are waiting on "waiton" we need to add "waiton/,waiton/b%" to test-patt ;; is this satisfied by merely appending "/" to the waiton name added to the list? ;; ;; This approach causes all of the items in an upstream test to be run ;; if we have this waiton already processed once we can analzye it for extending ;; tests to be run, since we can't properly process waitons unless they have been ;; initially added we add them again to be processed on second round AND add the hed ;; back in to also be processed on second round ;; | > > > > > > > > > > > > > > > > > > > > > > > > | | | | | | > | | | > | | | | > > > > > > > > > > > > > > | | | 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 | (if (and waiton (not (member waiton test-names))) (let* ((waiton-record (hash-table-ref/default test-records waiton #f)) (waiton-tconfig (if waiton-record (vector-ref waiton-record 1) #f)) (waiton-itemized (and waiton-tconfig (or (hash-table-ref/default waiton-tconfig "items" #f) (hash-table-ref/default waiton-tconfig "itemstable" #f)))) (itemmaps (tests:get-itemmaps config)) ;; (configf:lookup config "requirements" "itemmap")) (mode (tests:get-mode config)) (new-test-patts (tests:extend-test-patts test-patts hed waiton itemmaps))) (debug:print-info 0 #f "Test " waiton " has " (if waiton-record "a" "no") " waiton-record and" (if waiton-itemized " " " no ") "items") ;;(debug:print-info 0 "BB> Test is "hed" test-patts is "test-patts) ;;(debug:print-info 0 "BB> waiton is " waiton " mode is " mode" and new-test-patts is "new-test-patts) ;; need to account for test-patt here, if I am test "a", selected with a test-patt of "hed/b%" ;; and we are waiting on "waiton" we need to add "waiton/,waiton/b%" to test-patt ;; is this satisfied by merely appending "/" to the waiton name added to the list? ;; ;; This approach causes all of the items in an upstream test to be run ;; if we have this waiton already processed once we can analzye it for extending ;; tests to be run, since we can't properly process waitons unless they have been ;; initially added we add them again to be processed on second round AND add the hed ;; back in to also be processed on second round ;; ;;(debug:print-info 0 "BB> remaining tests: "tal) (let ((hed-depended-on-by-remaining-test ;; BB>> don't set testpatt if hed is waited on by another test in testnames (foldr (lambda (remaining-test previous-result) (let ((dependencies-on-remaining-test (hash-table-ref/default test-deps remaining-test '())) (mode (tests:get-mode config))) ;;(debug:print-info 0 "BB> remaining-test="remaining-test" dependencies-on-remaining-test: "dependencies-on-remaining-test) (or previous-result (if (or (not (equal? "itemwait" mode)) (member hed dependencies-on-remaining-test)) #t #f)))) #f tal))) ;;(debug:print-info 0 "BB> hed="hed" hed-depended-on-by-remaining-test="hed-depended-on-by-remaining-test) (if (and waiton-tconfig (not hed-depended-on-by-remaining-test)) (begin (set! test-names (cons waiton test-names)) ;; need to process this one, only add once the waiton tconfig read (if waiton-itemized (begin (debug:print-info 0 #f "New test patts: " new-test-patts ", prev test patts: " test-patts) (set! required-tests (cons (conc waiton "/") required-tests)) ;;(debug:print-info 0 "BB> set1 test-patts <- " test-patts) (set! test-patts new-test-patts)) (begin (debug:print-info 0 #f "Adding non-itemized test " waiton " to required-tests") (set! required-tests (cons waiton required-tests)) ;;(debug:print-info 0 "BB> set2 test-patts <- " test-patts) (set! test-patts new-test-patts)))) (begin (debug:print-info 0 #f "No testconfig info yet for " waiton ", setting up to re-process it") (set! tal (append (cons waiton tal)(list hed)))))) ;; (cons (conc waiton "/") required-tests)) ;; NOPE: didn't work. required needs to be plain test names. Try tacking on to test-patts ;; - doesn't work ;; (set! test-patts (conc test-patts "," waiton "/")) ;; (set! test-names (cons waiton test-names))))) ;; was an append, now a cons ))) (delete-duplicates (append waitons waitors))) ;; remember deps (hash-table-set! test-deps hed (delete-duplicates (append waitons waitors (hash-table-ref/default test-deps hed '())))) ;; (print "INFO::: test-deps") ;; (pp (hash-table->alist test-deps)) ;; (debugger-start start: 21) ;; (debugger-trace-var "waiton processing" "") ;; (debugger-trace-var "test-deps" (hash-table->alist test-deps)) ;; (debugger-pauser) (let ((remtests (delete-duplicates (append waitons tal)))) (if (not (null? remtests)) (begin ;; (debug:print-info 0 #f "Preprocessing continues for " (string-intersperse remtests ", ")) (loop (car remtests)(cdr remtests)))))))) (if (not (null? required-tests)) (debug:print-info 1 #f "Adding \"" (string-intersperse required-tests " ") "\" to the run queue")) ;; NOTE: these are all parent tests, items are not expanded yet. (debug:print-info 4 #f "test-records=" (hash-table->alist test-records)) (let ((reglen (configf:lookup *configdat* "setup" "runqueue"))) (if (> (length (hash-table-keys test-records)) 0) (let* ((keep-going #t) (run-queue-retries 5) (th1 (make-thread (lambda () (runs:run-tests-queue run-id runname test-records keyvals flags test-patts required-tests (any->number reglen) all-tests-registry test-deps)) ;; (handle-exceptions ;; exn ;; (begin ;; (print-call-chain (current-error-port)) ;; (debug:print 0 #f "ERROR: failure in runs:run-tests-queue thread, error: " ((condition-property-accessor 'exn 'message) exn)) ;; (if (> run-queue-retries 0) ;; (begin |
︙ | ︙ | |||
496 497 498 499 500 501 502 | "\n testmode: " testmode "\n (member 'toplevel testmode): " (member 'toplevel testmode) "\n (null? non-completed): " (null? non-completed) "\n reruns: " reruns "\n items: " items "\n can-run-more: " can-run-more) | | | 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 | "\n testmode: " testmode "\n (member 'toplevel testmode): " (member 'toplevel testmode) "\n (null? non-completed): " (null? non-completed) "\n reruns: " reruns "\n items: " items "\n can-run-more: " can-run-more) ;; ;; lets use the debugger eh? ;; (debugger-start start: 2) ;; (debugger-trace-var "runs:expand-items" "") ;; (debugger-trace-var "can-run-more" can-run-more) ;; (debugger-trace-var "hed" hed) ;; (debugger-trace-var "prereqs-not-met" (runs:pretty-string prereqs-not-met)) ;; (debugger-pauser) |
︙ | ︙ | |||
928 929 930 931 932 933 934 | ;; every time though the loop increment the test/itempatt val. ;; when the min is > max-allowed and none running then force exit ;; (define *max-tries-hash* (make-hash-table)) ;; test-records is a hash table testname:item_path => vector < testname testconfig waitons priority items-info ... > | | > | 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 | ;; every time though the loop increment the test/itempatt val. ;; when the min is > max-allowed and none running then force exit ;; (define *max-tries-hash* (make-hash-table)) ;; test-records is a hash table testname:item_path => vector < testname testconfig waitons priority items-info ... > (define (runs:run-tests-queue run-id runname test-records keyvals flags test-patts required-tests reglen-in all-tests-registry test-deps) ;; At this point the list of parent tests is expanded ;; NB// Should expand items here and then insert into the run queue. (debug:print 5 #f "test-records: " test-records ", flags: " (hash-table->alist flags)) ;; Do mark-and-find clean up of db before starting runing of quue ;; ;; (rmt:find-and-mark-incomplete) (let ((run-info (rmt:get-run-info run-id)) (tests-info (mt:get-tests-for-run run-id #f '() '())) ;; qryvals: "id,testname,item_path")) (sorted-test-names (tests:sort-by-priority-and-waiton test-records)) (test-registry (make-hash-table)) (no-can-run (make-hash-table)) ;; test/test/patt => #t hash of tests that can not run (registry-mutex (make-mutex)) (num-retries 0) (max-retries (config-lookup *configdat* "setup" "maxretries")) (max-concurrent-jobs (let ((mcj (config-lookup *configdat* "setup" "max_concurrent_jobs"))) (if (and mcj (string->number mcj)) (string->number mcj) 1))) ;; length of the register queue ahead |
︙ | ︙ | |||
970 971 972 973 974 975 976 | (set! max-retries (if (and max-retries (string->number max-retries))(string->number max-retries) 100)) (let loop ((hed (car sorted-test-names)) (tal (cdr sorted-test-names)) (reg '()) ;; registered, put these at the head of tal (reruns '())) | > > | > | 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 | (set! max-retries (if (and max-retries (string->number max-retries))(string->number max-retries) 100)) (let loop ((hed (car sorted-test-names)) (tal (cdr sorted-test-names)) (reg '()) ;; registered, put these at the head of tal (reruns '())) (set! reruns '()) ;; force it to test impact!! (if (not (null? reruns))(debug:print-info 4 "reruns=" reruns)) ;; Here we mark any old defunct tests as incomplete. Do this every fifteen minutes ;; moving this to a parallel thread and just run it once. ;; (if (> (current-seconds)(+ last-time-incomplete 900)) (begin (set! last-time-incomplete (current-seconds)) |
︙ | ︙ | |||
1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 | ;; (debugger-trace-var "runs:run-tests-queue" "") ;; (debugger-trace-var "hed" hed) ;; (debugger-trace-var "tal" tal) ;; (debugger-trace-var "items" items) ;; (debugger-trace-var "item-path" item-path) ;; (debugger-trace-var "waitons" waitons) ;; (debugger-pauser) ;; check for hed in waitons => this would be circular, remove it and issue an ;; error (if (member test-name waitons) (begin (debug:print 0 #f "ERROR: test " test-name " has listed itself as a waiton, please correct this!") (set! waiton (filter (lambda (x)(not (equal? x hed))) waitons)))) (cond | > | > | > > > > > | 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 | ;; (debugger-trace-var "runs:run-tests-queue" "") ;; (debugger-trace-var "hed" hed) ;; (debugger-trace-var "tal" tal) ;; (debugger-trace-var "items" items) ;; (debugger-trace-var "item-path" item-path) ;; (debugger-trace-var "waitons" waitons) ;; (debugger-pauser) ;; (debugger-trace-var "no-can-run" (hash-table->alist no-can-run)) ;; check for hed in waitons => this would be circular, remove it and issue an ;; error (if (member test-name waitons) (begin (debug:print 0 #f "ERROR: test " test-name " has listed itself as a waiton, please correct this!") (set! waiton (filter (lambda (x)(not (equal? x hed))) waitons)))) (cond ;; hed, test-deps :: hed -> ( waitons ) ;; We want to catch tests that have waitons that are NOT in the queue and discard them IFF ;; they have been through the wringer 10 or more times ((and (list? waitons) (not (null? waitons)) (> (hash-table-ref/default *max-tries-hash* tfullname 0) 10) (not (null? (filter number? (map (lambda (waiton) (if (and (not (member waiton tal)) ;; this waiton is not in the list to be tried to run (not (member waiton reruns))) 1 #f)) waitons))))) ;; could do this more elegantly with a marker.... (debug:print 0 #f "WARNING: Marking test " tfullname " as not runnable. It is waiting on tests that cannot be run. Giving up now.") (hash-table-set! test-registry tfullname 'removed) (hash-table-set! no-can-run tfullname #t) (for-each (lambda (waiton) (hash-table-set! no-can-run waiton #t)) ;; NB// this does not account for itemmap and itemwait waitons)) ;; items is #f then the test is ok to be handed off to launch (but not before) ;; ((not items) (debug:print-info 4 #f "OUTER COND: (not items)") (if (and (not (tests:match test-patts (tests:testqueue-get-testname test-record) item-path required: required-tests)) (not (null? tal))) |
︙ | ︙ |
Modified tests.scm from [86baf587d1] to [3441ce090d].
︙ | ︙ | |||
89 90 91 92 93 94 95 96 97 98 99 100 101 102 | (itemmap-table (configf:get-section tconfig "itemmap"))) (append (if base-itemmap (list (list "%" base-itemmap)) '()) (if itemmap-table itemmap-table '())))) ;; given a list of itemmaps (testname . map), return the first match ;; (define (tests:lookup-itemmap itemmaps testname) (let ((best-matches (filter (lambda (itemmap) (tests:match (car itemmap) testname #f)) itemmaps))) | > > > > | 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 | (itemmap-table (configf:get-section tconfig "itemmap"))) (append (if base-itemmap (list (list "%" base-itemmap)) '()) (if itemmap-table itemmap-table '())))) (define (tests:get-mode tconfig) (let ((itemwait (configf:lookup tconfig "requirements" "mode"))) itemwait)) ;; given a list of itemmaps (testname . map), return the first match ;; (define (tests:lookup-itemmap itemmaps testname) (let ((best-matches (filter (lambda (itemmap) (tests:match (car itemmap) testname #f)) itemmaps))) |
︙ | ︙ |