Index: dashboard.scm ================================================================== --- dashboard.scm +++ dashboard.scm @@ -472,11 +472,11 @@ (iup:toggle state #:action (lambda (obj val) (mark-for-update) (if (eq? val 1) (hash-table-set! *state-ignore-hash* state #t) (hash-table-delete! *state-ignore-hash* state))))) - '("RUNNING" "COMPLETED" "INCOMPLETE" "LAUNCHED" "NOT_STARTED" "KILLED"))) + '("RUNNING" "COMPLETED" "INCOMPLETE" "LAUNCHED" "NOT_STARTED" "KILLED" "DELETED"))) (iup:valuator #:valuechanged_cb (lambda (obj) (let ((val (inexact->exact (round (/ (string->number (iup:attribute obj "VALUE")) 10)))) (oldmax (string->number (iup:attribute obj "MAX"))) (maxruns *tot-run-count*)) (set! *start-run-offset* val) Index: db.scm ================================================================== --- db.scm +++ db.scm @@ -67,16 +67,27 @@ (db:initialize db)) (db:set-sync db) db)) (define (open-run-close proc idb . params) - (let* ((db (if idb idb (open-db))) - (res #f)) - (db:set-sync db) - (set! res (apply proc db params)) - (if (not idb)(sqlite3:finalize! db)) - res)) + (let ((runner (lambda () + (let* ((db (if idb idb (open-db))) + (res #f)) + (db:set-sync db) + (set! res (apply proc db params)) + (if (not idb)(sqlite3:finalize! db)) + res)))) + (handle-exceptions + exn + (begin + (debug:print 0 "EXCEPTION: database probably overloaded?") + (debug:print 0 " " exn) + (print-call-chain) + (thread-sleep! (random 120)) + (debug:print 0 "trying db call one more time....") + (runner)) + (runner)))) (define *global-delta* 0) (define *last-global-delta-printed* 0) (define (open-run-close-measure proc idb . params) Index: launch.scm ================================================================== --- launch.scm +++ launch.scm @@ -123,11 +123,11 @@ (job-thread #f) (runit (lambda () ;; (let-values ;; (((pid exit-status exit-code) ;; (run-n-wait fullrunscript))) - + (open-run-close test-set-status! #f test-id "RUNNING" "n/a" #f #f) ;; if there is a runscript do it first (if fullrunscript (let ((pid (process-run fullrunscript))) (let loop ((i 0)) (let-values