Index: db.scm ================================================================== --- db.scm +++ db.scm @@ -1975,19 +1975,21 @@ (debug:print-info 11 *default-log-port* "db:get-runs END qrystr: " qrystr " target: " target " offset: " offset " limit: " count) res)) ;; TODO: Switch this to use max(update_time) from each run db? Then if using a server there is no disk traffic (using inmem db) ;; +;; NOTE: This DOESN'T (necessarily) get the real run ids, but the number of the .db!! + (define (db:get-changed-run-ids since-time) (let* ((dbdir (db:dbfile-path)) ;; (configf:lookup *configdat* "setup" "dbdir")) (alldbs (glob (conc dbdir "/.megatest/[0-9]*.db"))) (changed (filter (lambda (dbfile) (> (file-modification-time dbfile) since-time)) alldbs))) (delete-duplicates (map (lambda (dbfile) - (let* ((res (string-match ".*\\/(\\d)*\\.db" dbfile))) + (let* ((res (string-match ".*\\/(\\d\\d)\\.db" dbfile))) (if res (string->number (cadr res)) (begin (debug:print 2 *default-log-port* "WARNING: Failed to process " dbfile " for run-id") 0)))) @@ -4423,26 +4425,39 @@ ;; no transaction, allow the db to be accessed between the big queries (let* ((backcons (lambda (lst item)(cons item lst))) (all_tests '()) (all_test_steps '()) (all_test_data '()) - (run_ids (db:get-changed-run-ids since-time)) + (changed_run_dbs (db:get-changed-run-ids since-time)) ;; gets the rundb numbers + (all_run_ids + (db:with-db dbstruct #f #f + (lambda (dbdat db) + (sqlite3:fold-row backcons '() db "SELECT id FROM runs")) + ) + ) + (changed_run_ids (filter (lambda (run) (member (modulo run 100) changed_run_dbs)) all_run_ids)) + (run_ids + (db:with-db dbstruct #f #f + (lambda (dbdat db) + (sqlite3:fold-row backcons '() db "SELECT id FROM runs WHERE last_update>=?" since-time)) + ) + ) (run_stat_ids (db:with-db dbstruct #f #f (lambda (dbdat db) (sqlite3:fold-row backcons '() db "SELECT id FROM run_stats WHERE last_update>=?" since-time)) ) ) ) (for-each (lambda (run_id) - (set! all_tests + (set! all_tests (append (map (lambda (x) (cons x run_id)) (db:with-db dbstruct run_id #f (lambda (dbdat db) - (sqlite3:fold-row backcons '() db "SELECT id FROM tests WHERE last_update>=?" since-time) + (sqlite3:fold-row backcons '() db "SELECT id FROM tests WHERE run_id=? and last_update>=?" run_id since-time) ) ) ) all_tests ) ) @@ -4449,11 +4464,11 @@ (set! all_test_steps (append (map (lambda (x) (cons x run_id)) (db:with-db dbstruct run_id #f (lambda (dbdat db) - (sqlite3:fold-row backcons '() db "SELECT id FROM test_steps WHERE last_update>=?" since-time) + (sqlite3:fold-row backcons '() db "SELECT id FROM test_steps WHERE last_update>=?" since-time) ) ) ) all_test_steps ) ) @@ -4467,11 +4482,11 @@ ) ) all_test_data ) ) ) - run_ids + changed_run_ids ) (debug:print 2 *default-log-port* "run_ids = " run_ids) (debug:print 2 *default-log-port* "all_tests = " all_tests) `((runs . ,run_ids)