Overview
Comment: | Added simple-get-runs and get-all-but-most-recent-n-per-target |
---|---|
Downloads: | Tarball | ZIP archive | SQL archive |
Timelines: | family | ancestors | descendants | both | v1.64-run-utils | v1.64-keep-running-fix |
Files: | files | file ages | folders |
SHA1: |
1d0be734859e5e0ea2effb77096d7813 |
User & Date: | matt on 2017-09-20 00:47:26 |
Other Links: | branch diff | manifest | tags |
Context
2017-09-20
| ||
18:07 | Got basics working for intelligent removal check-in: 921f5f46c6 user: mrwellan tags: v1.64-run-utils, v1.64-keep-running-fix | |
00:47 | Added simple-get-runs and get-all-but-most-recent-n-per-target check-in: 1d0be73485 user: matt tags: v1.64-run-utils, v1.64-keep-running-fix | |
2017-09-14
| ||
17:10 | fixed issue where item gets stuck in keep_waiting status when prerequisite item failed check-in: 23745b4302 user: bjbarcla tags: v1.6431, v1.64-keep-running-fix | |
Changes
Modified api.scm from [c4438e36a1] to [1606bcad22].
︙ | ︙ | |||
46 47 48 49 50 51 52 53 54 55 56 57 58 59 | ;; register-run get-tests-tags get-tests-for-run get-test-id get-tests-for-runs-mindata get-run-name-from-id get-runs get-num-runs get-all-run-ids get-prev-run-ids get-run-ids-matching-target get-runs-by-patt get-steps-data get-steps-for-test | > | 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 | ;; register-run get-tests-tags get-tests-for-run get-test-id get-tests-for-runs-mindata get-run-name-from-id get-runs simple-get-runs get-num-runs get-all-run-ids get-prev-run-ids get-run-ids-matching-target get-runs-by-patt get-steps-data get-steps-for-test |
︙ | ︙ | |||
262 263 264 265 266 267 268 269 270 271 272 273 274 275 | ((get-run-info) (apply db:get-run-info dbstruct params)) ((get-run-status) (apply db:get-run-status dbstruct params)) ((set-run-status) (apply db:set-run-status dbstruct params)) ((get-tests-for-run) (apply db:get-tests-for-run dbstruct params)) ((get-test-id) (apply db:get-test-id dbstruct params)) ((get-tests-for-run-mindata) (apply db:get-tests-for-run-mindata dbstruct params)) ((get-runs) (apply db:get-runs dbstruct params)) ((get-num-runs) (apply db:get-num-runs dbstruct params)) ((get-all-run-ids) (db:get-all-run-ids dbstruct)) ((get-prev-run-ids) (apply db:get-prev-run-ids dbstruct params)) ((get-run-ids-matching-target) (apply db:get-run-ids-matching-target dbstruct params)) ((get-runs-by-patt) (apply db:get-runs-by-patt dbstruct params)) ((get-run-name-from-id) (apply db:get-run-name-from-id dbstruct params)) ((get-main-run-stats) (apply db:get-main-run-stats dbstruct params)) | > | 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 | ((get-run-info) (apply db:get-run-info dbstruct params)) ((get-run-status) (apply db:get-run-status dbstruct params)) ((set-run-status) (apply db:set-run-status dbstruct params)) ((get-tests-for-run) (apply db:get-tests-for-run dbstruct params)) ((get-test-id) (apply db:get-test-id dbstruct params)) ((get-tests-for-run-mindata) (apply db:get-tests-for-run-mindata dbstruct params)) ((get-runs) (apply db:get-runs dbstruct params)) ((simple-get-runs) (apply db:simple-get-runs dbstruct params)) ((get-num-runs) (apply db:get-num-runs dbstruct params)) ((get-all-run-ids) (db:get-all-run-ids dbstruct)) ((get-prev-run-ids) (apply db:get-prev-run-ids dbstruct params)) ((get-run-ids-matching-target) (apply db:get-run-ids-matching-target dbstruct params)) ((get-runs-by-patt) (apply db:get-runs-by-patt dbstruct params)) ((get-run-name-from-id) (apply db:get-run-name-from-id dbstruct params)) ((get-main-run-stats) (apply db:get-main-run-stats dbstruct params)) |
︙ | ︙ |
Modified db.scm from [879c965cf5] to [69e0a35eb5].
︙ | ︙ | |||
2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 | (set! res (cons (apply vector a x) res))) db qrystr ))) (debug:print-info 11 *default-log-port* "db:get-runs END qrystr: " qrystr " keypatts: " keypatts " offset: " offset " limit: " count) (vector header res))) ;; TODO: Switch this to use max(update_time) from each run db? Then if using a server there is no disk traffic (using inmem db) ;; (define (db:get-changed-run-ids since-time) (let* ((dbdir (db:dbfile-path)) ;; (configf:lookup *configdat* "setup" "dbdir")) (alldbs (glob (conc dbdir "/[0-9]*.db"))) (changed (filter (lambda (dbfile) (> (file-modification-time dbfile) since-time)) | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 | (set! res (cons (apply vector a x) res))) db qrystr ))) (debug:print-info 11 *default-log-port* "db:get-runs END qrystr: " qrystr " keypatts: " keypatts " offset: " offset " limit: " count) (vector header res))) (define-record simple-run target id runname state status owner event_time) (define-record-printer (simple-run x out) (fprintf out "#,(simple-run ~S ~S ~S ~S)" (simple-run-target x) (simple-run-id x) (simple-run-runname x) (time->string (seconds->local-time (simple-run-event_time x) )))) ;; simple get-runs ;; (define (db:simple-get-runs dbstruct runpatt count offset target) (let* ((res '()) (keys (db:get-keys dbstruct)) (runpattstr (db:patt->like "runname" runpatt)) (remfields (list "id" "runname" "state" "status" "owner" "event_time")) (targstr (string-intersperse keys "||'/'||")) (keystr (conc targstr " AS target," (string-intersperse remfields ","))) (qrystr (conc "SELECT " keystr " FROM runs WHERE (" runpattstr ") " ;; runname LIKE ? " ;; Generate: " AND x LIKE 'keypatt' ..." " AND target LIKE '" target "'" " AND state != 'deleted' ORDER BY event_time DESC " (if (number? count) (conc " LIMIT " count) "") (if (number? offset) (conc " OFFSET " offset) "")))) (debug:print-info 11 *default-log-port* "db:get-runs START qrystr: " qrystr " target: " target " offset: " offset " limit: " count) (db:with-db dbstruct #f #f (lambda (db) (sqlite3:for-each-row (lambda (target id runname state status owner event_time) (set! res (cons (make-simple-run target id runname state status owner event_time) res))) db qrystr ))) (debug:print-info 11 *default-log-port* "db:get-runs END qrystr: " qrystr " target: " target " offset: " offset " limit: " count) res)) ;; TODO: Switch this to use max(update_time) from each run db? Then if using a server there is no disk traffic (using inmem db) ;; (define (db:get-changed-run-ids since-time) (let* ((dbdir (db:dbfile-path)) ;; (configf:lookup *configdat* "setup" "dbdir")) (alldbs (glob (conc dbdir "/[0-9]*.db"))) (changed (filter (lambda (dbfile) (> (file-modification-time dbfile) since-time)) |
︙ | ︙ |
Modified db_records.scm from [ebae0b2ffd] to [6d9634427c].
︙ | ︙ | |||
145 146 147 148 149 150 151 152 153 154 155 156 157 158 | (define-inline (db:testmeta-set-author! vec val)(vector-set! vec 2 val)) (define-inline (db:testmeta-set-owner! vec val)(vector-set! vec 3 val)) (define-inline (db:testmeta-set-description! vec val)(vector-set! vec 4 val)) (define-inline (db:testmeta-set-reviewed! vec val)(vector-set! vec 5 val)) (define-inline (db:testmeta-set-iterated! vec val)(vector-set! vec 6 val)) (define-inline (db:testmeta-set-avg_runtime! vec val)(vector-set! vec 7 val)) (define-inline (db:testmeta-set-avg_disk! vec val)(vector-set! vec 8 val)) ;;====================================================================== ;; T E S T D A T A ;;====================================================================== (define (make-db:test-data)(make-vector 10)) (define-inline (db:test-data-get-id vec) (vector-ref vec 0)) (define-inline (db:test-data-get-test_id vec) (vector-ref vec 1)) | > > > > > > | 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 | (define-inline (db:testmeta-set-author! vec val)(vector-set! vec 2 val)) (define-inline (db:testmeta-set-owner! vec val)(vector-set! vec 3 val)) (define-inline (db:testmeta-set-description! vec val)(vector-set! vec 4 val)) (define-inline (db:testmeta-set-reviewed! vec val)(vector-set! vec 5 val)) (define-inline (db:testmeta-set-iterated! vec val)(vector-set! vec 6 val)) (define-inline (db:testmeta-set-avg_runtime! vec val)(vector-set! vec 7 val)) (define-inline (db:testmeta-set-avg_disk! vec val)(vector-set! vec 8 val)) ;;====================================================================== ;; S I M P L E R U N ;;====================================================================== ;; (defstruct id "runname" "state" "status" "owner" "event_time" ;;====================================================================== ;; T E S T D A T A ;;====================================================================== (define (make-db:test-data)(make-vector 10)) (define-inline (db:test-data-get-id vec) (vector-ref vec 0)) (define-inline (db:test-data-get-test_id vec) (vector-ref vec 1)) |
︙ | ︙ |
Modified rmt.scm from [677a774188] to [ca730b9b42].
︙ | ︙ | |||
706 707 708 709 710 711 712 713 714 715 716 717 718 719 | (define (rmt:delete-old-deleted-test-records) (rmt:send-receive 'delete-old-deleted-test-records #f '())) (define (rmt:get-runs runpatt count offset keypatts) (rmt:send-receive 'get-runs #f (list runpatt count offset keypatts))) (define (rmt:get-all-run-ids) (rmt:send-receive 'get-all-run-ids #f '())) (define (rmt:get-prev-run-ids run-id) (rmt:send-receive 'get-prev-run-ids #f (list run-id))) (define (rmt:lock/unlock-run run-id lock unlock user) | > > > | 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 | (define (rmt:delete-old-deleted-test-records) (rmt:send-receive 'delete-old-deleted-test-records #f '())) (define (rmt:get-runs runpatt count offset keypatts) (rmt:send-receive 'get-runs #f (list runpatt count offset keypatts))) (define (rmt:simple-get-runs runpatt count offset target) (rmt:send-receive 'simple-get-runs #f (list runpatt count offset target))) (define (rmt:get-all-run-ids) (rmt:send-receive 'get-all-run-ids #f '())) (define (rmt:get-prev-run-ids run-id) (rmt:send-receive 'get-prev-run-ids #f (list run-id))) (define (rmt:lock/unlock-run run-id lock unlock user) |
︙ | ︙ |
Modified runs.scm from [0a0d3ed157] to [48c6863f6a].
︙ | ︙ | |||
1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 | (let ((fullname (conc real-dir "/" f))) (if (not (string-search (regexp "testdat.db") f)) (runs:recursive-delete-with-error-msg fullname))) (+ 1 x)) 0 real-dir) ;; then the entire directory (runs:recursive-delete-with-error-msg real-dir)) ;; Remove runs ;; fields are passing in through ;; action: ;; 'remove-runs ;; 'set-state-status ;; | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 | (let ((fullname (conc real-dir "/" f))) (if (not (string-search (regexp "testdat.db") f)) (runs:recursive-delete-with-error-msg fullname))) (+ 1 x)) 0 real-dir) ;; then the entire directory (runs:recursive-delete-with-error-msg real-dir)) ;; cleanup often needs to remove all but the last N runs per target ;; ;; target-patts a1/b1/c1,a2/b2/c2 ... ;; ;; This will fail if called with empty target or a bad target (i.e. missing or extra fields) ;; (define (runs:get-hash-by-target target-patts runpatt) (let* ((targets (string-split target-patts ",")) (keys (rmt:get-keys)) (res-ht (make-hash-table))) ;; target -> ( runrecord1 runrecord2 ... ) (for-each (lambda (target-patt) (let ((runs (rmt:simple-get-runs runpatt #f #f target-patt))) (for-each (lambda (run) (let ((target (simple-run-target run))) (hash-table-set! res-ht target (cons run (hash-table-ref/default res-ht target '()))))) runs))) targets) res-ht)) (define (runs:get-all-but-most-recent-n-per-target target-patts runpatt num-to-keep) (let ((runs-ht (runs:get-hash-by-target target-patts runpatt))) (for-each (lambda (target) (let* ((runs (hash-table-ref runs-ht target)) (sorted (sort runs (lambda (a b)(> (simple-run-event_time a)(simple-run-event_time b))))) (to-remove (if (> (length sorted) num-to-keep) (take sorted (- (length sorted) num-to-keep)) '()))) (hash-table-set! runs-ht target to-remove) (print "Sorted: " (map simple-run-event_time sorted)) (print "Remove: " (map simple-run-event_time to-remove)))) (hash-table-keys runs-ht)) runs-ht)) ;; Remove runs ;; fields are passing in through ;; action: ;; 'remove-runs ;; 'set-state-status ;; |
︙ | ︙ |