Overview
Comment: | Merged in v1.55 changes |
---|---|
Downloads: | Tarball | ZIP archive | SQL archive |
Timelines: | family | ancestors | descendants | both | v1.60 |
Files: | files | file ages | folders |
SHA1: |
f870afe4d0e6e26877efb43d5d3ed7ed |
User & Date: | mrwellan on 2014-08-13 16:13:34 |
Other Links: | branch diff | manifest | tags |
Context
2014-08-19
| ||
23:21 | Delay opening the database until *after* the server is started check-in: 8262fac699 user: matt tags: v1.60 | |
2014-08-13
| ||
16:13 | Merged in v1.55 changes check-in: f870afe4d0 user: mrwellan tags: v1.60 | |
13:57 | Unset TARGET* vars before starting the real work in nbfake check-in: e89be5432c user: mrwellan tags: v1.55 | |
2014-07-25
| ||
08:00 | Merged in run-wait changes check-in: d2a718227d user: matt tags: v1.60 | |
Changes
Modified Makefile from [285bfe2cba] to [5f716d848b].
︙ | ︙ | |||
163 164 165 166 167 168 169 | csc -deploy $(CSCOPTS) $(OFILES) megatest.scm -o deploytarg mv deploytarg/deploytarg deploytarg/mtest deploytarg/dboard : $(OFILES) $(GOFILES) dashboard.scm deploytarg/apropos.so csc -deploy $(OFILES) $(GOFILES) dashboard.scm -o deploytarg mv deploytarg/deploytarg deploytarg/dboard | > > > > > > | 163 164 165 166 167 168 169 170 171 172 173 174 175 | csc -deploy $(CSCOPTS) $(OFILES) megatest.scm -o deploytarg mv deploytarg/deploytarg deploytarg/mtest deploytarg/dboard : $(OFILES) $(GOFILES) dashboard.scm deploytarg/apropos.so csc -deploy $(OFILES) $(GOFILES) dashboard.scm -o deploytarg mv deploytarg/deploytarg deploytarg/dboard DATASHAREO=configf.o common.o process.o datashare-testing/datashare : datashare.scm $(DATASHAREO) csc datashare.scm $(DATASHAREO) -o datashare-testing/datashare datashare : datashare-testing/datashare ./datashare-testing/datashare |
Modified common.scm from [e3f681efc8] to [8133b36933].
︙ | ︙ | |||
364 365 366 367 368 369 370 | (let ((newval (string->number (cadr match)))) (if (number? newval) (set! freespc newval)))))) (car df-results)) freespc)) (define (get-cpu-load) | > | | | | | | | | | | | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 | (let ((newval (string->number (cadr match)))) (if (number? newval) (set! freespc newval)))))) (car df-results)) freespc)) (define (get-cpu-load) (car (common:get-cpu-load))) ;; (let* ((load-res (cmd-run->list "uptime")) ;; (load-rx (regexp "load average:\\s+(\\d+)")) ;; (cpu-load #f)) ;; (for-each (lambda (l) ;; (let ((match (string-search load-rx l))) ;; (if match ;; (let ((newval (string->number (cadr match)))) ;; (if (number? newval) ;; (set! cpu-load newval)))))) ;; (car load-res)) ;; cpu-load)) ;; get cpu load by reading from /proc/loadavg, return all three values ;; (define (common:get-cpu-load) (with-input-from-file "/proc/loadavg" (lambda ()(list (read)(read)(read))))) (define (common:wait-for-cpuload maxload numcpus waitdelay #!key (count 1000)) (let* ((loadavg (common:get-cpu-load)) (first (car loadavg)) (next (cadr loadavg)) (adjload (* maxload numcpus)) (loadjmp (- first next))) (cond ((and (> first adjload) (> count 0)) (debug:print-info 0 "waiting " waitdelay " seconds due to load " first " exceeding max of " adjload) (thread-sleep! waitdelay) (common:wait-for-cpuload maxload numcpus waitdelay count: (- count 1))) ((and (> loadjmp numcpus) (> count 0)) (debug:print-info 0 "waiting " waitdelay " seconds due to load jump " loadjmp " > numcpus " numcpus) (thread-sleep! waitdelay) (common:wait-for-cpuload maxload numcpus waitdelay count: (- count 1)))))) (define (common:get-num-cpus) (with-input-from-file "/proc/cpuinfo" (lambda () (let loop ((numcpu 0) (inl (read-line))) (if (eof-object? inl) numcpu (loop (if (string-match "^processor\\s+:\\s+\\d+$" inl) (+ numcpu 1) numcpu) (read-line))))))) (define (get-uname . params) (let* ((uname-res (cmd-run->list (conc "uname " (if (null? params) "-a" (car params))))) (uname #f)) (if (null? (car uname-res)) "unknown" (caar uname-res)))) |
︙ | ︙ |
Modified dashboard.scm from [be49d754f0] to [3ab66e328b].
︙ | ︙ | |||
14 15 16 17 18 19 20 | (import (prefix iup iup:)) (use canvas-draw) (import canvas-draw-iup) (use sqlite3 srfi-1 posix regex regex-case srfi-69) (import (prefix sqlite3 sqlite3:)) | < | 14 15 16 17 18 19 20 21 22 23 24 25 26 27 | (import (prefix iup iup:)) (use canvas-draw) (import canvas-draw-iup) (use sqlite3 srfi-1 posix regex regex-case srfi-69) (import (prefix sqlite3 sqlite3:)) (declare (uses common)) (declare (uses margs)) (declare (uses keys)) (declare (uses items)) (declare (uses db)) (declare (uses configf)) |
︙ | ︙ |
Added datashare-testing/.datashare.config version [1d9aef34b1].
> > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 | # Read in the users vars first (so the offical data cannot be overridden [include datastore.config] [storagegroups] 1 eng /tmp/datastore/eng [areas] synthesis asic/synthesis verilog asic/verilog oalibs custom/oalibs [target] basepath #{getenv BASEPATH} [quality] 0 untested 1 lightly tested 2 tested 3 full QA |
Added datashare.scm version [c941a31a14].
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 | ;; Copyright 2006-2013, Matthew Welland. ;; ;; This program is made available under the GNU GPL version 2.0 or ;; greater. See the accompanying file COPYING for details. ;; ;; This program is distributed WITHOUT ANY WARRANTY; without even the ;; implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR ;; PURPOSE. (use ssax) (use sxml-serializer) (use sxml-modifications) (use regex) (use srfi-69) (use regex-case) (use posix) (use json) (use csv) (use srfi-18) (use format) (require-library iup) (import (prefix iup iup:)) (require-library ini-file) (import (prefix ini-file ini:)) (use canvas-draw) (import canvas-draw-iup) (use sqlite3 srfi-1 posix regex regex-case srfi-69) (import (prefix sqlite3 sqlite3:)) (declare (uses configf)) (include "megatest-fossil-hash.scm") ;; ;; GLOBALS ;; (define *datashare:current-tab-number* 0) (define datashare:help (conc "Usage: datashare [action [params ...]] Note: run datashare without parameters to start the gui. publish <area> <key> [group] : Publish data to share, use group to protect (i) get <area> <key> [destpath] : Get a link to data, put the link in destpath (ii) update <area> <key> : Update the link to data to the latest iteration. (i) Uses group ownership of files to be published for group if not specified (ii) Uses local path or looks up script to find path in configs Part of the Megatest tool suite. Learn more at http://www.kiatoa.com/fossils/megatest Version: " megatest-fossil-hash)) ;; " ;;====================================================================== ;; DB ;;====================================================================== (define (datashare:initialize-db db) (for-each (lambda (qry) (sqlite3:execute db qry)) (list "CREATE TABLE pkgs (id INTEGER PRIMARY KEY, area TEXT, key TEXT, iteration INTEGER, submitter TEXT, datetime TEXT, storegrp TEXT, datavol INTEGER, quality TEXT, disk_id INTEGER, comment TEXT);" "CREATE TABLE refs (id INTEGER PRIMARY KEY, pkg_id INTEGER, destlink TEXT);" "CREATE TABLE disks (id INTEGER PRIMARY KEY, storegrp TEXT, path TEXT);"))) ;; Create the sqlite db (define (datashare:open-db path) (if (and path (directory? path) (file-read-access? path)) (let* ((dbpath (conc path "/datashare.db")) (writeable (file-write-access? dbpath)) (dbexists (file-exists? dbpath)) (handler (make-busy-timeout 136000))) (handle-exceptions exn (begin (debug:print 2 "ERROR: problem accessing db " dbpath ((condition-property-accessor 'exn 'message) exn)) (exit)) (set! db (sqlite3:open-database dbpath))) (if *db-write-access* (sqlite3:set-busy-handler! db handler)) (if (not dbexists) (begin (datashare:initialize-db db))) db))) ;;====================================================================== ;; GUI ;;====================================================================== ;; The main menu (define (datashare:main-menu) (iup:menu ;; a menu is a special attribute to a dialog (think Gnome putting the menu at screen top) (iup:menu-item "Files" (iup:menu ;; Note that you can use either #:action or action: for options (iup:menu-item "Open" action: (lambda (obj) (iup:show (iup:file-dialog)) (print "File->open " obj))) (iup:menu-item "Save" #:action (lambda (obj)(print "File->save " obj))) (iup:menu-item "Exit" #:action (lambda (obj)(exit))))) (iup:menu-item "Tools" (iup:menu (iup:menu-item "Create new blah" #:action (lambda (obj)(print "Tools->new blah"))) ;; (iup:menu-item "Show dialog" #:action (lambda (obj) ;; (show message-window ;; #:modal? #t ;; ;; set positon using coordinates or center, start, top, left, end, bottom, right, parent-center, current ;; ;; #:x 'mouse ;; ;; #:y 'mouse ;; ) )))) (define (datashare:publish-view configdat) (let* ((label-size "50x") (areas-sel (iup:listbox #:expand "YES" #:dropdown "YES")) (version-val (iup:textbox #:expand "YES" #:size "50x")) (iteration (iup:textbox #:expand "YES" #:size "20x")) (comment (iup:textbox #:expand "YES")) (source-path (iup:textbox #:expand "YES")) (browse-btn (iup:button "Browse" #:size "40x" #:action (lambda (obj) (let* ((fd (iup:file-dialog #:dialogtype "DIR")) (top (iup:show fd #:modal? "YES"))) (iup:attribute-set! source-path "VALUE" (iup:attribute fd "VALUE")) (iup:destroy! fd)))))) (iup:vbox (iup:hbox (iup:label "Area:" #:size label-size) areas-sel) (iup:hbox (iup:label "Version:" #:size label-size) version-val (iup:label "Iteration:") iteration) (iup:hbox (iup:label "Comment:" #:size label-size) comment) (iup:hbox (iup:label "Source path:" #:size label-size) source-path browse-btn)))) (define (datashare:get-view configdat) (iup:vbox (iup:hbox (iup:button "Pushme" #:expand "YES" )))) (define (datashare:manage-view configdat) (iup:vbox (iup:hbox (iup:button "Pushme" #:expand "YES" )))) (define (datashare:gui configdat) (iup:show (iup:dialog #:title (conc "DataShare dashboard " (current-user-name) ":" (current-directory)) #:menu (datashare:main-menu) (let* ((tabs (iup:tabs #:tabchangepos-cb (lambda (obj curr prev) (set! *datashare:current-tab-number* curr)) (datashare:publish-view configdat) (datashare:get-view configdat) (datashare:manage-view configdat) ))) ;; (set! (iup:callback tabs tabchange-cb:) (lambda (a b c)(print "SWITCHED TO TAB: " a " " b " " c))) (iup:attribute-set! tabs "TABTITLE0" "Publish") (iup:attribute-set! tabs "TABTITLE1" "Get") (iup:attribute-set! tabs "TABTITLE2" "Manage") ;; (iup:attribute-set! tabs "BGCOLOR" "190 190 190") tabs))) (iup:main-loop)) ;;====================================================================== ;; MAIN ;;====================================================================== (define (datashare:load-config path) (let ((fname (conc path "/.datashare.config"))) (ini:property-separator-patt " * *") (ini:property-separator #\space) (if (file-exists? fname) ;; (ini:read-ini fname) (read-config fname #f #t) (make-hash-table)))) (define (main) (let* ((args (argv)) (prog (car args)) (rema (cdr args)) (configdat (datashare:load-config (pathname-directory prog)))) (cond ((eq? (length rema) 1) (case (string->symbol (car rema)) ((help -h -help --h --help) (print datashare:help)) (else (print "ERROR: Unrecognised command. Try \"datashare help\"")))) ((null? rema)(datashare:gui configdat)) ((>= (length rema) 2) (apply process-action (car rema)(cdr rema))) (else (print "ERROR: Unrecognised command. Try \"datashare help\""))))) (main) |
Modified db.scm from [8ff4cd7b4a] to [bf63eca3b8].
︙ | ︙ | |||
152 153 154 155 156 157 158 | (sqlite3:execute db "INSERT OR IGNORE INTO tests (id,run_id,testname,event_time,item_path,state,status) VALUES (?,?,'bogustest',strftime('%s','now'),'nowherepath','DELETED','n/a');" (* run-id 30000) ;; allow for up to 30k tests per run run-id) )) ;; add strings db to rundb, not in use yet (sqlite3:set-busy-handler! db handler) | | | 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 | (sqlite3:execute db "INSERT OR IGNORE INTO tests (id,run_id,testname,event_time,item_path,state,status) VALUES (?,?,'bogustest',strftime('%s','now'),'nowherepath','DELETED','n/a');" (* run-id 30000) ;; allow for up to 30k tests per run run-id) )) ;; add strings db to rundb, not in use yet (sqlite3:set-busy-handler! db handler) (sqlite3:execute db "PRAGMA synchronous = 0;"))) ;; was 0 but 0 is a gamble, changed back to 0 (dbr:dbstruct-set-rundb! dbstruct db) (dbr:dbstruct-set-inuse! dbstruct #t) (dbr:dbstruct-set-olddb! dbstruct olddb) ;; (dbr:dbstruct-set-run-id! dbstruct run-id) (if local (begin (dbr:dbstruct-set-localdb! dbstruct run-id db) ;; (dbr:dbstruct-set-inmem! dbstruct db) ;; direct access ... |
︙ | ︙ |
Modified runs.scm from [2caf2e1eb6] to [50b24e2f3b].
︙ | ︙ | |||
197 198 199 200 201 202 203 | (else #f)))) (list (not can-not-run-more) num-running num-running-in-jobgroup max-concurrent-jobs job-group-limit))))) ;; test-names: Comma separated patterns same as test-patts but used in selection ;; of tests to run. The item portions are not respected. ;; FIXME: error out if /patt specified ;; | | | 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 | (else #f)))) (list (not can-not-run-more) num-running num-running-in-jobgroup max-concurrent-jobs job-group-limit))))) ;; test-names: Comma separated patterns same as test-patts but used in selection ;; of tests to run. The item portions are not respected. ;; FIXME: error out if /patt specified ;; (define (runs:run-tests target runname test-patts user flags #!key (run-count 3)) ;; test-names (let* ((keys (keys:config-get-fields *configdat*)) (keyvals (keys:target->keyval keys target)) (run-id (rmt:register-run keyvals runname "new" "n/a" user)) ;; test-name))) (deferred '()) ;; delay running these since they have a waiton clause (runconfigf (conc *toppath* "/runconfigs.config")) (test-records (make-hash-table)) ;; need to process runconfigs before generating these lists |
︙ | ︙ | |||
338 339 340 341 342 343 344 | (if (not (null? required-tests)) (debug:print-info 1 "Adding " required-tests " to the run queue")) ;; NOTE: these are all parent tests, items are not expanded yet. (debug:print-info 4 "test-records=" (hash-table->alist test-records)) (let ((reglen (configf:lookup *configdat* "setup" "runqueue"))) (if (> (length (hash-table-keys test-records)) 0) | > | > > > > > > > > | > | 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 | (if (not (null? required-tests)) (debug:print-info 1 "Adding " required-tests " to the run queue")) ;; NOTE: these are all parent tests, items are not expanded yet. (debug:print-info 4 "test-records=" (hash-table->alist test-records)) (let ((reglen (configf:lookup *configdat* "setup" "runqueue"))) (if (> (length (hash-table-keys test-records)) 0) (begin (runs:run-tests-queue run-id runname test-records keyvals flags test-patts required-tests (any->number reglen) all-tests-registry) ;; if run-count > 0 call, set -preclean and -rerun STUCK/DEAD (if (> run-count 0) (begin (if (not (hash-table-ref/default flags "-preclean" #f)) (hash-table-set! flags "-preclean" #t)) (if (not (hash-table-ref/default flags "-rerun" #f)) (hash-table-set! flags "-rerun" "STUCK/DEAD,n/a,ZERO_ITEMS")) (runs:run-tests target runname test-patts user flags run-count: (- run-count 1))))) (debug:print-info 0 "No tests to run"))) (debug:print-info 4 "All done by here") )) ;; loop logic. These are used in runs:run-tests-queue to make it a bit more readable. ;; ;; If reg not full and have items in tal then loop with (car tal)(cdr tal) reg reruns ;; If reg is full (i.e. length >= n ;; loop with (car reg) tal (cdr reg) reruns |
︙ | ︙ | |||
575 576 577 578 579 580 581 | (num-running-in-jobgroup (list-ref run-limits-info 2)) (max-concurrent-jobs (list-ref run-limits-info 3)) (job-group-limit (list-ref run-limits-info 4)) (prereqs-not-met (rmt:get-prereqs-not-met run-id waitons item-path testmode itemmap: itemmap)) ;; (prereqs-not-met (mt:lazy-get-prereqs-not-met run-id waitons item-path mode: testmode itemmap: itemmap)) (fails (runs:calc-fails prereqs-not-met)) (non-completed (runs:calc-not-completed prereqs-not-met)) | | > > > > | 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 | (num-running-in-jobgroup (list-ref run-limits-info 2)) (max-concurrent-jobs (list-ref run-limits-info 3)) (job-group-limit (list-ref run-limits-info 4)) (prereqs-not-met (rmt:get-prereqs-not-met run-id waitons item-path testmode itemmap: itemmap)) ;; (prereqs-not-met (mt:lazy-get-prereqs-not-met run-id waitons item-path mode: testmode itemmap: itemmap)) (fails (runs:calc-fails prereqs-not-met)) (non-completed (runs:calc-not-completed prereqs-not-met)) (loop-list (list hed tal reg reruns)) ;; configure the load runner (numcpus (common:get-num-cpus)) (maxload (string->number (or (configf:lookup *configdat* "jobtools" "maxload") "3"))) (waitdelay (string->number (or (configf:lookup *configdat* "jobtools" "waitdelay") "60")))) (debug:print-info 4 "have-resources: " have-resources " prereqs-not-met: (" (string-intersperse (map (lambda (t) (if (vector? t) (conc (db:test-get-state t) "/" (db:test-get-status t)) (conc " WARNING: t is not a vector=" t ))) prereqs-not-met) ", ") ") fails: " fails) |
︙ | ︙ | |||
674 675 676 677 678 679 680 681 682 683 684 685 686 687 | (or (null? prereqs-not-met) (and (eq? testmode 'toplevel) (null? non-completed)))) ;; (hash-table-delete! *max-tries-hash* (runs:make-full-test-name test-name item-path)) ;; we are going to reset all the counters for test retries by setting a new hash table ;; this means they will increment only when nothing can be run (set! *max-tries-hash* (make-hash-table)) (run:test run-id run-info keyvals runname test-record flags #f test-registry all-tests-registry) (hash-table-set! test-registry (runs:make-full-test-name test-name item-path) 'running) (runs:shrink-can-run-more-tests-count) ;; DELAY TWEAKER (still needed?) ;; (thread-sleep! *global-delta*) (if (or (not (null? tal))(not (null? reg))) (list (runs:queue-next-hed tal reg reglen regfull) (runs:queue-next-tal tal reg reglen regfull) | > > > > | 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 | (or (null? prereqs-not-met) (and (eq? testmode 'toplevel) (null? non-completed)))) ;; (hash-table-delete! *max-tries-hash* (runs:make-full-test-name test-name item-path)) ;; we are going to reset all the counters for test retries by setting a new hash table ;; this means they will increment only when nothing can be run (set! *max-tries-hash* (make-hash-table)) ;; well, first lets see if cpu load throttling is enabled. If so wait around until the ;; average cpu load is under the threshold before continuing (if (configf:lookup *configdat* "jobtools" "maxload") ;; only gate if maxload is specified (common:wait-for-cpuload maxload numcpus waitdelay)) (run:test run-id run-info keyvals runname test-record flags #f test-registry all-tests-registry) (hash-table-set! test-registry (runs:make-full-test-name test-name item-path) 'running) (runs:shrink-can-run-more-tests-count) ;; DELAY TWEAKER (still needed?) ;; (thread-sleep! *global-delta*) (if (or (not (null? tal))(not (null? reg))) (list (runs:queue-next-hed tal reg reglen regfull) (runs:queue-next-tal tal reg reglen regfull) |
︙ | ︙ | |||
1032 1033 1034 1035 1036 1037 1038 | ((not (null? tal)) (debug:print-info 4 "I'm pretty sure I shouldn't get here.")) ((not (null? reg)) ;; could we get here with leftovers? (debug:print-info 0 "Have leftovers!") (loop (car reg)(cdr reg) '() reruns)) (else (debug:print-info 4 "Exiting loop with...\n hed=" hed "\n tal=" tal "\n reruns=" reruns)) | | | > | | > | > | | | > | | | | | | | | > | | | 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 | ((not (null? tal)) (debug:print-info 4 "I'm pretty sure I shouldn't get here.")) ((not (null? reg)) ;; could we get here with leftovers? (debug:print-info 0 "Have leftovers!") (loop (car reg)(cdr reg) '() reruns)) (else (debug:print-info 4 "Exiting loop with...\n hed=" hed "\n tal=" tal "\n reruns=" reruns)) ))) ;; now *if* -run-wait we wait for all tests to be done ;; Now wait for any RUNNING tests to complete (if in run-wait mode) (let wait-loop ((num-running (rmt:get-count-tests-running-for-run-id run-id)) (prev-num-running 0)) ;; (debug:print 0 "num-running=" num-running ", prev-num-running=" prev-num-running) (if (and (or (args:get-arg "-run-wait") (equal? (configf:lookup *configdat* "setup" "run-wait") "yes")) (> num-running 0)) (begin ;; Here we mark any old defunct tests as incomplete. Do this every fifteen minutes ;; (debug:print 0 "Got here eh! num-running=" num-running " (> num-running 0) " (> num-running 0)) (if (> (current-seconds)(+ last-time-incomplete 900)) (begin (debug:print-info 0 "Marking stuck tests as INCOMPLETE while waiting for run " run-id ". Running as pid " (current-process-id) " on " (get-host-name)) (set! last-time-incomplete (current-seconds)) (cdb:remote-run db:find-and-mark-incomplete #f))) (if (not (eq? num-running prev-num-running)) (debug:print-info 0 "run-wait specified, waiting on " num-running " tests in RUNNING, REMOTEHOSTSTART or LAUNCHED state at " (time->string (seconds->local-time (current-seconds))))) (thread-sleep! 15) ;; (wait-loop (rmt:get-count-tests-running-for-run-id run-id) num-running)))) (wait-loop (rmt:get-count-tests-running-for-run-id run-id) num-running)))) ;; LET* ((test-record ;; we get here on "drop through". All done! (debug:print-info 1 "All tests launched"))) (define (runs:calc-fails prereqs-not-met) (filter (lambda (test) (and (vector? test) ;; not (string? test)) (equal? (db:test-get-state test) "COMPLETED") |
︙ | ︙ | |||
1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 | (test-waitons (tests:testqueue-get-waitons test-record)) (test-conf (tests:testqueue-get-testconfig test-record)) (itemdat (tests:testqueue-get-itemdat test-record)) (test-path (hash-table-ref all-tests-registry test-name)) ;; (conc *toppath* "/tests/" test-name)) ;; could use tests:get-testconfig here ... (force (hash-table-ref/default flags "-force" #f)) (rerun (hash-table-ref/default flags "-rerun" #f)) (keepgoing (hash-table-ref/default flags "-keepgoing" #f)) (item-path "") (db #f) (full-test-name #f)) ;; setting itemdat to a list if it is #f (if (not itemdat)(set! itemdat '())) (set! item-path (item-list->path itemdat)) | > | 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 | (test-waitons (tests:testqueue-get-waitons test-record)) (test-conf (tests:testqueue-get-testconfig test-record)) (itemdat (tests:testqueue-get-itemdat test-record)) (test-path (hash-table-ref all-tests-registry test-name)) ;; (conc *toppath* "/tests/" test-name)) ;; could use tests:get-testconfig here ... (force (hash-table-ref/default flags "-force" #f)) (rerun (hash-table-ref/default flags "-rerun" #f)) (keepgoing (hash-table-ref/default flags "-keepgoing" #f)) (incomplete-timeout (string->number (or (configf:lookup *configdat* "setup" "incomplete-timeout") "x"))) (item-path "") (db #f) (full-test-name #f)) ;; setting itemdat to a list if it is #f (if (not itemdat)(set! itemdat '())) (set! item-path (item-list->path itemdat)) |
︙ | ︙ | |||
1260 1261 1262 1263 1264 1265 1266 | (process-signal (current-process-id) signal/kill)))))))) ((KILLED) (debug:print 1 "NOTE: " full-test-name " is already running or was explictly killed, use -force to launch it.") (hash-table-set! test-registry (runs:make-full-test-name test-name test-path) 'DONOTRUN)) ;; KILLED)) ((LAUNCHED REMOTEHOSTSTART RUNNING) (if (> (- (current-seconds)(+ (db:test-get-event_time testdat) (db:test-get-run_duration testdat))) | > | | 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 | (process-signal (current-process-id) signal/kill)))))))) ((KILLED) (debug:print 1 "NOTE: " full-test-name " is already running or was explictly killed, use -force to launch it.") (hash-table-set! test-registry (runs:make-full-test-name test-name test-path) 'DONOTRUN)) ;; KILLED)) ((LAUNCHED REMOTEHOSTSTART RUNNING) (if (> (- (current-seconds)(+ (db:test-get-event_time testdat) (db:test-get-run_duration testdat))) (or incomplete-timeout 6000)) ;; i.e. no update for more than 6000 seconds (begin (debug:print 0 "WARNING: Test " test-name " appears to be dead. Forcing it to state INCOMPLETE and status STUCK/DEAD") (tests:test-set-status! run-id test-id "INCOMPLETE" "STUCK/DEAD" "" #f)) ;; (tests:test-set-status! test-id "INCOMPLETE" "STUCK/DEAD" "" #f)) (debug:print 2 "NOTE: " test-name " is already running"))) (else (debug:print 0 "ERROR: Failed to launch test " full-test-name ". Unrecognised state " (test:get-state testdat)) |
︙ | ︙ |
Modified tests/fullrun/config/mt_include_1.config from [9cabad0848] to [e2e84dc027].
︙ | ︙ | |||
11 12 13 14 15 16 17 | # workhosts localhost hermes # launcher exec nbfake # launcher nbfake launcher loadrunner # launcher echo # launcher nbfind # launcher nodanggood | | > > > > > > | 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 | # workhosts localhost hermes # launcher exec nbfake # launcher nbfake launcher loadrunner # launcher echo # launcher nbfind # launcher nodanggood # launcher loadrunner launcher nbfake # maxload *per cpu* maxload 4 # default waitdelay is 60 seconds waitdelay 15 ## use "xterm -e csi -- " as a launcher to examine the launch environment. ## exit with (exit) ## get a shell with (system "bash") # launcher xterm -e csi -- |
Modified tests/fullrun/megatest.config from [13fd353d7e] to [d00a35aaf5].
︙ | ︙ | |||
23 24 25 26 27 28 29 30 31 32 33 34 35 36 | 1 #{get misc parent}/simplerun/tests [setup] # Set launchwait to no to use the more agressive code that does not wait for the launch to complete before proceeding # this may save a few milliseconds on launching tests # launchwait no waivercommentpatt ^WW\d+ [a-z].* # Use http instead of direct filesystem access # transport http # transport fs # If set to "default" the old code is used. Otherwise defaults to 200 or uses # numeric value given. | > > > > > | 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 | 1 #{get misc parent}/simplerun/tests [setup] # Set launchwait to no to use the more agressive code that does not wait for the launch to complete before proceeding # this may save a few milliseconds on launching tests # launchwait no waivercommentpatt ^WW\d+ [a-z].* incomplete-timeout 1 # yes, anything else is no run-wait yes # Use http instead of direct filesystem access # transport http # transport fs # If set to "default" the old code is used. Otherwise defaults to 200 or uses # numeric value given. |
︙ | ︙ |
Modified utils/Makefile_latest.installall from [2749919870] to [a5be37ec2b].
︙ | ︙ | |||
41 42 43 44 45 46 47 | IUPBRANCH=iup-3.10.1 # Eggs to install (straightforward ones) EGGS=matchable readline apropos base64 regex-literals format regex-case test coops trace csv \ dot-locking posix-utils posix-extras directory-utils hostinfo tcp-server rpc csv-xml fmt \ json md5 awful http-client spiffy uri-common intarweb spiffy-request-vars \ spiffy-directory-listing ssax sxml-serializer sxml-modifications sql-de-lite \ | | | 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 | IUPBRANCH=iup-3.10.1 # Eggs to install (straightforward ones) EGGS=matchable readline apropos base64 regex-literals format regex-case test coops trace csv \ dot-locking posix-utils posix-extras directory-utils hostinfo tcp-server rpc csv-xml fmt \ json md5 awful http-client spiffy uri-common intarweb spiffy-request-vars \ spiffy-directory-listing ssax sxml-serializer sxml-modifications sql-de-lite \ srfi-19 refdb ini-file # # Derived variables # ifeq ($(PROXY),) PROX:= |
︙ | ︙ |
Added utils/loadrunner.scm.notfinished version [a8651ba3f3].
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 | ;; Copyright 2006-2013, Matthew Welland. ;; ;; This program is made available under the GNU GPL version 2.0 or ;; greater. See the accompanying file COPYING for details. ;; ;; This program is distributed WITHOUT ANY WARRANTY; without even the ;; implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR ;; PURPOSE. (use ssax) (use sxml-serializer) (use sxml-modifications) (use regex) (use srfi-69) (use regex-case) (use posix) (use json) (use csv) (use srfi-18) (use format) (require-library iup) (import (prefix iup iup:)) (require-library ini-file) (import (prefix ini-file ini:)) (use canvas-draw) (import canvas-draw-iup) (use sqlite3 srfi-1 posix regex regex-case srfi-69) (import (prefix sqlite3 sqlite3:)) (include "megatest-fossil-hash.scm") ;; ;; GLOBALS ;; (define *loadrunner:current-tab-number* 0) (define loadrunner:unrecognised-command "ERROR: Unrecognised command or missing params. Try \"loadrunner help\"") (define loadrunner:help (conc "Usage: loadrunner [action [params ...]] Note: run loadrunner without parameters to start the gui. run cmd [params ..] : Run cmd params ... when system load drops process : Process the queue Part of the Megatest tool suite. Learn more at http://www.kiatoa.com/fossils/megatest Version: " megatest-fossil-hash)) ;; " ;;====================================================================== ;; DB ;;====================================================================== (define (loadrunner:initialize-db db) (for-each (lambda (qry) (sqlite3:execute db qry)) (list "CREATE TABLE pkgs (id INTEGER PRIMARY KEY, cmd TEXT, datetime TEXT);"))) ;; Create the sqlite db (define (loadrunner:open-db path) (if (and path (directory? path) (file-read-access? path)) (let* ((dbpath (conc path "/loadrunner.db")) (writeable (file-write-access? dbpath)) (dbexists (file-exists? dbpath)) (handler (make-busy-timeout 136000))) (handle-exceptions exn (begin (debug:print 2 "ERROR: problem accessing db " dbpath ((condition-property-accessor 'exn 'message) exn)) (exit)) (set! db (sqlite3:open-database dbpath))) (if *db-write-access* (sqlite3:set-busy-handler! db handler)) (if (not dbexists) (begin (loadrunner:initialize-db db))) db))) ;;====================================================================== ;; GUI ;;====================================================================== ;; The main menu (define (loadrunner:main-menu) (iup:menu ;; a menu is a special attribute to a dialog (think Gnome putting the menu at screen top) (iup:menu-item "Files" (iup:menu ;; Note that you can use either #:action or action: for options (iup:menu-item "Open" action: (lambda (obj) (iup:show (iup:file-dialog)) (print "File->open " obj))) (iup:menu-item "Save" #:action (lambda (obj)(print "File->save " obj))) (iup:menu-item "Exit" #:action (lambda (obj)(exit))))) (iup:menu-item "Tools" (iup:menu (iup:menu-item "Create new blah" #:action (lambda (obj)(print "Tools->new blah"))) ;; (iup:menu-item "Show dialog" #:action (lambda (obj) ;; (show message-window ;; #:modal? #t ;; ;; set positon using coordinates or center, start, top, left, end, bottom, right, parent-center, current ;; ;; #:x 'mouse ;; ;; #:y 'mouse ;; ) )))) (define (loadrunner:publish-view) (iup:vbox (iup:hbox (iup:button "Pushme" #:expand "YES" )))) (define (loadrunner:get-view) (iup:vbox (iup:hbox (iup:button "Pushme" #:expand "YES" )))) (define (loadrunner:manage-view) (iup:vbox (iup:hbox (iup:button "Pushme" #:expand "YES" )))) (define (loadrunner:gui) (iup:show (iup:dialog #:title (conc "Loadrunner dashboard " (current-user-name) ":" (current-directory)) #:menu (loadrunner:main-menu) (let* ((tabs (iup:tabs #:tabchangepos-cb (lambda (obj curr prev) (set! *loadrunner:current-tab-number* curr)) (loadrunner:publish-view) (loadrunner:get-view) (loadrunner:manage-view) ))) ;; (set! (iup:callback tabs tabchange-cb:) (lambda (a b c)(print "SWITCHED TO TAB: " a " " b " " c))) (iup:attribute-set! tabs "TABTITLE0" "Publish") (iup:attribute-set! tabs "TABTITLE1" "Get") (iup:attribute-set! tabs "TABTITLE2" "Manage") ;; (iup:attribute-set! tabs "BGCOLOR" "190 190 190") tabs))) (iup:main-loop)) ;;====================================================================== ;; MAIN ;;====================================================================== (define (loadrunner:load-config path) (let ((fname (conc path "/.loadrunner.config"))) (ini:property-separator-patt " * *") (ini:property-separator #\space) (if (file-exists? fname) (ini:read fname) '()))) (define (main) (let* ((args (argv)) (prog (car args)) (rema (cdr args)) (conf (loadrunner:load-config (pathname-directory prog)))) ;; ( ????? (cond ((eq? (length rema) 1) (case (string->symbol (car rema)) ((process)(loadrunner:process-queue)) ((pause) (loadrunner:pause-queue (cdr rema))) ((help -h -help --h --help) (print loadrunner:help)) (else (print loadrunner:unrecognised-command)))) ((null? rema)(loadrunner:gui)) ((>= (length rema) 2) (case (string->symbol (car rema)) ((run) (loadrunner:process-cmd (cdr rema))) ((remove) (loadrunner:remove-cmds (cdr rema))) (else (print loadrunner:unrecognised-command)))) (else (print loadrunner:unrecognised-command))))) (main) |
Modified utils/nbfake from [6962f03891] to [99a526d022].
1 2 3 4 5 6 7 8 9 10 11 12 13 | #!/bin/bash # Can't always trust $PWD CURRWD=`pwd` if [[ $TARGETHOST_LOGF == "" ]]; then TARGETHOST_LOGF=NBFAKE-`date +%GWW%V.%u_%T` fi echo "#======================================================================" echo "# NBFAKE Running command:" echo "# \"$*\"" echo "#======================================================================" if [[ $TARGETHOST == "" ]]; then | > > > | | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 | #!/bin/bash # Can't always trust $PWD CURRWD=`pwd` if [[ $TARGETHOST_LOGF == "" ]]; then TARGETHOST_LOGF=NBFAKE-`date +%GWW%V.%u_%T` fi echo "#======================================================================" echo "# NBFAKE Running command:" echo "# \"$*\"" echo "#======================================================================" if [[ $TARGETHOST == "" ]]; then unset TARGETHOST TARGETHOST_LOGF_TEMP=$TARGETHOST_LOGF unset TARGETHOST_LOGF sh -c "cd $CURRWD;export DISPLAY=$DISPLAY; export PATH=$PATH; nohup $* > $TARGETHOST_LOGF_TEMP 2>&1 &" else ssh -n -f $TARGETHOST "sh -c \"cd $CURRWD;export DISPLAY=$DISPLAY; export PATH=$PATH; nohup $* > $TARGETHOST_LOGF 2>&1 &\"" fi |