Overview
Comment: | Added config and switch to control using cache |
---|---|
Downloads: | Tarball | ZIP archive | SQL archive |
Timelines: | family | ancestors | descendants | both | cache-control |
Files: | files | file ages | folders |
SHA1: |
c9471f325e217246b779711d0d91441b |
User & Date: | matt on 2017-03-21 22:19:27 |
Other Links: | branch diff | manifest | tags |
Context
2017-03-21
| ||
23:01 | bumped O level to 4 check-in: 97bb5db048 user: matt tags: cache-control | |
22:19 | Added config and switch to control using cache check-in: c9471f325e user: matt tags: cache-control | |
11:46 | Added defence against bad sqlite3 handles to the finalizer check-in: 79058725e7 user: matt tags: v1.63 | |
Changes
Modified common.scm from [31405ca571] to [644f3dd988].
︙ | ︙ | |||
559 560 561 562 563 564 565 | (if *toppath* (pathname-file *toppath*) (pathname-file (current-directory))))) (define (common:get-db-tmp-area) (if *db-cache-path* *db-cache-path* | > | | | | | | > | 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 | (if *toppath* (pathname-file *toppath*) (pathname-file (current-directory))))) (define (common:get-db-tmp-area) (if *db-cache-path* *db-cache-path* (if *toppath* (let ((dbpath (create-directory (conc "/tmp/" (current-user-name) "/megatest_localdb/" (common:get-testsuite-name) "/" (string-translate *toppath* "/" ".")) #t))) (set! *db-cache-path* dbpath) dbpath) #f))) (define (common:get-area-path-signature) (message-digest-string (md5-primitive) *toppath*)) ;;====================================================================== ;; E X I T H A N D L I N G ;;====================================================================== |
︙ | ︙ | |||
1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 | ;; (define (common:on-homehost?) (let ((hh (common:get-homehost))) (if hh (cdr hh) #f))) ;;====================================================================== ;; M I S C L I S T S ;;====================================================================== ;; items in lista are matched value and position in listb ;; return the remaining items in listb or #f ;; | > > > > > > > | 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 | ;; (define (common:on-homehost?) (let ((hh (common:get-homehost))) (if hh (cdr hh) #f))) ;; do we honor the caches of the config files? ;; (define (common:use-cache?) (not (or (args:get-arg "-no-cache") (and *configdat* (equal? (configf:lookup *configdat* "setup" "use-cache") "no"))))) ;;====================================================================== ;; M I S C L I S T S ;;====================================================================== ;; items in lista are matched value and position in listb ;; return the remaining items in listb or #f ;; |
︙ | ︙ |
Modified dashboard.scm from [9c476b54db] to [20957064b0].
︙ | ︙ | |||
97 98 99 100 101 102 103 | (exit))) (if (args:get-arg "-h") (begin (print help) (exit))) | < < < < > > > > | 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 | (exit))) (if (args:get-arg "-h") (begin (print help) (exit))) ;; TODO: Move this inside (main) ;; (if (not (launch:setup)) (begin (print "Failed to find megatest.config, exiting") (exit 1))) (if (not (common:on-homehost?)) (begin (debug:print 0 *default-log-port* "WARNING: Current policy requires running dashboard on homehost: " (common:get-homehost)))) ;; RA => Might require revert for filters ;; create a watch dog to move changes from lt/.db/*.db to megatest.db ;; ;;;(if (file-write-access? (conc *toppath* "/megatest.db")) ;;(debug:print-info 13 *default-log-port* "Before common:watchdog spawn") (thread-start! (make-thread common:watchdog "Watchdog thread")) ;;(debug:print-info 13 *default-log-port* "After common:watchdog spawn") |
︙ | ︙ |
Modified launch.scm from [946add6293] to [d0b975cae1].
︙ | ︙ | |||
758 759 760 761 762 763 764 | (mutex-unlock! *launch-setup-mutex*) *toppath*) (let ((res (launch:setup-body force: force areapath: areapath))) (mutex-unlock! *launch-setup-mutex*) res))) (define (launch:setup-body #!key (force #f) (areapath #f)) | > | | 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 | (mutex-unlock! *launch-setup-mutex*) *toppath*) (let ((res (launch:setup-body force: force areapath: areapath))) (mutex-unlock! *launch-setup-mutex*) res))) (define (launch:setup-body #!key (force #f) (areapath #f)) (let* ((use-cache (common:use-cache?)) (toppath (or *toppath* areapath (getenv "MT_RUN_AREA_HOME"))) ;; preserve toppath (runname (common:args-get-runname)) (target (common:args-get-target)) (linktree (common:get-linktree)) (sections (if target (list "default" target) #f)) ;; for runconfigs (mtconfig (or (args:get-arg "-config") "megatest.config")) ;; allow overriding megatest.config (rundir (if (and runname target linktree)(conc linktree "/" target "/" runname) #f)) (mtcachef (and rundir (conc rundir "/" ".megatest.cfg-" megatest-version "-" megatest-fossil-hash))) |
︙ | ︙ | |||
780 781 782 783 784 785 786 | ;; (print "runname: " runname " target: " target " mtcachef: " mtcachef " rccachef: " rccachef) (set! *toppath* toppath) ;; This is needed when we are running as a test using CMDINFO as a datasource (cond ;; data was read and cached and available in *configstatus*, toppath has already been set ((eq? *configstatus* 'fulldata) *toppath*) ;; if mtcachef exists just read it, however we need to assume toppath is available in $MT_RUN_AREA_HOME | | | 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 | ;; (print "runname: " runname " target: " target " mtcachef: " mtcachef " rccachef: " rccachef) (set! *toppath* toppath) ;; This is needed when we are running as a test using CMDINFO as a datasource (cond ;; data was read and cached and available in *configstatus*, toppath has already been set ((eq? *configstatus* 'fulldata) *toppath*) ;; if mtcachef exists just read it, however we need to assume toppath is available in $MT_RUN_AREA_HOME ((and mtcachef (file-exists? mtcachef) (get-environment-variable "MT_RUN_AREA_HOME") use-cache) (set! *configdat* (configf:read-alist mtcachef)) (set! *runconfigdat* (configf:read-alist rccachef)) (set! *configinfo* (list *configdat* (get-environment-variable "MT_RUN_AREA_HOME"))) (set! *configstatus* 'fulldata) (set! *toppath* (get-environment-variable "MT_RUN_AREA_HOME")) *toppath*) ;; we have all the info needed to fully process runconfigs and megatest.config |
︙ | ︙ | |||
891 892 893 894 895 896 897 898 899 900 901 902 903 904 | (directory-exists? *toppath*)) (begin (setenv "MT_RUN_AREA_HOME" *toppath*) (setenv "MT_TESTSUITENAME" (common:get-testsuite-name))) (begin (debug:print-error 0 *default-log-port* "failed to find the top path to your Megatest area.") ;;(exit 1) #f )) *toppath*)) (define (get-best-disk confdat testconfig) (let* ((disks (or (and testconfig (hash-table-ref/default testconfig "disks" #f)) (hash-table-ref/default confdat "disks" #f))) | > | 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 | (directory-exists? *toppath*)) (begin (setenv "MT_RUN_AREA_HOME" *toppath*) (setenv "MT_TESTSUITENAME" (common:get-testsuite-name))) (begin (debug:print-error 0 *default-log-port* "failed to find the top path to your Megatest area.") ;;(exit 1) (set! *toppath* #f) ;; force it to be false so we return #f #f )) *toppath*)) (define (get-best-disk confdat testconfig) (let* ((disks (or (and testconfig (hash-table-ref/default testconfig "disks" #f)) (hash-table-ref/default confdat "disks" #f))) |
︙ | ︙ |
Modified megatest.scm from [08c15272e3] to [2814b7c2ec].
︙ | ︙ | |||
85 86 87 88 89 90 91 92 93 94 95 96 97 98 | -lock : lock run specified by target and runname -unlock : unlock run specified by target and runname -set-run-status status : sets status for run to status, requires -target and -runname -get-run-status : gets status for run specified by target and runname -run-wait : wait on run specified by target and runname -preclean : remove the existing test directory before running the test -clean-cache : remove the cached megatest.config and runconfig.config files Selectors (e.g. use for -runtests, -remove-runs, -set-state-status, -list-runs etc.) -target key1/key2/... : run for key1, key2, etc. -reqtarg key1/key2/... : run for key1, key2, etc. but key1/key2 must be in runconfig -testpatt patt1/patt2,patt3/... : % is wildcard -runname : required, name for this particular test run -state : Applies to runs, tests or steps depending on context | > | 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 | -lock : lock run specified by target and runname -unlock : unlock run specified by target and runname -set-run-status status : sets status for run to status, requires -target and -runname -get-run-status : gets status for run specified by target and runname -run-wait : wait on run specified by target and runname -preclean : remove the existing test directory before running the test -clean-cache : remove the cached megatest.config and runconfig.config files -no-cache : do not use the cached config files. Selectors (e.g. use for -runtests, -remove-runs, -set-state-status, -list-runs etc.) -target key1/key2/... : run for key1, key2, etc. -reqtarg key1/key2/... : run for key1, key2, etc. but key1/key2 must be in runconfig -testpatt patt1/patt2,patt3/... : % is wildcard -runname : required, name for this particular test run -state : Applies to runs, tests or steps depending on context |
︙ | ︙ | |||
300 301 302 303 304 305 306 307 308 309 310 311 312 313 | "-summarize-items" "-gui" "-daemonize" "-preclean" "-rerun-clean" "-rerun-all" "-clean-cache" "-cache-db" "-use-db-cache" ;; misc "-repl" "-lock" "-unlock" "-list-servers" | > | 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 | "-summarize-items" "-gui" "-daemonize" "-preclean" "-rerun-clean" "-rerun-all" "-clean-cache" "-no-cache" "-cache-db" "-use-db-cache" ;; misc "-repl" "-lock" "-unlock" "-list-servers" |
︙ | ︙ | |||
849 850 851 852 853 854 855 | (let* ((rundir (if (and (getenv "MT_LINKTREE")(getenv "MT_TARGET")(getenv "MT_RUNNAME")) (conc (getenv "MT_LINKTREE") "/" (getenv "MT_TARGET") "/" (getenv "MT_RUNNAME")) #f)) (cfgf (if rundir (conc rundir "/.runconfig." megatest-version "-" megatest-fossil-hash) #f))) (if (and cfgf (file-exists? cfgf) | | > | 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 | (let* ((rundir (if (and (getenv "MT_LINKTREE")(getenv "MT_TARGET")(getenv "MT_RUNNAME")) (conc (getenv "MT_LINKTREE") "/" (getenv "MT_TARGET") "/" (getenv "MT_RUNNAME")) #f)) (cfgf (if rundir (conc rundir "/.runconfig." megatest-version "-" megatest-fossil-hash) #f))) (if (and cfgf (file-exists? cfgf) (file-write-access? cfgf) (common:use-cache?)) (configf:read-alist cfgf) (let* ((keys (rmt:get-keys)) (target (common:args-get-target)) (key-vals (if target (keys:target->keyval keys target) #f)) (sections (if target (list "default" target) #f)) (data (begin (setenv "MT_RUN_AREA_HOME" *toppath*) |
︙ | ︙ |
Modified tests.scm from [31439bf084] to [bccd138868].
︙ | ︙ | |||
1142 1143 1144 1145 1146 1147 1148 | ;; if .testconfig exists in test directory read and return it ;; else if have cached copy in *testconfigs* return it IFF there is a section "have fulldata" ;; else read the testconfig file ;; if have path to test directory save the config as .testconfig and return it ;; (define (tests:get-testconfig test-name item-path test-registry system-allowed #!key (force-create #f)) | > | | > | 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 | ;; if .testconfig exists in test directory read and return it ;; else if have cached copy in *testconfigs* return it IFF there is a section "have fulldata" ;; else read the testconfig file ;; if have path to test directory save the config as .testconfig and return it ;; (define (tests:get-testconfig test-name item-path test-registry system-allowed #!key (force-create #f)) (let* ((use-cache (common:use-cache?)) (cache-path (tests:get-test-path-from-environment)) (cache-file (and cache-path (conc cache-path "/.testconfig"))) (cache-exists (and cache-file (not force-create) ;; if force-create then pretend there is no cache to read (file-exists? cache-file))) (cached-dat (if (and (not force-create) cache-exists use-cache) (handle-exceptions exn #f ;; any issues, just give up with the cached version and re-read (configf:read-alist cache-file)) #f)) (test-full-name (if (and item-path (not (string-null? item-path))) (conc test-name "/" item-path) |
︙ | ︙ |
Modified tests/fdktestqa/testqa/megatest.config from [200e742890] to [d32541500d].
1 2 3 4 5 6 7 8 9 10 11 12 | [setup] testcopycmd cp --remove-destination -rlv TEST_SRC_PATH/. TEST_TARG_PATH/. >> TEST_TARG_PATH/mt_launch.log 2>> TEST_TARG_PATH/mt_launch.log # launchwait no launch-delay 0 # All these are overridden in ../fdk.config # [jobtools] # launcher nbfake # launcher bsub -q priority -o $MT_TEST_RUN_DIR/openlava.log [include ../fdk.config] | > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 | [setup] testcopycmd cp --remove-destination -rlv TEST_SRC_PATH/. TEST_TARG_PATH/. >> TEST_TARG_PATH/mt_launch.log 2>> TEST_TARG_PATH/mt_launch.log # launchwait no launch-delay 0 [server] runtime 180 # All these are overridden in ../fdk.config # [jobtools] # launcher nbfake # launcher bsub -q priority -o $MT_TEST_RUN_DIR/openlava.log [include ../fdk.config] |
︙ | ︙ |
Modified tests/fdktestqa/testqa/runsuite.sh from [5c9574fe6e] to [b83309f619].
1 2 | #!/bin/bash | | | | | < | | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 | #!/bin/bash # (cd ../../..;make && make install) || exit 1 # export PATH=$PWD/../../../bin:$PATH for i in a b c d e f;do # g h i j k l m n o p q r s t u v w x y z;do viewscreen megatest -run -testpatt % -target a/b -runname w$(date +%U.%u.%H)$i done echo "" > num-running.log while true; do foo=$(megatest -list-runs % | grep RUNNING | wc -l) echo "Num running at `date` $foo" echo "$foo at `date`" >> num-running.log sleep 10 done |
Modified tests/fullrun/megatest.config from [353b25ebc0] to [b0b07a12af].
︙ | ︙ | |||
236 237 238 239 240 241 242 243 244 245 246 247 248 249 | [archive-disks] # Archives will be organised under these paths like this: # <testsuite>/<creationdate> # Within the archive the data is structured like this: # <target>/<runname>/<test>/ disk0 /tmp/#{getenv USER}/adisk1 # Uncomment these to emulate a job queue with a long time (look in bin/sleeprunner for the time) [jobtools] launcher #{scheme (case (string->symbol (conc (getenv "datapath"))) \ ((none) "nbfake") \ ((openlava) "bsub -o $MT_LINKTREE/$MT_TARGET/$MT_RUNNAME.$MT_TESTNAME-$MT_ITEM_PATH.log") \ ((sleeprunner) "sleeprunner") \ | > | 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 | [archive-disks] # Archives will be organised under these paths like this: # <testsuite>/<creationdate> # Within the archive the data is structured like this: # <target>/<runname>/<test>/ disk0 /tmp/#{getenv USER}/adisk1 disk1 /mfs/tmp/archive # Uncomment these to emulate a job queue with a long time (look in bin/sleeprunner for the time) [jobtools] launcher #{scheme (case (string->symbol (conc (getenv "datapath"))) \ ((none) "nbfake") \ ((openlava) "bsub -o $MT_LINKTREE/$MT_TARGET/$MT_RUNNAME.$MT_TESTNAME-$MT_ITEM_PATH.log") \ ((sleeprunner) "sleeprunner") \ |
︙ | ︙ |
Modified tests/fullrun/runconfigs.config from [6612b4b8e1] to [ad7302ff42].
1 2 3 | [default] SOMEVAR This should show up in SOMEVAR3 VARNOVAL | | > | 1 2 3 4 5 6 7 8 9 10 11 12 | [default] SOMEVAR This should show up in SOMEVAR3 VARNOVAL VARNOVAL_WITHSPACE QUICK % # target based getting of config file, look at afs.config and nfs.config [include #{getenv fsname}.config] [include #{getenv MT_RUN_AREA_HOME}/common_runconfigs.config] # #{system echo 'VACKYVAR #{shell pwd}' > $MT_RUN_AREA_HOME/configs/$USER.config} |
︙ | ︙ |
Modified tests/fullrun/tests/test_mt_vars/testconfig from [0083ae639f] to [cd1b423cdc].
︙ | ︙ | |||
50 51 52 53 54 55 56 | (expect:required in "LogFileBody" = 1 "VARNOVAL" #/VARNOVAL/) [test_meta] author matt owner bob description This test runs a single ezstep which is expected to pass, no logpro file. | | | 50 51 52 53 54 55 56 57 58 | (expect:required in "LogFileBody" = 1 "VARNOVAL" #/VARNOVAL/) [test_meta] author matt owner bob description This test runs a single ezstep which is expected to pass, no logpro file. tags quick,first,single reviewed 09/10/2011, by Matt |