Index: archive.scm ================================================================== --- archive.scm +++ archive.scm @@ -386,6 +386,83 @@ (run-n-wait bup-exe params: bup-restore-params print-cmd: #f) ;; (mutex-unlock! bup-mutex) (mt:test-set-state-status-by-id run-id test-id "COMPLETED" #f #f))) (debug:print-error 0 *default-log-port* "No archive path in the record for run-id=" run-id " test-id=" test-id)))) (filter vector? tests)))) - + +(define (common:get-youngest-test tests) + (if (null? tests) + #f + (let ((res #f)) + (for-each + (lambda (test-dat) + (let ((event-time (db:test-get-event_time test-dat))) + (if (or (not res) + (> event-time (db:test-get-event_time res))) + (set! res test-dat)))) + tests) + res))) + +;; from an archive get a specific path - works ONLY with bup for now +;; +(define (archive:bup-get-data archive-command run-id-in run-name-in tests rp-mutex bup-mutex) + (if (null? tests) + (debug:print-info 0 *default-log-port* "get-data called with no matching tests to operate on.") + + (let* ((bup-exe (or (configf:lookup *configdat* "archive" "bup") "bup")) + (linktree (common:get-linktree)) ;; (configf:lookup *configdat* "setup" "linktree"))) + ;; (test-dat (common:get-youngest-test tests)) + (destpath (args:get-arg "-dest"))) + (cond + ((null? tests) + (debug:print-error 0 *default-log-port* + "No test matching provided target, runname pattern and test pattern found.")) + ((file-exists? destpath) + (debug:print-error 0 *default-log-port* + "Destination path alread exists! Please remove it before running get.")) + (else + (let loop ((rem-tests tests)) + (let* ((test-dat (common:get-youngest-test rem-tests)) + (item-path (db:test-get-item-path test-dat)) + (test-name (db:test-get-testname test-dat)) + (test-id (db:test-get-id test-dat)) + (run-id (db:test-get-run_id test-dat)) + (run-name (rmt:get-run-name-from-id run-id)) + (keyvals (rmt:get-key-val-pairs run-id)) + (target (string-intersperse (map cadr keyvals) "/")) + + (toplevel/children (and (db:test-get-is-toplevel test-dat) + (> (rmt:test-toplevel-num-items run-id test-name) 0))) + (test-partial-path (conc target "/" run-name "/" + (db:test-make-full-name test-name item-path))) + ;; note the trailing slash to get the dir inspite of it being a link + (test-path (conc linktree "/" test-partial-path)) + (archive-block-id (db:test-get-archived test-dat)) + (archive-block-info (rmt:test-get-archive-block-info archive-block-id)) + (archive-path (if (vector? archive-block-info) + (vector-ref archive-block-info 2) + #f)) + (archive-internal-path (conc (common:get-testsuite-name) "-" run-id + "/latest/" test-partial-path))) + + (if (and archive-path ;; no point in proceeding if there is no actual archive + (not toplevel/children)) + (begin + (let* ((bup-restore-params (list "-d" archive-path "restore" "-C" (or destpath "data") + ;; " " ;; What is the empty string for? + archive-internal-path))) + (debug:print-info 0 *default-log-port* "Restoring archived data to " (or destpath "data") + " from archive in " archive-path " ... " archive-internal-path) + (run-n-wait bup-exe params: bup-restore-params print-cmd: #t))) + (let ((new-rem-tests (filter (lambda (tdat) + (or (not (eq? (db:test-get-id tdat) test-id)) + (not (eq? (db:test-get-run_id tdat) run-id)))) + rem-tests) )) + (debug:print-info 0 *default-log-port* + "No archive path in the record for run-id=" run-id + " test-id=" test-id ", skipping.") + (if (null? new-rem-tests) + (begin + (debug:print-info 0 *default-log-port* "No archives found for " target "/" run-name "...") + #f) + (loop new-rem-tests))))))))))) + Index: docs/manual/howto.txt ================================================================== --- docs/manual/howto.txt +++ docs/manual/howto.txt @@ -13,196 +13,213 @@ // You should have received a copy of the GNU General Public License // along with Megatest. If not, see . // // Copyright 2006-2012, Matthew Welland. -How To Do Things ----------------- - -Process Runs -~~~~~~~~~~~~ - -Remove Runs -^^^^^^^^^^^ - -From the dashboard click on the button (PASS/FAIL...) for one of the tests. From the test control panel that -comes up push the clean test button. The command field will be prefilled with a template command for removing -that test. You can edit the command, for example change the argument to -testpatt to "%" to remove all tests. - -.Remove the test diskperf and all it's items ----------------- -megatest -remove-runs -target ubuntu/nfs/none -runname ww28.1a -testpatt diskperf/% -v ----------------- - -.Remove all tests for all runs and all targets ----------------- -megatest -remove-runs -target %/%/% -runname % -testpatt % -v ----------------- - -Archive Runs -^^^^^^^^^^^^ - -Megatest supports using the bup backup tool (https://bup.github.io/) to archive your tests for efficient storage -and retrieval. Archived data can be rapidly retrieved if needed. The metadata for the run (PASS/FAIL status, run -durations, time stamps etc.) are all preserved in the megatest database. - -For setup information see the Archiving topic in the reference section of this manual. - -To Archive -++++++++++ - -Hint: use the test control panel to create a template command by pushing the "Archive Tests" button. - -.Archive a full run ----------------- -megatest -target ubuntu/nfs/none -runname ww28.1a -archive save-remove -testpatt % ----------------- - -To Restore -++++++++++ - -.Retrieve a single test ----------------- -megatest -target ubuntu/nfs/none -runname ww28.1a -archive restore -testpatt diskperf/% ----------------- - -Hint: You can browse the archive using bup commands directly. - ----------------- -bup -d /path/to/bup/archive ftp ----------------- - -Submit jobs to Host Types based on Test Name -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.In megatest.config ------------------------- -[host-types] -general ssh #{getbgesthost general} -nbgeneral nbjob run JOBCOMMAND -log $MT_LINKTREE/$MT_TARGET/$MT_RUNNAME.$MT_TESTNAME-$MT_ITEM_PATH.lgo - -[hosts] -general cubian xena - -[launchers] -envsetup general -xor/%/n 4C16G -% nbgeneral - -[jobtools] -launcher bsub -# if defined and not "no" flexi-launcher will bypass launcher unless there is no -# match. -flexi-launcher yes ------------------------- - -Tricks ------- - -This section is a compendium of a various useful tricks for debugging, -configuring and generally getting the most out of Megatest. - -Limiting your running jobs -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The following example will limit a test in the jobgroup "group1" to no more than 10 tests simultaneously. - -In your testconfig: - ----------------- -[test_meta] -jobgroup group1 ----------------- - -In your megatest.config: - ---------------- -[jobgroups] -group1 10 -custdes 4 ---------------- - -Debugging Tricks ----------------- - -Examining The Environment -~~~~~~~~~~~~~~~~~~~~~~~~~ - -Test Control Panel - xterm -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -From the dashboard click on a test PASS/FAIL button. This brings up a test control panel. Aproximately near the center left of the -window there is a button "Start Xterm". Push this to get an xterm with the full context and environment loaded for that test. You can run -scripts or ezsteps by copying from the testconfig (hint, load up the testconfig in a separate gvim or emacs window). This is the easiest way -to debug your tests. - -During Config File Processing -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -It is often helpful to know the content of variables in various -contexts as Megatest does the actions needed to run your tests. A handy technique is to force the startup of an xterm in the context being examined. - -For example, if an item list is not being generated as expected you -can inject the startup of an xterm as if it were an item: - -.Original items table ------------------ -[items] -CELLNAME [system getcellname.sh] ------------------ - -.Items table modified for debug ------------------ -[items] -DEBUG [system xterm] -CELLNAME [system getcellnames.sh] ------------------ - -When this test is run an xterm will pop up. In that xterm the -environment is exactly that in which the script "getcellnames.sh" -would run. You can now debug the script to find out why it isn't -working as expected. - -Organising Your Tests and Tasks -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The default location "tests" for storing tests can be extended by -adding to your tests-paths section. - ----------------------------- -[misc] -parent #{shell dirname $(readlink -f .)} - -[tests-paths] -1 #{get misc parent}/simplerun/tests ----------------------------- - -The above example shows how you can use addition sections in your -config file to do complex processing. By putting results of relatively -slow operations into variables the processing of your configs can be -kept fast. - -Alternative Method for Running your Job Script -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.Directly running job in testconfig -------------------- -[setup] -runscript main.csh -------------------- - -The runscript method is essentially a brute force way to run scripts where the -user is responsible for setting STATE and STATUS and managing the details of running a test. - -Debugging Server Problems -~~~~~~~~~~~~~~~~~~~~~~~~~ - -Some handy Unix commands to track down issues with servers not -communicating with your test manager processes. Please put in tickets -at https://www.kiatoa.com/fossils/megatest if you have problems with -servers getting stuck. - ----------------- -sudo lsof -i -sudo netstat -lptu -sudo netstat -tulpn +How To Do Things +---------------- + +Process Runs +~~~~~~~~~~~~ + +Remove Runs +^^^^^^^^^^^ + +From the dashboard click on the button (PASS/FAIL...) for one of the tests. From the test control panel that +comes up push the clean test button. The command field will be prefilled with a template command for removing +that test. You can edit the command, for example change the argument to -testpatt to "%" to remove all tests. + +.Remove the test diskperf and all it's items +---------------- +megatest -remove-runs -target ubuntu/nfs/none -runname ww28.1a -testpatt diskperf/% -v +---------------- + +.Remove all tests for all runs and all targets +---------------- +megatest -remove-runs -target %/%/% -runname % -testpatt % -v +---------------- + +Archive Runs +^^^^^^^^^^^^ + +Megatest supports using the bup backup tool (https://bup.github.io/) to archive your tests for efficient storage +and retrieval. Archived data can be rapidly retrieved if needed. The metadata for the run (PASS/FAIL status, run +durations, time stamps etc.) are all preserved in the megatest database. + +For setup information see the Archiving topic in the reference section of this manual. + +To Archive +++++++++++ + +Hint: use the test control panel to create a template command by pushing the "Archive Tests" button. + +.Archive a full run +---------------- +megatest -target ubuntu/nfs/none -runname ww28.1a -archive save-remove -testpatt % +---------------- + +To Restore +++++++++++ + +.Retrieve a single test +---------------- +megatest -target ubuntu/nfs/none -runname ww28.1a -archive restore -testpatt diskperf/% +---------------- + +Hint: You can browse the archive using bup commands directly. + +---------------- +bup -d /path/to/bup/archive ftp +---------------- + +Pass Data from Test to Test +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.To save the data call archive save within your test: +---------------- +megatest -archive save +---------------- + +.To retrieve the data call archive get using patterns as needed +---------------- +# Put the retrieved data into /tmp +DESTPATH=/tmp/$USER/$MT_TARGET/$MT_RUN_NAME/$MT_TESTNAME/$MT_ITEMPATH/my_data +mkdir -p $DESTPATH +megatest -archive get -runname % -dest $DESTPATH +---------------- + + +Submit jobs to Host Types based on Test Name +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.In megatest.config +------------------------ +[host-types] +general ssh #{getbgesthost general} +nbgeneral nbjob run JOBCOMMAND -log $MT_LINKTREE/$MT_TARGET/$MT_RUNNAME.$MT_TESTNAME-$MT_ITEM_PATH.lgo + +[hosts] +general cubian xena + +[launchers] +envsetup general +xor/%/n 4C16G +% nbgeneral + +[jobtools] +launcher bsub +# if defined and not "no" flexi-launcher will bypass launcher unless there is no +# match. +flexi-launcher yes +------------------------ + +Tricks +------ + +This section is a compendium of a various useful tricks for debugging, +configuring and generally getting the most out of Megatest. + +Limiting your running jobs +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The following example will limit a test in the jobgroup "group1" to no more than 10 tests simultaneously. + +In your testconfig: + +---------------- +[test_meta] +jobgroup group1 +---------------- + +In your megatest.config: + +--------------- +[jobgroups] +group1 10 +custdes 4 +--------------- + +Debugging Tricks +---------------- + +Examining The Environment +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Test Control Panel - xterm +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +From the dashboard click on a test PASS/FAIL button. This brings up a test control panel. Aproximately near the center left of the +window there is a button "Start Xterm". Push this to get an xterm with the full context and environment loaded for that test. You can run +scripts or ezsteps by copying from the testconfig (hint, load up the testconfig in a separate gvim or emacs window). This is the easiest way +to debug your tests. + +During Config File Processing +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +It is often helpful to know the content of variables in various +contexts as Megatest does the actions needed to run your tests. A handy technique is to force the startup of an xterm in the context being examined. + +For example, if an item list is not being generated as expected you +can inject the startup of an xterm as if it were an item: + +.Original items table +----------------- +[items] +CELLNAME [system getcellname.sh] +----------------- + +.Items table modified for debug +----------------- +[items] +DEBUG [system xterm] +CELLNAME [system getcellnames.sh] +----------------- + +When this test is run an xterm will pop up. In that xterm the +environment is exactly that in which the script "getcellnames.sh" +would run. You can now debug the script to find out why it isn't +working as expected. + +Organising Your Tests and Tasks +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The default location "tests" for storing tests can be extended by +adding to your tests-paths section. + +---------------------------- +[misc] +parent #{shell dirname $(readlink -f .)} + +[tests-paths] +1 #{get misc parent}/simplerun/tests +---------------------------- + +The above example shows how you can use addition sections in your +config file to do complex processing. By putting results of relatively +slow operations into variables the processing of your configs can be +kept fast. + +Alternative Method for Running your Job Script +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.Directly running job in testconfig +------------------- +[setup] +runscript main.csh +------------------- + +The runscript method is essentially a brute force way to run scripts where the +user is responsible for setting STATE and STATUS and managing the details of running a test. + +Debugging Server Problems +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Some handy Unix commands to track down issues with servers not +communicating with your test manager processes. Please put in tickets +at https://www.kiatoa.com/fossils/megatest if you have problems with +servers getting stuck. + +---------------- +sudo lsof -i +sudo netstat -lptu +sudo netstat -tulpn ---------------- Index: docs/manual/megatest_manual.html ================================================================== --- docs/manual/megatest_manual.html +++ docs/manual/megatest_manual.html @@ -1452,10 +1452,26 @@ 
bup -d /path/to/bup/archive ftp
+ +
+

Pass Data from Test to Test

+
+
To save the data call archive save within your test:
+
+
megatest -archive save
+
+
+
To retrieve the data call archive get using patterns as needed
+
+
# Put the retrieved data into /tmp
+DESTPATH=/tmp/$USER/$MT_TARGET/$MT_RUN_NAME/$MT_TESTNAME/$MT_ITEMPATH/my_data
+mkdir -p $DESTPATH
+megatest -archive get -runname % -dest $DESTPATH
+

Submit jobs to Host Types based on Test Name

In megatest.config
@@ -2366,10 +2382,29 @@
Propagate environment to next step
$MT_MEGATEST -env2file .ezsteps/${stepname}
+
+
+

Scripts

+
+
Specifying scripts inline (best used for only simple scripts)
+
+
[scripts]
+loaddb #!/bin/bash
+  sqlite3 $1 <<EOF
+  .mode tabs
+  .import $2 data
+  .q
+  EOF
+
+

The above snippet results in the creation of an executable script +called "loaddb" in the test directory. NOTE: every line in the script +must be prefixed with the exact same number of spaces. Lines beginning +with a # will not work as expected. Currently you cannot indent +intermediate lines.

Full example with ezsteps, logpro rules, scripts etc.
# You can include a common file
 #
@@ -3134,10 +3169,10 @@
Index: docs/manual/reference.txt ================================================================== --- docs/manual/reference.txt +++ docs/manual/reference.txt @@ -464,14 +464,12 @@ Complex mapping example ~~~~~~~~~~~~~~~~~~~~~~~ - // image::itemmap.png[] image::complex-itemmap.png[] - We accomplish this by configuring the testconfigs of our tests C D and E as follows: .Testconfig for Test E has ---------------------- @@ -514,14 +512,10 @@ itemstable ~~~~~~~~~~ An alternative to defining items is the itemstable section. This lets you define the itempath in a table format rather than specifying components and relying on getting all permutations of those components. - - - - Dynamic Flow Dependency Tree ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .Autogeneration waiton list for dynamic flow dependency trees @@ -652,10 +646,30 @@ .Propagate environment to next step ---------------------------- $MT_MEGATEST -env2file .ezsteps/${stepname} ---------------------------- +Scripts +~~~~~~~ + +.Specifying scripts inline (best used for only simple scripts) +---------------------------- +[scripts] +loaddb #!/bin/bash + sqlite3 $1 < -target-patt -dumpmode -list-test-time : list time requered to complete each test in a run. It following following arguments @@ -299,10 +299,11 @@ "-rerun" "-days" "-rename-run" "-to" + "-dest" ;; values and messages ":category" ":variable" ":value" ":expected" @@ -1052,11 +1053,11 @@ (and (eq? action 'archive) ;; if it is an archive command fallback to MT_TEST_NAME and MT_ITEMPATH (common:get-full-test-name)) (and (eq? action 'kill-runs) "%/%") ;; I'm just guessing that this is correct :( (common:args-get-testpatt #f) ;; (args:get-arg "-testpatt"))) - ))) ;; + ))) ;; (cond ((not target) (debug:print-error 0 *default-log-port* "Missing required parameter for " action ", you must specify -target or -reqtarg") (exit 1)) @@ -1875,15 +1876,22 @@ ;; Archive tests ;;====================================================================== ;; Archive tests matching target, runname, and testpatt (if (args:get-arg "-archive") ;; else do a general-run-call - (general-run-call - "-archive" - "Archive" - (lambda (target runname keys keyvals) - (operate-on 'archive target-in: target runname-in: runname )))) + (begin + ;; for the archive get we need to preserve the starting dir as part of the target path + (if (and (args:get-arg "-dest") + (not (equal? (substring (args:get-arg "-dest") 0 1) "/"))) + (let ((newpath (conc (current-directory) "/" (args:get-arg "-dest")))) + (debug:print-info 1 *default-log-port* "Preserving original path to destination, was " (args:get-arg "-dest") ", now " newpath) + (hash-table-set! args:arg-hash "-dest" newpath))) + (general-run-call + "-archive" + "Archive" + (lambda (target runname keys keyvals) + (operate-on 'archive target-in: target runname-in: runname ))))) ;;====================================================================== ;; Extract a spreadsheet from the runs database ;;====================================================================== Index: runs.scm ================================================================== --- runs.scm +++ runs.scm @@ -2105,11 +2105,12 @@ (states (if state (string-split state ",") '())) (statuses (if status (string-split status ",") '())) (state-status (if (string? new-state-status) (string-split new-state-status ",") '(#f #f))) (rp-mutex (make-mutex)) (bup-mutex (make-mutex)) - (keep-records (args:get-arg "-keep-records"))) ;; used in conjunction with -remove-runs to keep the records, TODO: consolidate this with "mode". + (keep-records (args:get-arg "-keep-records")) ;; used in conjunction with -remove-runs to keep the records, TODO: consolidate this with "mode". + (test-records '())) ;; for tasks that we wish to operate on all tests in one fell swoop (let* ((write-access-actions '(remove-runs set-state-status archive run-wait kill-runs)) (dbfile (conc *toppath* "/megatest.db")) (readonly-mode (not (file-write-access? dbfile)))) (when (and readonly-mode @@ -2139,11 +2140,11 @@ (run-name (db:get-value-by-header run header "runname")) (tests (if (not (equal? run-state "locked")) (proc-get-tests run-id) '())) (lasttpath "/does/not/exist/I/hope") - (lastrealpath "/does/not/exist/I/hope") + (lastrealpath "/does/not/exist/I/hope") (worker-thread #f)) (debug:print-info 4 *default-log-port* "runs:operate-on run=" run ", header=" header) (if (not (null? tests)) (begin (case action @@ -2166,23 +2167,29 @@ action) ((run-wait) (debug:print 1 *default-log-port* "Waiting for run " runkey ", run=" runnamepatt " to complete")) ((archive) (debug:print 1 *default-log-port* "Archiving/restoring (" (args:get-arg "-archive") ") data for run: " runkey " " (db:get-value-by-header run header "runname")) - (set! worker-thread - (make-thread - (lambda () - (case (string->symbol (args:get-arg "-archive")) - ((save save-remove keep-html) - (archive:run-bup (args:get-arg "-archive") run-id run-name tests rp-mutex bup-mutex)) - ((restore) - (archive:bup-restore (args:get-arg "-archive") run-id run-name tests rp-mutex bup-mutex)) - (else - (debug:print-error 0 *default-log-port* "unrecognised sub command to -archive. Run \"megatest\" to see help") - (exit)))) - "archive-bup-thread")) - (thread-start! worker-thread)) + (let ((op (string->symbol (args:get-arg "-archive")))) + (set! worker-thread + (make-thread + (lambda () + (case op + ((save save-remove keep-html) + (archive:run-bup op run-id run-name tests rp-mutex bup-mutex)) + ((restore) + (archive:bup-restore op run-id run-name tests rp-mutex bup-mutex)) + ((get) ;;; NOTE: This is a special case. We wish to operate on ALL tests in one go + (set! test-records (append tests test-records))) + (else + (debug:print-error 0 *default-log-port* "unrecognised sub command " op " for -archive. Run \"megatest\" to see help") + (exit)))) + "archive-bup-thread")) + (thread-start! worker-thread) + (if (eq? op 'get) + (thread-join! worker-thread)) ;; we need the test-records set to not overlap + )) (else (debug:print-info 0 *default-log-port* "action not recognised " action))) ;; actions that operate on one test at a time can be handled below ;; @@ -2394,10 +2401,11 @@ (loop (car tal)(cdr tal)))) ))) ) (if worker-thread (thread-join! worker-thread))) (common:join-backgrounded-threads)))) + ;; remove the run if zero tests remain (if (eq? action 'remove-runs) (let* ((run-id (db:get-value-by-header run header "id")) ;; NB// masks run-id from above? (remtests (mt:get-tests-for-run run-id #f '("DELETED") '("n/a") not-in: #t))) (if (null? remtests) ;; no more tests remaining @@ -2423,14 +2431,16 @@ (runs:recursive-delete-with-error-msg realpath) ))))) )) runs) - ;; (sqlite3:finalize! (db:delay-if-busy tdbdat)) + ;; special case - archive get + (if (equal? (args:get-arg "-archive") "get") + (archive:bup-get-data "get" #f #f test-records rp-mutex bup-mutex)) + ) + #t ) -#t -) (define (runs:remove-test-directory test mode) ;; remove-data-only) (let* ((run-dir (db:test-get-rundir test)) ;; run dir is from the link tree (real-dir (if (common:file-exists? run-dir) ;; (resolve-pathname run-dir)