Index: common.scm ================================================================== --- common.scm +++ common.scm @@ -162,10 +162,11 @@ ;; (define *max-cache-size* 0) (define *logged-in-clients* (make-hash-table)) (define *server-id* #f) (define *server-info* #f) ;; good candidate for easily convert to non-global (define *time-to-exit* #f) +(define *time-to-exit-cleanup-thunk-list '()) (define *server-run* #t) (define *run-id* #f) (define *server-kind-run* (make-hash-table)) (define *home-host* #f) ;; (define *total-non-write-delay* 0) @@ -890,10 +891,18 @@ (define (std-exit-procedure) ;;(common:telemetry-log-close) (on-exit (lambda () 0)) + + (for-each (lambda (thunk) + (handle-exceptions + exn + (debug:print 0 *default-log-port* "WARNING: exception while running on-exit thunk item") + (thunk))) + *time-to-exit-cleanup-thunk-list*) + ;;(debug:print-info 13 *default-log-port* "std-exit-procedure called; *time-to-exit*="*time-to-exit*) (let ((no-hurry (if *time-to-exit* ;; hurry up #f (begin (set! *time-to-exit* #t) ADDED docs/manual/debug_techniques.txt Index: docs/manual/debug_techniques.txt ================================================================== --- /dev/null +++ docs/manual/debug_techniques.txt @@ -0,0 +1,89 @@ +// This file is part of Megatest. +// +// Megatest is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Megatest is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with Megatest. If not, see . +// +// Copyright 2006-2012, Matthew Welland. + +Study Plan +---------- + +Megatest is an extensive program with a lot to learn. Following are some paths through the material to smooth the learning path. + +Basic Concepts (suggest you pick these up on the way) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* Components of automation; run, test, iteration +* Selectors; target, runname, and testpatt + +Running Testsuites or Automation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* Using the dashboard gui (recommended) +** Using the "Runs" panel. +** Using the "Run Control" panel. +** Using a test control panel +** The Right Mouse Button menu +** Debug features +*** xterm +**** pstree +**** log files; mt_copy.log, mt_launch.log +**** variables; megatest.csh, megatest.sh +**** testconfig dump, *testconfig +*** State/status buttons +*** Run, Clean, KillReq +*** ReRunClean +* Using the command line +** Getting help; megatest -h, megatest -manual +** Starting runs; megatest -run +*** Selection controls; -target, -runname and -testpatt + +Writing Tests and Flows +~~~~~~~~~~~~~~~~~~~~~~~ + +* environment variables (table 5) +* tests/_testname_/testconfig link:megatest_manual.html#_the_testconfig_file[testconfig details] +** ezsteps and logpro section +** iteration (one test applied to many inputs), items, itemstable link:megatest_manual.html#_iteration[test iteration] +** dependencies, waiton, itemmatch, itemwait link:megatest_manual.html#_requirements_section[test requirements] +** miscellaneous; mode toplevel, runtimelim, skip on file or on running, waiver propagation +* megatest areas +** megatest.config +** runconfigs.config +** config language features; include, shell, system, scheme, rp|realpath, getenv, get, rget, scriptinc link:megatest.html#_config_file_helpers[config file helpers] + +Advanced Topics +~~~~~~~~~~~~~~~ + +* Removing and keeping runs selectively link:megatest_manual.html#_managing_old_runs[managing runs] +* Subruns link:megatest_manual.html#_nested_runs[nested runs] +* Config file features link:megatest_manual.html#_config_file_helpers[config file features] +* HTML output with -generate-html +* Triggers, post run, state/status +* MTLOWESTLOAD +* flexilauncher +* env delta and testconfig +* capturing test data, extracting values from logpro and using them for pass/fail +* mtutil, postgres connection, packets for cross-site/cross-user control (e.g. mcrun). + +Maintenance and Troubleshooting +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* cleanup-db, database structure of Megatest 1.6x +* archiving +* homehost management +* show-runconfig +* show-config +* show with -debug 0,9 +* load management + Index: docs/manual/megatest_manual.html ================================================================== --- docs/manual/megatest_manual.html +++ docs/manual/megatest_manual.html @@ -898,69 +898,11 @@ which can launch jobs on local and remote Linux hosts. Currently megatest uses the network filesystem to call home to your master sqlite3 database. Megatest has been used with the Intel Netbatch and lsf (also known as openlava) batch systems and it should be straightforward to use it with other similar systems.

- - - -
-

Overview

-
-
-

Stand-alone Megatest Area

-

A single, stand-alone, Megatest based testsuite or "area" is -sufficient for most validation, automation and build problems.

-
-
-Static -
-
-

Megatest is designed as a distributed or decoupled system. This means -you can run the areas stand-alone with no additional -infrastructure. I.e. there are no databases, web servers or other -centralized resources needed. However as your needs grow you can -integrate multiple areas into a bigger system.

-
-

Component Descriptions

-
    -
  1. -

    -Multi-area dashboard and xterm. A gui (the dashboard) is usually the - best option for controlling and launching runs but all operations - can also be done from the commandline. Note: The not yet released - multi-area dashboard replaces the old dashboard for browsing and - controlling runs but for managing a single area the old dashboard - works very well. -

    -
    2. -
  2. -

    -Area/testsuite. This is your testsuite or automation definition and - consists of the information in megatest.config, runconfigs.config - and your testconfigs along with any custom scripting that can’t be - done with the native Megatest features. -

    -
    3. -
  3. -

    -If your testsuite or build automation is too large to run on a - single instance you can distribute your jobs into a compute server - pool. The only current requirements are password-less ssh access and - a network filesystem. -

    -
    4. -
-
-
-
-

Full System Architecture

-
-
-Static -
-
+

this page intentionally left blank

Installation

@@ -1399,191 +1341,11 @@ reviewed never

This test runs a single step called "stepname1" which runs a script "stepname.sh". Note that although it is common to put the actions needed for a test step into a script it is not necessary.

- - - -
-

How To Do Things

-
-
-

Process Runs

-
-

Remove Runs

-

From the dashboard click on the button (PASS/FAIL…) for one of the tests. From the test control panel that -comes up push the clean test button. The command field will be prefilled with a template command for removing -that test. You can edit the command, for example change the argument to -testpatt to "%" to remove all tests.

-
-
Remove the test diskperf and all it’s items
-
-
megatest -remove-runs -target ubuntu/nfs/none -runname ww28.1a -testpatt diskperf/% -v
-
-
-
Remove all tests for all runs and all targets
-
-
megatest -remove-runs -target %/%/% -runname % -testpatt % -v
-
-
-
-

Archive Runs

-

Megatest supports using the bup backup tool (https://bup.github.io/) to archive your tests for efficient storage -and retrieval. Archived data can be rapidly retrieved if needed. The metadata for the run (PASS/FAIL status, run -durations, time stamps etc.) are all preserved in the megatest database.

-

For setup information see the Archiving topic in the reference section of this manual.

-
-
To Archive
-

Hint: use the test control panel to create a template command by pushing the "Archive Tests" button.

-
-
Archive a full run
-
-
megatest -target ubuntu/nfs/none -runname ww28.1a -archive save-remove -testpatt %
-
-
-
-
To Restore
-
-
Retrieve a single test
-
-
megatest -target ubuntu/nfs/none -runname ww28.1a -archive restore -testpatt diskperf/%
-
-

Hint: You can browse the archive using bup commands directly.

-
-
-
bup -d /path/to/bup/archive ftp
-
-
-
-
-
-

Submit jobs to Host Types based on Test Name

-
-
In megatest.config
-
-
[host-types]
-general ssh #{getbgesthost general}
-nbgeneral nbjob run JOBCOMMAND -log $MT_LINKTREE/$MT_TARGET/$MT_RUNNAME.$MT_TESTNAME-$MT_ITEM_PATH.lgo
-
-[hosts]
-general cubian xena
-
-[launchers]
-envsetup general
-xor/%/n 4C16G
-% nbgeneral
-
-[jobtools]
-launcher bsub
-# if defined and not "no" flexi-launcher will bypass launcher unless there is no
-# match.
-flexi-launcher yes
-
-
-
-
-
-

Tricks

-
-

This section is a compendium of a various useful tricks for debugging, -configuring and generally getting the most out of Megatest.

-
-

Limiting your running jobs

-

The following example will limit a test in the jobgroup "group1" to no more than 10 tests simultaneously.

-

In your testconfig:

-
-
-
[test_meta]
-jobgroup group1
-
-

In your megatest.config:

-
-
-
[jobgroups]
-group1 10
-custdes 4
-
-
-
-
-
-

Debugging Tricks

-
-
-

Examining The Environment

-
-

Test Control Panel - xterm

-

From the dashboard click on a test PASS/FAIL button. This brings up a test control panel. Aproximately near the center left of the -window there is a button "Start Xterm". Push this to get an xterm with the full context and environment loaded for that test. You can run -scripts or ezsteps by copying from the testconfig (hint, load up the testconfig in a separate gvim or emacs window). This is the easiest way -to debug your tests.

-
-
-

During Config File Processing

-

It is often helpful to know the content of variables in various -contexts as Megatest does the actions needed to run your tests. A handy technique is to force the startup of an xterm in the context being examined.

-

For example, if an item list is not being generated as expected you -can inject the startup of an xterm as if it were an item:

-
-
Original items table
-
-
[items]
-CELLNAME [system getcellname.sh]
-
-
-
Items table modified for debug
-
-
[items]
-DEBUG [system xterm]
-CELLNAME [system getcellnames.sh]
-
-

When this test is run an xterm will pop up. In that xterm the -environment is exactly that in which the script "getcellnames.sh" -would run. You can now debug the script to find out why it isn’t -working as expected.

-
-
-

Organising Your Tests and Tasks

-

The default location "tests" for storing tests can be extended by -adding to your tests-paths section.

-
-
-
[misc]
-parent #{shell dirname $(readlink -f .)}
-
-[tests-paths]
-1 #{get misc parent}/simplerun/tests
-
-

The above example shows how you can use addition sections in your -config file to do complex processing. By putting results of relatively -slow operations into variables the processing of your configs can be -kept fast.

-
-
-

Alternative Method for Running your Job Script

-
-
Directly running job in testconfig
-
-
[setup]
-runscript main.csh
-
-

The runscript method is essentially a brute force way to run scripts where the -user is responsible for setting STATE and STATUS and managing the details of running a test.

-
-
-
-

Debugging Server Problems

-

Some handy Unix commands to track down issues with servers not -communicating with your test manager processes. Please put in tickets -at https://www.kiatoa.com/fossils/megatest if you have problems with -servers getting stuck.

-
-
-
sudo lsof -i
-sudo netstat -lptu
-sudo netstat -tulpn
-
+

include:howto.txt[]

Reference

@@ -1789,11 +1551,11 @@

Job Submission Control

-

Submit jobs to Host Types based on Test Name

+

Submit jobs to Host Types based on Test Name

In megatest.config
[host-types]
 general   nbfake
@@ -2658,10 +2420,290 @@
+
+
+

Study Plan

+
+

Megatest is an extensive program with a lot to learn. Following are some paths through the material to smooth the learning path.

+
+

Basic Concepts (suggest you pick these up on the way)

+
    +
  • +

    +Components of automation; run, test, iteration +

    +
    • +
  • +

    +Selectors; target, runname, and testpatt +

    +
    • +
+
+
+

Running Testsuites or Automation

+
    +
  • +

    +Using the dashboard gui (recommended) +

    +
      +
    • +

      +Using the "Runs" panel. +

      +
      • +
    • +

      +Using the "Run Control" panel. +

      +
      • +
    • +

      +Using a test control panel +

      +
      • +
    • +

      +The Right Mouse Button menu +

      +
      • +
    • +

      +Debug features +

      +
        +
      • +

        +xterm +

        +
          +
        • +

          +pstree +

          +
          • +
        • +

          +log files; mt_copy.log, mt_launch.log +

          +
          • +
        • +

          +variables; megatest.csh, megatest.sh +

          +
          • +
        • +

          +testconfig dump, *testconfig +

          +
          • +
        +
        • +
      • +

        +State/status buttons +

        +
        • +
      • +

        +Run, Clean, KillReq +

        +
        • +
      • +

        +ReRunClean +

        +
        • +
      +
      • +
    +
    • +
  • +

    +Using the command line +

    +
      +
    • +

      +Getting help; megatest -h, megatest -manual +

      +
      • +
    • +

      +Starting runs; megatest -run +

      +
        +
      • +

        +Selection controls; -target, -runname and -testpatt +

        +
        • +
      +
      • +
    +
    • +
+
+
+

Writing Tests and Flows

+
    +
  • +

    +environment variables (table 5) +

    +
    • +
  • +

    +tests/testname/testconfig testconfig details +

    +
      +
    • +

      +ezsteps and logpro section +

      +
      • +
    • +

      +iteration (one test applied to many inputs), items, itemstable test iteration +

      +
      • +
    • +

      +dependencies, waiton, itemmatch, itemwait test requirements +

      +
      • +
    • +

      +miscellaneous; mode toplevel, runtimelim, skip on file or on running, waiver propagation +

      +
      • +
    +
    • +
  • +

    +megatest areas +

    +
      +
    • +

      +megatest.config +

      +
      • +
    • +

      +runconfigs.config +

      +
      • +
    • +

      +config language features; include, shell, system, scheme, rp|realpath, getenv, get, rget, scriptinc config file helpers +

      +
      • +
    +
    • +
+
+
+

Advanced Topics

+
    +
  • +

    +Removing and keeping runs selectively managing runs +

    +
    • +
  • +

    +Subruns nested runs +

    +
    • +
  • +

    +Config file features config file features +

    +
    • +
  • +

    +HTML output with -generate-html +

    +
    • +
  • +

    +Triggers, post run, state/status +

    +
    • +
  • +

    +MTLOWESTLOAD +

    +
    • +
  • +

    +flexilauncher +

    +
    • +
  • +

    +env delta and testconfig +

    +
    • +
  • +

    +capturing test data, extracting values from logpro and using them for pass/fail +

    +
    • +
  • +

    +mtutil, postgres connection, packets for cross-site/cross-user control (e.g. mcrun). +

    +
    • +
+
+
+

Maintenance and Troubleshooting

+
    +
  • +

    +cleanup-db, database structure of Megatest 1.6x +

    +
    • +
  • +

    +archiving +

    +
    • +
  • +

    +homehost management +

    +
    • +
  • +

    +show-runconfig +

    +
    • +
  • +

    +show-config +

    +
    • +
  • +

    +show with -debug 0,9 +

    +
    • +
  • +

    +load management +

    +
    • +
+
+

Megatest Internals

@@ -3033,10 +3075,10 @@
Index: docs/manual/megatest_manual.txt ================================================================== --- docs/manual/megatest_manual.txt +++ docs/manual/megatest_manual.txt @@ -110,13 +110,15 @@ include::study_plan.txt[] // :leveloffset: 0 include::writing_tests.txt[] -include::howto.txt[] +include:howto.txt[] include::reference.txt[] + +include::debug_techniques.txt[] Megatest Internals ------------------ ADDED docs/manual/overview.txt Index: docs/manual/overview.txt ================================================================== --- /dev/null +++ docs/manual/overview.txt @@ -0,0 +1,1 @@ +this page intentionally left blank Index: tests.scm ================================================================== --- tests.scm +++ tests.scm @@ -1547,40 +1547,46 @@ (let* ((treg (or test-registry (tests:get-all))) (test-path (or (hash-table-ref/default treg test-name #f) (conc *toppath* "/tests/" test-name))) (test-configf (conc test-path "/testconfig")) - (testexists (let loop ((tries-left 7)) + (testexists (let loop ((tries-left 14)) (cond ( (and (common:file-exists? test-configf)(file-read-access? test-configf)) #t) ( (common:file-exists? test-configf) (debug:print 0 *default-log-port* "WARNING: Cannot read testconfig file: "test-configf) #f) ( (and wait-a-minute (> tries-left 0)) + (debug:print 0 *default-log-port* "WARNING: Cannot read testconfig. Sleeping 10 and will try again. Tries left: " tries-left) (thread-sleep! 10) (loop (sub1 tries-left))) (else - (debug:print 0 *default-log-port* "WARNING: testconfig file does not exist: "test-configf) + (debug:print 0 *default-log-port* "WARNING: testconfig file does not exist: "test-configf " [wait-a-minute=" wait-a-minute "]") #f)))) (tcfg (if testexists (read-config test-configf #f system-allowed environ-patt: (if system-allowed "pre-launch-env-vars" #f)) #f))) - (if (and tcfg cache-file) (hash-table-set! tcfg "have fulldata" #t)) ;; mark this as fully read data - (if tcfg (hash-table-set! *testconfigs* test-full-name tcfg)) - (if (and testexists - cache-file - (file-write-access? cache-path) - allow-write-cache) - (let ((tpath (conc cache-path "/.testconfig"))) - (debug:print-info 1 *default-log-port* "Caching testconfig for " test-name " in " tpath) - (if (and tcfg (not (common:in-running-test?))) - (configf:write-alist tcfg tpath)))) - tcfg)))))) + + (cond + ((testexists + (if (and tcfg cache-file) (hash-table-set! tcfg "have fulldata" #t)) ;; mark this as fully read data + (if tcfg (hash-table-set! *testconfigs* test-full-name tcfg)) ;; side effect + (if (and testexists + cache-file + (file-write-access? cache-path) + allow-write-cache) + (let ((tpath (conc cache-path "/.testconfig"))) + (debug:print-info 1 *default-log-port* "Caching testconfig for " test-name " in " tpath) + (if (and tcfg (not (common:in-running-test?))) + (configf:write-alist tcfg tpath)))) + tcfg)) + (else + #f)))))))) ;; sort tests by priority and waiton ;; Move test specific stuff to a test unit FIXME one of these days (define (tests:sort-by-priority-and-waiton test-records) (if (eq? (hash-table-size test-records) 0)