Index: Makefile
==================================================================
--- Makefile
+++ Makefile
@@ -4,11 +4,11 @@
 INSTALL=install
 SRCFILES = common.scm items.scm launch.scm \
            ods.scm runconfig.scm server.scm configf.scm \
            db.scm keys.scm margs.scm megatest-version.scm \
            process.scm runs.scm tasks.scm tests.scm genexample.scm \
-	   http-transport.scm filedb.scm \
+	   http-transport.scm nmsg-transport.scm filedb.scm \
            client.scm gutils.scm synchash.scm daemon.scm mt.scm dcommon.scm \
 	   tree.scm ezsteps.scm lock-queue.scm sdb.scm \
 	   rmt.scm api.scm tdb.scm portlogger.scm
 
 # Eggs to install (straightforward ones)

Index: api.scm
==================================================================
--- api.scm
+++ api.scm
@@ -49,89 +49,105 @@
     testmeta-get-record))
 
 ;; These are called by the server on recipt of /api calls
 ;;    - keep it simple, only return the actual result of the call, i.e. no meta info here
 ;;
-(define (api:execute-requests dbstruct cmd params)
-  (case (string->symbol cmd)
-    ;; SERVERS
-    ((start-server)                 (apply server:kind-run params))
-    ((kill-server)                  (set! *server-run* #f))
-
-    ;; KEYS
-    ((get-key-val-pairs)            (apply db:get-key-val-pairs dbstruct params))
-    ((get-keys)                     (db:get-keys dbstruct))
-
-    ;; TESTS
-    ((test-toplevel-num-items)         (apply db:test-toplevel-num-items dbstruct params))
-    ((get-test-info-by-id)	       (apply db:get-test-info-by-id dbstruct params))
-    ((test-get-rundir-from-test-id)    (apply db:test-get-rundir-from-test-id dbstruct params))
-    ((test-set-state-status-by-id)     (apply db:test-set-state-status-by-id dbstruct params))
-    ((get-count-tests-running)         (apply db:get-count-tests-running dbstruct params))
-    ((get-count-tests-running-in-jobgroup) (apply db:get-count-tests-running-in-jobgroup dbstruct params))
-    ((delete-test-records)             (apply db:delete-test-records dbstruct params))
-    ;; ((delete-test-step-records)        (apply db:delete-test-step-records dbstruct params))
-    ((delete-old-deleted-test-records) (apply db:delete-old-deleted-test-records dbstruct params))
-    ((test-set-status-state)           (apply db:test-set-status-state dbstruct params))
-    ((get-previous-test-run-record)    (apply db:get-previous-test-run-record dbstruct params))
-    ((get-matching-previous-test-run-records)(apply db:get-matching-previous-test-run-records dbstruct params))
-    ((test-get-logfile-info)           (apply db:test-get-logfile-info dbstruct params))
-    ((test-get-records-for-index-file)  (apply db:test-get-records-for-index-file dbstruct params))
-    ((get-testinfo-state-status)       (apply db:get-testinfo-state-status dbstruct params))
-    ((test-set-top-process-pid)        (apply db:test-set-top-process-pid dbstruct params))
-    ((test-get-top-process-pid)        (apply db:test-get-top-process-pid dbstruct params))
-    ((test-get-paths-matching-keynames-target-new) (apply db:test-get-paths-matching-keynames-target-new dbstruct params))
-    ((get-prereqs-not-met)             (apply db:get-prereqs-not-met dbstruct params))
-    ((roll-up-pass-fail-counts)        (apply db:roll-up-pass-fail-counts dbstruct params))
-    ((update-fail-pass-counts)         (apply db:general-call dbstruct 'update-pass-fail-counts params))
-    ((get-count-tests-running-for-run-id) (apply db:get-count-tests-running-for-run-id dbstruct params))
-
-    ;; RUNS
-    ((get-run-info)                 (apply db:get-run-info dbstruct params))
-    ((get-run-status)               (apply db:get-run-status dbstruct params))
-    ((set-run-status)               (apply db:set-run-status dbstruct params))
-    ((register-run)                 (apply db:register-run dbstruct params))
-    ((set-tests-state-status)       (apply db:set-tests-state-status dbstruct params))
-    ((get-tests-for-run)            (apply db:get-tests-for-run dbstruct params))
-    ((get-test-id)                  (apply db:get-test-id dbstruct params))
-    ((get-tests-for-runs-mindata)   (apply db:get-tests-for-runs-mindata dbstruct params))
-    ((get-run-name-from-id)         (apply db:get-run-name-from-id dbstruct params))
-    ((delete-run)                   (apply db:delete-run dbstruct params))
-    ((get-runs)                     (apply db:get-runs dbstruct params))
-    ((get-all-run-ids)              (db:get-all-run-ids dbstruct))
-    ((get-prev-run-ids)             (apply db:get-prev-run-ids dbstruct params))
-    ((get-run-ids-matching-target)  (apply db:get-run-ids-matching-target dbstruct params))
-    ((get-runs-by-patt)             (apply db:get-runs-by-patt dbstruct params))
-    ((lock/unlock-run)              (apply db:lock/unlock-run dbstruct params))
-    ((update-run-event_time)        (apply db:update-run-event_time dbstruct params))
-    ((find-and-mark-incomplete)     (apply db:find-and-mark-incomplete dbstruct params))
-
-    ;; STEPS
-    ((teststep-set-status!)         (apply db:teststep-set-status! dbstruct params))
-
-    ;; TEST DATA
-    ((test-data-rollup)             (apply db:test-data-rollup dbstruct params))
-    ((csv->test-data)               (apply db:csv->test-data dbstruct params))
-    ((get-steps-data)               (apply db:get-steps-data dbstruct params))
-
-    ;; MISC
-    ((login)                        (apply db:login dbstruct params))
-    ((general-call)                 (let ((stmtname   (car params))
-					  (run-id     (cadr params))
-					  (realparams (cddr params)))
-				      (db:with-db dbstruct run-id #t ;; these are all for modifying the db
-						  (lambda (db)
-						    (db:general-call db stmtname realparams)))))
-    ((sync-inmem->db)               (db:sync-touched dbstruct run-id force-sync: #t))
-    ((sdb-qry)                      (apply sdb:qry params))
-
-    ;; TESTMETA
-    ((testmeta-get-record)       (apply db:testmeta-get-record dbstruct params))
-    ((testmeta-add-record)       (apply db:testmeta-add-record dbstruct params))
-    ((testmeta-update-field)     (apply db:testmeta-update-field dbstruct params))
-    (else
-     (list "ERROR" 0))))
+;;    - returns #( flag result )
+;;
+(define (api:execute-requests dbstruct dat)
+  (handle-exceptions
+   exn
+   (let ((call-chain (get-call-chain)))
+     (print-call-chain (current-error-port))
+     (debug:print 0 " message: " ((condition-property-accessor 'exn 'message) exn))       
+     (vector #f (vector exn call-chain dat))) ;; return some stuff for debug if an exception happens
+   (if (not (vector? dat))                    ;; it is an error to not receive a vector
+       (vector #f #f "remote must be called with a vector")       
+       (vector                                   ;; return a vector + the returned data structure
+	#t 
+	(let ((cmd    (vector-ref dat 0))
+	      (params (vector-ref dat 1)))
+	  (case (if (symbol? cmd)
+		    cmd
+		    (string->symbol cmd))
+	    ;; SERVERS
+	    ((start-server)                 (apply server:kind-run params))
+	    ((kill-server)                  (set! *server-run* #f))
+	    
+	    ;; KEYS
+	    ((get-key-val-pairs)            (apply db:get-key-val-pairs dbstruct params))
+	    ((get-keys)                     (db:get-keys dbstruct))
+
+	    ;; TESTS
+	    ((test-toplevel-num-items)         (apply db:test-toplevel-num-items dbstruct params))
+	    ((get-test-info-by-id)	       (apply db:get-test-info-by-id dbstruct params))
+	    ((test-get-rundir-from-test-id)    (apply db:test-get-rundir-from-test-id dbstruct params))
+	    ((test-set-state-status-by-id)     (apply db:test-set-state-status-by-id dbstruct params))
+	    ((get-count-tests-running)         (apply db:get-count-tests-running dbstruct params))
+	    ((get-count-tests-running-in-jobgroup) (apply db:get-count-tests-running-in-jobgroup dbstruct params))
+	    ((delete-test-records)             (apply db:delete-test-records dbstruct params))
+	    ;; ((delete-test-step-records)        (apply db:delete-test-step-records dbstruct params))
+	    ((delete-old-deleted-test-records) (apply db:delete-old-deleted-test-records dbstruct params))
+	    ((test-set-status-state)           (apply db:test-set-status-state dbstruct params))
+	    ((get-previous-test-run-record)    (apply db:get-previous-test-run-record dbstruct params))
+	    ((get-matching-previous-test-run-records)(apply db:get-matching-previous-test-run-records dbstruct params))
+	    ((test-get-logfile-info)           (apply db:test-get-logfile-info dbstruct params))
+	    ((test-get-records-for-index-file)  (apply db:test-get-records-for-index-file dbstruct params))
+	    ((get-testinfo-state-status)       (apply db:get-testinfo-state-status dbstruct params))
+	    ((test-set-top-process-pid)        (apply db:test-set-top-process-pid dbstruct params))
+	    ((test-get-top-process-pid)        (apply db:test-get-top-process-pid dbstruct params))
+	    ((test-get-paths-matching-keynames-target-new) (apply db:test-get-paths-matching-keynames-target-new dbstruct params))
+	    ((get-prereqs-not-met)             (apply db:get-prereqs-not-met dbstruct params))
+	    ((roll-up-pass-fail-counts)        (apply db:roll-up-pass-fail-counts dbstruct params))
+	    ((update-fail-pass-counts)         (apply db:general-call dbstruct 'update-pass-fail-counts params))
+	    ((get-count-tests-running-for-run-id) (apply db:get-count-tests-running-for-run-id dbstruct params))
+
+	    ;; RUNS
+	    ((get-run-info)                 (apply db:get-run-info dbstruct params))
+	    ((get-run-status)               (apply db:get-run-status dbstruct params))
+	    ((set-run-status)               (apply db:set-run-status dbstruct params))
+	    ((register-run)                 (apply db:register-run dbstruct params))
+	    ((set-tests-state-status)       (apply db:set-tests-state-status dbstruct params))
+	    ((get-tests-for-run)            (apply db:get-tests-for-run dbstruct params))
+	    ((get-test-id)                  (apply db:get-test-id dbstruct params))
+	    ((get-tests-for-runs-mindata)   (apply db:get-tests-for-runs-mindata dbstruct params))
+	    ((get-run-name-from-id)         (apply db:get-run-name-from-id dbstruct params))
+	    ((delete-run)                   (apply db:delete-run dbstruct params))
+	    ((get-runs)                     (apply db:get-runs dbstruct params))
+	    ((get-all-run-ids)              (db:get-all-run-ids dbstruct))
+	    ((get-prev-run-ids)             (apply db:get-prev-run-ids dbstruct params))
+	    ((get-run-ids-matching-target)  (apply db:get-run-ids-matching-target dbstruct params))
+	    ((get-runs-by-patt)             (apply db:get-runs-by-patt dbstruct params))
+	    ((lock/unlock-run)              (apply db:lock/unlock-run dbstruct params))
+	    ((update-run-event_time)        (apply db:update-run-event_time dbstruct params))
+	    ((find-and-mark-incomplete)     (apply db:find-and-mark-incomplete dbstruct params))
+
+	    ;; STEPS
+	    ((teststep-set-status!)         (apply db:teststep-set-status! dbstruct params))
+
+	    ;; TEST DATA
+	    ((test-data-rollup)             (apply db:test-data-rollup dbstruct params))
+	    ((csv->test-data)               (apply db:csv->test-data dbstruct params))
+	    ((get-steps-data)               (apply db:get-steps-data dbstruct params))
+
+	    ;; MISC
+	    ((login)                        (apply db:login dbstruct params))
+	    ((general-call)                 (let ((stmtname   (car params))
+						  (run-id     (cadr params))
+						  (realparams (cddr params)))
+					      (db:with-db dbstruct run-id #t ;; these are all for modifying the db
+							  (lambda (db)
+							    (db:general-call db stmtname realparams)))))
+	    ((sync-inmem->db)               (let ((run-id (car params)))
+					      (db:sync-touched dbstruct run-id force-sync: #t)))
+	    ((sdb-qry)                      (apply sdb:qry params))
+	    ((ping)                         (current-process-id))
+
+	    ;; TESTMETA
+	    ((testmeta-get-record)       (apply db:testmeta-get-record dbstruct params))
+	    ((testmeta-add-record)       (apply db:testmeta-add-record dbstruct params))
+	    ((testmeta-update-field)     (apply db:testmeta-update-field dbstruct params))))))))
 
 ;; http-server  send-response
 ;;                 api:process-request
 ;;                    db:*
 ;;
@@ -138,18 +154,19 @@
 ;; NB// Runs on the server as part of the server loop
 ;;
 (define (api:process-request dbstruct $) ;; the $ is the request vars proc
   (let* ((cmd     ($ 'cmd))
 	 (paramsj ($ 'params))
-	 (params  (db:string->obj paramsj)) ;; (rmt:json-str->dat paramsj))
-	 (res     (api:execute-requests dbstruct cmd params)))
+	 (params  (db:string->obj paramsj transport: 'http)) ;; (rmt:json-str->dat paramsj))
+	 (resdat  (api:execute-requests dbstruct (vector cmd params))) ;; #( flag result )
+	 (res     (vector-ref resdat 1)))
 
     ;; This can be here but needs controls to ensure it doesn't run more than every 4 seconds
     ;; (rmt:dat->json-str
     ;;  (if (or (string? res)
     ;;          (list?   res)
     ;;          (number? res)
     ;;          (boolean? res))
     ;;      res 
     ;;      (list "ERROR, not string, list, number or boolean" 1 cmd params res)))))
-    (db:obj->string res)))
+    (db:obj->string res transport: 'http)))
 

Index: client.scm
==================================================================
--- client.scm
+++ client.scm
@@ -53,75 +53,90 @@
 ;;
 ;; client:setup
 ;;
 ;; lookup_server, need to remove *runremote* stuff
 ;;
-(define (client:setup run-id #!key (remaining-tries 100) (failed-connects 0))
+(define (client:setup run-id #!key (remaining-tries 2) (failed-connects 0))
   (debug:print-info 2 "client:setup remaining-tries=" remaining-tries)
   (let* ((tdbdat (tasks:open-db)))
     (if (<= remaining-tries 0)
 	(begin
 	  (debug:print 0 "ERROR: failed to start or connect to server for run-id " run-id)
 	  (exit 1))
-	(let ((host-info (hash-table-ref/default *runremote* run-id #f)))
-	  (if host-info
-	      (let* ((iface     (http-transport:server-dat-get-iface host-info))
-		     (port      (http-transport:server-dat-get-port  host-info))
-		     (start-res (http-transport:client-connect iface port))
-		     (ping-res  (rmt:login-no-auto-client-setup start-res run-id)))
-		(if ping-res   ;; sucessful login?
-		    (begin
-		      (debug:print-info 2 "client:setup, ping is good using host-info=" host-info ", remaining-tries=" remaining-tries)
-		      ;; Why add the close-connections here?
-		      ;; (http-transport:close-connections run-id)
-		      (hash-table-set! *runremote* run-id start-res)
-		      start-res)  ;; return the server info
-		    ;; have host info but no ping. shutdown the current connection and try again
-		    (begin    ;; login failed
-		      (debug:print-info 1 "client:setup, ping is bad for start-res=" start-res " and *runremote*=" host-info)
-		      (http-transport:close-connections run-id)
-		      (hash-table-delete! *runremote* run-id)
-		      (if (< remaining-tries 8)
-			  (thread-sleep! 5)
-			  (thread-sleep! 1))
-		      (client:setup run-id remaining-tries: (- remaining-tries 1)))))
-	      ;; YUK: rename server-dat here
-	      (let* ((server-dat (tasks:get-server (db:delay-if-busy tdbdat) run-id)))
-		(debug:print-info 4 "client:setup server-dat=" server-dat ", remaining-tries=" remaining-tries)
-		(if server-dat
-		    (let* ((iface     (tasks:hostinfo-get-interface server-dat))
-			   (port      (tasks:hostinfo-get-port      server-dat))
-			   (start-res (http-transport:client-connect iface port))
-			   (ping-res  (rmt:login-no-auto-client-setup start-res run-id)))
-		      (if (and start-res
-			       ping-res)
-			  (begin
-			    (hash-table-set! *runremote* run-id start-res)
-			    (debug:print-info 2 "connected to " (http-transport:server-dat-make-url start-res))
-			    start-res)
-			  (begin    ;; login failed but have a server record, clean out the record and try again
-			    (debug:print-info 0 "client:setup, login failed, will attempt to start server ... start-res=" start-res ", run-id=" run-id ", server-dat=" server-dat)
-			    (http-transport:close-connections run-id)
-			    (hash-table-delete! *runremote* run-id)
-			    (tasks:server-force-clean-run-record (db:delay-if-busy tdbdat)
-								 run-id 
-								 (tasks:hostinfo-get-interface server-dat)
-								 (tasks:hostinfo-get-port      server-dat)
-								 " client:setup (server-dat = #t)")
-			    (thread-sleep! 2)
-			    (server:try-running run-id)
-			    (thread-sleep! 10) ;; give server a little time to start up
-			    (client:setup run-id remaining-tries: (- remaining-tries 1)))))
-		    (begin    ;; no server registered
-		      (let ((num-available (tasks:num-in-available-state (db:dbdat-get-db tdbdat) run-id)))
-			(debug:print-info 0 "client:setup, no server registered, remaining-tries=" remaining-tries " num-available=" num-available)
-			(thread-sleep! 2) 
-			(if (< num-available 2)
-			    (begin
-			      (server:try-running run-id)))
-			(thread-sleep! 10) ;; give server a little time to start up
-			(client:setup run-id remaining-tries: (- remaining-tries 1)))))))))))
+	(let* ((server-dat (tasks:get-server (db:delay-if-busy tdbdat) run-id)))
+	  (debug:print-info 4 "client:setup server-dat=" server-dat ", remaining-tries=" remaining-tries)
+	  (if server-dat
+	      (let* ((iface     (tasks:hostinfo-get-interface server-dat))
+		     (hostname  (tasks:hostinfo-get-hostname  server-dat))
+		     (port      (tasks:hostinfo-get-port      server-dat))
+		     (start-res (case *transport-type*
+				  ((http)(http-transport:client-connect iface port))
+				  ((nmsg)(nmsg-transport:client-connect hostname port))))
+		     (ping-res  (case *transport-type* 
+				  ((http)(rmt:login-no-auto-client-setup start-res run-id))
+				  ((nmsg)(let ((logininfo (rmt:login-no-auto-client-setup start-res run-id)))
+ 					   (if logininfo
+ 					       (car (vector-ref logininfo 1))
+ 					       #f))))))
+		(if (and start-res
+			 ping-res)
+		    (begin
+		      (hash-table-set! *runremote* run-id start-res)
+		      (debug:print-info 2 "connected to " (http-transport:server-dat-make-url start-res))
+		      start-res)
+		    (begin    ;; login failed but have a server record, clean out the record and try again
+		      (debug:print-info 0 "client:setup, login failed, will attempt to start server ... start-res=" start-res ", run-id=" run-id ", server-dat=" server-dat)
+		      (case *transport-type* 
+			((http)(http-transport:close-connections run-id)))
+		      (hash-table-delete! *runremote* run-id)
+		      (tasks:server-force-clean-run-record (db:delay-if-busy tdbdat)
+							   run-id 
+							   (tasks:hostinfo-get-interface server-dat)
+							   (tasks:hostinfo-get-port      server-dat)
+							   " client:setup (server-dat = #t)")
+		      (server:try-running run-id)
+		      (thread-sleep! 5) ;; give server a little time to start up
+		      (client:setup run-id remaining-tries: (- remaining-tries 1)))))
+	      (begin    ;; no server registered
+		(let ((num-available (tasks:num-in-available-state (db:dbdat-get-db tdbdat) run-id)))
+		  (debug:print-info 0 "client:setup, no server registered, remaining-tries=" remaining-tries " num-available=" num-available)
+		  (if (< num-available 2)
+		      (server:try-running run-id))
+		  (thread-sleep! 5) ;; give server a little time to start up
+		  (client:setup run-id remaining-tries: (- remaining-tries 1)))))))))
+
+;; 	(let ((host-info (hash-table-ref/default *runremote* run-id #f)))
+;; 	  (if host-info ;; this is a bit circular. the host-info *is* the start-res FIXME
+;; 	      (let* ((iface     (http-transport:server-dat-get-iface host-info))
+;; 		     (port      (http-transport:server-dat-get-port  host-info))
+;; 		     (start-res (case *transport-type* 
+;; 				  ((http)(http-transport:client-connect iface port))
+;; 				  ((nmsg)(nmsg-transport:client-connect iface port)) ;; (http-transport:server-dat-get-socket host-info))
+;; 				  (else #f)))
+;; 		     (ping-res  (case *transport-type*
+;; 				  ((http)(rmt:login-no-auto-client-setup start-res run-id))
+;; 				  ((nmsg)(let ((logininfo (rmt:login-no-auto-client-setup start-res run-id)))
+;; 					   (if logininfo
+;; 					       (vector-ref (vector-ref logininfo 1) 1)
+;; 					       #f)))
+;; 				  (else #f))))
+;; 		(if ping-res   ;; sucessful login?
+;; 		    (begin
+;; 		      (debug:print-info 2 "client:setup, ping is good using host-info=" host-info ", remaining-tries=" remaining-tries)
+;; 		      start-res)  ;; return the server info
+;; 		    ;; have host info but no ping. shutdown the current connection and try again
+;; 		    (begin    ;; login failed
+;; 		      (debug:print-info 1 "client:setup, ping is bad for start-res=" start-res " and *runremote*=" host-info)
+;; 		      (case *transport-type*
+;; 			((http)(http-transport:close-connections run-id)))
+;; 		      (hash-table-delete! *runremote* run-id)
+;; 		      (if (< remaining-tries 8)
+;; 			  (thread-sleep! 5)
+;; 			  (thread-sleep! 1))
+;; 		      (client:setup run-id remaining-tries: (- remaining-tries 1)))))
+;; 	      ;; YUK: rename server-dat here
+;; 
 
 ;; keep this as a function to ease future 
 (define (client:start run-id server-info)
   (http-transport:client-connect (tasks:hostinfo-get-interface server-info)
 				 (tasks:hostinfo-get-port server-info)))

Index: common.scm
==================================================================
--- common.scm
+++ common.scm
@@ -65,12 +65,12 @@
 (define *db-access-allowed*   #t) ;; flag to allow access
 (define *db-access-mutex*     (make-mutex))
 
 ;; SERVER
 (define *my-client-signature* #f)
+;; (define *transport-type*    'nmsg)
 (define *transport-type*    'http)
-(define *rpc:listener*      #f) ;; if set up for server communication this will hold the tcp port
 (define *runremote*         (make-hash-table)) ;; if set up for server communication this will hold <host port>
 (define *max-cache-size*    0)
 (define *logged-in-clients* (make-hash-table))
 (define *client-non-blocking-mode* #f)
 (define *server-id*         #f)
@@ -225,10 +225,11 @@
   (let ((run-ids (hash-table-keys *db-local-sync*)))
     (if (and (not (null? run-ids))
 	     (configf:lookup *configdat* "setup" "megatest-db"))
 	(db:multi-db-sync run-ids 'new2old)))
   (if *dbstruct-db* (db:close-all *dbstruct-db*))
+  (if *inmemdb*     (db:close-all *inmemdb*))
   (if (and *megatest-db*
 	   (sqlite3:database? *megatest-db*))
       (begin
 	(sqlite3:interrupt! *megatest-db*)
 	(sqlite3:finalize! *megatest-db* #t)

Index: db.scm
==================================================================
--- db.scm
+++ db.scm
@@ -296,11 +296,11 @@
 	(refdb  (dbr:dbstruct-get-refdb dbstruct))
 	(olddb  (dbr:dbstruct-get-olddb dbstruct))
 	;; (runid  (dbr:dbstruct-get-run-id dbstruct))
 	)
     (debug:print-info 4 "Syncing for run-id: " run-id)
-    (mutex-lock! *http-mutex*)
+    ;; (mutex-lock! *http-mutex*)
     (if (eq? run-id 0)
 	;; runid equal to 0 is main.db
 	(if maindb
 	    (if (or (not (number? mtime))
 		    (not (number? stime))
@@ -325,16 +325,16 @@
 		(> mtime stime)
 		force-sync)
 	    (begin
 	      (db:delay-if-busy rundb)
 	      (db:delay-if-busy olddb)
+	      (dbr:dbstruct-set-stime! dbstruct (current-milliseconds))
 	      (let ((num-synced (db:sync-tables db:sync-tests-only inmem refdb rundb olddb)))
-		(dbr:dbstruct-set-stime! dbstruct (current-milliseconds))
-		(mutex-unlock! *http-mutex*)
+		;; (mutex-unlock! *http-mutex*)
 		num-synced)
 	      (begin
-		(mutex-unlock! *http-mutex*)
+		;; (mutex-unlock! *http-mutex*)
 		0))))))
 
 (define (db:close-main dbstruct)
   (let ((maindb (dbr:dbstruct-get-main dbstruct)))
     (if maindb
@@ -2286,26 +2286,26 @@
 ;;======================================================================
 ;; QUEUE UP META, TEST STATUS AND STEPS REMOTE ACCESS
 ;;======================================================================
 
 ;; NOTE: Can remove the regex and base64 encoding for zmq
-(define (db:obj->string obj)
-  (case *transport-type*
+(define (db:obj->string obj #!key (transport 'http))
+  (case transport
     ;; ((fs) obj)
     ((http fs)
      (string-substitute
       (regexp "=") "_"
       (base64:base64-encode 
        (z3:encode-buffer
 	(with-output-to-string
 	  (lambda ()(serialize obj)))))
       #t))
-    ((zmq)(with-output-to-string (lambda ()(serialize obj))))
+    ((zmq nmsg)(with-output-to-string (lambda ()(serialize obj))))
     (else obj)))
 
-(define (db:string->obj msg)
-  (case *transport-type*
+(define (db:string->obj msg #!key (transport 'http))
+  (case transport
     ;; ((fs) msg)
     ((http fs)
      (if (string? msg)
 	 (with-input-from-string 
 	     (z3:decode-buffer
@@ -2314,11 +2314,11 @@
 		(regexp "_") "=" msg #t)))
 	   (lambda ()(deserialize)))
 	 (begin
 	   (debug:print 0 "ERROR: reception failed. Received " msg " but cannot translate it.")
 	   #f))) ;; crude reply for when things go awry
-    ((zmq)(with-input-from-string msg (lambda ()(deserialize))))
+    ((zmq nmsg)(with-input-from-string msg (lambda ()(deserialize))))
     (else msg)))
 
 (define (db:test-set-status-state dbstruct run-id test-id status state msg)
   (let ((dbdat  (db:get-db dbstruct run-id)))
     (if (member state '("LAUNCHED" "REMOTEHOSTSTART"))

Index: docs/manual/megatest_manual.html
==================================================================
--- docs/manual/megatest_manual.html
+++ docs/manual/megatest_manual.html
@@ -1,1284 +1,1361 @@
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
-    "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
-<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en">
-<head>
-<meta http-equiv="Content-Type" content="application/xhtml+xml; charset=UTF-8" />
-<meta name="generator" content="AsciiDoc 8.6.9" />
-<title>The Megatest Users Manual</title>
-<style type="text/css">
-/* Shared CSS for AsciiDoc xhtml11 and html5 backends */
-
-/* Default font. */
-body {
-  font-family: Georgia,serif;
-}
-
-/* Title font. */
-h1, h2, h3, h4, h5, h6,
-div.title, caption.title,
-thead, p.table.header,
-#toctitle,
-#author, #revnumber, #revdate, #revremark,
-#footer {
-  font-family: Arial,Helvetica,sans-serif;
-}
-
-body {
-  margin: 1em 5% 1em 5%;
-}
-
-a {
-  color: blue;
-  text-decoration: underline;
-}
-a:visited {
-  color: fuchsia;
-}
-
-em {
-  font-style: italic;
-  color: navy;
-}
-
-strong {
-  font-weight: bold;
-  color: #083194;
-}
-
-h1, h2, h3, h4, h5, h6 {
-  color: #527bbd;
-  margin-top: 1.2em;
-  margin-bottom: 0.5em;
-  line-height: 1.3;
-}
-
-h1, h2, h3 {
-  border-bottom: 2px solid silver;
-}
-h2 {
-  padding-top: 0.5em;
-}
-h3 {
-  float: left;
-}
-h3 + * {
-  clear: left;
-}
-h5 {
-  font-size: 1.0em;
-}
-
-div.sectionbody {
-  margin-left: 0;
-}
-
-hr {
-  border: 1px solid silver;
-}
-
-p {
-  margin-top: 0.5em;
-  margin-bottom: 0.5em;
-}
-
-ul, ol, li > p {
-  margin-top: 0;
-}
-ul > li     { color: #aaa; }
-ul > li > * { color: black; }
-
-.monospaced, code, pre {
-  font-family: "Courier New", Courier, monospace;
-  font-size: inherit;
-  color: navy;
-  padding: 0;
-  margin: 0;
-}
-pre {
-  white-space: pre-wrap;
-}
-
-#author {
-  color: #527bbd;
-  font-weight: bold;
-  font-size: 1.1em;
-}
-#email {
-}
-#revnumber, #revdate, #revremark {
-}
-
-#footer {
-  font-size: small;
-  border-top: 2px solid silver;
-  padding-top: 0.5em;
-  margin-top: 4.0em;
-}
-#footer-text {
-  float: left;
-  padding-bottom: 0.5em;
-}
-#footer-badges {
-  float: right;
-  padding-bottom: 0.5em;
-}
-
-#preamble {
-  margin-top: 1.5em;
-  margin-bottom: 1.5em;
-}
-div.imageblock, div.exampleblock, div.verseblock,
-div.quoteblock, div.literalblock, div.listingblock, div.sidebarblock,
-div.admonitionblock {
-  margin-top: 1.0em;
-  margin-bottom: 1.5em;
-}
-div.admonitionblock {
-  margin-top: 2.0em;
-  margin-bottom: 2.0em;
-  margin-right: 10%;
-  color: #606060;
-}
-
-div.content { /* Block element content. */
-  padding: 0;
-}
-
-/* Block element titles. */
-div.title, caption.title {
-  color: #527bbd;
-  font-weight: bold;
-  text-align: left;
-  margin-top: 1.0em;
-  margin-bottom: 0.5em;
-}
-div.title + * {
-  margin-top: 0;
-}
-
-td div.title:first-child {
-  margin-top: 0.0em;
-}
-div.content div.title:first-child {
-  margin-top: 0.0em;
-}
-div.content + div.title {
-  margin-top: 0.0em;
-}
-
-div.sidebarblock > div.content {
-  background: #ffffee;
-  border: 1px solid #dddddd;
-  border-left: 4px solid #f0f0f0;
-  padding: 0.5em;
-}
-
-div.listingblock > div.content {
-  border: 1px solid #dddddd;
-  border-left: 5px solid #f0f0f0;
-  background: #f8f8f8;
-  padding: 0.5em;
-}
-
-div.quoteblock, div.verseblock {
-  padding-left: 1.0em;
-  margin-left: 1.0em;
-  margin-right: 10%;
-  border-left: 5px solid #f0f0f0;
-  color: #888;
-}
-
-div.quoteblock > div.attribution {
-  padding-top: 0.5em;
-  text-align: right;
-}
-
-div.verseblock > pre.content {
-  font-family: inherit;
-  font-size: inherit;
-}
-div.verseblock > div.attribution {
-  padding-top: 0.75em;
-  text-align: left;
-}
-/* DEPRECATED: Pre version 8.2.7 verse style literal block. */
-div.verseblock + div.attribution {
-  text-align: left;
-}
-
-div.admonitionblock .icon {
-  vertical-align: top;
-  font-size: 1.1em;
-  font-weight: bold;
-  text-decoration: underline;
-  color: #527bbd;
-  padding-right: 0.5em;
-}
-div.admonitionblock td.content {
-  padding-left: 0.5em;
-  border-left: 3px solid #dddddd;
-}
-
-div.exampleblock > div.content {
-  border-left: 3px solid #dddddd;
-  padding-left: 0.5em;
-}
-
-div.imageblock div.content { padding-left: 0; }
-span.image img { border-style: none; vertical-align: text-bottom; }
-a.image:visited { color: white; }
-
-dl {
-  margin-top: 0.8em;
-  margin-bottom: 0.8em;
-}
-dt {
-  margin-top: 0.5em;
-  margin-bottom: 0;
-  font-style: normal;
-  color: navy;
-}
-dd > *:first-child {
-  margin-top: 0.1em;
-}
-
-ul, ol {
-    list-style-position: outside;
-}
-ol.arabic {
-  list-style-type: decimal;
-}
-ol.loweralpha {
-  list-style-type: lower-alpha;
-}
-ol.upperalpha {
-  list-style-type: upper-alpha;
-}
-ol.lowerroman {
-  list-style-type: lower-roman;
-}
-ol.upperroman {
-  list-style-type: upper-roman;
-}
-
-div.compact ul, div.compact ol,
-div.compact p, div.compact p,
-div.compact div, div.compact div {
-  margin-top: 0.1em;
-  margin-bottom: 0.1em;
-}
-
-tfoot {
-  font-weight: bold;
-}
-td > div.verse {
-  white-space: pre;
-}
-
-div.hdlist {
-  margin-top: 0.8em;
-  margin-bottom: 0.8em;
-}
-div.hdlist tr {
-  padding-bottom: 15px;
-}
-dt.hdlist1.strong, td.hdlist1.strong {
-  font-weight: bold;
-}
-td.hdlist1 {
-  vertical-align: top;
-  font-style: normal;
-  padding-right: 0.8em;
-  color: navy;
-}
-td.hdlist2 {
-  vertical-align: top;
-}
-div.hdlist.compact tr {
-  margin: 0;
-  padding-bottom: 0;
-}
-
-.comment {
-  background: yellow;
-}
-
-.footnote, .footnoteref {
-  font-size: 0.8em;
-}
-
-span.footnote, span.footnoteref {
-  vertical-align: super;
-}
-
-#footnotes {
-  margin: 20px 0 20px 0;
-  padding: 7px 0 0 0;
-}
-
-#footnotes div.footnote {
-  margin: 0 0 5px 0;
-}
-
-#footnotes hr {
-  border: none;
-  border-top: 1px solid silver;
-  height: 1px;
-  text-align: left;
-  margin-left: 0;
-  width: 20%;
-  min-width: 100px;
-}
-
-div.colist td {
-  padding-right: 0.5em;
-  padding-bottom: 0.3em;
-  vertical-align: top;
-}
-div.colist td img {
-  margin-top: 0.3em;
-}
-
-@media print {
-  #footer-badges { display: none; }
-}
-
-#toc {
-  margin-bottom: 2.5em;
-}
-
-#toctitle {
-  color: #527bbd;
-  font-size: 1.1em;
-  font-weight: bold;
-  margin-top: 1.0em;
-  margin-bottom: 0.1em;
-}
-
-div.toclevel0, div.toclevel1, div.toclevel2, div.toclevel3, div.toclevel4 {
-  margin-top: 0;
-  margin-bottom: 0;
-}
-div.toclevel2 {
-  margin-left: 2em;
-  font-size: 0.9em;
-}
-div.toclevel3 {
-  margin-left: 4em;
-  font-size: 0.9em;
-}
-div.toclevel4 {
-  margin-left: 6em;
-  font-size: 0.9em;
-}
-
-span.aqua { color: aqua; }
-span.black { color: black; }
-span.blue { color: blue; }
-span.fuchsia { color: fuchsia; }
-span.gray { color: gray; }
-span.green { color: green; }
-span.lime { color: lime; }
-span.maroon { color: maroon; }
-span.navy { color: navy; }
-span.olive { color: olive; }
-span.purple { color: purple; }
-span.red { color: red; }
-span.silver { color: silver; }
-span.teal { color: teal; }
-span.white { color: white; }
-span.yellow { color: yellow; }
-
-span.aqua-background { background: aqua; }
-span.black-background { background: black; }
-span.blue-background { background: blue; }
-span.fuchsia-background { background: fuchsia; }
-span.gray-background { background: gray; }
-span.green-background { background: green; }
-span.lime-background { background: lime; }
-span.maroon-background { background: maroon; }
-span.navy-background { background: navy; }
-span.olive-background { background: olive; }
-span.purple-background { background: purple; }
-span.red-background { background: red; }
-span.silver-background { background: silver; }
-span.teal-background { background: teal; }
-span.white-background { background: white; }
-span.yellow-background { background: yellow; }
-
-span.big { font-size: 2em; }
-span.small { font-size: 0.6em; }
-
-span.underline { text-decoration: underline; }
-span.overline { text-decoration: overline; }
-span.line-through { text-decoration: line-through; }
-
-div.unbreakable { page-break-inside: avoid; }
-
-
-/*
- * xhtml11 specific
- *
- * */
-
-div.tableblock {
-  margin-top: 1.0em;
-  margin-bottom: 1.5em;
-}
-div.tableblock > table {
-  border: 3px solid #527bbd;
-}
-thead, p.table.header {
-  font-weight: bold;
-  color: #527bbd;
-}
-p.table {
-  margin-top: 0;
-}
-/* Because the table frame attribute is overriden by CSS in most browsers. */
-div.tableblock > table[frame="void"] {
-  border-style: none;
-}
-div.tableblock > table[frame="hsides"] {
-  border-left-style: none;
-  border-right-style: none;
-}
-div.tableblock > table[frame="vsides"] {
-  border-top-style: none;
-  border-bottom-style: none;
-}
-
-
-/*
- * html5 specific
- *
- * */
-
-table.tableblock {
-  margin-top: 1.0em;
-  margin-bottom: 1.5em;
-}
-thead, p.tableblock.header {
-  font-weight: bold;
-  color: #527bbd;
-}
-p.tableblock {
-  margin-top: 0;
-}
-table.tableblock {
-  border-width: 3px;
-  border-spacing: 0px;
-  border-style: solid;
-  border-color: #527bbd;
-  border-collapse: collapse;
-}
-th.tableblock, td.tableblock {
-  border-width: 1px;
-  padding: 4px;
-  border-style: solid;
-  border-color: #527bbd;
-}
-
-table.tableblock.frame-topbot {
-  border-left-style: hidden;
-  border-right-style: hidden;
-}
-table.tableblock.frame-sides {
-  border-top-style: hidden;
-  border-bottom-style: hidden;
-}
-table.tableblock.frame-none {
-  border-style: hidden;
-}
-
-th.tableblock.halign-left, td.tableblock.halign-left {
-  text-align: left;
-}
-th.tableblock.halign-center, td.tableblock.halign-center {
-  text-align: center;
-}
-th.tableblock.halign-right, td.tableblock.halign-right {
-  text-align: right;
-}
-
-th.tableblock.valign-top, td.tableblock.valign-top {
-  vertical-align: top;
-}
-th.tableblock.valign-middle, td.tableblock.valign-middle {
-  vertical-align: middle;
-}
-th.tableblock.valign-bottom, td.tableblock.valign-bottom {
-  vertical-align: bottom;
-}
-
-
-/*
- * manpage specific
- *
- * */
-
-body.manpage h1 {
-  padding-top: 0.5em;
-  padding-bottom: 0.5em;
-  border-top: 2px solid silver;
-  border-bottom: 2px solid silver;
-}
-body.manpage h2 {
-  border-style: none;
-}
-body.manpage div.sectionbody {
-  margin-left: 3em;
-}
-
-@media print {
-  body.manpage div#toc { display: none; }
-}
-
-
-</style>
-<script type="text/javascript">
-/*<![CDATA[*/
-var asciidoc = {  // Namespace.
-
-/////////////////////////////////////////////////////////////////////
-// Table Of Contents generator
-/////////////////////////////////////////////////////////////////////
-
-/* Author: Mihai Bazon, September 2002
- * http://students.infoiasi.ro/~mishoo
- *
- * Table Of Content generator
- * Version: 0.4
- *
- * Feel free to use this script under the terms of the GNU General Public
- * License, as long as you do not remove or alter this notice.
- */
-
- /* modified by Troy D. Hanson, September 2006. License: GPL */
- /* modified by Stuart Rackham, 2006, 2009. License: GPL */
-
-// toclevels = 1..4.
-toc: function (toclevels) {
-
-  function getText(el) {
-    var text = "";
-    for (var i = el.firstChild; i != null; i = i.nextSibling) {
-      if (i.nodeType == 3 /* Node.TEXT_NODE */) // IE doesn't speak constants.
-        text += i.data;
-      else if (i.firstChild != null)
-        text += getText(i);
-    }
-    return text;
-  }
-
-  function TocEntry(el, text, toclevel) {
-    this.element = el;
-    this.text = text;
-    this.toclevel = toclevel;
-  }
-
-  function tocEntries(el, toclevels) {
-    var result = new Array;
-    var re = new RegExp('[hH]([1-'+(toclevels+1)+'])');
-    // Function that scans the DOM tree for header elements (the DOM2
-    // nodeIterator API would be a better technique but not supported by all
-    // browsers).
-    var iterate = function (el) {
-      for (var i = el.firstChild; i != null; i = i.nextSibling) {
-        if (i.nodeType == 1 /* Node.ELEMENT_NODE */) {
-          var mo = re.exec(i.tagName);
-          if (mo && (i.getAttribute("class") || i.getAttribute("className")) != "float") {
-            result[result.length] = new TocEntry(i, getText(i), mo[1]-1);
-          }
-          iterate(i);
-        }
-      }
-    }
-    iterate(el);
-    return result;
-  }
-
-  var toc = document.getElementById("toc");
-  if (!toc) {
-    return;
-  }
-
-  // Delete existing TOC entries in case we're reloading the TOC.
-  var tocEntriesToRemove = [];
-  var i;
-  for (i = 0; i < toc.childNodes.length; i++) {
-    var entry = toc.childNodes[i];
-    if (entry.nodeName.toLowerCase() == 'div'
-     && entry.getAttribute("class")
-     && entry.getAttribute("class").match(/^toclevel/))
-      tocEntriesToRemove.push(entry);
-  }
-  for (i = 0; i < tocEntriesToRemove.length; i++) {
-    toc.removeChild(tocEntriesToRemove[i]);
-  }
-
-  // Rebuild TOC entries.
-  var entries = tocEntries(document.getElementById("content"), toclevels);
-  for (var i = 0; i < entries.length; ++i) {
-    var entry = entries[i];
-    if (entry.element.id == "")
-      entry.element.id = "_toc_" + i;
-    var a = document.createElement("a");
-    a.href = "#" + entry.element.id;
-    a.appendChild(document.createTextNode(entry.text));
-    var div = document.createElement("div");
-    div.appendChild(a);
-    div.className = "toclevel" + entry.toclevel;
-    toc.appendChild(div);
-  }
-  if (entries.length == 0)
-    toc.parentNode.removeChild(toc);
-},
-
-
-/////////////////////////////////////////////////////////////////////
-// Footnotes generator
-/////////////////////////////////////////////////////////////////////
-
-/* Based on footnote generation code from:
- * http://www.brandspankingnew.net/archive/2005/07/format_footnote.html
- */
-
-footnotes: function () {
-  // Delete existing footnote entries in case we're reloading the footnodes.
-  var i;
-  var noteholder = document.getElementById("footnotes");
-  if (!noteholder) {
-    return;
-  }
-  var entriesToRemove = [];
-  for (i = 0; i < noteholder.childNodes.length; i++) {
-    var entry = noteholder.childNodes[i];
-    if (entry.nodeName.toLowerCase() == 'div' && entry.getAttribute("class") == "footnote")
-      entriesToRemove.push(entry);
-  }
-  for (i = 0; i < entriesToRemove.length; i++) {
-    noteholder.removeChild(entriesToRemove[i]);
-  }
-
-  // Rebuild footnote entries.
-  var cont = document.getElementById("content");
-  var spans = cont.getElementsByTagName("span");
-  var refs = {};
-  var n = 0;
-  for (i=0; i<spans.length; i++) {
-    if (spans[i].className == "footnote") {
-      n++;
-      var note = spans[i].getAttribute("data-note");
-      if (!note) {
-        // Use [\s\S] in place of . so multi-line matches work.
-        // Because JavaScript has no s (dotall) regex flag.
-        note = spans[i].innerHTML.match(/\s*\[([\s\S]*)]\s*/)[1];
-        spans[i].innerHTML =
-          "[<a id='_footnoteref_" + n + "' href='#_footnote_" + n +
-          "' title='View footnote' class='footnote'>" + n + "</a>]";
-        spans[i].setAttribute("data-note", note);
-      }
-      noteholder.innerHTML +=
-        "<div class='footnote' id='_footnote_" + n + "'>" +
-        "<a href='#_footnoteref_" + n + "' title='Return to text'>" +
-        n + "</a>. " + note + "</div>";
-      var id =spans[i].getAttribute("id");
-      if (id != null) refs["#"+id] = n;
-    }
-  }
-  if (n == 0)
-    noteholder.parentNode.removeChild(noteholder);
-  else {
-    // Process footnoterefs.
-    for (i=0; i<spans.length; i++) {
-      if (spans[i].className == "footnoteref") {
-        var href = spans[i].getElementsByTagName("a")[0].getAttribute("href");
-        href = href.match(/#.*/)[0];  // Because IE return full URL.
-        n = refs[href];
-        spans[i].innerHTML =
-          "[<a href='#_footnote_" + n +
-          "' title='View footnote' class='footnote'>" + n + "</a>]";
-      }
-    }
-  }
-},
-
-install: function(toclevels) {
-  var timerId;
-
-  function reinstall() {
-    asciidoc.footnotes();
-    if (toclevels) {
-      asciidoc.toc(toclevels);
-    }
-  }
-
-  function reinstallAndRemoveTimer() {
-    clearInterval(timerId);
-    reinstall();
-  }
-
-  timerId = setInterval(reinstall, 500);
-  if (document.addEventListener)
-    document.addEventListener("DOMContentLoaded", reinstallAndRemoveTimer, false);
-  else
-    window.onload = reinstallAndRemoveTimer;
-}
-
-}
-asciidoc.install();
-/*]]>*/
-</script>
-</head>
-<body class="book">
-<div id="header">
-<h1>The Megatest Users Manual</h1>
-<span id="author">Matt Welland</span><br />
-<span id="email"><code>&lt;<a href="mailto:matt@kiatoa.com">matt@kiatoa.com</a>&gt;</code></span><br />
-<span id="revnumber">version 1.0,</span>
-<span id="revdate">April 2012</span>
-</div>
-<div id="content">
-<div class="sect1">
-<h2 id="_preface">Preface</h2>
-<div class="sectionbody">
-<div class="paragraph"><p>This book is organised as three sub-books; getting started, writing tests and reference.</p></div>
-<div class="sect2">
-<h3 id="_why_megatest">Why Megatest?</h3>
-<div class="paragraph"><p>The Megatest project was started for two reasons, the first was an
-immediate and pressing need for a generalized tool to manage a suite
-of regression tests and the second was the fact that the author had
-written or maintained several such tools at different companies over
-the years and it seemed a good thing to have a single open source
-tool, flexible enough to meet the needs of any team doing continuous
-integrating and or running a complex suite of tests for release
-qualification.</p></div>
-</div>
-<div class="sect2">
-<h3 id="_megatest_design_philosophy">Megatest Design Philosophy</h3>
-<div class="paragraph"><p>Megatest is intended to provide the minimum needed resources to make
-writing a suite of tests and tasks for implementing continuous build
-for software, design engineering or process control (via owlfs for
-example) without being specialized for any specific problem
-space. Megatest in of itself does not know what constitutes a PASS or
-FAIL of a test. In most cases megatest is best used in conjunction
-with logpro or a similar tool to parse, analyze and decide on the test
-outcome.</p></div>
-</div>
-<div class="sect2">
-<h3 id="_megatest_architecture">Megatest Architecture</h3>
-<div class="paragraph"><p>All data to specify the tests and configure the system is stored in
-plain text files. All system state is stored in an sqlite3
-database. Tests are launched using the launching system available for
-the distributed compute platform in use. A template script is provided
-which can launch jobs on local and remote Linux hosts. Currently
-megatest uses the network filesystem to call home to your master
-sqlite3 database.</p></div>
-</div>
-</div>
-</div>
-<h1 id="_road_map">Road Map</h1>
-<div class="paragraph"><p>Note 1: This road-map is tentative and subject to change without notice.</p></div>
-<div class="paragraph"><p>Note 2: Starting over. Old plan is commented out.</p></div>
-<div class="sect1">
-<h2 id="_current_items">Current Items</h2>
-<div class="sectionbody">
-<div class="sect2">
-<h3 id="_ww05_migrate_to_inmem_db">ww05 - migrate to inmem-db</h3>
-<div class="paragraph"><p>Keep as much the same as possible. Add internal reference to almost
-eliminate contention on db(s).</p></div>
-<div class="olist arabic"><ol class="arabic">
-<li>
-<p>
-Add internal reference db
-</p>
-</li>
-<li>
-<p>
-Verify that actions are accessing correct db
-</p>
-<div class="olist loweralpha"><ol class="loweralpha">
-<li>
-<p>
--runtests  - inmem
-</p>
-</li>
-<li>
-<p>
--list-runs - local (but not megatest.db)
-</p>
-</li>
-<li>
-<p>
-dashboard  - local (but not megatest.db)
-</p>
-</li>
-</ol></div>
-</li>
-<li>
-<p>
-Mirror db to /var/tmp&#8230;
-</p>
-</li>
-<li>
-<p>
-Dashboard read db from per-run db.
-</p>
-</li>
-<li>
-<p>
-Dashboard read db from /var/tmp
-</p>
-</li>
-<li>
-<p>
-Runs register in tasks table in monitor.db
-</p>
-</li>
-<li>
-<p>
-Server polls tasks table for next action (in addition?)
-</p>
-</li>
-<li>
-<p>
-Change run loop to execute in server, triggered by call to polling of tasks table
-</p>
-</li>
-</ol></div>
-</div>
-</div>
-</div>
-<h1 id="_getting_started">Getting Started</h1>
-<div class="openblock">
-<div class="title">Getting started with Megatest</div>
-<div class="content">
-<div class="paragraph"><p>How to install Megatest and set it up for running your regressions and continuous integration process.</p></div>
-</div></div>
-<div class="sect1">
-<h2 id="_installation">Installation</h2>
-<div class="sectionbody">
-<div class="sect2">
-<h3 id="_dependencies">Dependencies</h3>
-<div class="paragraph"><p>Chicken scheme and a number of "eggs" are required for building
-Megatest. See the script installall.sch in the utils directory of the
-distribution for a mostly automated way to install everything needed
-for building Megatest on Linux.</p></div>
-<div class="paragraph"><p><span class="footnote"><br />[An example footnote.]<br /></span></p></div>
-<div class="paragraph"><p>And now for something completely different: monkeys, lions and
-tigers (Bengal and Siberian) using the alternative syntax index
-entries.
-
-
-
-Note that multi-entry terms generate separate index entries.</p></div>
-<div class="paragraph"><p>Here are a couple of image examples: an <span class="image">
-<img src="images/smallnew.png" alt="images/smallnew.png" />
-</span>
-example inline image followed by an example block image:</p></div>
-<div class="imageblock">
-<div class="content">
-<img src="images/tiger.png" alt="Tiger image" />
-</div>
-<div class="title">Figure 1. Tiger block image</div>
-</div>
-<div class="paragraph"><p>Followed by an example table:</p></div>
-<div class="tableblock">
-<table rules="all"
-width="60%"
-frame="border"
-cellspacing="0" cellpadding="4">
-<caption class="title">Table 1. An example table</caption>
-<col width="50%" />
-<col width="50%" />
-<thead>
-<tr>
-<th align="left" valign="top"> Option          </th>
-<th align="left" valign="top"> Description</th>
-</tr>
-</thead>
-<tbody>
-<tr>
-<td align="left" valign="top"><p class="table">-a <em>USER GROUP</em></p></td>
-<td align="left" valign="top"><p class="table">Add <em>USER</em> to <em>GROUP</em>.</p></td>
-</tr>
-<tr>
-<td align="left" valign="top"><p class="table">-R <em>GROUP</em></p></td>
-<td align="left" valign="top"><p class="table">Disables access to <em>GROUP</em>.</p></td>
-</tr>
-</tbody>
-</table>
-</div>
-<div class="exampleblock">
-<div class="title">Example 1. An example example</div>
-<div class="content">
-<div class="paragraph"><p>Lorum ipum&#8230;</p></div>
-</div></div>
-</div>
-<div class="sect2">
-<h3 id="X1">Sub-section with Anchor</h3>
-<div class="paragraph"><p>Sub-section at level 2.</p></div>
-<div class="sect3">
-<h4 id="_chapter_sub_section">Chapter Sub-section</h4>
-<div class="paragraph"><p>Sub-section at level 3.</p></div>
-<div class="sect4">
-<h5 id="_chapter_sub_section_2">Chapter Sub-section</h5>
-<div class="paragraph"><p>Sub-section at level 4.</p></div>
-<div class="paragraph"><p>This is the maximum sub-section depth supported by the distributed
-AsciiDoc configuration.
-<span class="footnote"><br />[A second example footnote.]<br /></span></p></div>
-</div>
-</div>
-</div>
-</div>
-</div>
-<div class="sect1">
-<h2 id="_the_second_chapter">The Second Chapter</h2>
-<div class="sectionbody">
-<div class="paragraph"><p>An example link to anchor at start of the <a href="#X1">first sub-section</a>.</p></div>
-<div class="paragraph"><p>An example link to a bibliography entry <a href="#taoup">[taoup]</a>.</p></div>
-</div>
-</div>
-<h1 id="_writing_tests">Writing Tests</h1>
-<div class="sect1">
-<h2 id="_the_first_chapter_of_the_second_part">The First Chapter of the Second Part</h2>
-<div class="sectionbody">
-<div class="paragraph"><p>Chapters grouped into book parts are at level 1 and can contain
-sub-sections.</p></div>
-</div>
-</div>
-<h1 id="_how_to_do_things">How To Do Things</h1>
-<div class="sect1">
-<h2 id="_tricks">Tricks</h2>
-<div class="sectionbody">
-<div class="paragraph"><p>This section is a compendium of a various useful tricks for debugging,
-configuring and generally getting the most out of Megatest.</p></div>
-</div>
-</div>
-<div class="sect1">
-<h2 id="_limiting_your_running_jobs">Limiting your running jobs</h2>
-<div class="sectionbody">
-<div class="paragraph"><p>The following example will limit a test in the jobgroup "group1" to no more than 10 tests simultaneously.</p></div>
-<div class="paragraph"><p>In your testconfig:</p></div>
-<div class="listingblock">
-<div class="content">
-<pre><code>[test_meta]
-jobgroup group1</code></pre>
-</div></div>
-<div class="paragraph"><p>In your megatest.config:</p></div>
-<div class="listingblock">
-<div class="content">
-<pre><code>[jobgroups]
-group1 10
-custdes 4</code></pre>
-</div></div>
-</div>
-</div>
-<div class="sect1">
-<h2 id="_debugging_tricks">Debugging Tricks</h2>
-<div class="sectionbody">
-<div class="sect2">
-<h3 id="_examining_the_environment">Examining The Environment</h3>
-<div class="sect3">
-<h4 id="_during_config_file_processing">During Config File Processing</h4>
-</div>
-<div class="sect3">
-<h4 id="_organising_your_tests_and_tasks">Organising Your Tests and Tasks</h4>
-<div class="listingblock">
-<div class="content">
-<pre><code>[tests-paths]
-1 #{get misc parent}/simplerun/tests</code></pre>
-</div></div>
-<div class="listingblock">
-<div class="content">
-<pre><code>[setup]</code></pre>
-</div></div>
-<div class="paragraph"><p>The runscript method is a brute force way to run scripts where the
-user is responsible for setting STATE and STATUS</p></div>
-<div class="listingblock">
-<div class="content">
-<pre><code>runscript main.csh</code></pre>
-</div></div>
-</div>
-</div>
-<div class="sect2">
-<h3 id="_debugging_server_problems">Debugging Server Problems</h3>
-<div class="listingblock">
-<div class="content">
-<pre><code>sudo lsof -i
-sudo netstat -lptu
-sudo netstat -tulpn</code></pre>
-</div></div>
-</div>
-</div>
-</div>
-<h1 id="_reference">Reference</h1>
-<div class="sect1">
-<h2 id="_the_first_chapter_of_the_second_part_2">The First Chapter of the Second Part</h2>
-<div class="sectionbody">
-<div class="paragraph"><p>Chapters grouped into book parts are at level 1 and can contain
-sub-sections.</p></div>
-</div>
-</div>
-<div class="sect1">
-<h2 id="_the_testconfig_file">The testconfig File</h2>
-<div class="sectionbody">
-<div class="sect2">
-<h3 id="_setup_section">Setup section</h3>
-<div class="sect3">
-<h4 id="_header">Header</h4>
-<div class="listingblock">
-<div class="content">
-<pre><code>[setup]</code></pre>
-</div></div>
-<div class="paragraph"><p>The runscript method is a brute force way to run scripts where the
-user is responsible for setting STATE and STATUS</p></div>
-<div class="listingblock">
-<div class="content">
-<pre><code>runscript main.csh</code></pre>
-</div></div>
-</div>
-</div>
-<div class="sect2">
-<h3 id="_requirements_section">Requirements section</h3>
-<div class="sect3">
-<h4 id="_header_2">Header</h4>
-<div class="listingblock">
-<div class="content">
-<pre><code>[requirements]</code></pre>
-</div></div>
-</div>
-<div class="sect3">
-<h4 id="_wait_on_other_tests">Wait on Other Tests</h4>
-<div class="listingblock">
-<div class="content">
-<pre><code># A normal waiton waits for the prior tests to be COMPLETED
-# and PASS, CHECK or WAIVED
-waiton test1 test2</code></pre>
-</div></div>
-</div>
-<div class="sect3">
-<h4 id="_mode">Mode</h4>
-<div class="paragraph"><p>The default (i.e. if mode is not specified) is normal. All pre-dependent tests
-must be COMPLETED and PASS, CHECK or WAIVED before the test will start</p></div>
-<div class="listingblock">
-<div class="content">
-<pre><code>mode   normal</code></pre>
-</div></div>
-<div class="paragraph"><p>The toplevel mode requires only that the prior tests are COMPLETED.</p></div>
-<div class="listingblock">
-<div class="content">
-<pre><code>mode toplevel</code></pre>
-</div></div>
-<div class="paragraph"><p>A item based waiton will start items in a test when the
-same-named item is COMPLETED and PASS, CHECK or WAIVED
-in the prior test</p></div>
-<div class="listingblock">
-<div class="content">
-<pre><code>mode itemmatch</code></pre>
-</div></div>
-<div class="listingblock">
-<div class="content">
-<pre><code># With a toplevel test you may wish to generate your list
-# of tests to run dynamically
-#
-# waiton #{shell get-valid-tests-to-run.sh}</code></pre>
-</div></div>
-</div>
-<div class="sect3">
-<h4 id="_run_time_limit">Run time limit</h4>
-<div class="listingblock">
-<div class="content">
-<pre><code>runtimelim 1h 2m 3s  # this will automatically kill the test if it runs for more than 1h 2m and 3s</code></pre>
-</div></div>
-</div>
-<div class="sect3">
-<h4 id="_skip">Skip</h4>
-</div>
-<div class="sect3">
-<h4 id="_header_3">Header</h4>
-<div class="listingblock">
-<div class="content">
-<pre><code>[skip]</code></pre>
-</div></div>
-</div>
-<div class="sect3">
-<h4 id="_skip_on_still_running_tests">Skip on Still-running Tests</h4>
-<div class="listingblock">
-<div class="content">
-<pre><code># NB// If the prevrunning line exists with *any* value the test will
-# automatically SKIP if the same-named test is currently RUNNING
-
-prevrunning x</code></pre>
-</div></div>
-</div>
-<div class="sect3">
-<h4 id="_skip_if_a_file_exists">Skip if a File Exists</h4>
-<div class="listingblock">
-<div class="content">
-<pre><code>fileexists /path/to/a/file # skip if /path/to/a/file exists</code></pre>
-</div></div>
-</div>
-<div class="sect3">
-<h4 id="_controlled_waiver_propagation">Controlled waiver propagation</h4>
-<div class="paragraph"><p>If test is FAIL and previous test in run with same MT_TARGET is WAIVED then apply the following rules from the testconfig:
-If a waiver check is specified in the testconfig apply the check and if it passes then set this FAIL to WAIVED</p></div>
-<div class="paragraph"><p>Waiver check has two parts, 1) a list of waiver, rulename, filepatterns and 2) the rulename script spec (note that "diff" and "logpro" are predefined)</p></div>
-<div class="listingblock">
-<div class="content">
-<pre><code>###### EXAMPLE FROM testconfig #########
-# matching file(s) will be diff'd with previous run and logpro applied
-# if PASS or WARN result from logpro then WAIVER state is set
-#
-[waivers]
-# logpro_file    rulename      input_glob
-waiver_1         logpro        lookittmp.log
-
-[waiver_rules]
-
-# This builtin rule is the default if there is no &lt;waivername&gt;.logpro file
-# diff   diff %file1% %file2%
-
-# This builtin rule is applied if a &lt;waivername&gt;.logpro file exists
-# logpro diff %file1% %file2% | logpro %waivername%.logpro %waivername%.html</code></pre>
-</div></div>
-</div>
-</div>
-<div class="sect2">
-<h3 id="_ezsteps">Ezsteps</h3>
-<div class="paragraph"><p>To transfer the environment to the next step you can do the following:</p></div>
-<div class="listingblock">
-<div class="content">
-<pre><code>$MT_MEGATEST -env2file .ezsteps/${stepname}</code></pre>
-</div></div>
-</div>
-<div class="sect2">
-<h3 id="_triggers">Triggers</h3>
-<div class="paragraph"><p>In your testconfig triggers can be specified</p></div>
-<div class="listingblock">
-<div class="content">
-<pre><code>[triggers]
-
-# Call script running.sh when test goes to state=RUNNING, status=PASS
-RUNNING/PASS running.sh
-
-# Call script running.sh any time state goes to RUNNING
-RUNNING/ running.sh
-
-# Call script onpass.sh any time status goes to PASS
-PASS/ onpass.sh</code></pre>
-</div></div>
-<div class="paragraph"><p>Scripts called will have; test-id test-rundir trigger, added to the commandline.</p></div>
-<div class="paragraph"><p>HINT</p></div>
-<div class="paragraph"><p>To start an xterm (useful for debugging), use a command line like the following:</p></div>
-<div class="listingblock">
-<div class="content">
-<pre><code>[triggers]
-COMPLETED/ xterm -e bash -s --</code></pre>
-</div></div>
-<div class="admonitionblock">
-<table><tr>
-<td class="icon">
-<div class="title">Note</div>
-</td>
-<td class="content">There is a trailing space after the --</td>
-</tr></table>
-</div>
-</div>
-<div class="sect2">
-<h3 id="_megatest_internals">Megatest Internals</h3>
-<div class="imageblock graphviz">
-<div class="content">
-<img src="server.png" alt="server.png" />
-</div>
-</div>
-</div>
-</div>
-</div>
-<div class="sect1">
-<h2 id="_example_appendix">Appendix A: Example Appendix</h2>
-<div class="sectionbody">
-<div class="paragraph"><p>One or more optional appendixes go here at section level zero.</p></div>
-<div class="sect2">
-<h3 id="_appendix_sub_section">Appendix Sub-section</h3>
-<div class="admonitionblock">
-<table><tr>
-<td class="icon">
-<div class="title">Note</div>
-</td>
-<td class="content">Preface and appendix subsections start out of sequence at level
-2 (level 1 is skipped).  This only applies to multi-part book
-documents.</td>
-</tr></table>
-</div>
-</div>
-</div>
-</div>
-<div class="sect1">
-<h2 id="_example_bibliography">Example Bibliography</h2>
-<div class="sectionbody">
-<div class="paragraph"><p>The bibliography list is a style of AsciiDoc bulleted list.</p></div>
-<div class="ulist bibliography"><ul>
-<li>
-<p>
-<a id="taoup"></a>[taoup] Eric Steven Raymond. <em>The Art of Unix
-  Programming</em>. Addison-Wesley. ISBN 0-13-142901-9.
-</p>
-</li>
-<li>
-<p>
-<a id="walsh-muellner"></a>[walsh-muellner] Norman Walsh &amp; Leonard Muellner.
-  <em>DocBook - The Definitive Guide</em>. O&#8217;Reilly &amp; Associates. 1999.
-  ISBN 1-56592-580-7.
-</p>
-</li>
-</ul></div>
-</div>
-</div>
-<div class="sect1">
-<h2 id="_example_glossary">Example Glossary</h2>
-<div class="sectionbody">
-<div class="paragraph"><p>Glossaries are optional. Glossaries entries are an example of a style
-of AsciiDoc labeled lists.</p></div>
-<div class="dlist glossary"><dl>
-<dt>
-A glossary term
-</dt>
-<dd>
-<p>
-  The corresponding (indented) definition.
-</p>
-</dd>
-<dt>
-A second glossary term
-</dt>
-<dd>
-<p>
-  The corresponding (indented) definition.
-</p>
-</dd>
-</dl></div>
-</div>
-</div>
-<div class="sect1">
-<h2 id="_example_colophon">Example Colophon</h2>
-<div class="sectionbody">
-<div class="paragraph"><p>Text at the end of a book describing facts about its production.</p></div>
-</div>
-</div>
-<div class="sect1">
-<h2 id="_example_index">Example Index</h2>
-<div class="sectionbody">
-</div>
-</div>
-</div>
-<div id="footnotes"><hr /></div>
-<div id="footer">
-<div id="footer-text">
-Version 1.0<br />
-Last updated 2014-10-08 23:02:21 MST
-</div>
-</div>
-</body>
-</html>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
+    "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en">
+<head>
+<meta http-equiv="Content-Type" content="application/xhtml+xml; charset=UTF-8" />
+<meta name="generator" content="AsciiDoc 8.6.9" />
+<title>The Megatest Users Manual</title>
+<style type="text/css">
+/* Shared CSS for AsciiDoc xhtml11 and html5 backends */
+
+/* Default font. */
+body {
+  font-family: Georgia,serif;
+}
+
+/* Title font. */
+h1, h2, h3, h4, h5, h6,
+div.title, caption.title,
+thead, p.table.header,
+#toctitle,
+#author, #revnumber, #revdate, #revremark,
+#footer {
+  font-family: Arial,Helvetica,sans-serif;
+}
+
+body {
+  margin: 1em 5% 1em 5%;
+}
+
+a {
+  color: blue;
+  text-decoration: underline;
+}
+a:visited {
+  color: fuchsia;
+}
+
+em {
+  font-style: italic;
+  color: navy;
+}
+
+strong {
+  font-weight: bold;
+  color: #083194;
+}
+
+h1, h2, h3, h4, h5, h6 {
+  color: #527bbd;
+  margin-top: 1.2em;
+  margin-bottom: 0.5em;
+  line-height: 1.3;
+}
+
+h1, h2, h3 {
+  border-bottom: 2px solid silver;
+}
+h2 {
+  padding-top: 0.5em;
+}
+h3 {
+  float: left;
+}
+h3 + * {
+  clear: left;
+}
+h5 {
+  font-size: 1.0em;
+}
+
+div.sectionbody {
+  margin-left: 0;
+}
+
+hr {
+  border: 1px solid silver;
+}
+
+p {
+  margin-top: 0.5em;
+  margin-bottom: 0.5em;
+}
+
+ul, ol, li > p {
+  margin-top: 0;
+}
+ul > li     { color: #aaa; }
+ul > li > * { color: black; }
+
+.monospaced, code, pre {
+  font-family: "Courier New", Courier, monospace;
+  font-size: inherit;
+  color: navy;
+  padding: 0;
+  margin: 0;
+}
+pre {
+  white-space: pre-wrap;
+}
+
+#author {
+  color: #527bbd;
+  font-weight: bold;
+  font-size: 1.1em;
+}
+#email {
+}
+#revnumber, #revdate, #revremark {
+}
+
+#footer {
+  font-size: small;
+  border-top: 2px solid silver;
+  padding-top: 0.5em;
+  margin-top: 4.0em;
+}
+#footer-text {
+  float: left;
+  padding-bottom: 0.5em;
+}
+#footer-badges {
+  float: right;
+  padding-bottom: 0.5em;
+}
+
+#preamble {
+  margin-top: 1.5em;
+  margin-bottom: 1.5em;
+}
+div.imageblock, div.exampleblock, div.verseblock,
+div.quoteblock, div.literalblock, div.listingblock, div.sidebarblock,
+div.admonitionblock {
+  margin-top: 1.0em;
+  margin-bottom: 1.5em;
+}
+div.admonitionblock {
+  margin-top: 2.0em;
+  margin-bottom: 2.0em;
+  margin-right: 10%;
+  color: #606060;
+}
+
+div.content { /* Block element content. */
+  padding: 0;
+}
+
+/* Block element titles. */
+div.title, caption.title {
+  color: #527bbd;
+  font-weight: bold;
+  text-align: left;
+  margin-top: 1.0em;
+  margin-bottom: 0.5em;
+}
+div.title + * {
+  margin-top: 0;
+}
+
+td div.title:first-child {
+  margin-top: 0.0em;
+}
+div.content div.title:first-child {
+  margin-top: 0.0em;
+}
+div.content + div.title {
+  margin-top: 0.0em;
+}
+
+div.sidebarblock > div.content {
+  background: #ffffee;
+  border: 1px solid #dddddd;
+  border-left: 4px solid #f0f0f0;
+  padding: 0.5em;
+}
+
+div.listingblock > div.content {
+  border: 1px solid #dddddd;
+  border-left: 5px solid #f0f0f0;
+  background: #f8f8f8;
+  padding: 0.5em;
+}
+
+div.quoteblock, div.verseblock {
+  padding-left: 1.0em;
+  margin-left: 1.0em;
+  margin-right: 10%;
+  border-left: 5px solid #f0f0f0;
+  color: #888;
+}
+
+div.quoteblock > div.attribution {
+  padding-top: 0.5em;
+  text-align: right;
+}
+
+div.verseblock > pre.content {
+  font-family: inherit;
+  font-size: inherit;
+}
+div.verseblock > div.attribution {
+  padding-top: 0.75em;
+  text-align: left;
+}
+/* DEPRECATED: Pre version 8.2.7 verse style literal block. */
+div.verseblock + div.attribution {
+  text-align: left;
+}
+
+div.admonitionblock .icon {
+  vertical-align: top;
+  font-size: 1.1em;
+  font-weight: bold;
+  text-decoration: underline;
+  color: #527bbd;
+  padding-right: 0.5em;
+}
+div.admonitionblock td.content {
+  padding-left: 0.5em;
+  border-left: 3px solid #dddddd;
+}
+
+div.exampleblock > div.content {
+  border-left: 3px solid #dddddd;
+  padding-left: 0.5em;
+}
+
+div.imageblock div.content { padding-left: 0; }
+span.image img { border-style: none; vertical-align: text-bottom; }
+a.image:visited { color: white; }
+
+dl {
+  margin-top: 0.8em;
+  margin-bottom: 0.8em;
+}
+dt {
+  margin-top: 0.5em;
+  margin-bottom: 0;
+  font-style: normal;
+  color: navy;
+}
+dd > *:first-child {
+  margin-top: 0.1em;
+}
+
+ul, ol {
+    list-style-position: outside;
+}
+ol.arabic {
+  list-style-type: decimal;
+}
+ol.loweralpha {
+  list-style-type: lower-alpha;
+}
+ol.upperalpha {
+  list-style-type: upper-alpha;
+}
+ol.lowerroman {
+  list-style-type: lower-roman;
+}
+ol.upperroman {
+  list-style-type: upper-roman;
+}
+
+div.compact ul, div.compact ol,
+div.compact p, div.compact p,
+div.compact div, div.compact div {
+  margin-top: 0.1em;
+  margin-bottom: 0.1em;
+}
+
+tfoot {
+  font-weight: bold;
+}
+td > div.verse {
+  white-space: pre;
+}
+
+div.hdlist {
+  margin-top: 0.8em;
+  margin-bottom: 0.8em;
+}
+div.hdlist tr {
+  padding-bottom: 15px;
+}
+dt.hdlist1.strong, td.hdlist1.strong {
+  font-weight: bold;
+}
+td.hdlist1 {
+  vertical-align: top;
+  font-style: normal;
+  padding-right: 0.8em;
+  color: navy;
+}
+td.hdlist2 {
+  vertical-align: top;
+}
+div.hdlist.compact tr {
+  margin: 0;
+  padding-bottom: 0;
+}
+
+.comment {
+  background: yellow;
+}
+
+.footnote, .footnoteref {
+  font-size: 0.8em;
+}
+
+span.footnote, span.footnoteref {
+  vertical-align: super;
+}
+
+#footnotes {
+  margin: 20px 0 20px 0;
+  padding: 7px 0 0 0;
+}
+
+#footnotes div.footnote {
+  margin: 0 0 5px 0;
+}
+
+#footnotes hr {
+  border: none;
+  border-top: 1px solid silver;
+  height: 1px;
+  text-align: left;
+  margin-left: 0;
+  width: 20%;
+  min-width: 100px;
+}
+
+div.colist td {
+  padding-right: 0.5em;
+  padding-bottom: 0.3em;
+  vertical-align: top;
+}
+div.colist td img {
+  margin-top: 0.3em;
+}
+
+@media print {
+  #footer-badges { display: none; }
+}
+
+#toc {
+  margin-bottom: 2.5em;
+}
+
+#toctitle {
+  color: #527bbd;
+  font-size: 1.1em;
+  font-weight: bold;
+  margin-top: 1.0em;
+  margin-bottom: 0.1em;
+}
+
+div.toclevel0, div.toclevel1, div.toclevel2, div.toclevel3, div.toclevel4 {
+  margin-top: 0;
+  margin-bottom: 0;
+}
+div.toclevel2 {
+  margin-left: 2em;
+  font-size: 0.9em;
+}
+div.toclevel3 {
+  margin-left: 4em;
+  font-size: 0.9em;
+}
+div.toclevel4 {
+  margin-left: 6em;
+  font-size: 0.9em;
+}
+
+span.aqua { color: aqua; }
+span.black { color: black; }
+span.blue { color: blue; }
+span.fuchsia { color: fuchsia; }
+span.gray { color: gray; }
+span.green { color: green; }
+span.lime { color: lime; }
+span.maroon { color: maroon; }
+span.navy { color: navy; }
+span.olive { color: olive; }
+span.purple { color: purple; }
+span.red { color: red; }
+span.silver { color: silver; }
+span.teal { color: teal; }
+span.white { color: white; }
+span.yellow { color: yellow; }
+
+span.aqua-background { background: aqua; }
+span.black-background { background: black; }
+span.blue-background { background: blue; }
+span.fuchsia-background { background: fuchsia; }
+span.gray-background { background: gray; }
+span.green-background { background: green; }
+span.lime-background { background: lime; }
+span.maroon-background { background: maroon; }
+span.navy-background { background: navy; }
+span.olive-background { background: olive; }
+span.purple-background { background: purple; }
+span.red-background { background: red; }
+span.silver-background { background: silver; }
+span.teal-background { background: teal; }
+span.white-background { background: white; }
+span.yellow-background { background: yellow; }
+
+span.big { font-size: 2em; }
+span.small { font-size: 0.6em; }
+
+span.underline { text-decoration: underline; }
+span.overline { text-decoration: overline; }
+span.line-through { text-decoration: line-through; }
+
+div.unbreakable { page-break-inside: avoid; }
+
+
+/*
+ * xhtml11 specific
+ *
+ * */
+
+div.tableblock {
+  margin-top: 1.0em;
+  margin-bottom: 1.5em;
+}
+div.tableblock > table {
+  border: 3px solid #527bbd;
+}
+thead, p.table.header {
+  font-weight: bold;
+  color: #527bbd;
+}
+p.table {
+  margin-top: 0;
+}
+/* Because the table frame attribute is overriden by CSS in most browsers. */
+div.tableblock > table[frame="void"] {
+  border-style: none;
+}
+div.tableblock > table[frame="hsides"] {
+  border-left-style: none;
+  border-right-style: none;
+}
+div.tableblock > table[frame="vsides"] {
+  border-top-style: none;
+  border-bottom-style: none;
+}
+
+
+/*
+ * html5 specific
+ *
+ * */
+
+table.tableblock {
+  margin-top: 1.0em;
+  margin-bottom: 1.5em;
+}
+thead, p.tableblock.header {
+  font-weight: bold;
+  color: #527bbd;
+}
+p.tableblock {
+  margin-top: 0;
+}
+table.tableblock {
+  border-width: 3px;
+  border-spacing: 0px;
+  border-style: solid;
+  border-color: #527bbd;
+  border-collapse: collapse;
+}
+th.tableblock, td.tableblock {
+  border-width: 1px;
+  padding: 4px;
+  border-style: solid;
+  border-color: #527bbd;
+}
+
+table.tableblock.frame-topbot {
+  border-left-style: hidden;
+  border-right-style: hidden;
+}
+table.tableblock.frame-sides {
+  border-top-style: hidden;
+  border-bottom-style: hidden;
+}
+table.tableblock.frame-none {
+  border-style: hidden;
+}
+
+th.tableblock.halign-left, td.tableblock.halign-left {
+  text-align: left;
+}
+th.tableblock.halign-center, td.tableblock.halign-center {
+  text-align: center;
+}
+th.tableblock.halign-right, td.tableblock.halign-right {
+  text-align: right;
+}
+
+th.tableblock.valign-top, td.tableblock.valign-top {
+  vertical-align: top;
+}
+th.tableblock.valign-middle, td.tableblock.valign-middle {
+  vertical-align: middle;
+}
+th.tableblock.valign-bottom, td.tableblock.valign-bottom {
+  vertical-align: bottom;
+}
+
+
+/*
+ * manpage specific
+ *
+ * */
+
+body.manpage h1 {
+  padding-top: 0.5em;
+  padding-bottom: 0.5em;
+  border-top: 2px solid silver;
+  border-bottom: 2px solid silver;
+}
+body.manpage h2 {
+  border-style: none;
+}
+body.manpage div.sectionbody {
+  margin-left: 3em;
+}
+
+@media print {
+  body.manpage div#toc { display: none; }
+}
+
+
+</style>
+<script type="text/javascript">
+/*<![CDATA[*/
+var asciidoc = {  // Namespace.
+
+/////////////////////////////////////////////////////////////////////
+// Table Of Contents generator
+/////////////////////////////////////////////////////////////////////
+
+/* Author: Mihai Bazon, September 2002
+ * http://students.infoiasi.ro/~mishoo
+ *
+ * Table Of Content generator
+ * Version: 0.4
+ *
+ * Feel free to use this script under the terms of the GNU General Public
+ * License, as long as you do not remove or alter this notice.
+ */
+
+ /* modified by Troy D. Hanson, September 2006. License: GPL */
+ /* modified by Stuart Rackham, 2006, 2009. License: GPL */
+
+// toclevels = 1..4.
+toc: function (toclevels) {
+
+  function getText(el) {
+    var text = "";
+    for (var i = el.firstChild; i != null; i = i.nextSibling) {
+      if (i.nodeType == 3 /* Node.TEXT_NODE */) // IE doesn't speak constants.
+        text += i.data;
+      else if (i.firstChild != null)
+        text += getText(i);
+    }
+    return text;
+  }
+
+  function TocEntry(el, text, toclevel) {
+    this.element = el;
+    this.text = text;
+    this.toclevel = toclevel;
+  }
+
+  function tocEntries(el, toclevels) {
+    var result = new Array;
+    var re = new RegExp('[hH]([1-'+(toclevels+1)+'])');
+    // Function that scans the DOM tree for header elements (the DOM2
+    // nodeIterator API would be a better technique but not supported by all
+    // browsers).
+    var iterate = function (el) {
+      for (var i = el.firstChild; i != null; i = i.nextSibling) {
+        if (i.nodeType == 1 /* Node.ELEMENT_NODE */) {
+          var mo = re.exec(i.tagName);
+          if (mo && (i.getAttribute("class") || i.getAttribute("className")) != "float") {
+            result[result.length] = new TocEntry(i, getText(i), mo[1]-1);
+          }
+          iterate(i);
+        }
+      }
+    }
+    iterate(el);
+    return result;
+  }
+
+  var toc = document.getElementById("toc");
+  if (!toc) {
+    return;
+  }
+
+  // Delete existing TOC entries in case we're reloading the TOC.
+  var tocEntriesToRemove = [];
+  var i;
+  for (i = 0; i < toc.childNodes.length; i++) {
+    var entry = toc.childNodes[i];
+    if (entry.nodeName.toLowerCase() == 'div'
+     && entry.getAttribute("class")
+     && entry.getAttribute("class").match(/^toclevel/))
+      tocEntriesToRemove.push(entry);
+  }
+  for (i = 0; i < tocEntriesToRemove.length; i++) {
+    toc.removeChild(tocEntriesToRemove[i]);
+  }
+
+  // Rebuild TOC entries.
+  var entries = tocEntries(document.getElementById("content"), toclevels);
+  for (var i = 0; i < entries.length; ++i) {
+    var entry = entries[i];
+    if (entry.element.id == "")
+      entry.element.id = "_toc_" + i;
+    var a = document.createElement("a");
+    a.href = "#" + entry.element.id;
+    a.appendChild(document.createTextNode(entry.text));
+    var div = document.createElement("div");
+    div.appendChild(a);
+    div.className = "toclevel" + entry.toclevel;
+    toc.appendChild(div);
+  }
+  if (entries.length == 0)
+    toc.parentNode.removeChild(toc);
+},
+
+
+/////////////////////////////////////////////////////////////////////
+// Footnotes generator
+/////////////////////////////////////////////////////////////////////
+
+/* Based on footnote generation code from:
+ * http://www.brandspankingnew.net/archive/2005/07/format_footnote.html
+ */
+
+footnotes: function () {
+  // Delete existing footnote entries in case we're reloading the footnodes.
+  var i;
+  var noteholder = document.getElementById("footnotes");
+  if (!noteholder) {
+    return;
+  }
+  var entriesToRemove = [];
+  for (i = 0; i < noteholder.childNodes.length; i++) {
+    var entry = noteholder.childNodes[i];
+    if (entry.nodeName.toLowerCase() == 'div' && entry.getAttribute("class") == "footnote")
+      entriesToRemove.push(entry);
+  }
+  for (i = 0; i < entriesToRemove.length; i++) {
+    noteholder.removeChild(entriesToRemove[i]);
+  }
+
+  // Rebuild footnote entries.
+  var cont = document.getElementById("content");
+  var spans = cont.getElementsByTagName("span");
+  var refs = {};
+  var n = 0;
+  for (i=0; i<spans.length; i++) {
+    if (spans[i].className == "footnote") {
+      n++;
+      var note = spans[i].getAttribute("data-note");
+      if (!note) {
+        // Use [\s\S] in place of . so multi-line matches work.
+        // Because JavaScript has no s (dotall) regex flag.
+        note = spans[i].innerHTML.match(/\s*\[([\s\S]*)]\s*/)[1];
+        spans[i].innerHTML =
+          "[<a id='_footnoteref_" + n + "' href='#_footnote_" + n +
+          "' title='View footnote' class='footnote'>" + n + "</a>]";
+        spans[i].setAttribute("data-note", note);
+      }
+      noteholder.innerHTML +=
+        "<div class='footnote' id='_footnote_" + n + "'>" +
+        "<a href='#_footnoteref_" + n + "' title='Return to text'>" +
+        n + "</a>. " + note + "</div>";
+      var id =spans[i].getAttribute("id");
+      if (id != null) refs["#"+id] = n;
+    }
+  }
+  if (n == 0)
+    noteholder.parentNode.removeChild(noteholder);
+  else {
+    // Process footnoterefs.
+    for (i=0; i<spans.length; i++) {
+      if (spans[i].className == "footnoteref") {
+        var href = spans[i].getElementsByTagName("a")[0].getAttribute("href");
+        href = href.match(/#.*/)[0];  // Because IE return full URL.
+        n = refs[href];
+        spans[i].innerHTML =
+          "[<a href='#_footnote_" + n +
+          "' title='View footnote' class='footnote'>" + n + "</a>]";
+      }
+    }
+  }
+},
+
+install: function(toclevels) {
+  var timerId;
+
+  function reinstall() {
+    asciidoc.footnotes();
+    if (toclevels) {
+      asciidoc.toc(toclevels);
+    }
+  }
+
+  function reinstallAndRemoveTimer() {
+    clearInterval(timerId);
+    reinstall();
+  }
+
+  timerId = setInterval(reinstall, 500);
+  if (document.addEventListener)
+    document.addEventListener("DOMContentLoaded", reinstallAndRemoveTimer, false);
+  else
+    window.onload = reinstallAndRemoveTimer;
+}
+
+}
+asciidoc.install();
+/*]]>*/
+</script>
+</head>
+<body class="book">
+<div id="header">
+<h1>The Megatest Users Manual</h1>
+<span id="author">Matt Welland</span><br />
+<span id="email"><code>&lt;<a href="mailto:matt@kiatoa.com">matt@kiatoa.com</a>&gt;</code></span><br />
+<span id="revnumber">version 1.0,</span>
+<span id="revdate">April 2012</span>
+</div>
+<div id="content">
+<div class="sect1">
+<h2 id="_preface">Preface</h2>
+<div class="sectionbody">
+<div class="paragraph"><p>This book is organised as three sub-books; getting started, writing tests and reference.</p></div>
+<div class="sect2">
+<h3 id="_why_megatest">Why Megatest?</h3>
+<div class="paragraph"><p>The Megatest project was started for two reasons, the first was an
+immediate and pressing need for a generalized tool to manage a suite
+of regression tests and the second was the fact that the author had
+written or maintained several such tools at different companies over
+the years and it seemed a good thing to have a single open source
+tool, flexible enough to meet the needs of any team doing continuous
+integrating and or running a complex suite of tests for release
+qualification.</p></div>
+</div>
+<div class="sect2">
+<h3 id="_megatest_design_philosophy">Megatest Design Philosophy</h3>
+<div class="paragraph"><p>Megatest is intended to provide the minimum needed resources to make
+writing a suite of tests and tasks for implementing continuous build
+for software, design engineering or process control (via owlfs for
+example) without being specialized for any specific problem
+space. Megatest in of itself does not know what constitutes a PASS or
+FAIL of a test. In most cases megatest is best used in conjunction
+with logpro or a similar tool to parse, analyze and decide on the test
+outcome.</p></div>
+</div>
+<div class="sect2">
+<h3 id="_megatest_architecture">Megatest Architecture</h3>
+<div class="paragraph"><p>All data to specify the tests and configure the system is stored in
+plain text files. All system state is stored in an sqlite3
+database. Tests are launched using the launching system available for
+the distributed compute platform in use. A template script is provided
+which can launch jobs on local and remote Linux hosts. Currently
+megatest uses the network filesystem to call home to your master
+sqlite3 database.</p></div>
+</div>
+</div>
+</div>
+<h1 id="_road_map">Road Map</h1>
+<div class="paragraph"><p>Note 1: This road-map is tentative and subject to change without notice.</p></div>
+<div class="paragraph"><p>Note 2: Starting over. Old plan is commented out.</p></div>
+<div class="sect1">
+<h2 id="_current_items">Current Items</h2>
+<div class="sectionbody">
+<div class="sect2">
+<h3 id="_ww05_migrate_to_inmem_db">ww05 - migrate to inmem-db</h3>
+<div class="paragraph"><p>Keep as much the same as possible. Add internal reference to almost
+eliminate contention on db(s).</p></div>
+<div class="olist arabic"><ol class="arabic">
+<li>
+<p>
+Add internal reference db
+</p>
+</li>
+<li>
+<p>
+Verify that actions are accessing correct db
+</p>
+<div class="olist loweralpha"><ol class="loweralpha">
+<li>
+<p>
+-runtests  - inmem
+</p>
+</li>
+<li>
+<p>
+-list-runs - local (but not megatest.db)
+</p>
+</li>
+<li>
+<p>
+dashboard  - local (but not megatest.db)
+</p>
+</li>
+</ol></div>
+</li>
+<li>
+<p>
+Mirror db to /var/tmp&#8230;
+</p>
+</li>
+<li>
+<p>
+Dashboard read db from per-run db.
+</p>
+</li>
+<li>
+<p>
+Dashboard read db from /var/tmp
+</p>
+</li>
+<li>
+<p>
+Runs register in tasks table in monitor.db
+</p>
+</li>
+<li>
+<p>
+Server polls tasks table for next action (in addition?)
+</p>
+</li>
+<li>
+<p>
+Change run loop to execute in server, triggered by call to polling of tasks table
+</p>
+</li>
+</ol></div>
+</div>
+</div>
+</div>
+<h1 id="_getting_started">Getting Started</h1>
+<div class="openblock">
+<div class="title">Getting started with Megatest</div>
+<div class="content">
+<div class="paragraph"><p>How to install Megatest and set it up for running your regressions and continuous integration process.</p></div>
+</div></div>
+<div class="sect1">
+<h2 id="_installation">Installation</h2>
+<div class="sectionbody">
+<div class="sect2">
+<h3 id="_dependencies">Dependencies</h3>
+<div class="paragraph"><p>Chicken scheme and a number of "eggs" are required for building
+Megatest. See the script installall.sch in the utils directory of the
+distribution for a mostly automated way to install everything needed
+for building Megatest on Linux.</p></div>
+<div class="paragraph"><p><span class="footnote"><br />[An example footnote.]<br /></span></p></div>
+<div class="paragraph"><p>And now for something completely different: monkeys, lions and
+tigers (Bengal and Siberian) using the alternative syntax index
+entries.
+
+
+
+Note that multi-entry terms generate separate index entries.</p></div>
+<div class="paragraph"><p>Here are a couple of image examples: an <span class="image">
+<img src="images/smallnew.png" alt="images/smallnew.png" />
+</span>
+example inline image followed by an example block image:</p></div>
+<div class="imageblock">
+<div class="content">
+<img src="images/tiger.png" alt="Tiger image" />
+</div>
+<div class="title">Figure 1. Tiger block image</div>
+</div>
+<div class="paragraph"><p>Followed by an example table:</p></div>
+<div class="tableblock">
+<table rules="all"
+width="60%"
+frame="border"
+cellspacing="0" cellpadding="4">
+<caption class="title">Table 1. An example table</caption>
+<col width="50%" />
+<col width="50%" />
+<thead>
+<tr>
+<th align="left" valign="top"> Option          </th>
+<th align="left" valign="top"> Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td align="left" valign="top"><p class="table">-a <em>USER GROUP</em></p></td>
+<td align="left" valign="top"><p class="table">Add <em>USER</em> to <em>GROUP</em>.</p></td>
+</tr>
+<tr>
+<td align="left" valign="top"><p class="table">-R <em>GROUP</em></p></td>
+<td align="left" valign="top"><p class="table">Disables access to <em>GROUP</em>.</p></td>
+</tr>
+</tbody>
+</table>
+</div>
+<div class="exampleblock">
+<div class="title">Example 1. An example example</div>
+<div class="content">
+<div class="paragraph"><p>Lorum ipum&#8230;</p></div>
+</div></div>
+</div>
+<div class="sect2">
+<h3 id="X1">Sub-section with Anchor</h3>
+<div class="paragraph"><p>Sub-section at level 2.</p></div>
+<div class="sect3">
+<h4 id="_chapter_sub_section">Chapter Sub-section</h4>
+<div class="paragraph"><p>Sub-section at level 3.</p></div>
+<div class="sect4">
+<h5 id="_chapter_sub_section_2">Chapter Sub-section</h5>
+<div class="paragraph"><p>Sub-section at level 4.</p></div>
+<div class="paragraph"><p>This is the maximum sub-section depth supported by the distributed
+AsciiDoc configuration.
+<span class="footnote"><br />[A second example footnote.]<br /></span></p></div>
+</div>
+</div>
+</div>
+</div>
+</div>
+<div class="sect1">
+<h2 id="_the_second_chapter">The Second Chapter</h2>
+<div class="sectionbody">
+<div class="paragraph"><p>An example link to anchor at start of the <a href="#X1">first sub-section</a>.</p></div>
+<div class="paragraph"><p>An example link to a bibliography entry <a href="#taoup">[taoup]</a>.</p></div>
+</div>
+</div>
+<h1 id="_writing_tests">Writing Tests</h1>
+<div class="sect1">
+<h2 id="_the_first_chapter_of_the_second_part">The First Chapter of the Second Part</h2>
+<div class="sectionbody">
+<div class="paragraph"><p>Chapters grouped into book parts are at level 1 and can contain
+sub-sections.</p></div>
+</div>
+</div>
+<h1 id="_how_to_do_things">How To Do Things</h1>
+<div class="sect1">
+<h2 id="_tricks">Tricks</h2>
+<div class="sectionbody">
+<div class="paragraph"><p>This section is a compendium of a various useful tricks for debugging,
+configuring and generally getting the most out of Megatest.</p></div>
+</div>
+</div>
+<div class="sect1">
+<h2 id="_limiting_your_running_jobs">Limiting your running jobs</h2>
+<div class="sectionbody">
+<div class="paragraph"><p>The following example will limit a test in the jobgroup "group1" to no more than 10 tests simultaneously.</p></div>
+<div class="paragraph"><p>In your testconfig:</p></div>
+<div class="listingblock">
+<div class="content">
+<pre><code>[test_meta]
+jobgroup group1</code></pre>
+</div></div>
+<div class="paragraph"><p>In your megatest.config:</p></div>
+<div class="listingblock">
+<div class="content">
+<pre><code>[jobgroups]
+group1 10
+custdes 4</code></pre>
+</div></div>
+</div>
+</div>
+<div class="sect1">
+<h2 id="_debugging_tricks">Debugging Tricks</h2>
+<div class="sectionbody">
+<div class="sect2">
+<h3 id="_examining_the_environment">Examining The Environment</h3>
+<div class="sect3">
+<h4 id="_during_config_file_processing">During Config File Processing</h4>
+</div>
+<div class="sect3">
+<h4 id="_organising_your_tests_and_tasks">Organising Your Tests and Tasks</h4>
+<div class="listingblock">
+<div class="content">
+<pre><code>[tests-paths]
+1 #{get misc parent}/simplerun/tests</code></pre>
+</div></div>
+<div class="listingblock">
+<div class="content">
+<pre><code>[setup]</code></pre>
+</div></div>
+<div class="paragraph"><p>The runscript method is a brute force way to run scripts where the
+user is responsible for setting STATE and STATUS</p></div>
+<div class="listingblock">
+<div class="content">
+<pre><code>runscript main.csh</code></pre>
+</div></div>
+</div>
+</div>
+<div class="sect2">
+<h3 id="_debugging_server_problems">Debugging Server Problems</h3>
+<div class="listingblock">
+<div class="content">
+<pre><code>sudo lsof -i
+sudo netstat -lptu
+sudo netstat -tulpn</code></pre>
+</div></div>
+</div>
+</div>
+</div>
+<h1 id="_reference">Reference</h1>
+<div class="sect1">
+<h2 id="_the_testconfig_file">The testconfig File</h2>
+<div class="sectionbody">
+<div class="sect2">
+<h3 id="_setup_section">Setup section</h3>
+<div class="sect3">
+<h4 id="_header">Header</h4>
+<div class="listingblock">
+<div class="content">
+<pre><code>[setup]</code></pre>
+</div></div>
+<div class="paragraph"><p>The runscript method is a brute force way to run scripts where the
+user is responsible for setting STATE and STATUS</p></div>
+<div class="listingblock">
+<div class="content">
+<pre><code>runscript main.csh</code></pre>
+</div></div>
+</div>
+</div>
+<div class="sect2">
+<h3 id="_requirements_section">Requirements section</h3>
+<div class="sect3">
+<h4 id="_header_2">Header</h4>
+<div class="listingblock">
+<div class="content">
+<pre><code>[requirements]</code></pre>
+</div></div>
+</div>
+<div class="sect3">
+<h4 id="_wait_on_other_tests">Wait on Other Tests</h4>
+<div class="listingblock">
+<div class="content">
+<pre><code># A normal waiton waits for the prior tests to be COMPLETED
+# and PASS, CHECK or WAIVED
+waiton test1 test2</code></pre>
+</div></div>
+</div>
+<div class="sect3">
+<h4 id="_mode">Mode</h4>
+<div class="paragraph"><p>The default (i.e. if mode is not specified) is normal. All pre-dependent tests
+must be COMPLETED and PASS, CHECK or WAIVED before the test will start</p></div>
+<div class="listingblock">
+<div class="content">
+<pre><code>mode   normal</code></pre>
+</div></div>
+<div class="paragraph"><p>The toplevel mode requires only that the prior tests are COMPLETED.</p></div>
+<div class="listingblock">
+<div class="content">
+<pre><code>mode toplevel</code></pre>
+</div></div>
+<div class="paragraph"><p>A item based waiton will start items in a test when the
+same-named item is COMPLETED and PASS, CHECK or WAIVED
+in the prior test</p></div>
+<div class="listingblock">
+<div class="content">
+<pre><code>mode itemmatch</code></pre>
+</div></div>
+<div class="listingblock">
+<div class="content">
+<pre><code># With a toplevel test you may wish to generate your list
+# of tests to run dynamically
+#
+# waiton #{shell get-valid-tests-to-run.sh}</code></pre>
+</div></div>
+</div>
+<div class="sect3">
+<h4 id="_run_time_limit">Run time limit</h4>
+<div class="listingblock">
+<div class="content">
+<pre><code>runtimelim 1h 2m 3s  # this will automatically kill the test if it runs for more than 1h 2m and 3s</code></pre>
+</div></div>
+</div>
+<div class="sect3">
+<h4 id="_skip">Skip</h4>
+</div>
+<div class="sect3">
+<h4 id="_header_3">Header</h4>
+<div class="listingblock">
+<div class="content">
+<pre><code>[skip]</code></pre>
+</div></div>
+</div>
+<div class="sect3">
+<h4 id="_skip_on_still_running_tests">Skip on Still-running Tests</h4>
+<div class="listingblock">
+<div class="content">
+<pre><code># NB// If the prevrunning line exists with *any* value the test will
+# automatically SKIP if the same-named test is currently RUNNING
+
+prevrunning x</code></pre>
+</div></div>
+</div>
+<div class="sect3">
+<h4 id="_skip_if_a_file_exists">Skip if a File Exists</h4>
+<div class="listingblock">
+<div class="content">
+<pre><code>fileexists /path/to/a/file # skip if /path/to/a/file exists</code></pre>
+</div></div>
+</div>
+<div class="sect3">
+<h4 id="_controlled_waiver_propagation">Controlled waiver propagation</h4>
+<div class="paragraph"><p>If test is FAIL and previous test in run with same MT_TARGET is WAIVED then apply the following rules from the testconfig:
+If a waiver check is specified in the testconfig apply the check and if it passes then set this FAIL to WAIVED</p></div>
+<div class="paragraph"><p>Waiver check has two parts, 1) a list of waiver, rulename, filepatterns and 2) the rulename script spec (note that "diff" and "logpro" are predefined)</p></div>
+<div class="listingblock">
+<div class="content">
+<pre><code>###### EXAMPLE FROM testconfig #########
+# matching file(s) will be diff'd with previous run and logpro applied
+# if PASS or WARN result from logpro then WAIVER state is set
+#
+[waivers]
+# logpro_file    rulename      input_glob
+waiver_1         logpro        lookittmp.log
+
+[waiver_rules]
+
+# This builtin rule is the default if there is no &lt;waivername&gt;.logpro file
+# diff   diff %file1% %file2%
+
+# This builtin rule is applied if a &lt;waivername&gt;.logpro file exists
+# logpro diff %file1% %file2% | logpro %waivername%.logpro %waivername%.html</code></pre>
+</div></div>
+</div>
+</div>
+<div class="sect2">
+<h3 id="_ezsteps">Ezsteps</h3>
+<div class="paragraph"><p>To transfer the environment to the next step you can do the following:</p></div>
+<div class="listingblock">
+<div class="content">
+<pre><code>$MT_MEGATEST -env2file .ezsteps/${stepname}</code></pre>
+</div></div>
+</div>
+<div class="sect2">
+<h3 id="_triggers">Triggers</h3>
+<div class="paragraph"><p>In your testconfig triggers can be specified</p></div>
+<div class="listingblock">
+<div class="content">
+<pre><code>[triggers]
+
+# Call script running.sh when test goes to state=RUNNING, status=PASS
+RUNNING/PASS running.sh
+
+# Call script running.sh any time state goes to RUNNING
+RUNNING/ running.sh
+
+# Call script onpass.sh any time status goes to PASS
+PASS/ onpass.sh</code></pre>
+</div></div>
+<div class="paragraph"><p>Scripts called will have; test-id test-rundir trigger, added to the commandline.</p></div>
+<div class="paragraph"><p>HINT</p></div>
+<div class="paragraph"><p>To start an xterm (useful for debugging), use a command line like the following:</p></div>
+<div class="listingblock">
+<div class="content">
+<pre><code>[triggers]
+COMPLETED/ xterm -e bash -s --</code></pre>
+</div></div>
+<div class="admonitionblock">
+<table><tr>
+<td class="icon">
+<div class="title">Note</div>
+</td>
+<td class="content">There is a trailing space after the --</td>
+</tr></table>
+</div>
+</div>
+</div>
+</div>
+<div class="sect1">
+<h2 id="_programming_api">Programming API</h2>
+<div class="sectionbody">
+<div class="paragraph"><p>These routines can be called from the megatest repl.</p></div>
+<div class="tableblock">
+<table rules="all"
+width="70%"
+frame="hsides"
+cellspacing="0" cellpadding="4">
+<caption class="title">Table 2. API Server Management Calls</caption>
+<col width="14%" />
+<col width="28%" />
+<col width="28%" />
+<col width="28%" />
+<thead>
+<tr>
+<th align="center" valign="top">API Call                  </th>
+<th align="left" valign="top"> Purpose comments   </th>
+<th align="left" valign="top"> Returns                 </th>
+<th align="left" valign="top"> Comments</th>
+</tr>
+</thead>
+<tfoot>
+<tr>
+<td align="center" valign="top"><p class="table">(rmt:login        run-id)</p></td>
+<td align="left" valign="top"><p class="table"><code>Verify the the version, testsuite area etc. are correct.</code></p></td>
+<td align="left" valign="top"><p class="table"><code>#( #t "successful login" )</code></p></td>
+<td align="left" valign="top"><p class="table"><code></code></p></td>
+</tr>
+</tfoot>
+<tbody>
+<tr>
+<td align="center" valign="top"><p class="table">(rmt:start-server run-id)</p></td>
+<td align="left" valign="top"><p class="table"><code></code></p></td>
+<td align="left" valign="top"><p class="table"><code>#( success/fail   n/a )</code></p></td>
+<td align="left" valign="top"><p class="table"><code></code></p></td>
+</tr>
+<tr>
+<td align="center" valign="top"><p class="table">(rmt:kill-server  run-id)</p></td>
+<td align="left" valign="top"><p class="table"><code></code></p></td>
+<td align="left" valign="top"><p class="table"><code>#( success/fail   n/a )</code></p></td>
+<td align="left" valign="top"><p class="table"><code>Works only if the server is still reachable</code></p></td>
+</tr>
+</tbody>
+</table>
+</div>
+<div class="tableblock">
+<table rules="all"
+width="70%"
+frame="hsides"
+cellspacing="0" cellpadding="4">
+<caption class="title">Table 3. API Keys Related Calls</caption>
+<col width="14%" />
+<col width="28%" />
+<col width="28%" />
+<col width="28%" />
+<thead>
+<tr>
+<th align="center" valign="top">API Call                        </th>
+<th align="left" valign="top"> Purpose comments   </th>
+<th align="left" valign="top"> Returns                 </th>
+<th align="left" valign="top"> Comments</th>
+</tr>
+</thead>
+<tfoot>
+<tr>
+<td align="center" valign="top"><p class="table">(rmt:get-key-val-pairs run-id)</p></td>
+<td align="left" valign="top"><p class="table"><code></code></p></td>
+<td align="left" valign="top"><p class="table"><code>#t=success/#f=fail</code></p></td>
+<td align="left" valign="top"><p class="table"><code>Works only if the server is still reachable</code></p></td>
+</tr>
+</tfoot>
+<tbody>
+<tr>
+<td align="center" valign="top"><p class="table">(rmt:get-keys run-id)</p></td>
+<td align="left" valign="top"><p class="table"><code></code></p></td>
+<td align="left" valign="top"><p class="table"><code>( key1 key2 &#8230; )</code></p></td>
+<td align="left" valign="top"><p class="table"><code></code></p></td>
+</tr>
+</tbody>
+</table>
+</div>
+<div class="sect2">
+<h3 id="_megatest_internals">Megatest Internals</h3>
+<div class="imageblock graphviz">
+<div class="content">
+<img src="server.png" alt="server.png" />
+</div>
+</div>
+</div>
+</div>
+</div>
+<div class="sect1">
+<h2 id="_example_appendix">Appendix A: Example Appendix</h2>
+<div class="sectionbody">
+<div class="paragraph"><p>One or more optional appendixes go here at section level zero.</p></div>
+<div class="sect2">
+<h3 id="_appendix_sub_section">Appendix Sub-section</h3>
+<div class="admonitionblock">
+<table><tr>
+<td class="icon">
+<div class="title">Note</div>
+</td>
+<td class="content">Preface and appendix subsections start out of sequence at level
+2 (level 1 is skipped).  This only applies to multi-part book
+documents.</td>
+</tr></table>
+</div>
+</div>
+</div>
+</div>
+<div class="sect1">
+<h2 id="_example_bibliography">Example Bibliography</h2>
+<div class="sectionbody">
+<div class="paragraph"><p>The bibliography list is a style of AsciiDoc bulleted list.</p></div>
+<div class="ulist bibliography"><ul>
+<li>
+<p>
+<a id="taoup"></a>[taoup] Eric Steven Raymond. <em>The Art of Unix
+  Programming</em>. Addison-Wesley. ISBN 0-13-142901-9.
+</p>
+</li>
+<li>
+<p>
+<a id="walsh-muellner"></a>[walsh-muellner] Norman Walsh &amp; Leonard Muellner.
+  <em>DocBook - The Definitive Guide</em>. O&#8217;Reilly &amp; Associates. 1999.
+  ISBN 1-56592-580-7.
+</p>
+</li>
+</ul></div>
+</div>
+</div>
+<div class="sect1">
+<h2 id="_example_glossary">Example Glossary</h2>
+<div class="sectionbody">
+<div class="paragraph"><p>Glossaries are optional. Glossaries entries are an example of a style
+of AsciiDoc labeled lists.</p></div>
+<div class="dlist glossary"><dl>
+<dt>
+A glossary term
+</dt>
+<dd>
+<p>
+  The corresponding (indented) definition.
+</p>
+</dd>
+<dt>
+A second glossary term
+</dt>
+<dd>
+<p>
+  The corresponding (indented) definition.
+</p>
+</dd>
+</dl></div>
+</div>
+</div>
+<div class="sect1">
+<h2 id="_example_colophon">Example Colophon</h2>
+<div class="sectionbody">
+<div class="paragraph"><p>Text at the end of a book describing facts about its production.</p></div>
+</div>
+</div>
+<div class="sect1">
+<h2 id="_example_index">Example Index</h2>
+<div class="sectionbody">
+</div>
+</div>
+</div>
+<div id="footnotes"><hr /></div>
+<div id="footer">
+<div id="footer-text">
+Version 1.0<br />
+Last updated 2014-11-25 21:11:28 MST
+</div>
+</div>
+</body>
+</html>

Index: docs/manual/reference.txt
==================================================================
--- docs/manual/reference.txt
+++ docs/manual/reference.txt
@@ -1,15 +1,9 @@
 
 Reference
 =========
 
-The First Chapter of the Second Part
-------------------------------------
-
-Chapters grouped into book parts are at level 1 and can contain
-sub-sections.
-
 The testconfig File
 -------------------
 
 Setup section
 ~~~~~~~~~~~~~
@@ -178,8 +172,31 @@
 [triggers]
 COMPLETED/ xterm -e bash -s -- 
 -----------------
 
 NOTE: There is a trailing space after the --
+
+Programming API
+---------------
+
+These routines can be called from the megatest repl. 
+
+.API Server Management Calls
+[width="70%",cols="^,2m,2m,2m",frame="topbot",options="header,footer"]
+|======================
+|API Call                  | Purpose comments   | Returns                 | Comments
+|(rmt:start-server run-id) |                    | #( success/fail   n/a ) | 
+|(rmt:kill-server  run-id) |                    | #( success/fail   n/a ) | Works only if the server is still reachable
+|(rmt:login        run-id) | Verify the the version, testsuite area etc. are correct. | #( #t "successful login" ) |
+|======================
+
+.API Keys Related Calls
+[width="70%",cols="^,2m,2m,2m",frame="topbot",options="header,footer"]
+|======================
+|API Call                        | Purpose comments   | Returns                 | Comments
+|(rmt:get-keys run-id)           |                    | ( key1 key2 ... )       | 
+| (rmt:get-key-val-pairs run-id) |                    | #t=success/#f=fail      | Works only if the server is still reachable
+|======================
+
 
 :numbered!:
 

Index: http-transport.scm
==================================================================
--- http-transport.scm
+++ http-transport.scm
@@ -243,11 +243,12 @@
 			 (http-transport:server-dat-get-api-req serverdat)
 			 (begin
 			   (debug:print 0 "FATAL ERROR: http-transport:client-api-send-receive called with no server info")
 			   (exit 1))))
 	 (res        #f)
-	 (success    #t))
+	 (success    #t)
+	 (sparams    (db:obj->string params transport: 'http)))
     (handle-exceptions
      exn
      (if (> numretries 0)
 	 (begin
 	   (mutex-unlock! *http-mutex*)
@@ -255,11 +256,11 @@
 	   (handle-exceptions
 	    exn
 	    (debug:print 0 "WARNING: closing connections failed. Server at " fullurl " almost certainly dead")
 	    (close-all-connections!))
 	   (debug:print 0 "WARNING: Failed to communicate with server, trying again, numretries left: " numretries)
-	   (http-transport:client-api-send-receive run-id serverdat cmd params numretries: (- numretries 1)))
+	   (http-transport:client-api-send-receive run-id serverdat cmd sparams numretries: (- numretries 1)))
 	 (begin
 	   (mutex-unlock! *http-mutex*)
 	   (tasks:kill-server-run-id run-id)
 	   #f))
      (begin
@@ -276,25 +277,27 @@
 			      (mutex-lock! *http-mutex*)
 			      ;; (condition-case (with-input-from-request "http://localhost"; #f read-lines)
 			      ;;					       ((exn http client-error) e (print e)))
 			      (set! res (vector
 					 success
-					 (handle-exceptions
-					  exn
-					  (begin
-					    (set! success #f)
-					    (debug:print 0 "WARNING: failure in with-input-from-request to " fullurl ". Killing associated server to allow clean retry.")
-					    (debug:print 0 " message: " ((condition-property-accessor 'exn 'message) exn))
-					    (hash-table-delete! *runremote* run-id)
-					    ;; (tasks:kill-server-run-id run-id)  ;; better to kill the server in the logic that called this routine.
-					    #f)
-					  (with-input-from-request ;; was dat
-					   fullurl 
-					   (list (cons 'key "thekey")
-						 (cons 'cmd cmd)
-						 (cons 'params params))
-					   read-string))))
+					 (db:string->obj 
+					  (handle-exceptions
+					   exn
+					   (begin
+					     (set! success #f)
+					     (debug:print 0 "WARNING: failure in with-input-from-request to " fullurl ". Killing associated server to allow clean retry.")
+					     (debug:print 0 " message: " ((condition-property-accessor 'exn 'message) exn))
+					     (hash-table-delete! *runremote* run-id)
+					     ;; (tasks:kill-server-run-id run-id)  ;; better to kill the server in the logic that called this routine.
+					     #f)
+					   (with-input-from-request ;; was dat
+					    fullurl 
+					    (list (cons 'key "thekey")
+						  (cons 'cmd cmd)
+						  (cons 'params sparams))
+					    read-string))
+					  transport: 'http)))
 			      ;; Shouldn't this be a call to the managed call-all-connections stuff above?
 			      (close-all-connections!)
 			      (mutex-unlock! *http-mutex*)
 			      ))
 	      (time-out     (lambda ()
@@ -318,17 +321,18 @@
 	  (close-connection! api-dat)
 	  #t)
 	#f)))
 
 
-(define (make-http-transport:server-dat)(make-vector 5))
+(define (make-http-transport:server-dat)(make-vector 6))
 (define (http-transport:server-dat-get-iface         vec)    (vector-ref  vec 0))
 (define (http-transport:server-dat-get-port          vec)    (vector-ref  vec 1))
 (define (http-transport:server-dat-get-api-uri       vec)    (vector-ref  vec 2))
 (define (http-transport:server-dat-get-api-url       vec)    (vector-ref  vec 3))
 (define (http-transport:server-dat-get-api-req       vec)    (vector-ref  vec 4))
 (define (http-transport:server-dat-get-last-access   vec)    (vector-ref  vec 5))
+(define (http-transport:server-dat-get-socket        vec)    (vector-ref  vec 6))
 
 (define (http-transport:server-dat-make-url vec)
   (if (and (http-transport:server-dat-get-iface vec)
 	   (http-transport:server-dat-get-port  vec))
       (conc "http://" 
@@ -355,10 +359,11 @@
 ;;
 (define (http-transport:keep-running server-id run-id)
   ;; if none running or if > 20 seconds since 
   ;; server last used then start shutdown
   ;; This thread waits for the server to come alive
+  (debug:print-info 0 "Starting the sync-back, keep alive thread in server for run-id=" run-id)
   (let* ((tdbdat      (tasks:open-db))
 	 (server-info (let loop ((start-time (current-seconds))
 				 (changed    #t)
 				 (last-sdat  "not this"))
                         (let ((sdat #f))
@@ -445,18 +450,12 @@
       ;; no_traffic, no running tests, if server 0, no running servers
       ;;
       ;; (let ((wait-on-running (configf:lookup *configdat* "server" "wait-on-running"))) ;; wait on running tasks (if not true then exit on time out)
       ;;
       (if (and *server-run*
-	       ;; (or
 	       (> (+ last-access server-timeout)
 		  (current-seconds)))
-;;		   (and (eq? run-id 0)
-;;			(> (tasks:num-servers-non-zero-running tdb) 0))
-;;		   (and (not (eq? run-id 0)) ;; only makes sense in non-zero run-id servers
-;;			(> (db:get-count-tests-actually-running *inmemdb* run-id) 0))
-;;		   ))
 	  (begin
 	    (debug:print-info 0 "Server continuing, seconds since last db access: " (- (current-seconds) last-access))
 	    ;;
 	    ;; Consider implementing some smarts here to re-insert the record or kill self is
 	    ;; the db indicates so

Index: newdashboard.scm
==================================================================
--- newdashboard.scm
+++ newdashboard.scm
@@ -78,10 +78,14 @@
 ;;     (begin
 ;;       (set! *runremote* (string-split (args:get-arg "-host" ":")))
 ;;       (client:launch))
 ;;     (client:launch))
 
+;; ease debugging by loading ~/.dashboardrc
+(let ((debugcontrolf (conc (get-environment-variable "HOME") "/.dashboardrc")))
+  (if (file-exists? debugcontrolf)
+      (load debugcontrolf)))
 
 (define *dbdir* (conc (configf:lookup *configdat* "setup" "linktree") "/.db"))
 (define *dbstruct-local*  (make-dbr:dbstruct path:  *dbdir*
 					     local: #t))
 (define *db-file-path* (db:dbfile-path 0))

ADDED   nmsg-transport.scm
Index: nmsg-transport.scm
==================================================================
--- /dev/null
+++ nmsg-transport.scm
@@ -0,0 +1,358 @@
+
+;; Copyright 2006-2012, Matthew Welland.
+;; 
+;;  This program is made available under the GNU GPL version 2.0 or
+;;  greater. See the accompanying file COPYING for details.
+;; 
+;;  This program is distributed WITHOUT ANY WARRANTY; without even the
+;;  implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+;;  PURPOSE.
+
+(require-extension (srfi 18) extras tcp s11n)
+
+(use sqlite3 srfi-1 posix regex regex-case srfi-69 hostinfo md5 message-digest)
+(import (prefix sqlite3 sqlite3:))
+
+(use nanomsg)
+
+(declare (unit nmsg-transport))
+
+(declare (uses common))
+(declare (uses db))
+(declare (uses tests))
+(declare (uses tasks)) ;; tasks are where stuff is maintained about what is running.
+(declare (uses server))
+
+(include "common_records.scm")
+(include "db_records.scm")
+
+;; Transition to pub --> sub with pull <-- push
+;;
+;;   1. client sends request to server via push to the pull port
+;;   2. server puts request in queue or processes immediately as appropriate
+;;   3. server puts responses from completed requests into pub port 
+;;
+;; TODO
+;;
+;; Done Tested
+;; [x]  [ ]    1. Add columns pullport pubport to servers table
+;; [x]  [ ]    2. Add rm of monitor.db if older than 11/12/2012 
+;; [x]  [ ]    3. Add create of pullport and pubport with finding of available ports
+;; [x]  [ ]    4. Add client compose of request
+;; [x]  [ ]        - name of client: testname/itempath-test_id-hostname 
+;; [x]  [ ]        - name of request: callname, params
+;; [x]  [ ]        - request key: f(clientname, callname, params)
+;; [x]  [ ]    5. Add processing of subscription hits
+;; [x]  [ ]        - done when get key 
+;; [x]  [ ]        - return results
+;; [x]  [ ]    6. Add timeout processing
+;; [x]  [ ]        - after 60 seconds
+;; [ ]  [ ]            i. check server alive, connect to new if necessary
+;; [ ]  [ ]           ii. resend request
+;; [ ]  [ ]    7. Turn self ping back on
+
+(define (nmsg-transport:make-server-url hostport #!key (bindall #f))
+  (if (not hostport)
+      #f
+      (conc "tcp://" (if bindall "*" (car hostport)) ":" (cadr hostport))))
+
+(define *server-loop-heart-beat* (current-seconds))
+(define *heartbeat-mutex* (make-mutex))
+
+;;======================================================================
+;; S E R V E R
+;;======================================================================
+
+(define (nmsg-transport:run dbstruct hostn run-id server-id #!key (retrynum 1000))
+  (debug:print 2 "Attempting to start the server ...")
+  (let* ((start-port      (portlogger:open-run-close portlogger:find-port))
+	 (server-thread   (make-thread (lambda ()
+					 (nmsg-transport:try-start-server dbstruct run-id start-port server-id))
+				       "server thread"))
+	 (tdbdat          (tasks:open-db)))
+    (thread-start! server-thread)
+    (thread-sleep! 0.1)
+    (if (nmsg-transport:ping hostn start-port timeout: 2 expected-key: (current-process-id))
+	(let ((interface (if (equal? hostn "-")(get-host-name) hostn)))
+	  (tasks:server-set-interface-port (db:delay-if-busy tdbdat) server-id interface start-port)
+	  (tasks:server-set-state! (db:delay-if-busy tdbdat) server-id "dbprep")
+	  (set! *server-info* (list hostn start-port)) ;; probably not needed anymore? currently used by keep-running
+	  (thread-sleep! 3) ;; give some margin for queries to complete before switching from file based access to server based access
+	  ;; (set! *inmemdb*  dbstruct)
+	  (tasks:server-set-state! (db:delay-if-busy tdbdat) server-id "running")
+	  (thread-start! (make-thread
+			  (lambda ()(nmsg-transport:keep-running server-id run-id))
+			  "keep running"))
+	  (thread-join! server-thread))
+	(if (> retrynum 0)
+	    (begin
+	      (debug:print 0 "WARNING: Failed to connect to server (self) on host " hostn ":" start-port ", trying again.")
+	      (tasks:server-delete-record (db:delay-if-busy tdbdat) server-id "failed to start, never received server alive signature")
+	      (portlogger:open-run-close portlogger:set-failed start-port)
+	      (nmsg-transport:run dbstruct hostn run-id server-id))
+	    (begin
+	      (debug:print 0 "ERROR: could not find an open port to start server on. Giving up")
+	      (exit 1))))))
+
+(define (nmsg-transport:try-start-server dbstruct run-id portnum server-id)
+  (let ((repsoc (nn-socket 'rep)))
+    (nn-bind repsoc (conc "tcp://*:" portnum))
+    (let loop ((msg-in (nn-recv repsoc)))
+      (let* ((dat    (db:string->obj msg-in transport: 'nmsg)))
+	(debug:print 0 "server, received: " dat)
+	(let ((result (api:execute-requests dbstruct dat)))
+	  (debug:print 0 "server, sending: " result)
+	  (nn-send repsoc (db:obj->string result  transport: 'nmsg)))
+	(loop (nn-recv repsoc))))))
+
+;; all routes though here end in exit ...
+;;
+(define (nmsg-transport:launch run-id)
+  (let* ((tdbdat   (tasks:open-db))
+	 (dbstruct (db:setup run-id))
+	 (hostn    (or (args:get-arg "-server") "-")))
+    (set! *run-id*   run-id)
+    (set! *inmemdb* dbstruct)
+    ;; with nbfake daemonize isn't really needed
+    ;;
+    ;; (if (args:get-arg "-daemonize")
+    ;;     (begin
+    ;;       (daemon:ize)
+    ;;       (if *alt-log-file* ;; we should re-connect to this port, I think daemon:ize disrupts it
+    ;;           (begin
+    ;;     	(current-error-port *alt-log-file*)
+    ;;     	(current-output-port *alt-log-file*)))))
+    (if (server:check-if-running run-id)
+	(begin
+	  (debug:print-info 0 "Server for run-id " run-id " already running")
+	  (exit 0)))
+    (let loop ((server-id (tasks:server-lock-slot (db:delay-if-busy tdbdat) run-id))
+	       (remtries  4))
+      (if (not server-id)
+	  (if (> remtries 0)
+	      (begin
+		(thread-sleep! 2)
+		(if (not (server:check-if-running run-id))
+		    (loop (tasks:server-lock-slot (db:delay-if-busy tdbdat) run-id)
+			  (- remtries 1))
+		    (begin
+		      (debug:print-info 0 "Another server took the slot, exiting")
+		      (exit 0))))
+	      (begin
+		;; since we didn't get the server lock we are going to clean up and bail out
+		(debug:print-info 2 "INFO: server pid=" (current-process-id) ", hostname=" (get-host-name) " not starting due to other candidates ahead in start queue")
+		(tasks:server-delete-records-for-this-pid (db:delay-if-busy tdbdat) " http-transport:launch")
+		))
+	  ;; locked in a server id, try to start up
+	  (nmsg-transport:run dbstruct hostn run-id server-id))
+      (set! *didsomething* #t)
+      (exit))))
+
+;;======================================================================
+;; S E R V E R   U T I L I T I E S 
+;;======================================================================
+
+(define (nmsg-transport:mk-signature)
+  (message-digest-string (md5-primitive) 
+			 (with-output-to-string
+			   (lambda ()
+			     (write (list (current-directory)
+					  (argv)))))))
+
+;;======================================================================
+;; C L I E N T S
+;;======================================================================
+
+;; ping the server at host:port
+;;   return the open socket if successful (return-socket == #t)
+;;   expect the key expected-key returned in payload
+;;   send our-key or #f as payload
+;;
+(define (nmsg-transport:ping hostn port #!key (timeout 3)(return-socket #t)(expected-key #f)(our-key #f)(socket #f))
+  ;; send a random number along with pid and check that we get it back
+  (let* ((host    (if (or (not hostn)
+			  (equal? hostn "-")) ;; use localhost
+		      (get-host-name)
+		      hostn))
+	 (req     (or socket
+		      (let ((soc (nn-socket 'req)))
+			(nn-connect soc (conc "tcp://" host ":" port))
+			soc)))
+	 (success #t)
+	 (dat     (vector "ping" our-key))
+	 (result  (condition-case 
+		   (nmsg-transport:client-api-send-receive-raw req dat timeout: timeout)
+		   ((timeout)(set! success #f) #f)))
+	 (key     (if success 
+		      (vector-ref result 1)
+		      #f)))
+    (debug:print 0 "success=" success ", key=" key ", expected-key=" expected-key ", equal? " (equal? key expected-key))
+    (if (and success
+	     (or (not expected-key) ;; just getting a reply is good enough then
+		 (equal? key expected-key)))
+	(if return-socket
+	    req
+	    (begin
+	      (if (not socket)(nn-close req)) ;; don't want a side effect of closing socket if handed it
+	      #t))
+	(begin
+	  (if (not socket)(nn-close req)) ;; failed to ping, close socket as side effect
+	  #f))))
+
+;; send data to server, wait max of timeout seconds for a response.
+;; return #( success/fail result )
+;;
+;; for effiency it is easier to do the obj->string and string->obj here.
+;;
+(define (nmsg-transport:client-api-send-receive-raw socreq indat #!key (enable-send #t)(timeout 25))
+  (let* ((success     #f)
+	 (result      #f)
+	 (keepwaiting #t)
+	 (dat         (db:obj->string indat transport: 'nmsg))
+	 (send-recv   (make-thread
+		       (lambda ()
+			 (nn-send socreq dat)
+			 (let* ((res (nn-recv socreq)))
+			   (set! success #t)
+			   (set! result (db:string->obj res transport: 'nmsg))))
+		       "send-recv"))
+	 (timeout     (make-thread
+		       (lambda ()
+			 (let loop ((count 0))
+			   (thread-sleep! 1)
+			   (debug:print-info 1 "send-receive-raw, still waiting after " count " seconds...")
+			   (if (and keepwaiting (< count timeout)) ;; yes, this is very aproximate
+			       (loop (+ count 1))))
+			 (if keepwaiting
+			     (begin
+			       (print "timeout waiting for ping")
+			       (thread-terminate! send-recv))))
+		       "timeout")))
+    ;; replace with condition-case?
+    (handle-exceptions
+     exn
+     (set! result "timeout")
+     (thread-start! timeout)
+     (thread-start! send-recv)
+     (thread-join! send-recv)
+     (if success (thread-terminate! timeout)))
+    ;; raise timeout error if timed out
+    (if success
+	(if (and (vector? result)
+		 (vector-ref result 0)) ;; did it fail at the server?
+	    result                ;; nope, all good
+	    (begin
+	      (debug:print 0 "ERROR: error occured at server, info=" (vector-ref result 2))
+	      (debug:print 0 " client call chain:")
+	      (print-call-chain (current-error-port))
+	      (debug:print 0 " server call chain:")
+	      (pp (vector-ref result 1) (current-error-port))
+	      (signal (vector-ref result 0))))
+	(signal (make-composite-condition
+		 (make-property-condition 'timeout 'message "nmsg-transport:client-api-send-receive-raw timed out talking to server"))))))
+
+;; run nmsg-transport:keep-running in a parallel thread to monitor that the db is being 
+;; used and to shutdown after sometime if it is not.
+;;
+(define (nmsg-transport:keep-running server-id run-id)
+  ;; if none running or if > 20 seconds since 
+  ;; server last used then start shutdown
+  ;; This thread waits for the server to come alive
+  (let* ((server-info (let loop ()
+                        (let ((sdat #f))
+                          (mutex-lock! *heartbeat-mutex*)
+                          (set! sdat *server-info*)
+                          (mutex-unlock! *heartbeat-mutex*)
+                          (if sdat 
+			      (begin
+				(debug:print-info 0 "keep-running got sdat=" sdat)
+				sdat)
+                              (begin
+                                (thread-sleep! 0.5)
+                                (loop))))))
+         (iface       (car server-info))
+         (port        (cadr server-info))
+         (last-access 0)
+	 (tdbdat      (tasks:open-db))
+	 (server-timeout (let ((tmo (configf:lookup  *configdat* "server" "timeout")))
+			   (if (and (string? tmo)
+				    (string->number tmo))
+			       (* 60 60 (string->number tmo))
+			       ;; (* 3 24 60 60) ;; default to three days
+			       (* 60 1)         ;; default to one minute
+			       ;; (* 60 60 25)      ;; default to 25 hours
+			       ))))
+    (print "Keep-running got server pid " server-id ", using iface " iface " and port " port)
+    (let loop ((count 0))
+      (thread-sleep! 4) ;; no need to do this very often
+      ;; NB// sync currently does NOT return queue-length
+      (let () ;; (queue-len (cdb:client-call server-info 'sync #t 1)))
+      ;; (print "Server running, count is " count)
+        (if (< count 1) ;; 3x3 = 9 secs aprox
+            (loop (+ count 1)))
+        
+        (mutex-lock! *heartbeat-mutex*)
+        (set! last-access *last-db-access*)
+        (mutex-unlock! *heartbeat-mutex*)
+	(db:sync-touched *inmemdb* run-id force-sync: #t)
+        (if (and *server-run*
+	       (> (+ last-access server-timeout)
+		  (current-seconds)))
+            (begin
+              (debug:print-info 0 "Server continuing, seconds since last db access: " (- (current-seconds) last-access))
+              (loop 0))
+            (begin
+              (debug:print-info 0 "Starting to shutdown the server.")
+              (set! *time-to-exit* #t)
+	      (db:sync-touched *inmemdb* run-id force-sync: #t)
+              (tasks:server-delete-record (db:delay-if-busy tdbdat) server-id " http-transport:keep-running")
+              (debug:print-info 0 "Server shutdown complete. Exiting")
+              (exit)
+	      ))))))
+
+;;======================================================================
+;; C L I E N T S
+;;======================================================================
+
+(define (nmsg-transport:client-connect iface portnum)
+  (let* ((reqsoc      (nmsg-transport:ping iface portnum return-socket: #t)))
+    (vector iface portnum #f #f #f (current-seconds) reqsoc)))
+
+;; returns result, there is no sucess/fail flag - handled via excpections
+;;
+(define (nmsg-transport:client-api-send-receive run-id connection-info cmd param #!key (remtries 5))
+  ;; NB// In the html version of this routine there is a call to 
+  ;;      tasks:kill-server-run-id when there is an exception
+  (mutex-lock! *http-mutex*)
+  (let* ((packet  (vector cmd param))
+	 (reqsoc  (http-transport:server-dat-get-socket connection-info))
+	 (res     (nmsg-transport:client-api-send-receive-raw reqsoc packet)))
+;;	 (status  (vector-ref rawres 0))
+;;	 (result  (vector-ref rawres 1)))
+    (mutex-unlock! *http-mutex*)
+    res)) ;; (vector status (if status (db:string->obj result transport: 'nmsg) result))))
+	
+;;======================================================================
+;; J U N K 
+;;======================================================================
+
+;; DO NOT USE
+;;
+(define (nmsg-transport:client-signal-handler signum)
+  (handle-exceptions
+   exn
+   (debug:print " ... exiting ...")
+   (let ((th1 (make-thread (lambda ()
+			     (if (not *received-response*)
+				 (receive-message* *runremote*))) ;; flush out last call if applicable
+			   "eat response"))
+	 (th2 (make-thread (lambda ()
+			     (debug:print 0 "ERROR: Received ^C, attempting clean exit. Please be patient and wait a few seconds before hitting ^C again.")
+			     (thread-sleep! 3) ;; give the flush three seconds to do it's stuff
+			     (debug:print 0 "       Done.")
+			     (exit 4))
+			   "exit on ^C timer")))
+     (thread-start! th2)
+     (thread-start! th1)
+     (thread-join! th2))))
+

Index: rmt.scm
==================================================================
--- rmt.scm
+++ rmt.scm
@@ -13,10 +13,11 @@
 
 (declare (unit rmt))
 (declare (uses api))
 (declare (uses tdb))
 (declare (uses http-transport))
+(declare (uses nmsg-transport))
 
 ;;
 ;; THESE ARE ALL CALLED ON THE CLIENT SIDE!!!
 ;;
 
@@ -63,63 +64,73 @@
 	;;  ;; (and (not (rmt:write-frequency-over-limit? cmd run-id))
 	(if (tasks:server-running-or-starting? (db:delay-if-busy (tasks:open-db)) run-id)
 	    (client:setup run-id)
 	    #f))))
 
-;; cmd is a symbol
-;; vars is a json string encoding the parameters for the call
-;;
-(define (rmt:send-receive cmd rid params #!key (attemptnum 0))
+(define (rmt:send-receive cmd rid params #!key (attemptnum 1)) ;; start attemptnum at 1 so the modulo below works as expected
   ;; clean out old connections
   (mutex-lock! *db-multi-sync-mutex*)
-  (let ((expire-time (- (current-seconds) 60)))
-    (for-each 
-     (lambda (run-id)
-       (let ((connection (hash-table-ref/default *runremote* run-id #f)))
-	 (if (and connection 
-		  (< (http-transport:server-dat-get-last-access connection) expire-time))
-	     (begin
-	       (debug:print-info 0 "Discarding connection to server for run-id " run-id ", too long between accesses")
-	       (hash-table-delete! *runremote* run-id)))))
-     (hash-table-keys *runremote*)))
+  ;; (let ((expire-time (- (current-seconds) 60)))
+  ;;   (for-each 
+  ;;    (lambda (run-id)
+  ;;      (let ((connection (hash-table-ref/default *runremote* run-id #f)))
+  ;;        (if (and connection 
+  ;;       	  (< (http-transport:server-dat-get-last-access connection) expire-time))
+  ;;            (begin
+  ;;              (debug:print-info 0 "Discarding connection to server for run-id " run-id ", too long between accesses")
+  ;;              ;; SHOULD CLOSE THE CONNECTION HERE
+  ;;              (hash-table-delete! *runremote* run-id)))))
+  ;;    (hash-table-keys *runremote*)))
   (mutex-unlock! *db-multi-sync-mutex*)
   (let* ((run-id          (if rid rid 0))
-	 (connection-info (rmt:get-connection-info run-id))
-	 (jparams         (db:obj->string params)))
+	 (connection-info (rmt:get-connection-info run-id)))
+    ;; the nmsg method does the encoding under the hood (the http method should be changed to do this also)
     (if connection-info
 	;; use the server if have connection info
-	(let* ((dat     (http-transport:client-api-send-receive run-id connection-info cmd jparams))
-	       (res     (if (and dat (vector? dat)) (vector-ref dat 1) #f))
-	       (success (if (and dat (vector? dat)) (vector-ref dat 0) #f)))
+	(let* ((dat     (case *transport-type*
+			  ((http)(http-transport:client-api-send-receive run-id connection-info cmd params))
+			  ((nmsg)(condition-case
+				  (nmsg-transport:client-api-send-receive run-id connection-info cmd params)
+				  ((timeout)(vector #f "timeout talking to server"))))
+			  (else  (exit))))
+	       (success (if (and dat (vector? dat)) (vector-ref dat 0) #f))
+	       (res     (if (and dat (vector? dat)) (vector-ref dat 1) #f)))
 	  (http-transport:server-dat-update-last-access connection-info)
 	  (if success
-	      (db:string->obj res)
-	      ;; (if (< attemptnum 100)
-	      ;;     (begin
-	      ;;       (hash-table-delete! *runremote* run-id)
-	      ;;       (thread-sleep! 0.5)
-	      ;;       (rmt:send-receive cmd rid params attempnum: (+ attemptnum 1)))
-	      ;;     (begin
-	      ;;       (print-call-chain (current-error-port))
-	      ;;       (debug:print 0 "ERROR: too many attempts to communicate have failed. Giving up. Kill your mtest processes and start over")
-	      ;;       (exit 1)))))
+	      (case *transport-type* 
+		((http) res) ;; (db:string->obj res))
+		((nmsg) res)) ;; (vector-ref res 1)))
 	      (begin ;; let ((new-connection-info (client:setup run-id)))
-		(debug:print 0 "WARNING: Communication failed, trying call to http-transport:client-api-send-receive again.")
+		(debug:print 0 "WARNING: Communication failed, trying call to rmt:send-receive again.")
+		;; (case *transport-type*
+		;;   ((nmsg)(nn-close (http-transport:server-dat-get-socket connection-info))))
 		(hash-table-delete! *runremote* run-id) ;; don't keep using the same connection
+		(if (eq? (modulo attemptnum 5) 0)
+		    (tasks:kill-server-run-id run-id tag: "api-send-receive-failed"))
+		(tasks:start-and-wait-for-server (tasks:open-db) run-id 15)
+		;; (nmsg-transport:client-api-send-receive run-id connection-info cmd param remtries: (- remtries 1))))))
 
 		;; no longer killing the server in http-transport:client-api-send-receive
 		;; may kill it here but what are the criteria?
 		;; start with three calls then kill server
-		(if (eq? attemptnum 3)(tasks:kill-server-run-id run-id))
-		(thread-sleep! 2)
+		;; (if (eq? attemptnum 3)(tasks:kill-server-run-id run-id))
+		;; (thread-sleep! 2)
 		(rmt:send-receive cmd run-id params attemptnum: (+ attemptnum 1)))))
-	(if (and (< attemptnum 10)
+	;; no connection info? try to start a server
+	(if (and (< attemptnum 15)
 		 (tasks:need-server run-id))
 	    (begin
+	      (hash-table-delete! *runremote* run-id)
 	      (tasks:start-and-wait-for-server (db:delay-if-busy (tasks:open-db)) run-id 10)
-	      (rmt:send-receive cmd rid params (+ attemptnum 1)))
-	    (rmt:open-qry-close-locally cmd run-id params)))))
+	      (client:setup run-id)
+	      (thread-sleep! (random 5)) ;; give some time to settle and minimize collison?
+	      (rmt:send-receive cmd rid params attemptnum: (+ attemptnum 1)))
+	    (begin
+	      (debug:print 0 "ERROR: Communication failed!")
+	      (exit)
+	      ;; (rmt:open-qry-close-locally cmd run-id params))))
+	      )))))
 
 (define (rmt:update-db-stats run-id rawcmd params duration)
   (mutex-lock! *db-stats-mutex*)
   (handle-exceptions
    exn
@@ -184,11 +195,12 @@
 			       (set! *dbstruct-db* db)
 			       db)))
 	 (db-file-path   (db:dbfile-path 0)))
     ;; (read-only      (not (file-read-access? db-file-path)))
     (let* ((start         (current-milliseconds))
-	   (res           (api:execute-requests dbstruct-local (symbol->string cmd) params))
+	   (resdat        (api:execute-requests dbstruct-local (symbol->string cmd) params))
+	   (res           (vector-ref resdat 1))
 	   (duration      (- (current-milliseconds) start)))
       (rmt:update-db-stats run-id cmd params duration)
       ;; mark this run as dirty if this was a write
       (if (not (member cmd api:read-only-queries))
 	  (let ((start-time (current-seconds)))
@@ -199,17 +211,19 @@
 	    (mutex-unlock! *db-multi-sync-mutex*)))
       res)))
 
 (define (rmt:send-receive-no-auto-client-setup connection-info cmd run-id params)
   (let* ((run-id   (if run-id run-id 0))
-	 (jparams  (db:obj->string params)) ;; (rmt:dat->json-str params))
-	 (dat      (http-transport:client-api-send-receive run-id connection-info cmd jparams)))
-    (if (and dat (vector-ref dat 0))
-	(db:string->obj (vector-ref dat 1))
-	(begin
-	  (debug:print 0 "ERROR: rmt:send-receive-no-auto-client-setup failed, attempting to continue. Got " dat)
-	  dat))))
+	 ;; (jparams  (db:obj->string params)) ;; (rmt:dat->json-str params))
+	 (res  	   (http-transport:client-api-send-receive run-id connection-info cmd params)))
+    (if (and res (vector-ref res 0))
+	res
+	#f)))
+;; 	(db:string->obj (vector-ref dat 1))
+;; 	(begin
+;; 	  (debug:print 0 "ERROR: rmt:send-receive-no-auto-client-setup failed, attempting to continue. Got " dat)
+;; 	  dat))))
 
 ;; Wrap json library for strings (why the ports crap in the first place?)
 (define (rmt:dat->json-str dat)
   (with-output-to-string 
     (lambda ()
@@ -242,14 +256,17 @@
 
 (define (rmt:login run-id)
   (rmt:send-receive 'login run-id (list *toppath* megatest-version run-id *my-client-signature*)))
 
 ;; This login does no retries under the hood - it acts a bit like a ping.
+;; Deprecated for nmsg-transport.
 ;;
 (define (rmt:login-no-auto-client-setup connection-info run-id)
-  (rmt:send-receive-no-auto-client-setup connection-info 'login run-id (list *toppath* megatest-version run-id *my-client-signature*)))
-  
+  (case *transport-type*
+    ((http)(rmt:send-receive-no-auto-client-setup connection-info 'login run-id (list *toppath* megatest-version run-id *my-client-signature*)))
+    ((nmsg)(nmsg-transport:client-api-send-receive run-id connection-info 'login (list *toppath* megatest-version run-id *my-client-signature*)))))
+
 ;; hand off a call to one of the db:queries statements
 ;; added run-id to make looking up the correct db possible 
 ;;
 (define (rmt:general-call stmtname run-id . params)
   (rmt:send-receive 'general-call run-id (append (list stmtname run-id) params)))

Index: runs.scm
==================================================================
--- runs.scm
+++ runs.scm
@@ -944,11 +944,11 @@
 	     (num-running (rmt:get-count-tests-running-for-run-id run-id)))
 
 	;; every couple minutes verify the server is there for this run
 	(if (and (common:low-noise-print 60 "try start server"  run-id)
 		 (tasks:need-server run-id))
-	    (tasks:start-and-wait-for-server tdbdat run-id 10))
+	    (tasks:start-and-wait-for-server tdbdat run-id 10)) ;; NOTE: delay and wait is done under the hood
 	
 	(if (> num-running 0)
 	  (set! last-time-some-running (current-seconds)))
 
       (if (> (current-seconds)(+ last-time-some-running 240))

Index: server.scm
==================================================================
--- server.scm
+++ server.scm
@@ -20,10 +20,11 @@
 (declare (uses common))
 (declare (uses db))
 (declare (uses tasks)) ;; tasks are where stuff is maintained about what is running.
 (declare (uses synchash))
 (declare (uses http-transport))
+(declare (uses nmsg-transport))
 (declare (uses launch))
 ;; (declare (uses zmq-transport))
 (declare (uses daemon))
 
 (include "common_records.scm")
@@ -47,11 +48,14 @@
 ;; all routes though here end in exit ...
 ;;
 ;; start_server
 ;;
 (define (server:launch run-id)
-  (http-transport:launch run-id))
+  (case *transport-type*
+    ((http)(http-transport:launch run-id))
+    ((nmsg)(nmsg-transport:launch run-id))
+    (else (debug:print 0 "ERROR: unknown server type " *transport-type*))))
 
 ;;======================================================================
 ;; Q U E U E   M A N A G E M E N T
 ;;======================================================================
 
@@ -136,13 +140,17 @@
 	;; note: client:start will set *runremote*. this needs to be changed
 	;;       also, client:start will login to the server, also need to change that.
 	;;
 	;; client:start returns #t if login was successful.
 	;;
-	(let ((res (server:ping-server run-id 
-				       (tasks:hostinfo-get-interface server)
-				       (tasks:hostinfo-get-port      server))))
+	(let ((res (case *transport-type*
+		     ((http)(server:ping-server run-id 
+						(tasks:hostinfo-get-interface server)
+						(tasks:hostinfo-get-port      server)))
+		     ((nmsg)(nmsg-transport:ping (tasks:hostinfo-get-interface server)
+						 (tasks:hostinfo-get-port      server)
+						 timeout: 2)))))
 	  ;; if the server didn't respond we must remove the record
 	  (if res
 	      #t
 	      (begin
 		(debug:print-info 0 "server at " server " not responding, removing record")

Index: tasks.scm
==================================================================
--- tasks.scm
+++ tasks.scm
@@ -185,11 +185,11 @@
    -1                         ;; pubport
    (random 1000)              ;; priority (used a tiebreaker on get-available)
    "available"                ;; state
    (common:version-signature) ;; mt_version
    -1                         ;; interface
-   "http"                     ;; transport
+   (conc *transport-type*)    ;; transport
    run-id
    ))
 
 (define (tasks:num-in-available-state mdb run-id)
   (let ((res 0))

ADDED   testnanomsg/basic-req-rep.scm
Index: testnanomsg/basic-req-rep.scm
==================================================================
--- /dev/null
+++ testnanomsg/basic-req-rep.scm
@@ -0,0 +1,3 @@
+(use nanomsg srfi-18 sqlite3 numbers)
+
+(define resp (nn-socket 'rep))

ADDED   testnanomsg/mockupclient.scm
Index: testnanomsg/mockupclient.scm
==================================================================
--- /dev/null
+++ testnanomsg/mockupclient.scm
@@ -0,0 +1,42 @@
+(use zmq posix numbers)
+
+(define cname "Bob")
+(define runtime 10)
+(let ((args (argv)))
+  (if (< (length args) 3)
+      (begin
+	(print "Usage: mockupclient clientname runtime")
+	(exit))
+      (begin
+	(set! cname (cadr args))
+	(set! runtime (string->number (caddr args))))))
+      
+;; (define start-delay (/ (random 100) 9))
+;; (define runtime     (+ 1 (/ (random 200) 2)))
+
+(print "Starting client " cname " with runtime " runtime)
+
+(include "mockupclientlib.scm")
+
+(set! endtime (+ (current-seconds) runtime))
+
+;; first ping the server to ensure we have a connection
+(if (server-ping cname 5)
+    (print "SUCCESS: Client " cname " connected to server")
+    (begin
+      (print "ERROR: Client " cname " failed ping of server, exiting")
+      (exit)))
+
+(let loop ()
+  (let ((x (random 15))
+	(varname (list-ref (list "hello" "goodbye" "saluton" "kiaorana")(random 4))))
+    (case x
+      ;; ((1)(dbaccess cname 'sync "nodat"    #f))
+      ((2 3 4 5)(dbaccess cname 'set varname (random 999)))
+      ((6 7 8 9 10)(print cname ": Get \"" varname "\" " (dbaccess cname 'get varname #f)))
+      (else
+       (thread-sleep! 0.011)))
+    (if (< (current-seconds) endtime)
+	(loop))))
+
+(print "Client " cname " all done!!")

ADDED   testnanomsg/mockupclientlib.scm
Index: testnanomsg/mockupclientlib.scm
==================================================================
--- /dev/null
+++ testnanomsg/mockupclientlib.scm
@@ -0,0 +1,58 @@
+(define reqs (nn-socket 'req))
+
+(connect-socket reqs "tcp://localhost:6563")
+
+(thread-sleep! 0.2)
+
+(define (server-ping cname timeout)
+  (let ((msg     (conc cname ":ping:" timeout))
+	(maxtime (+ (current-seconds) timeout)))
+    (print "pinging server from " cname " with timeout " timeout)
+    (let loop ((res   #f))
+      (if (< maxtime (current-seconds))
+	  #f ;; failed to ping
+	  (if (equal? res "Got ping")
+	      #t
+	      (begin
+		(print "Ping received from server " res)
+		(send-message push msg)
+		(thread-sleep! 0.1)
+		(loop (receive-message sub non-blocking: #t))))))))
+  
+(define (dbaccess cname cmd var val #!key (numtries 20))
+  (let* ((msg (conc cname ":" cmd ":" (if val (conc var " " val) var)))
+	 (res #f)
+	 (mtx1 (make-mutex))
+	 (do-access (lambda ()
+		      (let ((tmpres #f))
+			(print "Sending msg: " msg)
+			(send-message push msg)
+			(print "Message " msg " sent")
+			(print "Client " cname " waiting for response to " msg)
+			(print "Client " cname " received address " (receive-message* sub))
+			(set! tmpres (receive-message* sub))
+			(mutex-lock! mtx1)
+			(set! res tmpres)
+			(mutex-unlock! mtx1))))
+	 (th1 (make-thread do-access "do access"))
+	 (th2 (make-thread (lambda ()
+			     (let ((result #f))
+			       (mutex-lock! mtx1)
+			       (set! result res)
+			       (mutex-unlock! mtx1)
+			       (thread-sleep! 5)
+			       (if (not result)
+				   (if (> numtries 0)
+				       (begin
+					 (print "WARNING: access timed out for " cname ", trying again. Trys remaining=" numtries)
+					 (dbaccess cname cmd var val numtries: (- numtries 1)))
+				       (begin
+					 (print "ERROR: dbaccess timed out. Exiting")
+					 (exit)))))
+			     "timeout thread"))))
+    (thread-start! th1)
+    (thread-start! th2)
+    (thread-join! th1)
+    (if res (print "SUCCESS: received " res " with " numtries " remaining possible attempts"))
+    res))
+

ADDED   testnanomsg/mockupserver.scm
Index: testnanomsg/mockupserver.scm
==================================================================
--- /dev/null
+++ testnanomsg/mockupserver.scm
@@ -0,0 +1,146 @@
+;; pub/sub with envelope address
+;; Note that if you don't insert a sleep, the server will crash with SIGPIPE as soon
+;; as a client disconnects.  Also a remaining client may receive tons of
+;; messages afterward.
+
+(use nanomsg srfi-18 sqlite3 numbers)
+
+(define resp (nn-socket 'rep))
+(define cname "server")
+(define total-db-accesses 0)
+(define start-time (current-seconds))
+
+(nn-bind resp  "tcp://*:6563")
+
+(thread-sleep! 0.2)
+
+(define (open-db)
+  (let* ((dbpath    "mockup.db")
+	 (dbexists  (file-exists? dbpath))
+	 (db        (open-database dbpath)) ;; (never-give-up-open-db dbpath))
+	 (handler   (make-busy-timeout 10)))
+    (set-busy-handler! db handler)
+    (if (not dbexists)
+	(for-each
+	 (lambda (stmt)
+	   (execute db stmt))
+	 (list
+	  "PRAGMA SYNCHRONOUS=0;"
+	  "CREATE TABLE clients (id INTEGER PRIMARY KEY,name TEXT,num_accesses INTEGER DEFAULT 0);"
+	  "CREATE TABLE vars    (var TEXT,val TEXT,CONSTRAINT vars_constraint UNIQUE (var));")))
+    db))
+
+(define cid-cache (make-hash-table))
+
+(define (get-client-id db cname)
+  (let ((cid (hash-table-ref/default cid-cache cname #f)))
+    (if cid 
+	cid
+	(begin
+	  (execute db "INSERT OR REPLACE INTO clients (name) VALUES(?);" cname)
+	  (for-each-row 
+	   (lambda (id)
+	     (set! cid id))
+	   db
+	   "SELECT id FROM clients WHERE name=?;" cname)
+	  (hash-table-set! cid-cache cname cid)
+	  (set! total-db-accesses (+ total-db-accesses 2))
+	  cid))))
+
+(define (count-client db cname)
+  (let ((cid (get-client-id db cname)))
+    (execute db "UPDATE clients SET num_accesses=num_accesses+1 WHERE id=?;" cid)
+    (set! total-db-accesses (+ total-db-accesses 1))
+    ))
+
+(define db (open-db))
+;; (define queuelst '())
+;; (define mx1 (make-mutex))
+
+(define max-queue-len 0)
+
+(define (process-queue queuelst)
+  (let ((queuelen (length queuelst)))
+    (if (> queuelen max-queue-len)
+	(set! max-queue-len queuelen))
+    (for-each
+     (lambda (item)
+       (let ((cname (vector-ref item 1))
+	     (clcmd (vector-ref item 2))
+	     (cdata (vector-ref item 3)))
+	 (send-message pub cname send-more: #t)
+	 (send-message pub (case clcmd
+			     ((sync)
+			      (conc queuelen))
+			     ((set)
+			      (set! total-db-accesses (+ total-db-accesses 1))
+			      (apply execute db "INSERT OR REPLACE INTO vars (var,val) VALUES (?,?);" (string-split cdata))
+			      "ok")
+			     ((get)
+			      (set! total-db-accesses (+ total-db-accesses 1))
+			      (let ((res "noval"))
+				(for-each-row
+				 (lambda (val)
+				   (set! res val))
+				 db 
+				 "SELECT val FROM vars WHERE var=?;" cdata)
+				res))
+			     (else (conc "unk cmd: " clcmd))))))
+     queuelst)))
+
+;; SERVER THREAD
+(define th1 (make-thread 
+	     (lambda ()
+	       (let ((last-run 0)) ;; current-seconds when run last
+		 (let loop ((queuelst '()))
+		   (let* ((indat (receive-message* pull))
+			  (parts (string-split indat ":"))
+			  (cname (car parts))                   ;; client name
+			  (clcmd (string->symbol (cadr parts))) ;; client cmd
+			  (cdata (caddr parts))                 ;; client data
+			  (svect (vector (current-seconds) cname clcmd cdata))) ;; record for the queue
+		     ;; (print "Server received message: " indat)
+		     (count-client db cname)
+		     (case clcmd
+		       ((ping)
+			(print "Got ping from " cname)
+			(send-message pub cname send-more: #t)
+			(send-message pub "Got ping")
+			(loop queuelst))
+		       ((sync) ;; just process the queue
+			(print "Got sync from " cname)
+			(process-queue (cons svect queuelst))
+			(loop '()))
+		       ((get)
+			(process-queue (cons svect queuelst))
+			(loop '()))
+		       (else
+			(loop (cons svect queuelst))))))))
+	     "server thread"))
+
+(include "mockupclientlib.scm")
+
+;; SYNC THREAD
+;; send a sync to the pull port
+(define th2 (make-thread
+	     (lambda ()
+	       (let ((last-action-time (current-seconds)))
+		 (let loop ()
+		   (thread-sleep! 5)
+		   (let ((queuelen (string->number (dbaccess "server" 'sync "nada" #f)))
+			 (last-action-delta #f))
+		     (if (> queuelen 1)(set! last-action-time (current-seconds)))
+		     (set! last-action-delta (- (current-seconds) last-action-time))
+		     (print "Server: Got queuelen=" queuelen ", last-action-delta=" last-action-delta)
+		     (if (< last-action-delta 60)
+			 (loop)
+			 (print "Server exiting, 25 seconds since last access"))))))
+	     "sync thread"))
+
+(thread-start! th1)
+(thread-start! th2)
+(thread-join! th2)
+
+(let* ((run-time       (- (current-seconds) start-time))
+       (queries/second (/  total-db-accesses run-time)))
+  (print "Server exited! Total db accesses=" total-db-accesses " in " run-time " seconds for " queries/second " queries/second with max queue length of: " max-queue-len))

ADDED   testnanomsg/pipeline.scm
Index: testnanomsg/pipeline.scm
==================================================================
--- /dev/null
+++ testnanomsg/pipeline.scm
@@ -0,0 +1,25 @@
+;; watch nanomsg's pipeline load-balancer in action.
+(use nanomsg)
+
+(define push (nn-socket 'push))
+(define pull1 (nn-socket 'pull))
+(define pull2 (nn-socket 'pull))
+
+(nn-bind    push  "inproc://test")
+(nn-connect pull1 "inproc://test")
+(nn-connect pull2 "inproc://test")
+
+(nn-send push "a")
+(nn-send push "b")
+(nn-send push "c")
+(nn-send push "d")
+
+(define ((th sock))
+  (print (current-thread) ": " (nn-recv sock))
+  (print (current-thread) ": " (nn-recv sock))
+  (print (current-thread) " is done"))
+
+(thread-start! (th pull1))
+(thread-start! (th pull2))
+
+(thread-sleep! 1)

ADDED   testnanomsg/req-rep-client.scm
Index: testnanomsg/req-rep-client.scm
==================================================================
--- /dev/null
+++ testnanomsg/req-rep-client.scm
@@ -0,0 +1,30 @@
+;; watch nanomsg's pipeline load-balancer in action.
+(use nanomsg)
+
+(define req   (nn-socket 'req))
+
+(nn-connect req  "tcp://localhost:22022")
+
+;; (with-output-to-string (lambda ()(serialize obj)))
+(define (client-send-receive soc msg)
+  (nn-send soc msg)
+  (nn-recv soc))
+
+(define ((talk-to-server soc))
+  (let loop ((cnt 20))
+    (let ((name (list-ref '("Matt" "Tom" "Bob" "Jill" "James" "Jane")(random 6))))
+      (print "Sending " name)
+      (print (client-send-receive req name))
+      (if (> cnt 0)(loop (- cnt 1)))))
+  (print (client-send-receive req "quit"))
+  (nn-close req)
+  (exit))
+
+;; (thread-start! (lambda ()
+;; 		 (thread-sleep! 20)
+;; 		 (print "Give up on waiting for the server")
+;; 		 (nn-close req)
+;; 		 (exit)))
+
+(thread-join! (thread-start! (talk-to-server req)))
+

ADDED   testnanomsg/req-rep-server.scm
Index: testnanomsg/req-rep-server.scm
==================================================================
--- /dev/null
+++ testnanomsg/req-rep-server.scm
@@ -0,0 +1,90 @@
+;; watch nanomsg's pipeline load-balancer in action.
+(use nanomsg)
+
+;; (use trace)
+;; (trace nn-bind nn-socket nn-assert nn-recv nn-send thread-terminate! nn-close )
+
+(define port  22022)
+(define host  "127.0.0.1")
+
+(define rep   (nn-socket 'rep))
+
+(print "connecting, got: " (nn-bind    rep  (conc "tcp://" "*" ":" port)))
+
+(define (server soc)
+  (print "server starting")
+  (let loop ((msg-in (nn-recv soc)))
+    (print "server received: " msg-in)
+    (cond
+     ((equal? msg-in "quit")
+      (nn-send soc "Ok, quitting"))
+     ((and (>= (string-length msg-in) 4)
+	   (equal? (substring msg-in 0 4) "ping"))
+      (nn-send soc (conc (current-process-id)))
+      (loop (nn-recv soc)))
+     ;;((and (>= (string-length msg-in)
+     (else
+      (let ((this-task (random 15)))
+	(thread-sleep! this-task)
+	(nn-send soc (conc "hello " msg-in " this task took " this-task " seconds to complete"))
+	(loop (nn-recv soc)))))))
+
+(define (ping-self host port #!key (return-socket #t))
+  ;; send a random number along with pid and check that we get it back
+  (let* ((req     (nn-socket 'req))
+	 (key     "ping")
+	 (success #f)
+	 (keepwaiting #t)
+	 (ping    (make-thread
+		   (lambda ()
+		     (print "ping: sending string \"" key "\", expecting " (current-process-id))
+		     (nn-send req key)
+		     (let ((result  (nn-recv req)))
+		       (if (equal? (conc (current-process-id)) result)
+			   (begin
+			     (print "ping, success: received \"" result "\"")
+			     (set! success #t))
+			   (begin
+			     (print "ping, failed: received key \"" result "\"")
+			     (set! keepwaiting #f)
+			     (set! success #f)))))
+		   "ping"))
+	 (timeout (make-thread (lambda ()
+				 (let loop ((count 0))
+				   (thread-sleep! 1)
+				   (print "still waiting after count seconds...")
+				   (if (and keepwaiting (< count 10))
+				       (loop (+ count 1))))
+				 (if keepwaiting
+				     (begin
+				       (print "timeout waiting for ping")
+				       (thread-terminate! ping))))
+			       "timeout")))
+    (nn-connect req (conc "tcp://" host ":" port))
+    (handle-exceptions
+     exn
+     (begin
+       (print-call-chain)
+       (print 0 " message: " ((condition-property-accessor 'exn 'message) exn))
+       (print "exn=" (condition->list exn))
+       (print "ping failed to connect to " host ":" port))
+     (thread-start! timeout)
+     (thread-start! ping)
+     (thread-join! ping)
+     (if success (thread-terminate! timeout)))
+    (if return-socket
+	(if success req #f)
+	(begin
+	  (nn-close req)
+	  success))))
+
+(let ((server-thread (make-thread (lambda ()(server rep)) "server")))
+  (thread-start! server-thread)
+  ;; (thread-sleep! 1)
+  (if (ping-self host port)
+      (begin
+	(thread-join! server-thread)
+	(nn-close rep))
+      (print "ping failed")))
+
+(exit)

ADDED   testnanomsg/req-rep.scm
Index: testnanomsg/req-rep.scm
==================================================================
--- /dev/null
+++ testnanomsg/req-rep.scm
@@ -0,0 +1,30 @@
+;; watch nanomsg's pipeline load-balancer in action.
+(use nanomsg)
+
+(define req   (nn-socket 'req))
+(define rep   (nn-socket 'rep))
+
+(nn-bind    rep  "inproc://test")
+(nn-connect req  "inproc://test")
+
+(define (client-send-receive soc msg)
+  (nn-send soc msg)
+  (nn-recv soc))
+
+(define ((server soc))
+  (let loop ((msg-in (nn-recv soc)))
+    (if (not (equal? msg-in "quit"))
+	(begin
+	  (nn-send soc (conc "hello " msg-in))
+	  (loop (nn-recv soc))))))
+
+(thread-start! (server rep))
+
+(print (client-send-receive req "Matt"))
+(print (client-send-receive req "Tom"))
+
+;; (client-send-receive req "quit")
+
+(nn-close req)
+(nn-close rep)
+(exit)

Index: tests/Makefile
==================================================================
--- tests/Makefile
+++ tests/Makefile
@@ -34,11 +34,11 @@
 	cd ..;make -j && make install
 	cd fullrun;$(MEGATEST) -stop-server 0
 
 repl :
 	cd ..;make -j && make install
-	cd fullrun;$(MEGATEST) -repl
+	cd fullrun;$(MEGATEST) -:b -repl
 
 test0 : cleanprep
 	cd simplerun ; $(MEGATEST) -server - -debug $(DEBUG)
 
 test1 : cleanprep
@@ -116,27 +116,27 @@
 test9 : minsetup test9a test9b test9c test9d test9e
 
 test9a :
 	@echo Run super-simple mintest e, no waitons.
 	cd mintest;$(DASHBOARD)&
-	cd mintest;$(MEGATEST) -preclean -runtests e -target $(VER) :runname `date +%H.%M.%S` -debug $(DEBUG)
+	cd mintest;$(MEGATEST) -preclean -runtests e -target $(VER) -runname $(shell date +%H.%M.%S) -debug $(DEBUG)
 
 test9b :
 	@echo Run simple mintest d with one waiton c
-	cd mintest;$(MEGATEST) -preclean -runtests d -target $(VER) :runname `date +%H.%M.%S` -debug $(DEBUG)
+	cd mintest;$(MEGATEST) -preclean -runtests d -target $(VER) -runname `date +%H.%M.%S` -debug $(DEBUG)
 
 test9c :
 	@echo Run mintest a with full waiton chain a -> b -> c -> d -> e
-	cd mintest;$(MEGATEST) -preclean -runtests a -target $(VER) :runname `date +%H.%M.%S` -debug $(DEBUG)
+	cd mintest;$(MEGATEST) -preclean -runtests a -target $(VER) -runname `date +%H.%M.%S` -debug $(DEBUG)
 
 test9d :
 	@echo Run an itemized test with no items
-	cd mintest;$(MEGATEST) -preclean -runtests g -target $(VER) :runname `date +%H.%M.%S` -debug $(DEBUG)
+	cd mintest;$(MEGATEST) -preclean -runtests g -target $(VER) -runname `date +%H.%M.%S` -debug $(DEBUG)
 
 test9e :
 	@echo Run mintest a1 with full waiton chain with d1fail: a1 -> b1 -> c1 -> d1fail -> e1
-	cd mintest;$(MEGATEST) -preclean -runtests a1 -target $(VER) :runname `date +%H.%M.%S` -debug $(DEBUG)
+	cd mintest;$(MEGATEST) -preclean -runtests a1 -target $(VER) -runname `date +%H.%M.%S` -debug $(DEBUG)
 
 test10 :
 	@echo Run a bunch of different targets simultaneously
 	(cd fullrun;$(MEGATEST) -server - ;sleep 2)&
 	for targ in mint/btrfs/mintdir sunos/sshfs/loc; do \

Index: tests/fullrun/megatest.config
==================================================================
--- tests/fullrun/megatest.config
+++ tests/fullrun/megatest.config
@@ -133,18 +133,18 @@
 port 8080
 
 # This server will keep running this number of hours after last access. 
 # Three minutes is 0.05 hours
 # timeout 0.025
-timeout 0.1
+timeout 0.061
 
 # Server is required - slower but more resistant to Sqlite issues.
-# required yes
+required yes
 
 # Start server when average query takes longer than this
-server-query-threshold 100
-# 55500
+# server-query-threshold 55500
+server-query-threshold -1
 
 # daemonize yes
 # hostname #{scheme (get-host-name)}
 
 ## disks are:

Index: tests/mintest/megatest.config
==================================================================
--- tests/mintest/megatest.config
+++ tests/mintest/megatest.config
@@ -1,11 +1,11 @@
 [fields]
 X TEXT
 
 [setup]
 max_concurrent_jobs 50
-linktree #{getenv PWD}/linktree
+linktree #{getenv MT_RUN_AREA_HOME}/linktree
 transport http
 
 [server]
 port 8090
 

Index: tests/unittests/basicserver.scm
==================================================================
--- tests/unittests/basicserver.scm
+++ tests/unittests/basicserver.scm
@@ -14,15 +14,16 @@
 
 ;; NON Server tests go here
 
 (test #f #f (db:dbdat-get-path *db*))
 (test #f #f (db:get-run-name-from-id *db* run-id))
-(test #f '("SYSTEM" "RELEASE") (rmt:get-keys))
+;; (test #f '("SYSTEM" "RELEASE") (rmt:get-keys))
 
 ;; (exit)
 
 ;; Server tests go here 
+(for-each (lambda (run-id)
 (test #f #f (tasks:server-running-or-starting? (db:delay-if-busy (tasks:open-db)) run-id))
 (server:kind-run run-id)
 (test "did server start within 20 seconds?"
       #t
       (let loop ((remtries 20)
@@ -31,11 +32,11 @@
 							     run-id)))
 	(if running 
 	    (> running 0)
 	    (if (> remtries 0)
 		(begin
-		  (thread-sleep! 1.1)
+		  (thread-sleep! 1)
 		  (loop (- remtries 1)
 			(tasks:server-running-or-starting? (db:delay-if-busy
 							    (tasks:open-db))
 							   run-id)))))))
 
@@ -48,11 +49,12 @@
 	      (if (> remtries 0)
 		  (begin
 		    (thread-sleep! 1.1)
 		    (loop (- remtries 1)(tasks:get-server (db:delay-if-busy (tasks:open-db)) run-id)))
 		  res)))))
-
+)
+(list 0 1))
 (define user    (current-user-name))
 (define runname "mytestrun")
 (define keys    (rmt:get-keys))
 (define runinfo #f)
 (define keyvals '(("SYSTEM" "abc")("RELEASE" "def")))
@@ -100,16 +102,18 @@
 				(db:get-header run-info)
 				"runname")))
 
       ;; (vector header (vector "abc" "def" 1 "mytestrun" "new" "n/a" "matt" 1416280640.0))
 
+(for-each (lambda (run-id)
 ;; test killing server
 ;;
 (tasks:kill-server-run-id run-id)
 
 (test #f #f (tasks:server-running-or-starting? (db:delay-if-busy (tasks:open-db)) run-id))
-
+)
+(list 0 1))
 ;; (test #f #f (client:setup run-id))
 
 ;; (set! *transport-type* 'http)
 ;; 
 ;; (test "setup for run" #t (begin (launch:setup-for-run)

Index: utils/Makefile.installall
==================================================================
--- utils/Makefile.installall
+++ utils/Makefile.installall
@@ -145,10 +145,26 @@
 	cd sqlite-autoconf-$(SQLITE3_VERSION);./configure --prefix=$(PREFIX);make;make install
 
 $(PREFIX)/lib/sqlite3.so : $(PREFIX)/bin/sqlite3
 	CSC_OPTIONS="-I$(PREFIX)/include -L$(PREFIX)/lib" $(CHICKEN_INSTALL) $(PROX) sqlite3
 
+#======================================================================
+# N  A N O M S G
+#======================================================================
+
+nanomsg-0.5-beta.tar.gz :
+	wget http://download.nanomsg.org/nanomsg-0.5-beta.tar.gz
+
+nanomsg-0.5-beta/COPYING : nanomsg-0.5-beta.tar.gz
+	tar xfvz nanomsg-0.5-beta.tar.gz
+
+$(PREFIX)/bin/nanocat : nanomsg-0.5-beta/COPYING
+	cd nanomsg-0.5-beta;./configure --prefix=$(PREFIX);make;make install
+
+$(PREFIX)/lib/nanomsg.so : $(PREFIX)/bin/nanocat
+	CSC_OPTIONS="-I$(PREFIX)/include -L$(PREFIX)/lib" $(CHICKEN_INSTALL) $(PROX) nanomsg
+
 #======================================================================
 # M A T T S   U T I L S
 #======================================================================
 
 opensrc.fossil :

Index: utils/plot-code.scm
==================================================================
--- utils/plot-code.scm
+++ utils/plot-code.scm
@@ -1,7 +1,14 @@
 #!/mfs/pkgs/chicken/4.8.0.5/bin/csi -nbq
 
+;; Coming soon (right?) Usage: plot-code file1.scm,file2.scm "fun1,fun2,x*" *.scm > plot.dot
+;; Usage: plot-code file1.scm,file2.scm *.scm > plot.dot
+;;        dot -Tpdf plot.dot > plot.pdf
+;; first param is comma separated list of files to include in the map, use - to do all
+;; second param is list of regexs for functions to include in the map
+;; third param is list of files to scan
+
 (use regex srfi-69 srfi-13)
 
 (define targs #f) 
 (define files (cddddr (argv)))
 
@@ -15,10 +22,12 @@
 
 (define defn-rx (regexp "^\\s*\\(define\\s+\\(([^\\s\\)]+).*"))
 (define all-regexs (make-hash-table))
 
 (define all-fns '())
+
+;; for the se
 
 (define (print-err . data)
   (with-output-to-port (current-error-port)
     (lambda ()
       (apply print data))))
@@ -81,11 +90,32 @@
 			  newres)))
 	      (if (null? tal)
 		  res
 		  (loop (car tal)(cdr tal) res)))))
       '()))
-  
+
+;; (define mm-header #<<MMHEADER
+;; <map version="freeplane 1.2.0">
+;; <!--To view this file, download free mind mapping software Freeplane from http://freeplane.sourceforge.net -->
+;; 
+;; MMHEADER
+;; 
+;; (define (add-node text)
+;;   <node TEXT="homenode" ID="ID_1723255651" CREATED="1283093380553" MODIFIED="1417113442955"><hook NAME="MapStyle">
+;; )
+;; 
+;;  minimal mindmap file
+;;  <map version="freeplane 1.2.0">
+;;   <!--To view this file, download free mind mapping software Freeplane from http://freeplane.sourceforge.net -->
+;;   <node TEXT="homenode" ID="ID_1723255651" CREATED="1283093380553" MODIFIED="1417113442955">
+;;   <node TEXT="node1" POSITION="right" ID="ID_1810107939" CREATED="1417113473476" MODIFIED="1417113480425">
+;;   <node TEXT="node2" ID="ID_68133256" CREATED="1417113482134" MODIFIED="1417113484466"/>
+;;   <node TEXT="node3" ID="ID_1572284821" CREATED="1417113487785" MODIFIED="1417113491589"/>
+;;   </node>
+;;   </node>
+;;   </map>
+
 ;; Gather the usages
 (print "digraph G {")
 (define curr-cluster-num 0)
 (define function-calls '())
 

ADDED   utils/trace/trace.import.scm
Index: utils/trace/trace.import.scm
==================================================================
--- /dev/null
+++ utils/trace/trace.import.scm
@@ -0,0 +1,32 @@
+;;;; trace.import.scm - GENERATED BY CHICKEN 4.9.0.1 -*- Scheme -*-
+
+(eval '(import
+         scheme
+         chicken
+         csi
+         advice
+         extras
+         ports
+         data-structures
+         (except srfi-1 break)
+         miscmacros))
+(##sys#register-compiled-module
+  'trace
+  (list)
+  '((breakpoint . trace#breakpoint)
+    (trace . trace#trace)
+    (untrace . trace#untrace)
+    (break . trace#break)
+    (unbreak . trace#unbreak)
+    (trace-output-port . trace#trace-output-port)
+    (continue . trace#continue)
+    (c . trace#c)
+    (traced? . trace#traced?)
+    (trace-module . trace#trace-module)
+    (untrace-module . trace#untrace-module)
+    (trace-verbose . trace#trace-verbose)
+    (trace/untrace . trace#trace/untrace))
+  (list)
+  (list))
+
+;; END OF FILE

ADDED   utils/trace/trace.meta
Index: utils/trace/trace.meta
==================================================================
--- /dev/null
+++ utils/trace/trace.meta
@@ -0,0 +1,10 @@
+;;;; trace.meta -*- Scheme -*-
+
+
+((category tools)
+ (synopsis "tracing and breakpoints")
+ (author "felix winkelmann")
+ (license "public domain")
+ (needs advice				; don't we all?
+	miscmacros)
+ (files "tests/run.scm" "trace.meta" "trace.release-info" "trace.scm" "trace.setup") )

ADDED   utils/trace/trace.scm
Index: utils/trace/trace.scm
==================================================================
--- /dev/null
+++ utils/trace/trace.scm
@@ -0,0 +1,259 @@
+;;;; trace.scm
+
+
+(module trace (breakpoint 
+	       trace untrace
+	       break unbreak
+	       trace-output-port
+	       continue c 
+	       traced?
+	       trace-module untrace-module
+	       trace-verbose
+	       trace/untrace)
+	       
+(import scheme chicken csi)
+
+(use advice extras ports data-structures)
+(require-library srfi-1)
+(import (except srfi-1 break) miscmacros)
+
+
+(define *last-breakpoint* #f)
+(define *traced-procedures* '())
+(define *broken-procedures* '())
+(define *trace-indent-level* 0)
+
+(define trace-output-port (make-parameter (current-output-port)))
+(define trace-verbose (make-parameter #t))
+
+(define (break-entry name args)
+  ;; Does _not_ unwind!
+  (##sys#call-with-current-continuation
+   (lambda (c)
+     (let ((exn (##sys#make-structure
+		 'condition
+		 '(exn breakpoint)
+		 (list '(exn . message) "*** breakpoint ***"
+		       '(exn . arguments) (list (cons name args))
+		       '(exn . location) name
+		       '(exn . continuation) c) ) ) )
+       (set! *last-breakpoint* exn)
+       (signal exn) ) ) ) )
+
+(define (break-resume exn)
+  (let ((a (member '(exn . continuation) (##sys#slot exn 2))))
+    (if a
+	((cadr a) (void))
+	(error "condition has no continuation" exn) ) ) )
+
+(define (breakpoint #!optional (name 'breakpoint))
+  (break-entry name '()) )
+
+(define (trace-indent)
+  (let ((port (trace-output-port)))
+    (do ((i (fxmin 3 *trace-indent-level*) (fx- i 1)))
+	((fx<= i 0))
+      (write-char #\space port) )
+    (fprintf port "[~a] " *trace-indent-level*) ) )
+
+(define (traced-procedure-entry name args)
+  (let ((port (trace-output-port)))
+    (trace-indent)
+    (set! *trace-indent-level* (fx+ 1 *trace-indent-level*))
+    (write (cons name args) port)
+    (write ", Called from: " port)
+    (write (conc (car (reverse (get-call-chain)))))
+    (write-char #\newline port)
+    (flush-output port) ) )
+
+(define (traced-procedure-exit name results)
+  (let ((port (trace-output-port)))
+    (set! *trace-indent-level* (fx- *trace-indent-level* 1))
+    (trace-indent)
+    (fprintf port "~a -> " name)
+    (if results
+	(for-each
+	 (lambda (x)
+	   (write x port)
+	   (write-char #\space port) )
+	 results)
+	(display "(escaping)" port))
+    (write-char #\newline port)
+    (flush-output port) ) )
+
+(define (procedure-name proc)
+  (cond ((procedure-information proc) =>
+	 (lambda (info)
+	   (if (pair? info) (car info) info) ) )
+	(else '<unknown>)) )
+
+(define (do-trace procs)
+  (for-each
+   (lambda (s)
+     (ensure procedure? s)
+     (cond ((traced? s)
+	    (warning "procedure already traced" s) )
+	   (else
+	    (let ((name (procedure-name s)))
+	      (when (trace-verbose)
+		(fprintf (current-error-port) "; tracing ~a~%" name))
+	      (set! *traced-procedures* (cons (cons s name) *traced-procedures*))
+	      (advise 
+	       'around s
+	       (lambda (next args)
+		 (let ((results #f))
+		   (dynamic-wind
+		       (cut traced-procedure-entry name args)
+		       (lambda () 
+			 (call-with-values (cut apply next args)
+			   (lambda rs
+			     (set! results rs)
+			     (apply values rs))))
+		       (cut traced-procedure-exit name results))))
+	       '*trace*)))))
+   procs) )
+
+(define (do-untrace-all)
+  (define (unadvise* p)
+    (ignore-errors (unadvise p '*trace*)))
+  (for-each
+   (lambda (proc)
+     (let ((proc (car proc)))
+       (when (trace-verbose)
+	 (fprintf (current-error-port) "; untracing ~a~%" (procedure-name proc))
+	 (unadvise* proc))))
+   *traced-procedures*)
+  (set! *traced-procedures* '()))
+
+(define (do-untrace procs)
+  (for-each
+   (lambda (s)
+     (ensure procedure? s)
+     (let ((p (assq s *traced-procedures*)) 
+	   (name (procedure-name s)))
+       (cond ((not p) (warning "procedure not traced" name))
+	     (else
+	      (when (trace-verbose)
+		(fprintf (current-error-port) "; untracing ~a~%" name))
+	      (ignore-errors (unadvise s '*trace*))
+	      (set! *traced-procedures* 
+		(delete 
+		 p *traced-procedures* 
+		 eq?))))))
+   procs) )
+
+(define (do-break procs)
+  (for-each
+   (lambda (s)
+     (let ((name (procedure-name s)))
+       (ensure procedure? s)
+       (cond ((assq s *broken-procedures*)
+	      (warning "procedure already has break-point" name))
+	     (else
+	      (when (trace-verbose)
+		(fprintf (current-error-port) "; setting break-point in ~a~%" name))
+	      (set! *broken-procedures* (cons (cons s name) *broken-procedures*))
+	      (advise 
+	       'before s
+	       (lambda (args)
+		 (break-entry name args) )
+	       '*break*) ) )))
+   procs) )
+
+(define (do-unbreak procs)
+  (for-each
+   (lambda (s)
+     (ensure procedure? s)
+     (let ((p (assq s *broken-procedures*)) 
+	   (name (procedure-name s)))
+       (cond ((not p) (warning "procedure has no breakpoint" name))
+	     (else
+	      (when (trace-verbose)
+		(fprintf (current-error-port) "; removing break-point in ~a~%" name))
+	      (ignore-errors (unadvise s '*break*))
+	      (set! *broken-procedures* (delete p *broken-procedures* eq?) ) ) ) ) )
+   procs) )
+
+(define (do-unbreak-all)
+  (for-each
+   (lambda (bp)
+     (ignore-errors (unadvise (car bp) '*break*)))
+   *broken-procedures*)
+  (set! *broken-procedures* '())
+  (void))
+
+(define (trace . procs)
+  (cond ((null? procs)
+	 (when (pair? *traced-procedures*)
+	   (printf "Traced:~%~%")
+	   (for-each (lambda (p) (printf "  ~a~%" (cdr p))) *traced-procedures*)) )
+	(else
+	 (do-trace procs) ) ) )
+
+(define (untrace . procs)
+  (cond ((null? procs) (do-untrace-all))
+	(else (do-untrace procs)))
+  (void))
+
+(define (break . procs)
+  (cond ((null? procs)
+	 (when (pair? *broken-procedures*)
+	   (printf "Breakpoints:~%~%")
+	   (for-each (lambda (p) (printf "  ~a~%" (cdr p))) *broken-procedures*)) )
+	(else
+	 (do-break procs) ) ) )
+
+(define (unbreak . procs)
+  (cond ((null? procs) (do-unbreak-all))
+	(else (do-unbreak procs))))
+
+(define (continue #!optional (bp *last-breakpoint*))
+  (cond (*last-breakpoint*
+	 (let ((exn *last-breakpoint*))
+	   (set! *last-breakpoint* #f)
+	   (break-resume exn) ) )
+	(else (display "no breakpoint pending\n") ) ) )
+
+(define c continue)
+
+(define (traced? proc)
+  (assq proc *traced-procedures*))
+
+(define (trace/untrace . procs)
+  (for-each
+   (lambda (proc)
+     ((if (traced? proc) do-untrace do-trace) (list proc)))
+   procs))
+
+(define (walk-module mname proc)
+  (let* ((m (##sys#find-module mname))
+	 (exps (nth-value 1 (##sys#module-exports m))))
+    (for-each
+     (lambda (exp)
+       (let* ((realname (cdr exp))
+	      (prim (get realname '##core#primitive)))
+	 (if prim
+	     (warning "export is a core-library primitive - not traced" (car exp))
+	     (when (##sys#symbol-has-toplevel-binding? realname)
+	       (let ((val (##sys#slot realname 0)))
+		 (when (procedure? val)
+		   (proc val)))))))
+     exps)))
+
+(define (trace-module . mnames)
+  (for-each
+   (lambda (mname)
+     (walk-module mname trace))
+   mnames))
+
+(define (untrace-module . mnames)
+  (for-each
+   (lambda (mname)
+     (walk-module 
+      mname 
+      (lambda (proc)
+	(when (traced? proc)
+	  (do-untrace (list proc))))))
+   mnames))
+
+)

ADDED   utils/trace/trace.setup
Index: utils/trace/trace.setup
==================================================================
--- /dev/null
+++ utils/trace/trace.setup
@@ -0,0 +1,9 @@
+;;;; trace.setup -*- Scheme -*-
+
+
+(compile -s trace.scm -O3 -d1 -j trace)
+(compile -s trace.import.scm -O3 -d0)
+
+(install-extension
+ 'trace
+ '("trace.so" "trace.import.so"))