Index: dbmod.scm
==================================================================
--- dbmod.scm
+++ dbmod.scm
@@ -526,11 +526,11 @@
     ;; (sqlite3:finalize! dbh)
     res))
 
 ;; called before db is open?
 ;;
-(define (db:get-iam-server-lock dbh dbfname port)
+(define (db:get-iam-server-lock dbh dbfname host port)
   (sqlite3:with-transaction
    dbh
    (lambda ()
      (let* ((locker (db:get-locker dbh dbfname)))
        (if locker
@@ -556,13 +556,17 @@
 (define (db:steal-lock-db dbh dbfname port)
   (sqlite3:execute dbh "DELETE FROM locks WHERE lockname=?;" dbfname)
   (sqlite3:execute dbh "INSERT INTO locks (lockname,owner_pid,owner_host,owner_port) VALUES (?,?,?,?);" dbfname (current-process-id) (get-host-name) port)
   #t)
 
-(define (db:release-lock dbh dbfname)
+(define (db:release-lock-force dbh dbfname)
   (sqlite3:execute dbh "DELETE FROM locks WHERE lockname=?;" dbfname))
 
+;; release a lock if it matches
+(define (db:release-lock dbh dbfname host port)
+  (sqlite3:execute dbh "DELETE FROM locks WHERE lockname=? AND owner_host=? AND owner_port=?;" dbfname host port))
+
 ;;======================================================================
 ;; SQLITE3 HELPERS
 ;;======================================================================
 
 (define (db:general-sqlite-error-dump exn stmt . params)

Index: rmtmod.scm
==================================================================
--- rmtmod.scm
+++ rmtmod.scm
@@ -217,49 +217,58 @@
 ;;
 (define (rmt:open-main-connection remdat apath)
   (let* ((fullpath (db:dbname->path apath "/.db/main.db"))
 	 (conns    (remotedat-conns remdat))
 	 (conn     (hash-table-ref/default conns fullpath #f))) ;; TODO - create call for this
-    (if (and conn                                             ;; conn is NOT a socket, just saying ...
-	     (< (current-seconds) (conndat-expires conn)))
-	#t ;; we are current and good to go - we'll deal elsewhere with a server that was killed or died 
-	;; Below we will find or create and connect to main
-	(let* ((dbname         (db:run-id->dbname #f))
-	       (the-srv        (rmt:find-main-server apath dbname))
-	       (start-main-srv (lambda () ;; call IF there is no the-srv found
-				 (mutex-lock! *connstart-mutex*)
-				 (if (> (- (current-seconds) *last-main-start*) 5) ;; at least four seconds since last attempt to start main server
-				     (begin
-				       (api:run-server-process apath dbname)
-				       (set! *last-main-start* (current-seconds))
-				       (thread-sleep! 1)))
-				 (mutex-unlock! *connstart-mutex*)
-				 (rmt:open-main-connection remdat apath) ;; TODO: Add limit to number of tries
-				 )))
-	  (if (not the-srv) ;; have server, try connecting to it
-	      (start-main-srv)
-	      (let* ((srv-addr (server-address the-srv)) ;; need serv
-		     (ipaddr   (alist-ref 'ipaddr  the-srv))
-		     (port     (alist-ref 'port    the-srv))
-		     (srvkey   (alist-ref 'servkey the-srv))
-		     (fullpath (db:dbname->path apath dbname))
-		     
-		     (new-the-srv (make-conndat
-				   apath:   apath
-				   dbname:  dbname
-				   fullname: fullpath
-				   hostport: srv-addr
-				   socket: (open-nn-connection srv-addr)
-				   ipaddr: ipaddr
-				   port: port
-				   srvpkt: the-srv
-				   srvkey: srvkey ;; generated by rmt:get-signature on the server side
-				   lastmsg: (current-seconds)
-				   expires: (+ (current-seconds) 60) ;; this needs to be gathered during the ping
-				   )))
-		(hash-table-set! conns fullpath new-the-srv)))
-	  #t))))
+    (cond
+     ((and conn                                             ;; conn is NOT a socket, just saying ...
+	   (< (current-seconds) (conndat-expires conn)))
+      #t) ;; we are current and good to go - we'll deal elsewhere with a server that was killed or died 
+     ((and conn
+	   (>= (current-seconds)(conndat-expires conn)))
+      (debug:print-info 0 *default-log-port* "connection to "fullpath" server expired. Reconnecting.")
+      (if (conndat-socket conn)
+	  (nng-close! (conndat-socket conn)))
+      (hash-table-set! conns fullpath #f) ;; clean up
+      (rmt:open-main-connection remdat apath))
+     (else
+      ;; Below we will find or create and connect to main
+      (let* ((dbname         (db:run-id->dbname #f))
+	     (the-srv        (rmt:find-main-server apath dbname))
+	     (start-main-srv (lambda () ;; call IF there is no the-srv found
+			       (mutex-lock! *connstart-mutex*)
+			       (if (> (- (current-seconds) *last-main-start*) 5) ;; at least four seconds since last attempt to start main server
+				   (begin
+				     (api:run-server-process apath dbname)
+				     (set! *last-main-start* (current-seconds))
+				     (thread-sleep! 1)))
+			       (mutex-unlock! *connstart-mutex*)
+			       (rmt:open-main-connection remdat apath) ;; TODO: Add limit to number of tries
+			       )))
+	(if (not the-srv) ;; have server, try connecting to it
+	    (start-main-srv)
+	    (let* ((srv-addr (server-address the-srv)) ;; need serv
+		   (ipaddr   (alist-ref 'ipaddr  the-srv))
+		   (port     (alist-ref 'port    the-srv))
+		   (srvkey   (alist-ref 'servkey the-srv))
+		   (fullpath (db:dbname->path apath dbname))
+		   
+		   (new-the-srv (make-conndat
+				 apath:   apath
+				 dbname:  dbname
+				 fullname: fullpath
+				 hostport: srv-addr
+				 socket: (open-nn-connection srv-addr)
+				 ipaddr: ipaddr
+				 port: port
+				 srvpkt: the-srv
+				 srvkey: srvkey ;; generated by rmt:get-signature on the server side
+				 lastmsg: (current-seconds)
+				 expires: (+ (current-seconds) 60) ;; this needs to be gathered during the ping
+				 )))
+	      (hash-table-set! conns fullpath new-the-srv)))
+	#t)))))
 
 ;; NB// remdat is a remotedat struct
 ;;
 (define (rmt:general-open-connection remdat apath dbname #!key (num-tries 5))
   (assert (not (equal? dbname ".db/main.db")) "ERROR: general-open-connection should never be called with main as the db")
@@ -273,10 +282,15 @@
 	  (debug:print-info 0 *default-log-port* "Turning on logging to main, look in logs dir for main log.")
 	  (debug:print-logger rmt:log-to-main)))
     (cond
      ((or (not mconn) ;; no channel open to main?
 	  (< (conndat-expires mconn)(+ (current-seconds) 2))) ;; restablish connection if less than 2 seconds on the lease
+      (if mconn ;; previously opened - clean up NB// consolidate this with the similar code in open main above
+	  (begin
+	    (debug:print-info 0 *default-log-port* "Clearing out connection to main that has expired.")
+	    (nng-close! (conndat-socket mconn))
+	    (hash-table-set! conns fullname #f)))
       (rmt:open-main-connection remdat apath)
       (rmt:general-open-connection remdat apath mdbname))
      ((not (rmt:get-conn remdat apath dbname))                 ;; no channel open to dbname?     
       (let* ((res (rmt:send-receive-real remdat apath mdbname 'get-server `(,apath ,dbname))))
 	(case res
@@ -1506,11 +1520,12 @@
 
 (define (common:api-changed?)
   (not (equal? (substring (->string megatest-version) 0 4)
                (substring (conc (common:get-last-run-version)) 0 4))))
 
-(define (rmt:server-shutdown)
+;; host and port are used to ensure we are remove proper records
+(define (rmt:server-shutdown host port)
   (let ((dbfile   (servdat-dbfile *server-info*)))
     (debug:print-info 0 *default-log-port* "dbfile is "dbfile)
     (if dbfile
 	(let* ((am-server  (args:get-arg "-server"))
 	       (dbfile     (args:get-arg "-db"))
@@ -1540,14 +1555,15 @@
 		  (let ((pkt-file (conc (get-pkts-dir *toppath*)
 					"/" (servdat-uuid *server-info*)
 					".pkt")))
 		    (debug:print-info 0 *default-log-port* "removing pkt "pkt-file)
 		    (delete-file* pkt-file)
-		    (debug:print-info 0 *default-log-port* "Releasing lock for "dbfile)
-		    (db:with-lock-db (servdat-dbfile *server-info*)
-				     (lambda (dbh dbfile)
-				       (db:release-lock dbh dbfile))))
+		    (debug:print-info 0 *default-log-port* "Releasing lock (if any) for "dbfile ", host "host", port "port)
+		    (db:with-lock-db
+		     (servdat-dbfile *server-info*)
+		     (lambda (dbh dbfile)
+		       (db:release-lock dbh dbfile host port)))) ;; I'm not the server - should not have a lock to remove
 		  (let* ((sdat *server-info*) ;; we have a run-id server
 			 (host (servdat-host sdat))
 			 (port (servdat-port sdat))
 			 (uuid (servdat-uuid sdat))
 			 (res  (rmt:deregister-server remdat *toppath* host port uuid dbfile)))
@@ -1572,11 +1588,12 @@
 		  (let* ((start-time (current-seconds)))
 		    (if (and *server-info*
 			     *unclean-shutdown*)
 			(begin
 			  (debug:print-info 0 *default-log-port* "Unclean server exit, calling server-shtudown")
-			  (rmt:server-shutdown)))
+			  (rmt:server-shutdown (servdat-host *server-info*)
+					       (servdat-port *server-info*))))
 		    (debug:print-info 0 *default-log-port* "Shutdown activities completed in "(- (current-seconds) start-time)" seconds"))
 		  ;; (if *dbstruct-db* (db:close-all *dbstruct-db*)) ;; one second allocated
 		  #;(if (bdat-task-db *bdat*)    ;; TODO: Check that this is correct for task db
 		  (let ((db (cdr (bdat-task-db *bdat*))))
 		  (if (sqlite3:database? db)
@@ -1679,13 +1696,16 @@
 		(set! *db-last-access* (current-seconds))
 		(nng-send rep resdat)
 		(loop (nng-recv rep)))))))
     (debug:print-info 0 *default-log-port* "After server, should never see this")
     ;; server exit stuff here
-    (let* ((portnum (servdat-port *server-info*)))
+    (let* ((portnum (servdat-port *server-info*))
+	   (host    (servdat-host *server-info*)))
       (portlogger:open-run-close portlogger:set-port portnum "released")
-      (rmt:server-shutdown)
+      (if (not (equal? (get-host-name) host))
+	  (debug:print-info 0 *default-log-port* "Server shutdown called for host "host", but we are on "(get-host-name))
+	  (rmt:server-shutdown host portnum))
       ;; (bdat-time-to-exit-set! *bdat* #t) ;; tell on-exit to be fast as we've already cleaned up
       (portlogger:open-run-close portlogger:set-port port "released") ;; done in rmt:run
       ;; (debug:print-info 0 *default-log-port* "Max cached queries was    " *max-cache-size*)
       ;; (debug:print-info 0 *default-log-port* "Number of cached writes   " *number-of-writes*)
       ;; (debug:print-info 0 *default-log-port* "Average cached write time "
@@ -1794,72 +1814,29 @@
 
 (define (rmt:inc-requests-and-prep-to-close-all-connections)
   (mutex-lock! *http-mutex*)
   (set! *http-requests-in-progress* (+ 1 *http-requests-in-progress*)))
 
-;; careful closing of connections stored in *runremote*
-;;
-(define (rmt:close-connections #!key (area-dat #f))
-  (debug:print-info 0 *default-log-port* "rmt:close-connections doesn't do anything now!"))
-;;   (let* ((runremote  (or area-dat *runremote*))
-;; 	 (server-dat (if runremote
-;;                          (remote-conndat runremote)
-;;                          #f))) ;; (hash-table-ref/default *runremote* run-id #f)))
-;;     (if (vector? server-dat)
-;; 	(let ((api-dat (http-transport:server-dat-get-api-uri server-dat)))
-;; 	  (handle-exceptions
-;; 	    exn
-;; 	    (begin
-;; 	      (print-call-chain *default-log-port*)
-;; 	      (debug:print-error 0 *default-log-port* " closing connection failed with error: " ((condition-property-accessor 'exn 'message) exn) ", exn=" exn))
-;; 	    (close-connection! api-dat)
-;;             ;;(close-idle-connections!)
-;; 	    #t))
-;; 	#f)))
-
-
-
-;; initialize servdat for client side, setup needed parameters
-;; pass in #f as sdat-in to create sdat
-;;
-#;(define (servdat-init sdat-in iface port uuid)
-  (let* ((sdat (or sdat-in (make-servdat))))
-    
-    (assert #f "This is a bad idea.")
-
-    (if uuid (servdat-uuid-set! sdat uuid))
-    (servdat-host-set! sdat iface)
-    (servdat-port-set! sdat port)
-    (servdat-api-url-set! sdat (conc "http://" iface ":" port "/api"))
-    (servdat-api-uri-set! sdat (uri-reference (servdat-api-url sdat)))
-    (servdat-api-req-set! sdat (make-request method: 'POST
-					     uri: (servdat-api-uri sdat)))
-    ;; set up the http-client parameters
-    (max-retry-attempts 1)
-    ;; consider all requests indempotent
-    (retry-request? (lambda (request)
-		      #f))
-    (determine-proxy (constantly #f))
-   sdat))
-
 ;;======================================================================
 ;; NEW SERVER METHOD
 ;;======================================================================
 
 ;; only use for main.db - need to re-write some of this :(
 ;;
-(define (get-lock-db sdat dbfile port)
+(define (get-lock-db sdat dbfile host port)
+  (assert host "FATAL: get-lock-db called with host not set.")
+  (assert port "FATAL: get-lock-db called with port not set.")
   (let* ((dbh (db:open-run-db dbfile db:initialize-db)) ;; open-run-db creates a standard db with schema used by all situations
-	 (res (db:get-iam-server-lock dbh dbfile port)))
+	 (res (db:get-iam-server-lock dbh dbfile host port)))
     ;; res => list then already locked, check server is responsive
     ;;     => #t then sucessfully got the lock
     ;;     => #f reserved for future use as to indicate something went wrong
     (match res
       ((owner_pid owner_host owner_port event_time)
        (if (server-ready? owner_host owner_port "abc")
-	   #f
-	   (begin
+	   #f      ;; locked by someone else
+	   (begin  ;; locked by someone dead and gone
 	     (debug:print 0 *default-log-port* "WARNING: stale lock - have to steal it. This may fail.")
 	     (db:steal-lock-db dbh dbfile port))))
       (#t  #t) ;; placeholder so that we don't touch res if it is #t
       (else (set! res #f)))
     (sqlite3:finalize! dbh)
@@ -1912,38 +1889,14 @@
 (define (server-ready? host port key) ;; server-address is host:port
   (let* ((data (sexpr->string  `((cmd . ping)
 				 (key . ,key)
 				 (params . ()))))
 	 (res  (open-send-receive-nn (conc host ":" port) data)))
-    (string->sexpr res)))
-
-;; (let ((res (with-input-from-port i
-;; 		       read)))
-;; 	    (close-output-port o)
-;; 	    (close-input-port i)
-;; 	    res))
-;; 	    (if (string? res)
-;; 		(string->sexpr res)
-;; 		res)))
-;;     (begin ;; connection failed
-;; 	  (debug:print-info 0 *default-log-port* "Server at "host":"port" is not responding.")
-;; 	  #f))))
-
-;; (define (loop-test host port data) ;; server-address is host:port
-;;   ;; ping the server and ask it
-;;   ;; if it ready
-;;   ;; (let* ((sdat (servdat-init #f host port #f)))
-;;   ;;   (http-transport:send-receive sdat "abc" 'ping '())))
-;;   (let* ((payload (sexpr->string data))
-;; 	 (res     (with-input-from-request
-;; 		   (conc "http://"host":"port"/loop-test")
-;; 		   `((data . ,payload))
-;; 		   read-string)))
-;;   (string->sexpr res))
-;;   #f
-;;   )
-	      
+    (if res
+	(string->sexpr res)
+	res)))
+
 ; from the pkts return servers associated with dbpath
 ;; NOTE: Only one can be alive - have to check on each
 ;;       in the list of pkts returned
 ;;
 (define (get-viable-servers serv-pkts dbpath)
@@ -1961,14 +1914,11 @@
   (filter (lambda (pkt)
 	    (let* ((host (alist-ref 'host pkt))
 		   (port (alist-ref 'port pkt))
 		   (key  (alist-ref 'servkey  pkt))
 		   (pktz (alist-ref 'Z        pkt))
-		   (res  (handle-exceptions
-			     exn
-			     #f
-			   (server-ready? host port key))))
+		   (res  (server-ready? host port key)))
 	      (if res
 		  res
 		  (let* ((pktsdir (get-pkts-dir *toppath*))
 			 (pktpath (conc pktsdir"/"pktz".pkt")))
 		    (debug:print 0 *default-log-port* "WARNING: pkt with no server "pktpath)
@@ -2073,11 +2023,11 @@
 				       (debug:print-info 0 *default-log-port* "Attempting to remove bogus pkt file "pktfile)
 				       (delete-file* pktfile))))) ;; remove immediately instead of waiting for on-exit
 		(debug:print 0 *default-log-port* "best-srv-key: "best-srv-key", server-key: "server-key", i-am-srv: "i-am-srv)
 		;; am I the best-srv, compare server-keys to know
 		(if i-am-srv
-		    (if (get-lock-db sdat db-file (servdat-port sdat)) ;; (db:get-iam-server-lock *dbstruct-db* *toppath* run-id)
+		    (if (get-lock-db sdat db-file (servdat-host sdat)(servdat-port sdat)) ;; (db:get-iam-server-lock *dbstruct-db* *toppath* run-id)
 			(begin
 			  (debug:print-info 0 *default-log-port* "I'm the server!")
 			  (servdat-dbfile-set! sdat db-file)
 			  (servdat-status-set! sdat 'db-locked))
 			(begin
@@ -2194,14 +2144,14 @@
 	 (pkts-dir          (get-pkts-dir))
 	 (server-key        (rmt:get-signature)) ;; This servers key
 	 (is-main           (equal? (args:get-arg "-db") ".db/main.db"))
 	 (last-access       0)
 	 (server-timeout    (server:expiration-timeout))
-	 (shutdown-server-sequence (lambda (port)
+	 (shutdown-server-sequence (lambda (host port)
 				     (set! *unclean-shutdown* #f)
 				     (debug:print-info 0 *default-log-port* "Starting to shutdown the server. pid="(current-process-id))
-				     (rmt:server-shutdown)
+				     (rmt:server-shutdown host port)
 				     (portlogger:open-run-close portlogger:set-port port "released")
 				     (exit)))
 	 (timed-out?        (lambda ()
 			      (<= (+ last-access server-timeout)
 				  (current-seconds)))))
@@ -2232,17 +2182,19 @@
 		  (let ((res (rmt:register-server remdat
 						  *toppath* iface port
 						  server-key dbname)))
 		    (if res ;; we are the server
 			(servdat-status-set! *server-info* 'have-interface-and-db)
+			;; now check that the db locker is alive, clear it out if not
 			(let* ((serv-info (rmt:get-server-info *toppath* dbname)))
 			  (match serv-info
 			    ((host port servkey pid ipaddr apath dbpath)
 			     (if (not (server-ready? host port servkey))
 				 (begin
 				   (debug:print-info 0 *default-log-port* "Server registered but not alive. Removing and trying again.")
 				   (rmt:deregister-server remdat apath host port servkey dbpath) ;; servkey pid ipaddr apath dbpath)
+				   
 				   (loop (+ count 1) bad-sync-count start-time))))
 			    (else
 			     (debug:print 0 *default-log-port* "We are not the server for "dbname", exiting. Server info is: "serv-info)
 			     (exit)))))))
 	      (debug:print 0 *default-log-port*
@@ -2290,14 +2242,14 @@
 	      (db:print-current-query-stats)))
 	(let* ((hrs-since-start  (/ (- (current-seconds) server-start-time) 3600)))
 	  (cond
 	   ((not *server-run*)
 	    (debug:print-info 0 *default-log-port* "*server-run* set to #f. Shutting down.")
-	    (shutdown-server-sequence port))
+	    (shutdown-server-sequence (get-host-name) port))
 	   ((timed-out?)
 	    (debug:print-info 0 *default-log-port* "Server timed out. seconds since last db access: " (- (current-seconds) last-access))
-	    (shutdown-server-sequence port))
+	    (shutdown-server-sequence (get-host-name) port))
 	   ((and *server-run*
 		 (or (not (timed-out?))
 		     (if is-main ;; do not exit if there are other servers (keep main open until all others gone)
 			 (> (rmt:get-count-servers remdat *toppath*) 1)
 			 #f)))
@@ -2305,11 +2257,11 @@
 		(debug:print-info 0 *default-log-port* "Server continuing, seconds since last db access: " (- (current-seconds) last-access)))
 	    (loop 0 bad-sync-count (current-milliseconds)))
 	   (else
 	    (set! *unclean-shutdown* #f)
 	    (debug:print-info 0 *default-log-port* "Server timed out. seconds since last db access: " (- (current-seconds) last-access))
-	    (shutdown-server-sequence port)
+	    (shutdown-server-sequence (get-host-name) port)
 	    #;(debug:print-info 0 *default-log-port* "Sending 'quit to server, received: "
 			      (open-send-receive-nn (conc iface":"port)      ;; do this here and not in server-shutdown
 						    (sexpr->string 'quit)))
 	    )))))))
 

Index: tests/Makefile
==================================================================
--- tests/Makefile
+++ tests/Makefile
@@ -40,11 +40,11 @@
 TARGET     = "ubuntu/nfs/none"
 
 all : build unit test4
 # test1 test2 test3 test4 test5 test6 test7 test8 test9
 
-unit : basicserver.log server.log
+unit : basicserver.log server.log all-rmt.log
 # all-rmt.log all-api.log
 # runs.log misc.log tests.log
 
 # inter dependencies on the unit tests, I wish these could be "suggestions"
 all-rmt.log : all-api.log

Index: tests/unittests/all-rmt.scm
==================================================================
--- tests/unittests/all-rmt.scm
+++ tests/unittests/all-rmt.scm
@@ -29,74 +29,76 @@
 ;;   NTN - no test needed
 ;;   DEP - function is deprecated, no point in testing
 ;;   NED - function nested under others, no test needed.
 ;;   DEF - deferred
 
-(print "start dir: " (current-directory))
-       
-(define toppath (current-directory))
-
-(test #f #f (server:check-if-running toppath))           ;; these are used by server:start-and-wait
-(test #f #t (list? (server:get-list toppath)))
-(test #f '() (server:get-best '()))
-(test #f #t (common:simple-file-lock-and-wait "test.lock" expire-time: 15))
-(test #f "test.lock" (common:simple-file-release-lock "test.lock"))
-(test #f #t (server:get-best-guess-address (get-host-name)))
-(test #f #t (string? (common:get-homehost)))
-
-;; clean out any old running servers
-;;
-(let ((servers (server:get-list toppath)))
-  (print "Known servers: "  servers)
-  (if (not (null? servers))
-      (begin
-	(for-each
-	 (lambda (server)
-	   (let ((pid (list-ref server 4)))
-	     (thread-start!
-	      (make-thread
-	       (lambda ()
-		 (print "Attempting to kill server: " server)
-		 (print "Attempting to kill pid " pid)
-		 (system (conc "kill " pid))
-		 (thread-sleep! 2)
-		 (system (conc "kill -9 " pid)))
-	       (conc pid)))))
-	 servers)
-	(thread-sleep! 2))))
-;; let's start up a server the mechanical way
-(system "nbfake megatest -server -")
-(thread-sleep! 2)
-;; (test #f #t (string? (server:start-and-wait *toppath*)))
-
-(test "setup for run" #t (begin (launch:setup)
- 				(string? (getenv "MT_RUN_AREA_HOME"))))
-(test #f #t (client:setup-http toppath))
-(test #f #t (vector? (client:setup toppath)))
-
-(test #f #t (vector? (rmt:get-connection-info toppath))) ;; TODO: push areapath down.
-(test #f #t (string? (server:check-if-running ".")))
-;; DEF (test #f #f (rmt:send-receive-no-auto-client-setup *runremote* 'get-keys #f '()))
-;; DEF (rmt:kill-server run-id)
-;; DEF (rmt:start-server run-id)
-(test #f '(#t "successful login")(rmt:login #f))
-;; DEF (rmt:login-no-auto-client-setup connection-info)
-(test #f #t (pair? (rmt:get-latest-host-load (get-host-name))))
-
-;; get-latest-host-load does a lookup in the db, it won't return a useful value unless
-;; a test ran recently on host
-(test-batch rmt:get-latest-host-load
-            "rmt:get-latest-host-load"
-            (list (list "localhost"  #t (get-host-name))
-                  (list "not-a-host" #t "not-a-host"  ))
-            post-proc: pair?)
-                                           
-(test #f #t (list? (rmt:get-changed-record-ids 0)))
-
-(test #f #f (begin (runs:update-all-test_meta #f) #f))
-
-(test #f '("test1" "test2")(sort (alist-ref "tagtwo" (rmt:get-tests-tags) equal?) string<=))
+(import big-chicken rmtmod apimod runsmod)
+
+(print "start dir: " (current-directory))
+;;        
+(define toppath (current-directory))
+;; 
+;; (test #f #f (server:check-if-running toppath))           ;; these are used by server:start-and-wait
+;; (test #f #t (list? (server:get-list toppath)))
+;; (test #f '() (server:get-best '()))
+;; (test #f #t (common:simple-file-lock-and-wait "test.lock" expire-time: 15))
+;; (test #f "test.lock" (common:simple-file-release-lock "test.lock"))
+;; (test #f #t (server:get-best-guess-address (get-host-name)))
+;; (test #f #t (string? (common:get-homehost)))
+;; 
+;; ;; clean out any old running servers
+;; ;;
+;; (let ((servers (server:get-list toppath)))
+;;   (print "Known servers: "  servers)
+;;   (if (not (null? servers))
+;;       (begin
+;; 	(for-each
+;; 	 (lambda (server)
+;; 	   (let ((pid (list-ref server 4)))
+;; 	     (thread-start!
+;; 	      (make-thread
+;; 	       (lambda ()
+;; 		 (print "Attempting to kill server: " server)
+;; 		 (print "Attempting to kill pid " pid)
+;; 		 (system (conc "kill " pid))
+;; 		 (thread-sleep! 2)
+;; 		 (system (conc "kill -9 " pid)))
+;; 	       (conc pid)))))
+;; 	 servers)
+;; 	(thread-sleep! 2))))
+;; ;; let's start up a server the mechanical way
+;; (system "nbfake megatest -server -")
+;; (thread-sleep! 2)
+;; ;; (test #f #t (string? (server:start-and-wait *toppath*)))
+;; 
+;; (test "setup for run" #t (begin (launch:setup)
+;;  				(string? (getenv "MT_RUN_AREA_HOME"))))
+;; (test #f #t (client:setup-http toppath))
+;; (test #f #t (vector? (client:setup toppath)))
+;; 
+;; (test #f #t (vector? (rmt:get-connection-info toppath))) ;; TODO: push areapath down.
+;; (test #f #t (string? (server:check-if-running ".")))
+;; ;; DEF (test #f #f (rmt:send-receive-no-auto-client-setup *runremote* 'get-keys #f '()))
+;; ;; DEF (rmt:kill-server run-id)
+;; ;; DEF (rmt:start-server run-id)
+;; (test #f '(#t "successful login")(rmt:login #f))
+;; ;; DEF (rmt:login-no-auto-client-setup connection-info)
+;; (test #f #t (pair? (rmt:get-latest-host-load (get-host-name))))
+;; 
+;; ;; get-latest-host-load does a lookup in the db, it won't return a useful value unless
+;; ;; a test ran recently on host
+;; (test-batch rmt:get-latest-host-load
+;;             "rmt:get-latest-host-load"
+;;             (list (list "localhost"  #t (get-host-name))
+;;                   (list "not-a-host" #t "not-a-host"  ))
+;;             post-proc: pair?)
+;;                                            
+;; (test #f #t (list? (rmt:get-changed-record-ids 0)))
+;; 
+(test #f #f (begin (runs:update-all-test_meta #f) #f))
+ 
+(test #f '("test1" "test2")(sort (alist-ref "tagtwo" (rmt:get-tests-tags) equal?) string<=?))
 (test #f '() (rmt:get-key-val-pairs 0))
 (test #f '("SYSTEM" "RELEASE") (rmt:get-keys))
 (test #f '("SYSTEM" "RELEASE") (rmt:get-keys-write)) ;; dummy query to force server start
 (test #f '() (rmt:get-key-vals 1))
 (test #f (vector '("SYSTEM" "RELEASE") '()) (rmt:get-targets))
@@ -131,11 +133,11 @@
 (test #f #f (begin (rmt:set-state-status-and-roll-up-items 1 "foo" "" "COMPLETED" "FAIL" "Just yet another message") #f))
 (test #f #t (rmt:top-test-set-per-pf-counts 1 "foo"))
 (test #f '() (rmt:get-raw-run-stats 1))
 (test #f #t (vector? (rmt:get-run-info 1)))
 (test #f 0 (rmt:get-num-runs "%"))
-(define keypatts '(("SYSTEM" "ubuntu")("RELEASE" "v1.234")) )
+(define keypatts '(("SYSTEM" "ubuntu")("RELEASE" "v1.234")))
 (test #f 1 (rmt:register-run '(("SYSTEM" "ubuntu")("RELEASE" "v1.234")) "bar" "NEW" "JUSTFINE" "bobafett" "quick"))
 (test #f "bar" (rmt:get-run-name-from-id 1))
 (test #f #t (begin (rmt:delete-run 2) #t)) ;; delete a non-existant run
 (test #f #t (begin (rmt:update-run-stats 1 '()) #t))
 (test #f #t (begin (rmt:delete-old-deleted-test-records) #t))
@@ -166,50 +168,51 @@
 
 (test #f '(("Totals" "UNKNOWN" 1) ("bar" "UNKNOWN" 1)) (begin (rmt:get-run-stats)))
 (test #f #t (begin (rmt:set-run-state-status 1 "COMPLETE" "PASS") #t))
 (test #f '"COMPLETE" (rmt:get-run-state 1))
 (test #f '"PASS" (rmt:get-run-status 1))
-(test #f #t (begin (rmt:set-var "foo" "bar")#t))
-(test #f "bar" (rmt:get-var "foo"))
+(test #f #t (begin (rmt:set-var 1 "foo" "bar")#t))
+(test #f "bar" (rmt:get-var 1 "foo"))
 (test #f #t (begin (rmt:print-db-stats) #t))
-(test #f #t (begin (rmt:del-var "foo") #t))
-(test #f #f (rmt:get-var "foo"))
+(test #f #t (begin (rmt:del-var 1 "foo") #t))
+(test #f #f (rmt:get-var 1 "foo"))
 (test #f (vector #f #f #f #f #f #f #f #f #f #f #f #f) (rmt:get-data-info-by-id 1))
 (test #f '() (rmt:get-key-vals 1))
 (test #f "ubuntu/v1.234" (rmt:get-target 1))
 (print (rmt:get-run-info 1))
 (test #f '((runs) (tests) (test_steps) (test_data)) (rmt:get-run-record-ids "ubuntu/v1.234" 1 '("fail_count") "bar"))
-;; (rmt:find-and-mark-incomplete run-id ovr-deadtime)
-;; (rmt:get-main-run-stats run-id)
-;; (rmt:get-var varname)
-;; (rmt:set-var varname value)
-;; (rmt:find-and-mark-incomplete-all-runs #!key (ovr-deadtime #f))
-;; (rmt:get-previous-test-run-record run-id test-name item-path)
-;; (rmt:get-run-stats)
-;; (rmt:teststep-set-status! run-id test-id teststep-name state-in status-in comment logfile)
-;; (rmt:get-steps-for-test run-id test-id)
-;; (rmt:read-test-data run-id test-id categorypatt #!key (work-area #f)) 
-;; (rmt:testmeta-add-record testname)
-;; (rmt:testmeta-get-record testname)
-;; (rmt:testmeta-update-field test-name fld val)
-;; (rmt:test-data-rollup run-id test-id status)
-;; (rmt:csv->test-data run-id test-id csvdata)
-;; (rmt:tasks-find-task-queue-records target run-name test-patt state-patt action-patt)
-;; (rmt:tasks-add action owner target runname testpatt params)
-;; (rmt:tasks-set-state-given-param-key param-key new-state)
-;; (rmt:tasks-get-last target runname)
-;; (rmt:archive-get-allocations  testname itempath dneeded)
-;; (rmt:archive-register-block-name bdisk-id archive-path)
-;; (rmt:archive-allocate-testsuite/area-to-block block-id testsuite-name areakey)
-;; (rmt:archive-register-disk bdisk-name bdisk-path df)
-;; (rmt:test-set-archive-block-id run-id test-id archive-block-id)
-;; (rmt:test-get-archive-block-info archive-block-id)
-;; NED (rmt:open-qry-close-locally cmd run-id params #!key (remretries 5))
-;; NED (rmt:send-receive cmd rid params #!key (attemptnum 1)(area-dat #f)) ;; start attemptnum at 1 so the modulo below works as expected
-;; DEF (test #f #f (rmt:print-db-stats))
-;; DEF (rmt:get-max-query-average run-id)
-;; NED (rmt:general-call stmtname run-id . params)
-;; DEP (rmt:sdb-qry qry val run-id)
-;; DEF (rmt:runtests user run-id testpatt params)
-;; DEP (rmt:open-test-db-by-test-id run-id test-id #!key (work-area #f))
-;; DEP (rmt:synchash-get run-id proc synckey keynum params)
-;; DEP (test #f #f (rmt:update-pass-fail-counts 1 "foo"))
+;; ;; (rmt:find-and-mark-incomplete run-id ovr-deadtime)
+;; ;; (rmt:get-main-run-stats run-id)
+;; ;; (rmt:get-var varname)
+;; ;; (rmt:set-var varname value)
+;; ;; (rmt:find-and-mark-incomplete-all-runs #!key (ovr-deadtime #f))
+;; ;; (rmt:get-previous-test-run-record run-id test-name item-path)
+;; ;; (rmt:get-run-stats)
+;; ;; (rmt:teststep-set-status! run-id test-id teststep-name state-in status-in comment logfile)
+;; ;; (rmt:get-steps-for-test run-id test-id)
+;; ;; (rmt:read-test-data run-id test-id categorypatt #!key (work-area #f)) 
+;; ;; (rmt:testmeta-add-record testname)
+;; ;; (rmt:testmeta-get-record testname)
+;; ;; (rmt:testmeta-update-field test-name fld val)
+;; ;; (rmt:test-data-rollup run-id test-id status)
+;; ;; (rmt:csv->test-data run-id test-id csvdata)
+;; ;; (rmt:tasks-find-task-queue-records target run-name test-patt state-patt action-patt)
+;; ;; (rmt:tasks-add action owner target runname testpatt params)
+;; ;; (rmt:tasks-set-state-given-param-key param-key new-state)
+;; ;; (rmt:tasks-get-last target runname)
+;; ;; (rmt:archive-get-allocations  testname itempath dneeded)
+;; ;; (rmt:archive-register-block-name bdisk-id archive-path)
+;; ;; (rmt:archive-allocate-testsuite/area-to-block block-id testsuite-name areakey)
+;; ;; (rmt:archive-register-disk bdisk-name bdisk-path df)
+;; ;; (rmt:test-set-archive-block-id run-id test-id archive-block-id)
+;; ;; (rmt:test-get-archive-block-info archive-block-id)
+;; ;; NED (rmt:open-qry-close-locally cmd run-id params #!key (remretries 5))
+;; ;; NED (rmt:send-receive cmd rid params #!key (attemptnum 1)(area-dat #f)) ;; start attemptnum at 1 so the modulo below works as expected
+;; ;; DEF (test #f #f (rmt:print-db-stats))
+;; ;; DEF (rmt:get-max-query-average run-id)
+;; ;; NED (rmt:general-call stmtname run-id . params)
+;; ;; DEP (rmt:sdb-qry qry val run-id)
+;; ;; DEF (rmt:runtests user run-id testpatt params)
+;; ;; DEP (rmt:open-test-db-by-test-id run-id test-id #!key (work-area #f))
+;; ;; DEP (rmt:synchash-get run-id proc synckey keynum params)
+;; ;; DEP (test #f #f (rmt:update-pass-fail-counts 1 "foo"))
+;; 

Index: tests/unittests/server.scm
==================================================================
--- tests/unittests/server.scm
+++ tests/unittests/server.scm
@@ -20,11 +20,11 @@
 
 
 ;; Run like this:
 ;;
 ;;  (cd ..;make && make install) && ./rununittest.sh server 1;(cd simplerun;megatest -stop-server 0)
-(import rmtmod trace http-client apimod dbmod
+(import big-chicken rmtmod trace http-client apimod dbmod
 	launchmod)
 
 (trace-call-sites #t)
 (trace
    
@@ -64,24 +64,27 @@
 (test #f ".db/2.db" (list-ref (rmt:send-receive-real *remotedat* *toppath* ".db/main.db" 'get-server `(,apath ,dbname))
 			      6))
 
 (thread-sleep! 2)
 (test #f #t (rmt:general-open-connection *remotedat* *toppath* ".db/2.db"))
-(test #f #t (list? (rmt:get-servers-info *toppath*)))
-
-(test #f '("SYSTEM" "RELEASE") (rmt:get-keys))
-(test #f 1 (rmt:send-receive 'register-run #f (list keyvals "run1" "new" "n/a" "justme" #f)))
-;; (print "Got here.")
-
-(test #f 1 (rmt:send-receive 'register-run 1 (list keyvals "run1" "new" "n/a" "justme" #f)))
-
-(test #f 2 (rmt:register-run keyvals "run2" "new" "n/a" "justme" #f))
-;; (test #f 2 (rmt:deregister-server *remotedat* *toppath* iface port server-key dbname
-
-(test #f 2 (rmt:get-count-servers *remotedat* *toppath*))
-
-(test #f "run2" (rmt:get-run-name-from-id 2))
-(test #f #f     (rmt:send-receive 'get-test-info-by-id 2 '(2 1)))
-      
-(test #f #t     (rmt:general-call 'update-cpuload-diskfree 2 1.5 1e6 1))
+
+;; (let loop ((end-time (+ (current-seconds) 61)))
+  (test #f #t (list? (rmt:get-servers-info *toppath*)))
+
+  (test #f '("SYSTEM" "RELEASE") (rmt:get-keys))
+  (test #f 1 (rmt:send-receive 'register-run #f (list keyvals "run1" "new" "n/a" "justme" #f)))
+  ;; (print "Got here.")
+
+  (test #f 1 (rmt:send-receive 'register-run 1 (list keyvals "run1" "new" "n/a" "justme" #f)))
+
+  (test #f 2 (rmt:register-run keyvals "run2" "new" "n/a" "justme" #f))
+  ;; (test #f 2 (rmt:deregister-server *remotedat* *toppath* iface port server-key dbname
+
+  (test #f 2 (rmt:get-count-servers *remotedat* *toppath*))
+
+  (test #f "run2" (rmt:get-run-name-from-id 2))
+  (test #f #f     (rmt:send-receive 'get-test-info-by-id 2 '(2 1)))
+  
+  (test #f #t     (rmt:general-call 'update-cpuload-diskfree 2 1.5 1e6 1))
+;;  (if (< (current-seconds) end-time)(loop end-time)))
 
 (exit)