Index: Makefile
==================================================================
--- Makefile
+++ Makefile
@@ -5,11 +5,11 @@
 SRCFILES = common.scm items.scm launch.scm \
            ods.scm runconfig.scm server.scm configf.scm \
            db.scm keys.scm margs.scm megatest-version.scm \
            process.scm runs.scm tasks.scm tests.scm genexample.scm \
 	   fs-transport.scm http-transport.scm \
-           client.scm gutils.scm synchash.scm daemon.scm
+           client.scm gutils.scm synchash.scm daemon.scm mt.scm
 
 GUISRCF  = dashboard-tests.scm dashboard-guimonitor.scm 
 
 OFILES   = $(SRCFILES:%.scm=%.o)
 GOFILES  = $(GUISRCF:%.scm=%.o)

Index: dashboard.scm
==================================================================
--- dashboard.scm
+++ dashboard.scm
@@ -29,10 +29,11 @@
 (declare (uses runs))
 (declare (uses dashboard-tests))
 (declare (uses dashboard-guimonitor))
 ;; (declare (uses dashboard-main))
 (declare (uses megatest-version))
+(declare (uses mt))
 
 (include "common_records.scm")
 (include "db_records.scm")
 (include "run_records.scm")
 
@@ -201,11 +202,11 @@
 	    ;; 
 	    ;; trim runs to only those that are changing often here
 	    ;; 
 	    (for-each (lambda (run)
 			(let* ((run-id   (db:get-value-by-header run header "id"))
-			       (tests    (let ((tsts (cdb:remote-run db:get-tests-for-run #f run-id testnamepatt states statuses)))
+			       (tests    (let ((tsts (mt:get-tests-for-run run-id testnamepatt states statuses)))
 					   (if *tests-sort-reverse* (reverse tsts) tsts)))
 			       (key-vals (cdb:remote-run db:get-key-vals #f run-id)))
 			  ;; Not sure this is needed?
 			  (set! referenced-run-ids (cons run-id referenced-run-ids))
 			  (if (> (length tests) maxtests)

Index: db.scm
==================================================================
--- db.scm
+++ db.scm
@@ -1,7 +1,7 @@
 ;;======================================================================
-;; Copyright 2006-2012, Matthew Welland.
+;; Copyright 2006-2013, Matthew Welland.
 ;; 
 ;;  This program is made available under the GNU GPL version 2.0 or
 ;;  greater. See the accompanying file COPYING for details.
 ;; 
 ;;  This program is distributed WITHOUT ANY WARRANTY; without even the
@@ -27,10 +27,11 @@
 (declare (uses common))
 (declare (uses keys))
 (declare (uses ods))
 (declare (uses fs-transport))
 (declare (uses client))
+(declare (uses mt))
 
 (include "common_records.scm")
 (include "db_records.scm")
 (include "key_records.scm")
 (include "run_records.scm")
@@ -766,13 +767,12 @@
 
 ;; states and statuses are lists, turn them into ("PASS","FAIL"...) and use NOT IN
 ;; i.e. these lists define what to NOT show.
 ;; states and statuses are required to be lists, empty is ok
 ;; not-in #t = above behaviour, #f = must match
-(define (db:get-tests-for-run db run-id testpatt states statuses 
-			      #!key (not-in #t)
-			      (sort-by #f) ;; 'rundir 'event_time
+(define (db:get-tests-for-run db run-id testpatt states statuses offset limit not-in sort-by
+			      #!key
 			      (qryvals "id,run_id,testname,state,status,event_time,host,cpuload,diskfree,uname,rundir,item_path,run_duration,final_logf,comment")
 			      )
   (debug:print-info 11 "db:get-tests-for-run START run-id=" run-id ", testpatt=" testpatt ", states=" states ", statuses=" statuses ", not-in=" not-in ", sort-by=" sort-by)
   (let* ((res '())
 	 ;; if states or statuses are null then assume match all when not-in is false
@@ -795,13 +795,18 @@
 				" FROM tests WHERE run_id=? AND state != 'DELETED' "
 				(if states-qry   (conc " AND " states-qry)   "")
 				(if statuses-qry (conc " AND " statuses-qry) "")
 				(if tests-match-qry (conc " AND (" tests-match-qry ") ") "")
 				(case sort-by
-				  ((rundir)     " ORDER BY length(rundir) DESC;")
-				  ((event_time) " ORDER BY event_time ASC;")
-				  (else         ";"))
+				  ((rundir)     " ORDER BY length(rundir) DESC ")
+				  ((event_time) " ORDER BY event_time ASC ")
+				  (else         (if (string? sort-by)
+						    (conc " ORDER BY " sort-by) 
+						    "")))
+				(if limit  (conc " LIMIT " limit)   "")
+				(if offset (conc " OFFSET " offset) "")
+				";"
 			 )))
     (debug:print-info 8 "db:get-tests-for-run qry=" qry)
     (sqlite3:for-each-row 
      (lambda (a . b) ;; id run-id testname state status event-time host cpuload diskfree uname rundir item-path run-duration final-logf comment)
        (set! res (cons (apply vector a b) res))) ;; id run-id testname state status event-time host cpuload diskfree uname rundir item-path run-duration final-logf comment) res)))
@@ -2035,11 +2040,11 @@
 	     (result         '()))
 	(for-each 
 	 (lambda (waitontest-name)
 	   ;; by getting the tests with matching name we are looking only at the matching test 
 	   ;; and related sub items
-	   (let ((tests             (cdb:remote-run db:get-tests-for-run #f run-id waitontest-name '() '()))
+	   (let ((tests             (mt:get-tests-for-run run-id waitontest-name '() '()))
 		 (ever-seen         #f)
 		 (parent-waiton-met #f)
 		 (item-waiton-met   #f))
 	     (for-each 
 	      (lambda (test)

Index: megatest.scm
==================================================================
--- megatest.scm
+++ megatest.scm
@@ -538,11 +538,11 @@
 			 (hash-table-set! seen targetstr #t)
 			 ;; (print "[" targetstr "]"))))
 			 (print targetstr))))
 	       (if (not db-targets)
 		   (let* ((run-id (db:get-value-by-header run header "id"))
-			  (tests  (cdb:remote-run db:get-tests-for-run #f run-id testpatt '() '())))
+			  (tests  (mt:get-tests-for-run run-id testpatt '() '())))
 		     (print "Run: " targetstr "/" (db:get-value-by-header run header "runname") 
 			    " status: " (db:get-value-by-header run header "state")
 			    " run-id: " run-id ", number tests: " (length tests))
 		     (for-each 
 		      (lambda (test)

ADDED   mt.scm
Index: mt.scm
==================================================================
--- /dev/null
+++ mt.scm
@@ -0,0 +1,50 @@
+;; Copyright 2006-2013, Matthew Welland.
+;; 
+;;  This program is made available under the GNU GPL version 2.0 or
+;;  greater. See the accompanying file COPYING for details.
+;; 
+;;  This program is distributed WITHOUT ANY WARRANTY; without even the
+;;  implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+;;  PURPOSE.
+
+
+(use sqlite3 srfi-1 posix regex regex-case srfi-69 dot-locking (srfi 18) posix-extras directory-utils)
+(import (prefix sqlite3 sqlite3:))
+
+(declare (unit mt))
+(declare (uses db))
+(declare (uses common))
+(declare (uses items))
+(declare (uses runconfig))
+(declare (uses tests))
+(declare (uses server))
+(declare (uses runs))
+
+(include "common_records.scm")
+(include "key_records.scm")
+(include "db_records.scm")
+(include "run_records.scm")
+(include "test_records.scm")
+
+;; This is the Megatest API. All generally "useful" routines will be wrapped or extended
+;; here.
+
+
+(define (mt:get-tests-for-run run-id testpatt states status #!key (not-in #f) (sort-by #f))
+  (let loop ((testsdat (cdb:remote-run db:get-tests-for-run #f run-id testpatt states status 0 500 not-in sort-by))
+	     (res      '())
+	     (offset   0)
+	     (limit    500))
+    (let* ((full-list (append res testsdat))
+	   (have-more (eq? (length testsdat) limit)))
+      (if have-more 
+	  (let ((new-offset (+ offset limit)))
+	    (debug:print-info 0 "More than " limit " tests, have " (length full-list) " tests so far.")
+	    (loop (cdb:remote-run db:get-tests-for-run #f run-id testpatt states status new-offset limit not-in sort-by)
+		  full-list
+		  new-offset
+		  limit))
+	  full-list))))
+
+(define (mt:get-prereqs-not-met run-id waitons ref-item-path #!key (mode 'normal))
+  (db:get-prereqs-not-met run-id waitons ref-item-path mode: mode))

Index: run-tests-queue-new.scm
==================================================================
--- run-tests-queue-new.scm
+++ run-tests-queue-new.scm
@@ -3,11 +3,11 @@
 (define (runs:run-tests-queue-new run-id runname test-records keyvals flags test-patts required-tests reglen)
   ;; At this point the list of parent tests is expanded 
   ;; NB// Should expand items here and then insert into the run queue.
   (debug:print 5 "test-records: " test-records ", flags: " (hash-table->alist flags))
   (let ((run-info              (cdb:remote-run db:get-run-info #f run-id))
-	(tests-info            (cdb:remote-run db:get-tests-for-run #f run-id #f '() '())) ;;  qryvals: "id,testname,item_path"))
+	(tests-info            (mt:get-tests-for-run run-id #f '() '())) ;;  qryvals: "id,testname,item_path"))
 	(sorted-test-names     (tests:sort-by-priority-and-waiton test-records))
 	(test-registry         (make-hash-table))
 	(registry-mutex        (make-mutex))
 	(num-retries           0)
 	(max-retries           (config-lookup *configdat* "setup" "maxretries"))
@@ -86,11 +86,11 @@
 		     (have-resources          (car run-limits-info))
 		     (num-running             (list-ref run-limits-info 1))
 		     (num-running-in-jobgroup (list-ref run-limits-info 2))
 		     (max-concurrent-jobs     (list-ref run-limits-info 3))
 		     (job-group-limit         (list-ref run-limits-info 4))
-		     (prereqs-not-met         (db:get-prereqs-not-met run-id waitons item-path mode: testmode))
+		     (prereqs-not-met         (mt:get-prereqs-not-met run-id waitons item-path mode: testmode))
 		     (fails                   (runs:calc-fails prereqs-not-met))
 		     (non-completed           (runs:calc-not-completed prereqs-not-met)))
 		(debug:print-info 8 "have-resources: " have-resources " prereqs-not-met: " 
 			     (string-intersperse 
 			      (map (lambda (t)
@@ -241,11 +241,11 @@
 	     ;;    - but only do that if resources exist to kick off the job
 	     ((or (procedure? items)(eq? items 'have-procedure))
 	      (let ((can-run-more    (cdb:remote-run runs:can-run-more-tests #f jobgroup max-concurrent-jobs)))
 		(if (and (list? can-run-more)
 			 (car can-run-more))
-		    (let* ((prereqs-not-met (db:get-prereqs-not-met run-id waitons item-path mode: testmode))
+		    (let* ((prereqs-not-met (mt:get-prereqs-not-met run-id waitons item-path mode: testmode))
 			   (fails           (runs:calc-fails prereqs-not-met))
 			   (non-completed   (runs:calc-not-completed prereqs-not-met)))
 		      (debug:print-info 8 "can-run-more: " can-run-more
 					"\n testname:        " hed
 					"\n prereqs-not-met: " (runs:pretty-string prereqs-not-met)

Index: runs.scm
==================================================================
--- runs.scm
+++ runs.scm
@@ -1,7 +1,7 @@
 
-;; Copyright 2006-2012, Matthew Welland.
+;; Copyright 2006-2013, Matthew Welland.
 ;; 
 ;;  This program is made available under the GNU GPL version 2.0 or
 ;;  greater. See the accompanying file COPYING for details.
 ;; 
 ;;  This program is distributed WITHOUT ANY WARRANTY; without even the
@@ -18,10 +18,11 @@
 (declare (uses common))
 (declare (uses items))
 (declare (uses runconfig))
 (declare (uses tests))
 (declare (uses server))
+(declare (uses mt))
 
 (include "common_records.scm")
 (include "key_records.scm")
 (include "db_records.scm")
 (include "run_records.scm")
@@ -565,16 +566,16 @@
      (lambda (run)
        (let ((runkey (string-intersperse (map (lambda (k)
 						(db:get-value-by-header run header k)) keys) "/"))
 	     (dirs-to-remove (make-hash-table))
 	     (proc-get-tests (lambda (run-id)
-			       (cdb:remote-run db:get-tests-for-run db run-id
-					       testpatt states statuses
-					       not-in:  #f
-					       sort-by: (case action
-							  ((remove-runs) 'rundir)
-							  (else          'event_time))))))
+			      (mt:get-tests-for-run run-id
+						    testpatt states statuses
+						    not-in:  #f
+						    sort-by: (case action
+							       ((remove-runs) 'rundir)
+							       (else          'event_time))))))
 	 (let* ((run-id    (db:get-value-by-header run header "id"))
 		(run-state (db:get-value-by-header run header "state"))
 		(tests     (if (not (equal? run-state "locked"))
 			       (proc-get-tests run-id)
 			       '()))
@@ -670,11 +671,13 @@
 					    ))
 				    (if (not (null? tal))
 					(loop (car tal)(cdr tal))))))
 			     ((set-state-status)
 			      (debug:print-info 2 "new state " (car state-status) ", new status " (cadr state-status))
-			      (cdb:remote-run db:test-set-state-status-by-id db (db:test-get-id test) (car state-status)(cadr state-status) #f))
+			      (cdb:remote-run db:test-set-state-status-by-id db (db:test-get-id test) (car state-status)(cadr state-status) #f)
+			      (if (not (null? tal))
+				  (loop (car tal)(cdr tal))))
 			     ((run-wait)
 			      (debug:print-info 2 "still waiting, " (length tests) " tests still running")
 			      (thread-sleep! 10)
 			      (let ((new-tests (proc-get-tests run-id)))
 				(if (null? new-tests)
@@ -681,11 +684,11 @@
 				    (debug:print-info 1 "Run completed according to zero tests matching provided criteria.")
 				    (loop (car new-tests)(cdr new-tests))))))))
 		   )))
 	   ;; remove the run if zero tests remain
 	   (if (eq? action 'remove-runs)
-	       (let ((remtests (cdb:remote-run db:get-tests-for-run db (db:get-value-by-header run header "id") #f '("DELETED") '("n/a") not-in: #t)))
+	       (let ((remtests (mt:get-tests-for-run (db:get-value-by-header run header "id") #f '("DELETED") '("n/a") not-in: #t)))
 		 (if (null? remtests) ;; no more tests remaining
 		     (let* ((dparts  (string-split lasttpath "/"))
 			    (runpath (conc "/" (string-intersperse 
 						(take dparts (- (length dparts) 1))
 						"/"))))
@@ -820,11 +823,11 @@
 (define (runs:rollup-run keys runname user keyvals)
   (debug:print 4 "runs:rollup-run, keys: " keys " :runname " runname " user: " user)
   (let* ((db              #f)
 	 (new-run-id      (cdb:remote-run db:register-run #f keyvals runname "new" "n/a" user))
 	 (prev-tests      (cdb:remote-run test:get-matching-previous-test-run-records db new-run-id "%" "%"))
-	 (curr-tests      (cdb:remote-run db:get-tests-for-run db new-run-id "%/%" '() '()))
+	 (curr-tests      (mt:get-tests-for-run new-run-id "%/%" '() '()))
 	 (curr-tests-hash (make-hash-table)))
     (cdb:remote-run db:update-run-event_time db new-run-id)
     ;; index the already saved tests by testname and itemdat in curr-tests-hash
     (for-each
      (lambda (testdat)
@@ -848,11 +851,11 @@
 	 (apply sqlite3:execute 
 		db 
 		(conc "INSERT OR REPLACE INTO tests (run_id,testname,state,status,event_time,host,cpuload,diskfree,uname,rundir,item_path,run_duration,final_logf,comment) "
 		      "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?);")
 		new-run-id (cddr (vector->list testdat)))
-	 (set! new-testdat (car (cdb:remote-run db:get-tests-for-run db new-run-id (conc testname "/" item-path) '() '())))
+	 (set! new-testdat (car (mt:get-tests-for-run new-run-id (conc testname "/" item-path) '() '())))
 	 (hash-table-set! curr-tests-hash full-name new-testdat) ;; this could be confusing, which record should go into the lookup table?
 	 ;; Now duplicate the test steps
 	 (debug:print 4 "Copying records in test_steps from test_id=" (db:test-get-id testdat) " to " (db:test-get-id new-testdat))
 	 (cdb:remote-run 
 	  (lambda ()

Index: tests.scm
==================================================================
--- tests.scm
+++ tests.scm
@@ -133,11 +133,11 @@
 	  ;; if found then return that matching test record
 	  (debug:print 4 "selstr: " selstr ", qrystr: " qrystr ", keyvals: " keyvals ", previous run ids found: " prev-run-ids)
 	  (if (null? prev-run-ids) #f
 	      (let loop ((hed (car prev-run-ids))
 			 (tal (cdr prev-run-ids)))
-		(let ((results (db:get-tests-for-run db hed (conc test-name "/" item-path)'() '())))
+		(let ((results (db:get-tests-for-run db hed (conc test-name "/" item-path)'() '() #f #f #f #f)))
 		  (debug:print 4 "Got tests for run-id " run-id ", test-name " test-name ", item-path " item-path ": " results)
 		  (if (and (null? results)
 			   (not (null? tal)))
 		      (loop (car tal)(cdr tal))
 		      (if (null? results) #f
@@ -144,12 +144,15 @@
 			  (car results))))))))))
     
 ;; get the previous records for when these tests were run where all keys match but runname
 ;; NB// Merge this with test:get-previous-test-run-records? This one looks for all matching tests
 ;; can use wildcards. Also can likely be factored in with get test paths?
+;;
+;; Run this remotely!!
+;;
 (define (test:get-matching-previous-test-run-records db run-id test-name item-path)
-  (let* ((keys    (cdb:remote-run db:get-keys #f))
+  (let* ((keys    (db:get-keys db))
 	 (selstr  (string-intersperse (map (lambda (x)(vector-ref x 0)) keys) ","))
 	 (qrystr  (string-intersperse (map (lambda (x)(conc (vector-ref x 0) "=?")) keys) " AND "))
 	 (keyvals #f)
 	 (tests-hash (make-hash-table)))
     ;; first look up the key values from the run selected by run-id
@@ -171,11 +174,11 @@
 	  (debug:print 4 "selstr: " selstr ", qrystr: " qrystr ", keyvals: " keyvals 
 		       ", previous run ids found: " prev-run-ids)
 	  (if (null? prev-run-ids) '()  ;; no previous runs? return null
 	      (let loop ((hed (car prev-run-ids))
 			 (tal (cdr prev-run-ids)))
-		(let ((results (cdb:remote-run db:get-tests-for-run #f hed (conc test-name "/" item-path) '() '())))
+		(let ((results (db:get-tests-for-run db hed (conc test-name "/" item-path) '() '() #f #f #f #f)))
 		  (debug:print 4 "Got tests for run-id " run-id ", test-name " test-name 
 			       ", item-path " item-path " results: " (intersperse results "\n"))
 		  ;; Keep only the youngest of any test/item combination
 		  (for-each 
 		   (lambda (testdat)

Index: tests/Makefile
==================================================================
--- tests/Makefile
+++ tests/Makefile
@@ -19,12 +19,16 @@
 TARGET     = "ubuntu/nfs/none"
 
 all : test1 test2 test3 test4 test5
 
 server :
-	(cd ..;make;make install) && \
-	(cd fullrun;../../bin/megatest -server - -debug 22 &) 
+	cd ..;make;make install
+	cd fullrun;../../bin/megatest -server - -debug 22 &
+
+stopserver :
+	cd ..;make && make install
+	cd fullrun;$(MEGATEST) -stop-server 0
 
 test0 : cleanprep
 	cd simplerun ; $(MEGATEST) -server - -debug $(DEBUG)
 
 test1 : cleanprep