Index: Makefile
==================================================================
--- Makefile
+++ Makefile
@@ -36,11 +36,12 @@
            subrun.scm archive.scm env.scm		\
            diff-report.scm cgisetup/models/pgdb.scm
 
 # module source files
 MSRCFILES = dbfile.scm debugprint.scm mtargs.scm commonmod.scm dbmod.scm \
-            tcp-transportmod.scm rmtmod.scm portlogger.scm apimod.scm
+            tcp-transportmod.scm rmtmod.scm portlogger.scm apimod.scm \
+	    configfmod.scm processmod.scm
 
 transport-mode.scm : transport-mode.scm.template
 	cp transport-mode.scm.template transport-mode.scm
 
 dashboard-transport-mode.scm : dashboard-transport-mode.scm.template
@@ -49,14 +50,17 @@
 mtest : transport-mode.scm
 dboard : dashboard-transport-mode.scm
 
 # dbmod.import.o is just a hack here
 mofiles/portlogger.o : mofiles/dbmod.o
+process.o : mofiles/processmod.o
+mofiles/configfmod.o : mofiles/processmod.o
+mofiles/processmod.o : mofiles/commonmod.o
 
 mofiles/dbfile.o     : \
        mofiles/debugprint.o mofiles/commonmod.o 
-mofiles/apimod.o : mofiles/commonmod.o mofiles/tcp-transportmod.o
+mofiles/apimod.o : mofiles/commonmod.o mofiles/tcp-transportmod.o mofiles/configfmod.o
 mofiles/dbmod.o : mofiles/dbfile.o
 mofiles/api.o : mofiles/apimod.o
 mofiles/commonmod.o : mofiles/debugprint.o
 configf.o : commonmod.import.o
 mofiles/dbfile.o : mofiles/debugprint.o
@@ -181,10 +185,11 @@
 # Special dependencies for the includes
 $(MOFILE) $(MOIMPFILES) : megatest-fossil-hash.scm
 
 mofiles/commonmod.o : mofiles/debugprint.o megatest-fossil-hash.scm
 common.o : mofiles/commonmod.o
+mofiles/configfmod.o : mofiles/commonmod.o
 
 # mofiles/dbmod.o : mofiles/configfmod.o
 
 # commonmod.o dashboard.o megatest.o tcmt.o apimod.o : megatest-fossil-hash.scm
 

Index: common.scm
==================================================================
--- common.scm
+++ common.scm
@@ -77,27 +77,10 @@
                                               (print-error-message exn) ))))
      (debug:print-info 0 *default-log-port* "    -- continuing after nonfatal condition...")
      #f)
    (thunk)))
 
-(define getenv get-environment-variable)
-(define (safe-setenv key val)
-  (if (or (substring-index "!" key)
-	  (substring-index ":" key)  ;; variables containing : are for internal use and cannot be environment variables.
-	  (substring-index "." key)) ;; periods are not allowed in environment variables
-      (debug:print-error 4 *default-log-port* "skip setting internal use only variables containing \":\" or starting with \"!\"")
-      (if (and (string? val)
-	       (string? key))
-	  (handle-exceptions
-	      exn
-	      (debug:print-error 0 *default-log-port* "bad value for setenv, key=" key ", value=" val ", exn=" exn)
-	    (setenv key val))
-	  (debug:print-error 0 *default-log-port* "bad value for setenv, key=" key ", value=" val))))
-
-(define home (getenv "HOME"))
-(define user (getenv "USER"))
-
 
 ;; returns list of fd count, socket count
 (define (get-file-descriptor-count #!key  (pid (current-process-id )))
   (list
     (length (glob (conc "/proc/" pid "/fd/*")))
@@ -1185,46 +1168,10 @@
       rtestpatt)
      (else 
       (debug:print-info 0 *default-log-port* "using testpatt " args-testpatt " rtestpatt:" rtestpatt)
       args-testpatt))))
 
-(define (common:false-on-exception thunk #!key (message #f))
-  (handle-exceptions exn
-                     (begin
-                       (if message
-                           (debug:print-info 0 *default-log-port* message))
-                       #f) (thunk) ))
-
-(define (common:file-exists? path-string #!key (silent #f))
-  ;;;; TODO: catch permission denied exceptions and emit appropriate warnings. Might be able to get rid of with chicken 5?
-  (common:false-on-exception (lambda () (file-exists? path-string))
-                             message: (if (not silent)
-                                          (conc "Unable to access path: " path-string)
-                                          #f)
-                             ))
-
-(define (common:directory-exists? path-string)
-  ;;;; TODO: catch permission denied exceptions and emit appropriate warnings
-  (common:false-on-exception (lambda () (directory-exists? path-string))
-                             message: (conc "Unable to access path: " path-string)
-                             ))
-
-;;======================================================================
-;; does the directory exist and do we have write access?
-;;
-;;    returns the directory or #f
-;;
-(define (common:directory-writable? path-string)
-  (handle-exceptions
-      exn
-    (begin
-      (debug:print 0 *default-log-port* "Failed to identify access to " path-string ", exn=" exn)
-      #f)
-   (if (and (directory-exists? path-string)
-            (file-write-access? path-string))
-       path-string
-       #f)))
 
 (define (common:get-linktree)
   (or (getenv "MT_LINKTREE")
       (if *configdat*
 	  (configf:lookup *configdat* "setup" "linktree")
@@ -1551,55 +1498,14 @@
   (apply max
 	 (map
 	  common:lazy-modification-time 
 	  file-list))))
 
-;;======================================================================
-;; return a nice clean pathname made absolute
-(define (common:nice-path dir)
-  (let ((match (string-match "^(~[^\\/]*)(\\/.*|)$" dir)))
-    (if match ;; using ~ for home?
-	(common:nice-path (conc (common:read-link-f (cadr match)) "/" (caddr match)))
-	(normalize-pathname (if (absolute-pathname? dir)
-				dir
-				(conc (current-directory) "/" dir))))))
-
 ;;======================================================================
 ;; make "nice-path" available in config files and the repl
 (define nice-path common:nice-path)
 
-(define (common:read-link-f path)
-  (handle-exceptions
-      exn
-      (begin
-	(debug:print-error 0 *default-log-port* "command \"/bin/readlink -f " path "\" failed. exn=" exn)
-	path) ;; just give up
-    (with-input-from-pipe
-	(conc "/bin/readlink -f " path)
-      (lambda ()
-	(read-line)))))
-
-;; for reasons I don't understand multiple calls to real-path in parallel threads
-;; must be protected by mutexes
-;;
-(define (common:real-path inpath)
-  ;; (process:cmd-run-with-stderr->list "readlink" "-f" inpath)) ;; cmd . params)
-  ;; (let-values 
-  ;;  (((inp oup pid) (process "readlink" (list "-f" inpath))))
-  ;;  (with-input-from-port inp
-  ;;    (let loop ((inl (read-line))
-  ;;       	(res #f))
-  ;;      (print "inl=" inl)
-  ;;      (if (eof-object? inl)
-  ;;          (begin
-  ;;            (close-input-port inp)
-  ;;            (close-output-port oup)
-  ;;            ;; (process-wait pid)
-  ;;            res)
-  ;;          (loop (read-line) inl))))))
-  (with-input-from-pipe (conc "readlink -f " inpath) read-line))
-
 ;;======================================================================
 ;; returns *effective load* (not normalized)
 ;;
 (define (common:get-intercept onemin fivemin)
   (if (< onemin fivemin) ;; load is decreasing, just use the onemin load
@@ -3030,50 +2936,10 @@
 	   `(,(alist-ref 't x) . ,(string->number (alist-ref 'D x))))
 	 pkts)
     (lambda (a b)(> (cdr a)(cdr b))))      ;; sort descending
    (lambda (a b)(equal? (car a)(car b))))) ;; remove duplicates by target
 
-;;======================================================================
-;; accept an alist or hash table containing envvar/env value pairs (value of #f causes unset) 
-;;   execute thunk in context of environment modified as per this list
-;;   restore env to prior state then return value of eval'd thunk.
-;;   ** this is not thread safe **
-(define (common:with-env-vars delta-env-alist-or-hash-table thunk)
-  (let* ((delta-env-alist (if (hash-table? delta-env-alist-or-hash-table)
-                              (hash-table->alist delta-env-alist-or-hash-table)
-                              delta-env-alist-or-hash-table))
-         (restore-thunks
-          (filter
-           identity
-           (map (lambda (env-pair)
-                  (let* ((env-var     (car env-pair))
-                         (new-val     (let ((tmp (cdr env-pair)))
-                                        (if (list? tmp) (car tmp) tmp)))
-                         (current-val (get-environment-variable env-var))
-                         (restore-thunk
-                          (cond
-                           ((not current-val) (lambda () (unsetenv env-var)))
-                           ((not (string? new-val)) #f)
-                           ((eq? current-val new-val) #f)
-                           (else 
-                            (lambda () (setenv env-var current-val))))))
-                    ;;(when (not (string? new-val))
-                    ;;    (debug:print 0 *default-log-port* " PROBLEM: not a string: "new-val"\n from env-alist:\n"delta-env-alist)
-                    ;;    (pp delta-env-alist)
-                    ;;    (exit 1))
-                        
-                    
-                    (cond
-                     ((not new-val)  ;; modify env here
-                      (unsetenv env-var))
-                     ((string? new-val)
-                      (setenv env-var new-val)))
-                    restore-thunk))
-                delta-env-alist))))
-    (let ((rv (thunk)))
-      (for-each (lambda (x) (x)) restore-thunks) ;; restore env to original state
-      rv)))
 
 (define *common:thread-punchlist* (make-hash-table))
 (define (common:send-thunk-to-background-thread thunk #!key (name #f))
   ;;(BB> "launched thread " name)
   ;; we need a unique name for the thread.

Index: commonmod.scm
==================================================================
--- commonmod.scm
+++ commonmod.scm
@@ -124,10 +124,111 @@
 	(begin
 	  (hash-table-set! *common:denoise* key currtime)
 	  #t)
 	#f)))
 
+;; environment vars handy stuff from common.scm
+;;
+(define getenv get-environment-variable)
+(define (safe-setenv key val)
+  (if (or (substring-index "!" key)
+	  (substring-index ":" key)  ;; variables containing : are for internal use and cannot be environment variables.
+	  (substring-index "." key)) ;; periods are not allowed in environment variables
+      (debug:print-error 4 *default-log-port* "skip setting internal use only variables containing \":\" or starting with \"!\"")
+      (if (and (string? val)
+	       (string? key))
+	  (handle-exceptions
+	      exn
+	      (debug:print-error 0 *default-log-port* "bad value for setenv, key=" key ", value=" val ", exn=" exn)
+	    (setenv key val))
+	  (debug:print-error 0 *default-log-port* "bad value for setenv, key=" key ", value=" val))))
+
+;;======================================================================
+;; accept an alist or hash table containing envvar/env value pairs (value of #f causes unset) 
+;;   execute thunk in context of environment modified as per this list
+;;   restore env to prior state then return value of eval'd thunk.
+;;   ** this is not thread safe **
+(define (common:with-env-vars delta-env-alist-or-hash-table thunk)
+  (let* ((delta-env-alist (if (hash-table? delta-env-alist-or-hash-table)
+                              (hash-table->alist delta-env-alist-or-hash-table)
+                              delta-env-alist-or-hash-table))
+         (restore-thunks
+          (filter
+           identity
+           (map (lambda (env-pair)
+                  (let* ((env-var     (car env-pair))
+                         (new-val     (let ((tmp (cdr env-pair)))
+                                        (if (list? tmp) (car tmp) tmp)))
+                         (current-val (get-environment-variable env-var))
+                         (restore-thunk
+                          (cond
+                           ((not current-val) (lambda () (unsetenv env-var)))
+                           ((not (string? new-val)) #f)
+                           ((eq? current-val new-val) #f)
+                           (else 
+                            (lambda () (setenv env-var current-val))))))
+                    ;;(when (not (string? new-val))
+                    ;;    (debug:print 0 *default-log-port* " PROBLEM: not a string: "new-val"\n from env-alist:\n"delta-env-alist)
+                    ;;    (pp delta-env-alist)
+                    ;;    (exit 1))
+                        
+                    
+                    (cond
+                     ((not new-val)  ;; modify env here
+                      (unsetenv env-var))
+                     ((string? new-val)
+                      (setenv env-var new-val)))
+                    restore-thunk))
+                delta-env-alist))))
+    (let ((rv (thunk)))
+      (for-each (lambda (x) (x)) restore-thunks) ;; restore env to original state
+      rv)))
+
+(define home (getenv "HOME"))
+(define user (getenv "USER"))
+
+;;======================================================================
+;; return a nice clean pathname made absolute
+(define (common:nice-path dir)
+  (let ((match (string-match "^(~[^\\/]*)(\\/.*|)$" dir)))
+    (if match ;; using ~ for home?
+	(common:nice-path (conc (common:read-link-f (cadr match)) "/" (caddr match)))
+	(normalize-pathname (if (absolute-pathname? dir)
+				dir
+				(conc (current-directory) "/" dir))))))
+
+(define (common:read-link-f path)
+  (handle-exceptions
+      exn
+      (begin
+	(debug:print-error 0 *default-log-port* "command \"/bin/readlink -f " path "\" failed. exn=" exn)
+	path) ;; just give up
+    (with-input-from-pipe
+	(conc "/bin/readlink -f " path)
+      (lambda ()
+	(read-line)))))
+
+;; for reasons I don't understand multiple calls to real-path in parallel threads
+;; must be protected by mutexes
+;;
+(define (common:real-path inpath)
+  ;; (process:cmd-run-with-stderr->list "readlink" "-f" inpath)) ;; cmd . params)
+  ;; (let-values 
+  ;;  (((inp oup pid) (process "readlink" (list "-f" inpath))))
+  ;;  (with-input-from-port inp
+  ;;    (let loop ((inl (read-line))
+  ;;       	(res #f))
+  ;;      (print "inl=" inl)
+  ;;      (if (eof-object? inl)
+  ;;          (begin
+  ;;            (close-input-port inp)
+  ;;            (close-output-port oup)
+  ;;            ;; (process-wait pid)
+  ;;            res)
+  ;;          (loop (read-line) inl))))))
+  (with-input-from-pipe (conc "readlink -f " inpath) read-line))
+
 ;; KEEP THIS ONE
 ;;
 ;; client:get-signature
 
 (define *my-client-signature* #f)
@@ -138,10 +239,47 @@
 	(set! *my-client-signature* sig)
 	*my-client-signature*)))
 
 (define *server-info* #f)
 (define *toppath* #f)
+
+(define (common:false-on-exception thunk #!key (message #f))
+  (handle-exceptions exn
+                     (begin
+                       (if message
+                           (debug:print-info 0 *default-log-port* message))
+                       #f) (thunk) ))
+
+(define (common:file-exists? path-string #!key (silent #f))
+  ;;;; TODO: catch permission denied exceptions and emit appropriate warnings. Might be able to get rid of with chicken 5?
+  (common:false-on-exception (lambda () (file-exists? path-string))
+                             message: (if (not silent)
+                                          (conc "Unable to access path: " path-string)
+                                          #f)
+                             ))
+
+(define (common:directory-exists? path-string)
+  ;;;; TODO: catch permission denied exceptions and emit appropriate warnings
+  (common:false-on-exception (lambda () (directory-exists? path-string))
+                             message: (conc "Unable to access path: " path-string)
+                             ))
+
+;;======================================================================
+;; does the directory exist and do we have write access?
+;;
+;;    returns the directory or #f
+;;
+(define (common:directory-writable? path-string)
+  (handle-exceptions
+      exn
+    (begin
+      (debug:print 0 *default-log-port* "Failed to identify access to " path-string ", exn=" exn)
+      #f)
+   (if (and (directory-exists? path-string)
+            (file-write-access? path-string))
+       path-string
+       #f)))
 
 ;;======================================================================
 ;; config file utils
 ;;======================================================================
 

Index: configf.scm
==================================================================
--- configf.scm
+++ configf.scm
@@ -31,92 +31,67 @@
 (declare (uses mtargs))
 (declare (uses mtargs.import))
 (declare (uses common))
 (declare (uses commonmod))
 (declare (uses commonmod.import))
+(declare (uses configfmod))
+(declare (uses configfmod.import))
 (declare (uses dbfile))
 (declare (uses dbfile.import))
 (declare (uses dbmod))
 (declare (uses dbmod.import))
 
 (import commonmod
+	configfmod
 	(prefix mtargs args:)
 	debugprint)
 
 (include "common_records.scm")
 
-;; return list (path fullpath configname)
-(define (find-config configname #!key (toppath #f))
-  (if toppath
-      (let ((cfname (conc toppath "/" configname)))
-	(if (common:file-exists? cfname)
-	    (list toppath cfname configname)
-	    (list #f      #f     #f)))
-      (let* ((cwd (string-split (current-directory) "/")))
-	(let loop ((dir cwd))
-	  (let* ((path     (conc "/" (string-intersperse dir "/")))
-		 (fullpath (conc path "/" configname)))
-	    (if (common:file-exists? fullpath)
-		(list path fullpath configname)
-		(let ((remcwd (take dir (- (length dir) 1))))
-		  (if (null? remcwd)
-		      (list #f #f #f) ;;  #f #f) 
-		  (loop remcwd)))))))))
-
-(define (configf:assoc-safe-add alist key val #!key (metadata #f))
-  (let ((newalist (filter (lambda (x)(not (equal? key (car x)))) alist)))
-    (append newalist (list (if metadata
-			       (list key val metadata)
-			       (list key val))))))
-
-;; this is used in megatestqa/ext.scm.
-;; remove it from here and there by 12/31/21
-;; (define config:assoc-safe-add configf:assoc-safe-add)
-
-(define (configf:section-var-set! cfgdat section-name var value #!key (metadata #f))
-  (hash-table-set! cfgdat section-name
-		   (configf:assoc-safe-add
-		    (hash-table-ref/default cfgdat section-name '())
-		    var value metadata: metadata)))
-
-(define (configf:eval-string-in-environment str)
-  ;; (if (or (string-null? str)
-  ;;	  (equal? "!" (substring str 0 1))) ;; null string or starts with ! are preserved but NOT set in the environment
-      str
-      (handle-exceptions
-       exn
-       (begin
-	 (debug:print-error 0 *default-log-port* "problem evaluating \"" str "\" in the shell environment, exn=" exn)
-	 #f)
-       (let ((cmdres (process:cmd-run->list (conc "echo " str))))
-	 (if (null? cmdres) ""
-	     (caar cmdres))))) ;; )
-
-;;======================================================================
-;; Make the regexp's needed globally available
-;;======================================================================
-
-(define configf:include-rx (regexp "^\\[include\\s+(.*)\\]\\s*$"))
-(define configf:script-rx  (regexp "^\\[scriptinc\\s+(\\S+)([^\\]]*)\\]\\s*$")) ;; include output from a script
-(define configf:section-rx (regexp "^\\[(.*)\\]\\s*$"))
-(define configf:blank-l-rx (regexp "^\\s*$"))
-(define configf:key-sys-pr (regexp "^(\\S+)\\s+\\[system\\s+(\\S+.*)\\]\\s*$"))
-(define configf:key-val-pr (regexp "^(\\S+)(\\s+(.*)|())$"))
-(define configf:key-no-val (regexp "^(\\S+)(\\s*)$"))
-(define configf:comment-rx (regexp "^\\s*#.*"))
-(define configf:cont-ln-rx (regexp "^(\\s+)(\\S+.*)$"))
-(define configf:settings   (regexp "^\\[configf:settings\\s+(\\S+)\\s+(\\S+)]\\s*$"))
-
-;; read a line and process any #{ ... } constructs
-
-(define configf:var-expand-regex (regexp "^(.*)#\\{(scheme|system|shell|getenv|get|runconfigs-get|rget|scm|sh|rp|gv|g|mtrah)\\s+([^\\}\\{]*)\\}(.*)"))
-
-(define (configf:system ht cmd)
-  (system cmd)
-  )
-
-(define configf:imports "(import commonmod (prefix mtargs args:))")
+(define (configf:write-alist cdat fname)
+  (if (not (common:faux-lock fname))
+      (debug:print 0 *default-log-port* "INFO: Could not get lock on " fname))
+  (let* ((dat  (configf:config->alist cdat))
+         (res
+          (begin
+            (with-output-to-file fname ;; first write out the file
+              (lambda ()
+                (pp dat)))
+            
+            (if (common:file-exists? fname)   ;; now verify it is readable
+                (if (configf:read-alist fname)
+                    #t ;; data is good.
+                    (begin
+                      (handle-exceptions
+			  exn
+			(begin
+			  (debug:print 0 *default-log-port* "deleting " fname " failed, exn=" exn)
+			  #f)
+			(debug:print 0 *default-log-port* "WARNING: content " dat " for cache " fname " is not readable. Deleting generated file.")
+			(delete-file fname))
+                      #f))
+                #f))))
+    (common:faux-unlock fname)
+    res))
+
+;; pathenvvar will set the named var to the path of the config
+(define (find-and-read-config fname #!key (environ-patt #f)(given-toppath #f)(pathenvvar #f))
+  (let* ((curr-dir   (current-directory))
+         (configinfo (find-config fname toppath: given-toppath))
+	 (toppath    (car configinfo))
+	 (configfile (cadr configinfo))
+	 (set-fields (lambda (curr-section next-section ht path)
+		       (let ((field-names (if ht (common:get-fields ht) '()))
+			     (target      (or (getenv "MT_TARGET")(args:get-arg "-reqtarg")(args:get-arg "-target"))))
+			 (debug:print-info 9 *default-log-port* "set-fields with field-names=" field-names " target=" target " curr-section=" curr-section " next-section=" next-section " path=" path " ht=" ht)
+			 (if (not (null? field-names))(keys:target-set-args field-names target #f))))))
+    (if toppath (change-directory toppath)) 
+    (if (and toppath pathenvvar)(setenv pathenvvar toppath))
+    (let ((configdat  (if configfile 
+			  (read-config configfile #f #t environ-patt: environ-patt post-section-procs: (list (cons "^fields$" set-fields)) #f))))
+      (if toppath (change-directory curr-dir))
+      (list configdat toppath configfile fname))))
 
 (define (configf:process-line l ht allow-system #!key (linenum #f))
   (let loop ((res l))
     (if (string? res)
 	(let ((matchdat (string-search configf:var-expand-regex res)))
@@ -170,27 +145,10 @@
 			 (debug:print-info 9 *default-log-port* "for line \"" l "\"\n command:  " cmd " took " delta " seconds to run with output:\n   " result)))))
 		(loop (conc prestr result poststr)))
 	      res))
 	res)))
 
-;; Run a shell command and return the output as a string
-(define (shell cmd)
-  (let* ((output (process:cmd-run->list cmd))
-	 (res    (car output))
-	 (status (cadr output)))
-    (if (equal? status 0)
-	(let ((outres (string-intersperse 
-		       res
-		       "\n")))
-	  (debug:print-info 4 *default-log-port* "shell result:\n" outres)
-	  outres)
-	(begin ;; why is this printing to error-port and not using debug:print? -mrw-
-	  (with-output-to-port (current-error-port)
-	    (lambda ()
-	      (print "ERROR: " cmd " returned bad exit code " status)))
-	  ""))))
-
 ;; this was inline but I'm pretty sure that is a hold over from when it was *very* simple ...
 ;;
 (define (configf:read-line p ht allow-processing settings)
   (let loop ((inl (read-line p)))
     (let ((cont-line (and (string? inl)
@@ -214,62 +172,10 @@
 	    (if (and (string? res)  ;; must set to "no" to force NOT trimming trailing spaces
 		     (not (equal? (hash-table-ref/default settings "trim-trailing-spaces" "yes") "no")))
 		(string-substitute "\\s+$" "" res)
 		res))))))
 
-(define (configf:cfgdat->env-alist section cfgdat-ht allow-system)
-  (filter
-   (lambda (pair)
-     (let* ((var (car pair))
-            (val (cdr pair)))
-       (cons var
-             (cond
-              ((and allow-system (procedure? val)) ;; if we decided to use something other than #t or #f for allow-system ('return-procs or 'return-string) , this may become problematic
-               (val))
-              ((procedure? val) #f)
-              ((string? val) val)
-              (else "#f")))))
-   (append
-    (hash-table-ref/default cfgdat-ht "default" '())
-    (if (equal? section "default") '() (hash-table-ref/default cfgdat-ht section '())))))
-
-(define (calc-allow-system allow-system section sections)
-  (if sections
-      (and (or (equal? "default" section)
-	       (member section sections))
-	   allow-system) ;; account for sections and return allow-system as it might be a symbol such as return-strings
-      allow-system))
-    
-;; given a config hash and a section name, apply that section to all matching sections (using wildcard % or regex if /..../)
-;; remove the section when done so that there is no downstream clobbering
-;;
-(define (configf:apply-wildcards ht section-name)
-  (if (hash-table-exists? ht section-name)
-      (let* ((vars  (hash-table-ref ht section-name))
-	     (rxstr (if (string-contains section-name "%")
-			(string-substitute (regexp "%") ".*" section-name)
-			(string-substitute (regexp "^/(.*)/$") "\\1" section-name)))
-	     (rx    (regexp rxstr)))
-	;; (print "\nsection-name: " section-name " rxstr: " rxstr)
-        (for-each
-         (lambda (section)
-	   (if section
-	       (let ((same-section (string=? section-name section))
-		     (rx-match     (string-match rx section)))
-		 ;; (print "section: " section " vars: " vars " same-section: " same-section " rx-match: " rx-match)
-		 (if (and (not same-section) rx-match)
-		     (for-each
-		      (lambda (bundle)
-			;; (print "bundle: " bundle)
-			(let ((key  (car bundle))
-			      (val  (cadr bundle))
-			      (meta (if (> (length bundle) 2)(caddr bundle) #f)))
-			  (hash-table-set! ht section (configf:assoc-safe-add (hash-table-ref ht section) key val metadata: meta))))
-		      vars)))))
-         (hash-table-keys ht))))
-  ht)
-
 ;; read a config file, returns hash table of alists
 
 ;; read a config file, returns hash table of alists
 ;; adds to ht if given (must be #f otherwise)
 ;; allow-system:
@@ -505,232 +411,17 @@
 		     (set! var-flag #f)
 		     (loop (configf:read-line inp res (calc-allow-system allow-system curr-section-name sections) settings) curr-section-name #f #f))))
           ) ;; end loop
         )))
   
-;; pathenvvar will set the named var to the path of the config
-(define (find-and-read-config fname #!key (environ-patt #f)(given-toppath #f)(pathenvvar #f))
-  (let* ((curr-dir   (current-directory))
-         (configinfo (find-config fname toppath: given-toppath))
-	 (toppath    (car configinfo))
-	 (configfile (cadr configinfo))
-	 (set-fields (lambda (curr-section next-section ht path)
-		       (let ((field-names (if ht (common:get-fields ht) '()))
-			     (target      (or (getenv "MT_TARGET")(args:get-arg "-reqtarg")(args:get-arg "-target"))))
-			 (debug:print-info 9 *default-log-port* "set-fields with field-names=" field-names " target=" target " curr-section=" curr-section " next-section=" next-section " path=" path " ht=" ht)
-			 (if (not (null? field-names))(keys:target-set-args field-names target #f))))))
-    (if toppath (change-directory toppath)) 
-    (if (and toppath pathenvvar)(setenv pathenvvar toppath))
-    (let ((configdat  (if configfile 
-			  (read-config configfile #f #t environ-patt: environ-patt post-section-procs: (list (cons "^fields$" set-fields)) #f))))
-      (if toppath (change-directory curr-dir))
-      (list configdat toppath configfile fname))))
-
-(define (configf:lookup cfgdat section var)
-  (if (hash-table? cfgdat)
-      (let ((sectdat (hash-table-ref/default cfgdat section '())))
-	(if (null? sectdat)
-	    #f
-	    (let ((match (assoc var sectdat)))
-	      (if match ;; (and match (list? match)(> (length match) 1))
-		  (cadr match)
-		  #f))
-	    ))
-      #f))
-
-;; use to have definitive setting:
-;;  [foo]
-;;  var yes
-;;
-;;  (configf:var-is? cfgdat "foo" "var" "yes") => #t
-;;
-(define (configf:var-is? cfgdat section var expected-val)
-  (equal? (configf:lookup cfgdat section var) expected-val))
-
-;; redefines
-(define config-lookup configf:lookup)
-(define configf:read-file read-config)
-
-;; safely look up a value that is expected to be a number, return
-;; a default (#f unless provided)
-;;
-(define (configf:lookup-number cfgdat section varname #!key (default #f))
-  (let* ((val (configf:lookup cfgdat section varname))
-         (res (if val
-                  (string->number (string-substitute "\\s+" "" val #t))
-                  #f)))
-    (cond
-     (res  res)
-     (val  (debug:print 0 *default-log-port* "ERROR: no number found for [" section "], " varname ", got: " val))
-     (else default))))
-
-(define (configf:section-vars cfgdat section)
-  (let ((sectdat (hash-table-ref/default cfgdat section '())))
-    (if (null? sectdat)
-	'()
-	(map car sectdat))))
-
-(define (configf:get-section cfgdat section)
-  (hash-table-ref/default cfgdat section '()))
-
-(define (configf:set-section-var cfgdat section var val)
-  (let ((sectdat (configf:get-section cfgdat section)))
-    (hash-table-set! cfgdat section
-                     (configf:assoc-safe-add sectdat var val))))
-
-;;======================================================================
-;;(append (filter (lambda (x)(not (assoc var sectdat))) sectdat)
-;;	    (list var val))))
-
 (define (setup)
   (let* ((configf (find-config "megatest.config"))
 	 (config  (if configf (read-config configf #f #t) #f)))
     (if config
 	(setenv "RUN_AREA_HOME" (pathname-directory configf)))
     config))
 
-;;======================================================================
-;; Non destructive writing of config file
-;;======================================================================
-
-(define (configf:compress-multi-lines fdat)
-  ;; step 1.5 - compress any continued lines
-  (if (null? fdat) fdat
-	(let loop ((hed (car fdat))
-		   (tal (cdr fdat))
-		   (cur "")
-		   (led #f)
-		   (res '()))
-	  ;; ALL WHITESPACE LEADING LINES ARE TACKED ON!!
-	  ;;  1. remove led whitespace
-	  ;;  2. tack on to hed with "\n"
-	  (let ((match (string-match configf:cont-ln-rx hed)))
-	    (if match ;; blast! have to deal with a multiline
-		(let* ((lead (cadr match))
-		       (lval (caddr match))
-		       (newl (conc cur "\n" lval)))
-		  (if (not led)(set! led lead))
-		  (if (null? tal) 
-		      (set! fdat (append fdat (list newl)))
-		      (loop (car tal)(cdr tal) newl led res))) ;; NB// not tacking newl onto res
-		(let ((newres (if led 
-				  (append res (list cur hed))
-				  (append res (list hed)))))
-		  ;; prev was a multiline
-		  (if (null? tal)
-		      newres
-		      (loop (car tal)(cdr tal) "" #f newres))))))))
-
-;; note: I'm cheating a little here. I merely replace "\n" with "\n         "
-(define (configf:expand-multi-lines fdat)
-  ;; step 1.5 - compress any continued lines
-  (if (null? fdat) fdat
-      (let loop ((hed (car fdat))
-		 (tal (cdr fdat))
-		 (res '()))
-	(let ((newres (append res (list (string-substitute (regexp "\n") "\n         " hed #t)))))
-	  (if (null? tal)
-	      newres
-	      (loop (car tal)(cdr tal) newres))))))
-
-(define (configf:file->list fname)
-  (if (common:file-exists? fname)
-      (let ((inp (open-input-file fname)))
-	(let loop ((inl (read-line inp))
-		   (res '()))
-	  (if (eof-object? inl)
-	      (begin
-		(close-input-port inp)
-		(reverse res))
-	      (loop (read-line inp)(cons inl res)))))
-      '()))
-
-;;======================================================================
-;; Write a config
-;;   0. Given a refererence data structure "indat"
-;;   1. Open the output file and read it into a list
-;;   2. Flatten any multiline entries
-;;   3. Modify values per contents of "indat" and remove absent values
-;;   4. Append new values to the section (immediately after last legit entry)
-;;   5. Write out the new list 
-;;======================================================================
-
-(define (configf:write-config indat fname #!key (required-sections '()))
-  (let* (;; step 1: Open the output file and read it into a list
-	 (fdat       (configf:file->list fname))
-	 (refdat  (make-hash-table))
-	 (sechash (make-hash-table)) ;; current section hash, init with hash for "default" section
-	 (new     #f) ;; put the line to be used in new, if it is to be deleted the set new to #f
-	 (secname #f))
-
-    ;; step 2: Flatten multiline entries
-    (if (not (null? fdat))(set! fdat (configf:compress-multi-lines fdat)))
-
-    ;; step 3: Modify values per contents of "indat" and remove absent values
-    (if (not (null? fdat))
-	(let loop ((hed  (car fdat))
-		   (tal  (cadr fdat))
-		   (res  '())
-		   (lnum 0))
-	  (regex-case 
-	   hed
-	   (configf:comment-rx _                  (set! res (append res (list hed)))) ;; (loop (read-line inp) curr-section-name #f #f))
-	   (configf:blank-l-rx _                  (set! res (append res (list hed)))) ;; (loop (read-line inp) curr-section-name #f #f))
-	   (configf:section-rx ( x section-name ) (let ((section-hash (hash-table-ref/default refdat section-name #f)))
-					    (if (not section-hash)
-						(let ((newhash (make-hash-table)))
-						  (hash-table-set! refdat section-name newhash) ;; was refhash - not sure that refdat is correct here
-						  (set! sechash newhash))
-						(set! sechash section-hash))
-					    (set! new hed) ;; will append this at the bottom of the loop
-					    (set! secname section-name)
-					    ))
-	   ;; No need to process key cmd, let it fall though to key val
-	   (configf:key-val-pr ( x key val      )
-		       (let ((newval (configf:lookup indat secname key))) ;; was sec, bug or correct?
-			 ;; can handle newval == #f here => that means key is removed
-			 (cond 
-			  ((equal? newval val)
-			   (set! res (append res (list hed))))
-			  ((not newval) ;; key has been removed
-			   (set! new #f))
-			  ((not (equal? newval val))
-			     (hash-table-set! sechash key newval)
-			     (set! new (conc key " " newval)))
-			  (else
-			   (debug:print-error 0 *default-log-port* "problem parsing line number " lnum "\"" hed "\"")))))
-	   (else
-	    (debug:print-error 0 *default-log-port* "Problem parsing line num " lnum " :\n   " hed )))
-	  (if (not (null? tal))
-	      (loop (car tal)(cdr tal)(if new (append res (list new)) res)(+ lnum 1)))
-	  ;; drop to here when done processing, res contains modified list of lines
-	  (set! fdat res)))
-
-    ;; step 4: Append new values to the section
-    (for-each 
-     (lambda (section)
-       (let ((sdat   '()) ;; append needed bits here
-	     (svars  (configf:section-vars indat section)))
-	 (for-each 
-	  (lambda (var)
-	    (let ((val (configf:lookup refdat section var)))
-	      (if (not val) ;; this one is new
-		  (begin
-		    (if (null? sdat)(set! sdat (list (conc "[" section "]"))))
-		    (set! sdat (append sdat (list (conc var " " val))))))))
-	  svars)
-	 (set! fdat (append fdat sdat))))
-     (delete-duplicates (append required-sections (hash-table-keys indat))))
-
-    ;; step 5: Write out new file
-    (with-output-to-file fname 
-      (lambda ()
-	(for-each 
-	 (lambda (line)
-	   (print line))
-	 (configf:expand-multi-lines fdat))))))
-
 ;;======================================================================
 ;; refdb
 ;;======================================================================
 
 ;; reads a refdb into an assoc array of assoc arrays
@@ -760,95 +451,10 @@
 		   ;; (set! data (append data (list (list sheet-name ref-assoc))))))
 		   (set! data (cons (list sheet-name ref-assoc) data))))
 	       sheets)
 	      (list data "NO ERRORS"))))))
 
-;; map over all pairs in a three level hierarchial alist and apply a function to the keys/val
-;;
-(define (configf:map-all-hier-alist data proc #!key (initproc1 #f)(initproc2 #f)(initproc3 #f))
-  (for-each 
-   (lambda (sheetname)
-     (let* ((sheettmp  (assoc sheetname data))
-	    (sheetdat  (if sheettmp (cadr sheettmp) '())))
-       (if initproc1 (initproc1 sheetname))
-       (for-each 
-	(lambda (sectionname)
-	  (let* ((sectiontmp  (assoc sectionname sheetdat))
-		 (sectiondat  (if sectiontmp (cadr sectiontmp) '())))
-	    (if initproc2 (initproc2 sheetname sectionname))
-	    (for-each
-	     (lambda (varname)
-	       (let* ((valtmp (assoc varname sectiondat))
-		      (val    (if valtmp (cadr valtmp) "")))
-		 (proc sheetname sectionname varname val)))
-	     (map car sectiondat))))
-	(map car sheetdat))))
-   (map car data))
-  data)
-
-;;======================================================================
-;;  C O N F I G   T O / F R O M   A L I S T
-;;======================================================================
-
-(define (configf:config->alist cfgdat)
-  (hash-table->alist cfgdat))
-
-(define (configf:alist->config adat)
-  (let ((ht (make-hash-table)))
-    (for-each
-     (lambda (section)
-       (hash-table-set! ht (car section)(cdr section)))
-     adat)
-    ht))
-
-;; if 
-(define (configf:read-alist fname)
-  (handle-exceptions
-      exn
-    (begin
-      (debug:print 0 *default-log-port* "read of alist " fname " failed. exn=" exn)
-      #f)
-    (configf:alist->config
-     (with-input-from-file fname read))))
-
-(define (configf:write-alist cdat fname)
-  (if (not (common:faux-lock fname))
-      (debug:print 0 *default-log-port* "INFO: Could not get lock on " fname))
-  (let* ((dat  (configf:config->alist cdat))
-         (res
-          (begin
-            (with-output-to-file fname ;; first write out the file
-              (lambda ()
-                (pp dat)))
-            
-            (if (common:file-exists? fname)   ;; now verify it is readable
-                (if (configf:read-alist fname)
-                    #t ;; data is good.
-                    (begin
-                      (handle-exceptions
-			  exn
-			(begin
-			  (debug:print 0 *default-log-port* "deleting " fname " failed, exn=" exn)
-			  #f)
-			(debug:print 0 *default-log-port* "WARNING: content " dat " for cache " fname " is not readable. Deleting generated file.")
-			(delete-file fname))
-                      #f))
-                #f))))
-    (common:faux-unlock fname)
-    res))
-  
-;; convert hierarchial list to ini format
-;;
-(define (configf:config->ini data)
-  (map 
-   (lambda (section)
-     (let ((section-name (car section))
-	   (section-dat  (cdr section)))
-       (print "\n[" section-name "]")
-       (map (lambda (dat-pair)
-	      (let* ((var (car dat-pair))
-		     (val (cadr dat-pair))
-		     (fname (if (> (length dat-pair) 2)(caddr dat-pair) #f)))
-		(if fname (print "# " var "=>" fname))
-		(print var " " val)))
-	    section-dat))) ;;       (print "section-dat: " section-dat))
-   (hash-table->alist data)))
+
+
+;; redefines
+(define config-lookup configf:lookup)
+(define configf:read-file read-config)

Index: configfmod.scm
==================================================================
--- configfmod.scm
+++ configfmod.scm
@@ -17,59 +17,436 @@
 ;;     along with Megatest.  If not, see <http://www.gnu.org/licenses/>.
 
 ;;======================================================================
 
 (declare (unit configfmod))
-;; (declare (uses mtargs))
-;; (declare (uses debugprint))
-;; (declare (uses keysmod))
+(declare (uses debugprint))
+(declare (uses commonmod))
+(declare (uses processmod))
 
 (module configfmod
 *	
 
-(import srfi-1
-  
-;;        scheme
-;;
-;;	big-chicken        ;; more of a reminder than anything ...
-;;	chicken.base
-;;	chicken.condition
-;;	chicken.file
-;;	chicken.io
-;;	chicken.pathname
-;;	chicken.port
-;;	chicken.pretty-print
-;;	chicken.process
-;;	chicken.process-context
-;;	chicken.process-context.posix
-;;	chicken.sort
-;;	chicken.string
-;;	chicken.time
-;;	chicken.eval
-;;	
-;;	debugprint
-;;	(prefix mtargs args:)
-;;	pkts
-;;	keysmod
-;;
-;;	(prefix base64 base64:)
-;;	(prefix dbi dbi:)
-;;	(prefix sqlite3 sqlite3:)
-;;	(srfi 18)
-;;	directory-utils
-;;	format
-;;	matchable
-;;	md5
-;;	message-digest
-;;	regex
-;;	regex-case
-;;	sparse-vectors
-;;	srfi-1
-;;	srfi-13
-;;	srfi-69
-;;	stack
-;;	typed-records
-;;	z3
-	
-	)
+(import scheme
+        chicken
+	extras
+	files
+	ports
+	srfi-1
+	srfi-13
+	srfi-69
+	
+	posix
+	data-structures
+
+	regex
+	regex-case
+	
+	)
+
+(import debugprint
+	commonmod
+	processmod)
+
+;; return list (path fullpath configname)
+(define (find-config configname #!key (toppath #f))
+  (if toppath
+      (let ((cfname (conc toppath "/" configname)))
+	(if (common:file-exists? cfname)
+	    (list toppath cfname configname)
+	    (list #f      #f     #f)))
+      (let* ((cwd (string-split (current-directory) "/")))
+	(let loop ((dir cwd))
+	  (let* ((path     (conc "/" (string-intersperse dir "/")))
+		 (fullpath (conc path "/" configname)))
+	    (if (common:file-exists? fullpath)
+		(list path fullpath configname)
+		(let ((remcwd (take dir (- (length dir) 1))))
+		  (if (null? remcwd)
+		      (list #f #f #f) ;;  #f #f) 
+		  (loop remcwd)))))))))
+
+(define (configf:assoc-safe-add alist key val #!key (metadata #f))
+  (let ((newalist (filter (lambda (x)(not (equal? key (car x)))) alist)))
+    (append newalist (list (if metadata
+			       (list key val metadata)
+			       (list key val))))))
+
+;; this is used in megatestqa/ext.scm.
+;; remove it from here and there by 12/31/21
+;; (define config:assoc-safe-add configf:assoc-safe-add)
+
+(define (configf:section-var-set! cfgdat section-name var value #!key (metadata #f))
+  (hash-table-set! cfgdat section-name
+		   (configf:assoc-safe-add
+		    (hash-table-ref/default cfgdat section-name '())
+		    var value metadata: metadata)))
+
+(define (configf:eval-string-in-environment str)
+  ;; (if (or (string-null? str)
+  ;;	  (equal? "!" (substring str 0 1))) ;; null string or starts with ! are preserved but NOT set in the environment
+      str
+      (handle-exceptions
+       exn
+       (begin
+	 (debug:print-error 0 *default-log-port* "problem evaluating \"" str "\" in the shell environment, exn=" exn)
+	 #f)
+       (let ((cmdres (process:cmd-run->list (conc "echo " str))))
+	 (if (null? cmdres) ""
+	     (caar cmdres))))) ;; )
+
+;;======================================================================
+;; Make the regexp's needed globally available
+;;======================================================================
+
+(define configf:include-rx (regexp "^\\[include\\s+(.*)\\]\\s*$"))
+(define configf:script-rx  (regexp "^\\[scriptinc\\s+(\\S+)([^\\]]*)\\]\\s*$")) ;; include output from a script
+(define configf:section-rx (regexp "^\\[(.*)\\]\\s*$"))
+(define configf:blank-l-rx (regexp "^\\s*$"))
+(define configf:key-sys-pr (regexp "^(\\S+)\\s+\\[system\\s+(\\S+.*)\\]\\s*$"))
+(define configf:key-val-pr (regexp "^(\\S+)(\\s+(.*)|())$"))
+(define configf:key-no-val (regexp "^(\\S+)(\\s*)$"))
+(define configf:comment-rx (regexp "^\\s*#.*"))
+(define configf:cont-ln-rx (regexp "^(\\s+)(\\S+.*)$"))
+(define configf:settings   (regexp "^\\[configf:settings\\s+(\\S+)\\s+(\\S+)]\\s*$"))
+
+;; read a line and process any #{ ... } constructs
+
+(define configf:var-expand-regex (regexp "^(.*)#\\{(scheme|system|shell|getenv|get|runconfigs-get|rget|scm|sh|rp|gv|g|mtrah)\\s+([^\\}\\{]*)\\}(.*)"))
+
+(define (configf:system ht cmd)
+  (system cmd)
+  )
+
+(define configf:imports "(import commonmod (prefix mtargs args:))")
+
+;; Run a shell command and return the output as a string
+(define (shell cmd)
+  (let* ((output (process:cmd-run->list cmd))
+	 (res    (car output))
+	 (status (cadr output)))
+    (if (equal? status 0)
+	(let ((outres (string-intersperse 
+		       res
+		       "\n")))
+	  (debug:print-info 4 *default-log-port* "shell result:\n" outres)
+	  outres)
+	(begin ;; why is this printing to error-port and not using debug:print? -mrw-
+	  (with-output-to-port (current-error-port)
+	    (lambda ()
+	      (print "ERROR: " cmd " returned bad exit code " status)))
+	  ""))))
+
+(define (configf:cfgdat->env-alist section cfgdat-ht allow-system)
+  (filter
+   (lambda (pair)
+     (let* ((var (car pair))
+            (val (cdr pair)))
+       (cons var
+             (cond
+              ((and allow-system (procedure? val)) ;; if we decided to use something other than #t or #f for allow-system ('return-procs or 'return-string) , this may become problematic
+               (val))
+              ((procedure? val) #f)
+              ((string? val) val)
+              (else "#f")))))
+   (append
+    (hash-table-ref/default cfgdat-ht "default" '())
+    (if (equal? section "default") '() (hash-table-ref/default cfgdat-ht section '())))))
+
+(define (calc-allow-system allow-system section sections)
+  (if sections
+      (and (or (equal? "default" section)
+	       (member section sections))
+	   allow-system) ;; account for sections and return allow-system as it might be a symbol such as return-strings
+      allow-system))
+    
+;; given a config hash and a section name, apply that section to all matching sections (using wildcard % or regex if /..../)
+;; remove the section when done so that there is no downstream clobbering
+;;
+(define (configf:apply-wildcards ht section-name)
+  (if (hash-table-exists? ht section-name)
+      (let* ((vars  (hash-table-ref ht section-name))
+	     (rxstr (if (string-contains section-name "%")
+			(string-substitute (regexp "%") ".*" section-name)
+			(string-substitute (regexp "^/(.*)/$") "\\1" section-name)))
+	     (rx    (regexp rxstr)))
+	;; (print "\nsection-name: " section-name " rxstr: " rxstr)
+        (for-each
+         (lambda (section)
+	   (if section
+	       (let ((same-section (string=? section-name section))
+		     (rx-match     (string-match rx section)))
+		 ;; (print "section: " section " vars: " vars " same-section: " same-section " rx-match: " rx-match)
+		 (if (and (not same-section) rx-match)
+		     (for-each
+		      (lambda (bundle)
+			;; (print "bundle: " bundle)
+			(let ((key  (car bundle))
+			      (val  (cadr bundle))
+			      (meta (if (> (length bundle) 2)(caddr bundle) #f)))
+			  (hash-table-set! ht section (configf:assoc-safe-add (hash-table-ref ht section) key val metadata: meta))))
+		      vars)))))
+         (hash-table-keys ht))))
+  ht)
+
+(define (configf:lookup cfgdat section var)
+  (if (hash-table? cfgdat)
+      (let ((sectdat (hash-table-ref/default cfgdat section '())))
+	(if (null? sectdat)
+	    #f
+	    (let ((match (assoc var sectdat)))
+	      (if match ;; (and match (list? match)(> (length match) 1))
+		  (cadr match)
+		  #f))
+	    ))
+      #f))
+
+;; use to have definitive setting:
+;;  [foo]
+;;  var yes
+;;
+;;  (configf:var-is? cfgdat "foo" "var" "yes") => #t
+;;
+(define (configf:var-is? cfgdat section var expected-val)
+  (equal? (configf:lookup cfgdat section var) expected-val))
+
+;; safely look up a value that is expected to be a number, return
+;; a default (#f unless provided)
+;;
+(define (configf:lookup-number cfgdat section varname #!key (default #f))
+  (let* ((val (configf:lookup cfgdat section varname))
+         (res (if val
+                  (string->number (string-substitute "\\s+" "" val #t))
+                  #f)))
+    (cond
+     (res  res)
+     (val  (debug:print 0 *default-log-port* "ERROR: no number found for [" section "], " varname ", got: " val))
+     (else default))))
+
+(define (configf:section-vars cfgdat section)
+  (let ((sectdat (hash-table-ref/default cfgdat section '())))
+    (if (null? sectdat)
+	'()
+	(map car sectdat))))
+
+(define (configf:get-section cfgdat section)
+  (hash-table-ref/default cfgdat section '()))
+
+(define (configf:set-section-var cfgdat section var val)
+  (let ((sectdat (configf:get-section cfgdat section)))
+    (hash-table-set! cfgdat section
+                     (configf:assoc-safe-add sectdat var val))))
+
+;;======================================================================
+;;(append (filter (lambda (x)(not (assoc var sectdat))) sectdat)
+;;	    (list var val))))
+
+;;======================================================================
+;; Non destructive writing of config file
+;;======================================================================
+
+(define (configf:compress-multi-lines fdat)
+  ;; step 1.5 - compress any continued lines
+  (if (null? fdat) fdat
+	(let loop ((hed (car fdat))
+		   (tal (cdr fdat))
+		   (cur "")
+		   (led #f)
+		   (res '()))
+	  ;; ALL WHITESPACE LEADING LINES ARE TACKED ON!!
+	  ;;  1. remove led whitespace
+	  ;;  2. tack on to hed with "\n"
+	  (let ((match (string-match configf:cont-ln-rx hed)))
+	    (if match ;; blast! have to deal with a multiline
+		(let* ((lead (cadr match))
+		       (lval (caddr match))
+		       (newl (conc cur "\n" lval)))
+		  (if (not led)(set! led lead))
+		  (if (null? tal) 
+		      (set! fdat (append fdat (list newl)))
+		      (loop (car tal)(cdr tal) newl led res))) ;; NB// not tacking newl onto res
+		(let ((newres (if led 
+				  (append res (list cur hed))
+				  (append res (list hed)))))
+		  ;; prev was a multiline
+		  (if (null? tal)
+		      newres
+		      (loop (car tal)(cdr tal) "" #f newres))))))))
+
+;; note: I'm cheating a little here. I merely replace "\n" with "\n         "
+(define (configf:expand-multi-lines fdat)
+  ;; step 1.5 - compress any continued lines
+  (if (null? fdat) fdat
+      (let loop ((hed (car fdat))
+		 (tal (cdr fdat))
+		 (res '()))
+	(let ((newres (append res (list (string-substitute (regexp "\n") "\n         " hed #t)))))
+	  (if (null? tal)
+	      newres
+	      (loop (car tal)(cdr tal) newres))))))
+
+(define (configf:file->list fname)
+  (if (common:file-exists? fname)
+      (let ((inp (open-input-file fname)))
+	(let loop ((inl (read-line inp))
+		   (res '()))
+	  (if (eof-object? inl)
+	      (begin
+		(close-input-port inp)
+		(reverse res))
+	      (loop (read-line inp)(cons inl res)))))
+      '()))
+
+;;======================================================================
+;; Write a config
+;;   0. Given a refererence data structure "indat"
+;;   1. Open the output file and read it into a list
+;;   2. Flatten any multiline entries
+;;   3. Modify values per contents of "indat" and remove absent values
+;;   4. Append new values to the section (immediately after last legit entry)
+;;   5. Write out the new list 
+;;======================================================================
+
+(define (configf:write-config indat fname #!key (required-sections '()))
+  (let* (;; step 1: Open the output file and read it into a list
+	 (fdat       (configf:file->list fname))
+	 (refdat  (make-hash-table))
+	 (sechash (make-hash-table)) ;; current section hash, init with hash for "default" section
+	 (new     #f) ;; put the line to be used in new, if it is to be deleted the set new to #f
+	 (secname #f))
+
+    ;; step 2: Flatten multiline entries
+    (if (not (null? fdat))(set! fdat (configf:compress-multi-lines fdat)))
+
+    ;; step 3: Modify values per contents of "indat" and remove absent values
+    (if (not (null? fdat))
+	(let loop ((hed  (car fdat))
+		   (tal  (cadr fdat))
+		   (res  '())
+		   (lnum 0))
+	  (regex-case 
+	   hed
+	   (configf:comment-rx _                  (set! res (append res (list hed)))) ;; (loop (read-line inp) curr-section-name #f #f))
+	   (configf:blank-l-rx _                  (set! res (append res (list hed)))) ;; (loop (read-line inp) curr-section-name #f #f))
+	   (configf:section-rx ( x section-name ) (let ((section-hash (hash-table-ref/default refdat section-name #f)))
+					    (if (not section-hash)
+						(let ((newhash (make-hash-table)))
+						  (hash-table-set! refdat section-name newhash) ;; was refhash - not sure that refdat is correct here
+						  (set! sechash newhash))
+						(set! sechash section-hash))
+					    (set! new hed) ;; will append this at the bottom of the loop
+					    (set! secname section-name)
+					    ))
+	   ;; No need to process key cmd, let it fall though to key val
+	   (configf:key-val-pr ( x key val      )
+		       (let ((newval (configf:lookup indat secname key))) ;; was sec, bug or correct?
+			 ;; can handle newval == #f here => that means key is removed
+			 (cond 
+			  ((equal? newval val)
+			   (set! res (append res (list hed))))
+			  ((not newval) ;; key has been removed
+			   (set! new #f))
+			  ((not (equal? newval val))
+			     (hash-table-set! sechash key newval)
+			     (set! new (conc key " " newval)))
+			  (else
+			   (debug:print-error 0 *default-log-port* "problem parsing line number " lnum "\"" hed "\"")))))
+	   (else
+	    (debug:print-error 0 *default-log-port* "Problem parsing line num " lnum " :\n   " hed )))
+	  (if (not (null? tal))
+	      (loop (car tal)(cdr tal)(if new (append res (list new)) res)(+ lnum 1)))
+	  ;; drop to here when done processing, res contains modified list of lines
+	  (set! fdat res)))
+
+    ;; step 4: Append new values to the section
+    (for-each 
+     (lambda (section)
+       (let ((sdat   '()) ;; append needed bits here
+	     (svars  (configf:section-vars indat section)))
+	 (for-each 
+	  (lambda (var)
+	    (let ((val (configf:lookup refdat section var)))
+	      (if (not val) ;; this one is new
+		  (begin
+		    (if (null? sdat)(set! sdat (list (conc "[" section "]"))))
+		    (set! sdat (append sdat (list (conc var " " val))))))))
+	  svars)
+	 (set! fdat (append fdat sdat))))
+     (delete-duplicates (append required-sections (hash-table-keys indat))))
+
+    ;; step 5: Write out new file
+    (with-output-to-file fname 
+      (lambda ()
+	(for-each 
+	 (lambda (line)
+	   (print line))
+	 (configf:expand-multi-lines fdat))))))
+
+;; map over all pairs in a three level hierarchial alist and apply a function to the keys/val
+;;
+(define (configf:map-all-hier-alist data proc #!key (initproc1 #f)(initproc2 #f)(initproc3 #f))
+  (for-each 
+   (lambda (sheetname)
+     (let* ((sheettmp  (assoc sheetname data))
+	    (sheetdat  (if sheettmp (cadr sheettmp) '())))
+       (if initproc1 (initproc1 sheetname))
+       (for-each 
+	(lambda (sectionname)
+	  (let* ((sectiontmp  (assoc sectionname sheetdat))
+		 (sectiondat  (if sectiontmp (cadr sectiontmp) '())))
+	    (if initproc2 (initproc2 sheetname sectionname))
+	    (for-each
+	     (lambda (varname)
+	       (let* ((valtmp (assoc varname sectiondat))
+		      (val    (if valtmp (cadr valtmp) "")))
+		 (proc sheetname sectionname varname val)))
+	     (map car sectiondat))))
+	(map car sheetdat))))
+   (map car data))
+  data)
+
+;;======================================================================
+;;  C O N F I G   T O / F R O M   A L I S T
+;;======================================================================
+
+(define (configf:config->alist cfgdat)
+  (hash-table->alist cfgdat))
+
+(define (configf:alist->config adat)
+  (let ((ht (make-hash-table)))
+    (for-each
+     (lambda (section)
+       (hash-table-set! ht (car section)(cdr section)))
+     adat)
+    ht))
+
+;; if 
+(define (configf:read-alist fname)
+  (handle-exceptions
+      exn
+    (begin
+      (debug:print 0 *default-log-port* "read of alist " fname " failed. exn=" exn)
+      #f)
+    (configf:alist->config
+     (with-input-from-file fname read))))
+
+  
+;; convert hierarchial list to ini format
+;;
+(define (configf:config->ini data)
+  (map 
+   (lambda (section)
+     (let ((section-name (car section))
+	   (section-dat  (cdr section)))
+       (print "\n[" section-name "]")
+       (map (lambda (dat-pair)
+	      (let* ((var (car dat-pair))
+		     (val (cadr dat-pair))
+		     (fname (if (> (length dat-pair) 2)(caddr dat-pair) #f)))
+		(if fname (print "# " var "=>" fname))
+		(print var " " val)))
+	    section-dat))) ;;       (print "section-dat: " section-dat))
+   (hash-table->alist data)))
+
+
 )
 

Index: process.scm
==================================================================
--- process.scm
+++ process.scm
@@ -23,216 +23,10 @@
 ;;======================================================================
 
 (use regex directory-utils)
 (declare (unit process))
 (declare (uses debugprint))
-
-(import debugprint)
-
-(define (process:conservative-read port)
-  (let loop ((res ""))
-    (if (not (eof-object? (peek-char port)))
-	(loop (conc res (read-char port)))
-	res)))
-
-(define (process:cmd-run-with-stderr->list cmd . params)
-  ;; (print "Called with cmd=" cmd ", proc=" proc ", params=" params)
-;;  (handle-exceptions
-;;   exn
-;;   (begin
-;;     (print "ERROR:  Failed to run command: " cmd " " (string-intersperse params " "))
-;;     (print "       " ((condition-property-accessor 'exn 'message) exn))
-;;     #f)
-   (let-values (((fh fho pid fhe) (if (null? params)
-				      (process* cmd)
-				      (process* cmd params))))
-       (let loop ((curr (read-line fh))
-		  (result  '()))
-	 (let ((errstr (process:conservative-read fhe)))
-	   (if (not (string=? errstr ""))
-	       (set! result (append result (list errstr)))))
-       (if (not (eof-object? curr))
-	   (loop (read-line fh)
-		 (append result (list curr)))
-	   (begin
-	     (close-input-port fh)
-	     (close-input-port fhe)
-	     (close-output-port fho)
-	     result))))) ;; )
-
-(define (process:cmd-run-with-stderr-and-exitcode->list cmd . params)
-  ;; (print "Called with cmd=" cmd ", proc=" proc ", params=" params)
-;;  (handle-exceptions
-;;   exn
-;;   (begin
-;;     (print "ERROR:  Failed to run command: " cmd " " (string-intersperse params " "))
-;;     (print "       " ((condition-property-accessor 'exn 'message) exn))
-;;     #f)
-   (let-values (((fh fho pid fhe) (if (null? params)
-				      (process* cmd)
-				      (process* cmd params))))
-       (let loop ((curr (read-line fh))
-		  (result  '()))
-	 (let ((errstr (process:conservative-read fhe)))
-	   (if (not (string=? errstr ""))
-	       (set! result (append result (list errstr)))))
-       (if (not (eof-object? curr))
-	   (loop (read-line fh)
-		 (append result (list curr)))
-	   (begin
-             (let-values (((anotherpid normalexit? exitstatus)  (process-wait pid)))
-	     (close-input-port fh)
-	     (close-input-port fhe)
-	     (close-output-port fho)
-               (list result (if normalexit? exitstatus -1))))))))
-
-(define (process:cmd-run-proc-each-line cmd proc . params)
-  ;; (print "Called with cmd=" cmd ", proc=" proc ", params=" params)
-  (handle-exceptions
-   exn
-   (begin
-     (debug:print 0 *default-log-port* "ERROR:  Failed to run command: " cmd " " (string-intersperse params " "))
-     (debug:print 0 *default-log-port* " message: " ((condition-property-accessor 'exn 'message) exn))
-     (debug:print 5 *default-log-port* "exn=" (condition->list exn))
-     #f)
-   (let-values (((fh fho pid) (if (null? params)
-				  (process cmd)
-				  (process cmd params))))
-       (let loop ((curr (read-line fh))
-		(result  '()))
-       (if (not (eof-object? curr))
-	   (loop (read-line fh)
-		 (append result (list (proc curr))))
-	   (begin
-	     (close-input-port fh)
-	     ;;(close-input-port fhe)
-	     (close-output-port fho)
-	     result))))))
-
-(define (process:cmd-run-proc-each-line-alt cmd proc)
-  (let* ((fh (open-input-pipe cmd))
-         (res (port-proc->list fh proc))
-         (status (close-input-pipe fh)))
-    (if (eq? status 0) res #f)))
-
-(define (process:cmd-run->list cmd #!key (delta-env-alist-or-hash-table '()))
-  (common:with-env-vars
-   delta-env-alist-or-hash-table
-   (lambda ()
-     (let* ((fh (open-input-pipe cmd))
-            (res (port->list fh))
-            (status (close-input-pipe fh)))
-       (list res status)))))
-   
-(define (port->list fh)
-  (if (eof-object? fh) #f
-      (let loop ((curr (read-line fh))
-                 (result '()))
-        (if (not (eof-object? curr))
-            (loop (read-line fh)
-                  (append result (list curr)))
-            result))))
-
-(define (port-proc->list fh proc)
-  (if (eof-object? fh) #f
-      (let loop ((curr (proc (read-line fh)))
-                 (result '()))
-        (if (not (eof-object? curr))
-            (loop (let ((l (read-line fh)))
-                    (if (eof-object? l) l (proc l)))
-                  (append result (list curr)))
-            result))))
-
-;; here is an example line where the shell is sh or bash
-;; "find / -print 2&>1 > findall.log"
-(define (run-n-wait cmdline #!key (params #f)(print-cmd #f)(run-dir #f))
-  (if print-cmd 
-      (debug:print 0 *default-log-port* 
-		   (if (string? print-cmd)
-		       print-cmd
-		       "")
-		   (if run-dir (conc "Run in " run-dir ";") "")
-		   cmdline
-		   (if params
-		       (conc " " (string-intersperse params " "))
-		       "")))
-  (if (and run-dir
-	   (directory-exists? run-dir))
-      (push-directory run-dir))
-  (let ((pid (if params
-		 (process-run cmdline params)
-		 (process-run cmdline))))
-    (let loop ((i 0))
-      (let-values (((pid-val exit-status exit-code) (process-wait pid #t)))
-         (if (eq? pid-val 0)
-	     (begin
-	       (thread-sleep! 2)
-	       (loop (+ i 1)))
-	     (begin
-	       (if (and run-dir
-			(directory-exists? run-dir))
-		   (pop-directory))
-	       (values pid-val exit-status exit-code)))))))
-  
-;;======================================================================
-;; MISC PROCESS RELATED STUFF
-;;======================================================================
-
-(define (process:children proc)
-  (with-input-from-pipe
-   (conc "ps h --ppid " (current-process-id) " -o pid")
-   (lambda ()
-     (let loop ((inl (read-line))
-		(res '()))
-       (if (eof-object? inl)
-	   (reverse res)
-	   (let ((pid (string->number inl)))
-	     (if proc (proc pid))
-	     (loop (read-line) (cons pid res))))))))
-
-(define (process:alive? pid)
-  (handle-exceptions
-   exn
-   ;; possibly pid is a process not a child, look in /proc to see if it is running still
-   (common:file-exists? (conc "/proc/" pid))
-   (let-values (((rpid exit-type exit-signal)(process-wait pid #t)))
-       (and (number? rpid)
-	    (equal? rpid pid)))))
-
-(define (process:alive-on-host? host pid)
-  (let ((cmd (conc "ssh " host " ps -o pid= -p " pid)))
-    (common:generic-ssh
-     cmd
-   ;; 
-   ;; handle-exceptions
-   ;; 	exn
-   ;;  (begin
-   ;; 	(debug:print 0 *default-log-port* "failed to identify if process " pid ", on host " host " is alive. exn=" exn)
-   ;; 	#f) ;; anything goes wrong - assume the process in NOT running.
-   ;;  (with-input-from-pipe 
-   ;;   cmd
-     (lambda ()
-       (let loop ((inl (read-line)))
-	 (if (eof-object? inl)
-	     #f
-	     (let* ((clean-str (string-substitute "^[^\\d]*([0-9]+)[^\\d]*$" "\\1" inl))
-		    (innum     (string->number clean-str)))
-	       (and innum
-		    (eq? pid innum))))))
-     #f
-     (lambda ()
-       (debug:print 0 *default-log-port* "failed to identify if process "
-		    pid", on host "host" is alive. exn="exn)))))
-
-
-(define (process:get-sub-pids pid)
-  (with-input-from-pipe
-   (conc "pstree -A -p " pid) ;; | tr 'a-z\\-+`()\\.' ' ' " pid)
-   (lambda ()
-     (let loop ((inl (read-line))
-		(res '()))
-       (if (eof-object? inl)
-	   (reverse res)
-	   (let ((nums (map string->number
-			    (string-split-fields "\\d+" inl))))
-	     (loop (read-line)
-		   (append res nums))))))))
+(declare (uses processmod))
+
+(import debugprint
+	processmod)
+

ADDED   processmod.scm
Index: processmod.scm
==================================================================
--- /dev/null
+++ processmod.scm
@@ -0,0 +1,307 @@
+;;======================================================================
+;; Copyright 2017, Matthew Welland.
+;; 
+;; This file is part of Megatest.
+;; 
+;;     Megatest is free software: you can redistribute it and/or modify
+;;     it under the terms of the GNU General Public License as published by
+;;     the Free Software Foundation, either version 3 of the License, or
+;;     (at your option) any later version.
+;; 
+;;     Megatest is distributed in the hope that it will be useful,
+;;     but WITHOUT ANY WARRANTY; without even the implied warranty of
+;;     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;;     GNU General Public License for more details.
+;; 
+;;     You should have received a copy of the GNU General Public License
+;;     along with Megatest.  If not, see <http://www.gnu.org/licenses/>.
+
+;;======================================================================
+
+(declare (unit processmod))
+(declare (uses debugprint))
+(declare (uses commonmod))
+
+(use srfi-69)
+
+(module processmod
+	*
+
+(import scheme)
+(cond-expand
+ (chicken-4
+  
+  (import chicken
+	  ports
+	  
+	  (prefix sqlite3 sqlite3:)
+	  data-structures
+	  directory-utils
+	  extras
+	  files
+	  matchable
+	  md5
+	  message-digest
+	  pathname-expand
+	  posix
+	  posix-extras
+	  regex
+	  regex-case
+	  srfi-1
+	  srfi-18
+	  srfi-69
+	  typed-records
+
+	  debugprint
+	  commonmod
+	  )
+  (use srfi-69))
+ (chicken-5
+  (import (prefix sqlite3 sqlite3:)
+	  ;; data-structures
+	  ;; extras
+	  ;; files
+	  ;; posix
+	  ;; posix-extras
+	  chicken.base
+	  chicken.condition
+	  chicken.file
+	  chicken.file.posix
+	  chicken.io
+	  chicken.pathname
+	  chicken.port
+	  chicken.process
+	  chicken.process-context
+	  chicken.process-context.posix
+	  chicken.sort
+	  chicken.string
+	  chicken.time
+	  chicken.time.posix
+	  
+	  matchable
+	  md5
+	  message-digest
+	  pathname-expand
+	  regex
+	  regex-case
+	  srfi-1
+	  srfi-18
+	  srfi-69
+	  typed-records
+	  system-information
+
+	  debugprint
+	  commonmod
+  )))
+
+(define (process:conservative-read port)
+  (let loop ((res ""))
+    (if (not (eof-object? (peek-char port)))
+	(loop (conc res (read-char port)))
+	res)))
+
+(define (process:cmd-run-with-stderr->list cmd . params)
+  ;; (print "Called with cmd=" cmd ", proc=" proc ", params=" params)
+;;  (handle-exceptions
+;;   exn
+;;   (begin
+;;     (print "ERROR:  Failed to run command: " cmd " " (string-intersperse params " "))
+;;     (print "       " ((condition-property-accessor 'exn 'message) exn))
+;;     #f)
+   (let-values (((fh fho pid fhe) (if (null? params)
+				      (process* cmd)
+				      (process* cmd params))))
+       (let loop ((curr (read-line fh))
+		  (result  '()))
+	 (let ((errstr (process:conservative-read fhe)))
+	   (if (not (string=? errstr ""))
+	       (set! result (append result (list errstr)))))
+       (if (not (eof-object? curr))
+	   (loop (read-line fh)
+		 (append result (list curr)))
+	   (begin
+	     (close-input-port fh)
+	     (close-input-port fhe)
+	     (close-output-port fho)
+	     result))))) ;; )
+
+(define (process:cmd-run-with-stderr-and-exitcode->list cmd . params)
+  ;; (print "Called with cmd=" cmd ", proc=" proc ", params=" params)
+;;  (handle-exceptions
+;;   exn
+;;   (begin
+;;     (print "ERROR:  Failed to run command: " cmd " " (string-intersperse params " "))
+;;     (print "       " ((condition-property-accessor 'exn 'message) exn))
+;;     #f)
+   (let-values (((fh fho pid fhe) (if (null? params)
+				      (process* cmd)
+				      (process* cmd params))))
+       (let loop ((curr (read-line fh))
+		  (result  '()))
+	 (let ((errstr (process:conservative-read fhe)))
+	   (if (not (string=? errstr ""))
+	       (set! result (append result (list errstr)))))
+       (if (not (eof-object? curr))
+	   (loop (read-line fh)
+		 (append result (list curr)))
+	   (begin
+             (let-values (((anotherpid normalexit? exitstatus)  (process-wait pid)))
+	     (close-input-port fh)
+	     (close-input-port fhe)
+	     (close-output-port fho)
+               (list result (if normalexit? exitstatus -1))))))))
+
+(define (process:cmd-run-proc-each-line cmd proc . params)
+  ;; (print "Called with cmd=" cmd ", proc=" proc ", params=" params)
+  (handle-exceptions
+   exn
+   (begin
+     (debug:print 0 *default-log-port* "ERROR:  Failed to run command: " cmd " " (string-intersperse params " "))
+     (debug:print 0 *default-log-port* " message: " ((condition-property-accessor 'exn 'message) exn))
+     (debug:print 5 *default-log-port* "exn=" (condition->list exn))
+     #f)
+   (let-values (((fh fho pid) (if (null? params)
+				  (process cmd)
+				  (process cmd params))))
+       (let loop ((curr (read-line fh))
+		(result  '()))
+       (if (not (eof-object? curr))
+	   (loop (read-line fh)
+		 (append result (list (proc curr))))
+	   (begin
+	     (close-input-port fh)
+	     ;;(close-input-port fhe)
+	     (close-output-port fho)
+	     result))))))
+
+(define (process:cmd-run-proc-each-line-alt cmd proc)
+  (let* ((fh (open-input-pipe cmd))
+         (res (port-proc->list fh proc))
+         (status (close-input-pipe fh)))
+    (if (eq? status 0) res #f)))
+
+(define (process:cmd-run->list cmd #!key (delta-env-alist-or-hash-table '()))
+  (common:with-env-vars
+   delta-env-alist-or-hash-table
+   (lambda ()
+     (let* ((fh (open-input-pipe cmd))
+            (res (port->list fh))
+            (status (close-input-pipe fh)))
+       (list res status)))))
+   
+(define (port->list fh)
+  (if (eof-object? fh) #f
+      (let loop ((curr (read-line fh))
+                 (result '()))
+        (if (not (eof-object? curr))
+            (loop (read-line fh)
+                  (append result (list curr)))
+            result))))
+
+(define (port-proc->list fh proc)
+  (if (eof-object? fh) #f
+      (let loop ((curr (proc (read-line fh)))
+                 (result '()))
+        (if (not (eof-object? curr))
+            (loop (let ((l (read-line fh)))
+                    (if (eof-object? l) l (proc l)))
+                  (append result (list curr)))
+            result))))
+
+;; here is an example line where the shell is sh or bash
+;; "find / -print 2&>1 > findall.log"
+(define (run-n-wait cmdline #!key (params #f)(print-cmd #f)(run-dir #f))
+  (if print-cmd 
+      (debug:print 0 *default-log-port* 
+		   (if (string? print-cmd)
+		       print-cmd
+		       "")
+		   (if run-dir (conc "Run in " run-dir ";") "")
+		   cmdline
+		   (if params
+		       (conc " " (string-intersperse params " "))
+		       "")))
+  (if (and run-dir
+	   (directory-exists? run-dir))
+      (push-directory run-dir))
+  (let ((pid (if params
+		 (process-run cmdline params)
+		 (process-run cmdline))))
+    (let loop ((i 0))
+      (let-values (((pid-val exit-status exit-code) (process-wait pid #t)))
+         (if (eq? pid-val 0)
+	     (begin
+	       (thread-sleep! 2)
+	       (loop (+ i 1)))
+	     (begin
+	       (if (and run-dir
+			(directory-exists? run-dir))
+		   (pop-directory))
+	       (values pid-val exit-status exit-code)))))))
+  
+;;======================================================================
+;; MISC PROCESS RELATED STUFF
+;;======================================================================
+
+(define (process:children proc)
+  (with-input-from-pipe
+   (conc "ps h --ppid " (current-process-id) " -o pid")
+   (lambda ()
+     (let loop ((inl (read-line))
+		(res '()))
+       (if (eof-object? inl)
+	   (reverse res)
+	   (let ((pid (string->number inl)))
+	     (if proc (proc pid))
+	     (loop (read-line) (cons pid res))))))))
+
+(define (process:alive? pid)
+  (handle-exceptions
+   exn
+   ;; possibly pid is a process not a child, look in /proc to see if it is running still
+   (common:file-exists? (conc "/proc/" pid))
+   (let-values (((rpid exit-type exit-signal)(process-wait pid #t)))
+       (and (number? rpid)
+	    (equal? rpid pid)))))
+
+(define (process:alive-on-host? host pid)
+  (let ((cmd (conc "ssh " host " ps -o pid= -p " pid)))
+    (common:generic-ssh
+     cmd
+   ;; 
+   ;; handle-exceptions
+   ;; 	exn
+   ;;  (begin
+   ;; 	(debug:print 0 *default-log-port* "failed to identify if process " pid ", on host " host " is alive. exn=" exn)
+   ;; 	#f) ;; anything goes wrong - assume the process in NOT running.
+   ;;  (with-input-from-pipe 
+   ;;   cmd
+     (lambda ()
+       (let loop ((inl (read-line)))
+	 (if (eof-object? inl)
+	     #f
+	     (let* ((clean-str (string-substitute "^[^\\d]*([0-9]+)[^\\d]*$" "\\1" inl))
+		    (innum     (string->number clean-str)))
+	       (and innum
+		    (eq? pid innum))))))
+     #f
+     (lambda ()
+       (debug:print 0 *default-log-port* "failed to identify if process "
+		    pid", on host "host" is alive.")))))
+
+
+(define (process:get-sub-pids pid)
+  (with-input-from-pipe
+   (conc "pstree -A -p " pid) ;; | tr 'a-z\\-+`()\\.' ' ' " pid)
+   (lambda ()
+     (let loop ((inl (read-line))
+		(res '()))
+       (if (eof-object? inl)
+	   (reverse res)
+	   (let ((nums (map string->number
+			    (string-split-fields "\\d+" inl))))
+	     (loop (read-line)
+		   (append res nums))))))))
+
+
+)