Index: Makefile ================================================================== --- Makefile +++ Makefile @@ -32,12 +32,14 @@ # diff-report.scm cgisetup/models/pgdb.scm # module source files # MSRCFILES = # ftail.scm rmtmod.scm commonmod.scm removed -MSRCFILES = autoload.scm dbi.scm ducttape-lib.scm pkts.scm stml2.scm cookie.scm mutils.scm \ - mtargs.scm apimod.scm commonmod.scm dbmod.scm rmtmod.scm debugprint.scm +MSRCFILES = autoload.scm dbi.scm ducttape-lib.scm pkts.scm stml2.scm \ + cookie.scm mutils.scm mtargs.scm apimod.scm commonmod.scm \ + dbmod.scm rmtmod.scm debugprint.scm mtver.scm \ + csv-xml.scm servermod.scm hostinfo.scm # commonmod.scm dbmod.scm adjutant.scm ulex.scm \ # rmtmod.scm apimod.scm GUISRCF = dashboard-context-menu.scm dashboard-tests.scm \ @@ -60,11 +62,11 @@ # module dependencies mofiles/stml2.o : mofiles/dbi.o mofiles/dbi.o : mofiles/autoload.o mofiles/apimod.o mofiles/dbmod.o mofiles/rmtmod.o : mofiles/commonmod.o -mofiles/commonmod.o : mofiles/mtargs.o mofiles/debugprint.o +mofiles/commonmod.o : mofiles/mtargs.o mofiles/debugprint.o mofiles/megatest-version.o ADTLSCR=mt_laststep mt_runstep mt_ezstep HELPERS=$(addprefix $(PREFIX)/bin/,$(ADTLSCR)) DEPLOYHELPERS=$(addprefix deploytarg/,$(ADTLSCR)) MTESTHASH=$(shell fossil info|grep checkout:| awk '{print $$2}') Index: common.scm ================================================================== --- common.scm +++ common.scm @@ -730,55 +730,10 @@ (print-call-chain (current-error-port)) #f) (read (open-input-string (base64:base64-decode instr)))) (read (open-input-string (z3:decode-buffer (base64:base64-decode instr)))))) -;; dot-locking egg seems not to work, using this for now -;; if lock is older than expire-time then remove it and try again -;; to get the lock -;; -(define (common:simple-file-lock fname #!key (expire-time 300)) - (let ((fmod-time (handle-exceptions - ext - (current-seconds) - (file-modification-time fname)))) - (if (common:file-exists? fname) - (if (> (- (current-seconds) fmod-time) expire-time) - (begin - (handle-exceptions exn #f (delete-file* fname)) - (common:simple-file-lock fname expire-time: expire-time)) - #f) - (let ((key-string (conc (get-host-name) "-" (current-process-id)))) - (with-output-to-file fname - (lambda () - (print key-string))) - (thread-sleep! 0.25) - (if (common:file-exists? fname) - (handle-exceptions exn - #f - (with-input-from-file fname - (lambda () - (equal? key-string (read-line))))) - #f))))) - -(define (common:simple-file-lock-and-wait fname #!key (expire-time 300)) - (let ((end-time (+ expire-time (current-seconds)))) - (let loop ((got-lock (common:simple-file-lock fname expire-time: expire-time))) - (if got-lock - #t - (if (> end-time (current-seconds)) - (begin - (thread-sleep! 3) - (loop (common:simple-file-lock fname expire-time: expire-time))) - #f))))) - -(define (common:simple-file-release-lock fname) - (handle-exceptions - exn - #f ;; I don't really care why this failed (at least for now) - (delete-file* fname))) - ;;====================================================================== ;; S T A T E S A N D S T A T U S E S ;;====================================================================== ;; BBnote: *common:std-states* - dashboard filter control and test control state buttons defined here; used in set-fields-panel and dboard:make-controls @@ -1010,11 +965,12 @@ ;;====================================================================== ;; currently the primary job of the watchdog is to run the sync back to megatest.db from the db in /tmp ;; if we are on the homehost and we are a server (by definition we are on the homehost if we are a server) ;; (define (common:readonly-watchdog dbstruct) - (thread-sleep! 0.05) ;; delay for startup + (let ((just-testing 0.0501)) + (thread-sleep! just-testing)) ;; (/ 1 20)) ;; 0.051) ;; delay for startup (debug:print-info 13 *default-log-port* "common:readonly-watchdog entered.") ;; sync megatest.db to /tmp/.../megatst.db (let* ((sync-cool-off-duration 3) (golden-mtdb (dbr:dbstruct-mtdb dbstruct)) (golden-mtpath (db:dbdat-get-path golden-mtdb)) Index: commonmod.scm ================================================================== --- commonmod.scm +++ commonmod.scm @@ -17,26 +17,37 @@ ;; along with Megatest. If not, see . ;;====================================================================== (declare (unit commonmod)) +(declare (uses mtver)) (module commonmod * -(import scheme chicken.base +(import scheme + chicken.base + chicken.condition + chicken.file + chicken.time + chicken.file.posix + chicken.process-context.posix + chicken.io + chicken.string + (prefix sqlite3 sqlite3:) + system-information typed-records md5 message-digest regex - srfi-1 srfi-18 srfi-69 + mtver ) ;;====================================================================== ;; CONTENTS ;; @@ -44,128 +55,54 @@ ;; misc conversion, data manipulation functions ;; testsuite and area utilites ;; ;;====================================================================== -(include "megatest-version.scm") (include "megatest-fossil-hash.scm") -;; (define (get-full-version) -;; (conc megatest-version "-" megatest-fossil-hash)) -;; -;; (define (version-signature) -;; (conc megatest-version "-" (substring megatest-fossil-hash 0 4))) -;; -;; -;; ;;====================================================================== -;; ;; config file utils -;; ;;====================================================================== -;; -;; (define (lookup cfgdat section var) -;; (if (hash-table? cfgdat) -;; (let ((sectdat (hash-table-ref/default cfgdat section '()))) -;; (if (null? sectdat) -;; #f -;; (let ((match (assoc var sectdat))) -;; (if match ;; (and match (list? match)(> (length match) 1)) -;; (cadr match) -;; #f)) -;; )) -;; #f)) -;; -;; ;; returns var key1=val1; key2=val2 ... as alist -;; (define (get-key-list cfgdat section var) -;; ;; convert string a=1; b=2; c=a silly thing; d= -;; (let ((valstr (lookup cfgdat section var))) -;; (if valstr -;; (val->alist valstr) -;; '()))) ;; should it return empty list or #f to indicate not set? -;; -;; -;; (define (get-section cfgdat section) -;; (hash-table-ref/default cfgdat section '())) -;; -;; ;;====================================================================== -;; ;; misc conversion, data manipulation functions -;; ;;====================================================================== -;; -;; ;; if it looks like a number -> convert it to a number, else return it -;; ;; -;; (define (lazy-convert inval) -;; (let* ((as-num (if (string? inval)(string->number inval) #f))) -;; (or as-num inval))) -;; -;; ;; to '((a . 1)(b . 2)(c . "a silly thing")(d . "")) -;; ;; -;; (define (val->alist val #!key (convert #f)) -;; (let ((val-list (string-split-fields ";\\s*" val #:infix))) -;; (if val-list -;; (map (lambda (x) -;; (let ((f (string-split-fields "\\s*=\\s*" x #:infix))) -;; (case (length f) -;; ((0) `(,#f)) ;; null string case -;; ((1) `(,(string->symbol (car f)))) -;; ((2) `(,(string->symbol (car f)) . -;; ,(let ((inval (cadr f))) -;; (if convert (lazy-convert inval) inval)))) -;; (else f)))) -;; (filter (lambda (x) -;; (not (string-match "^\\s*" x))) -;; val-list)) -;; '()))) -;; -;; ;;====================================================================== -;; ;; testsuite and area utilites -;; ;;====================================================================== -;; -;; (define (get-testsuite-name toppath configdat) -;; (or (lookup configdat "setup" "area-name") -;; (lookup configdat "setup" "testsuite") -;; (get-environment-variable "MT_TESTSUITE_NAME") -;; (if (string? toppath) -;; (pathname-file toppath) -;; #f))) -;; -;; (define (get-area-path-signature toppath #!optional (short #f)) -;; (let ((res (message-digest-string (md5-primitive) toppath))) -;; (if short -;; (substring res 0 4) -;; res))) -;; -;; (define (get-area-name configdat toppath #!optional (short #f)) -;; ;; look up my area name in areas table (future) -;; ;; generate auto name -;; (conc (get-area-path-signature toppath short) -;; "-" -;; (get-testsuite-name toppath configdat))) -;; -;; ;; need generic find-record-with-var-nmatching-val -;; ;; -;; (define (path->area-record cfgdat path) -;; (let* ((areadat (get-cfg-areas cfgdat)) -;; (all (filter (lambda (x) -;; (let* ((keyvals (cdr x)) -;; (pth (alist-ref 'path keyvals))) -;; (equal? path pth))) -;; areadat))) -;; (if (null? all) -;; #f -;; (car all)))) ;; return first match -;; -;; ;; given a config return an alist of alists -;; ;; area-name => data -;; ;; -;; (define (get-cfg-areas cfgdat) -;; (let ((adat (get-section cfgdat "areas"))) -;; (map (lambda (entry) -;; `(,(car entry) . -;; ,(val->alist (cadr entry)))) -;; adat))) -;; -;; ;; (define (debug:print . params) #f) -;; ;; (define (debug:print-info . params) #f) -;; ;; -;; ;; (define (set-functions dbgp dbgpinfo) -;; ;; (set! debug:print dbgp) -;; ;; (set! debug:print-info dbgpinfo)) +;; dot-locking egg seems not to work, using this for now +;; if lock is older than expire-time then remove it and try again +;; to get the lock +;; +(define (common:simple-file-lock fname #!key (expire-time 300)) + (let ((fmod-time (handle-exceptions + ext + (current-seconds) + (file-modification-time fname)))) + (if (file-exists? fname) + (if (> (- (current-seconds) fmod-time) expire-time) + (begin + (handle-exceptions exn #f (delete-file* fname)) + (common:simple-file-lock fname expire-time: expire-time)) + #f) + (let ((key-string (conc (get-host-name) "-" (current-process-id)))) + (with-output-to-file fname + (lambda () + (print key-string))) + (thread-sleep! 0.251) + (if (file-exists? fname) + (handle-exceptions exn + #f + (with-input-from-file fname + (lambda () + (equal? key-string (read-line))))) + #f))))) + +(define (common:simple-file-lock-and-wait fname #!key (expire-time 300)) + (let ((end-time (+ expire-time (current-seconds)))) + (let loop ((got-lock (common:simple-file-lock fname expire-time: expire-time))) + (if got-lock + #t + (if (> end-time (current-seconds)) + (begin + (thread-sleep! 3) + (loop (common:simple-file-lock fname expire-time: expire-time))) + #f))))) + +(define (common:simple-file-release-lock fname) + (handle-exceptions + exn + #f ;; I don't really care why this failed (at least for now) + (delete-file* fname))) + ) Index: db.scm ================================================================== --- db.scm +++ db.scm @@ -16,97 +16,10 @@ ;; You should have received a copy of the GNU General Public License ;; along with Megatest. If not, see . ;; ;;====================================================================== -;;====================================================================== -;; Database access -;;====================================================================== - -;; dbstruct vector containing all the relevant dbs like main.db, megatest.db, run.db etc - -;; (use (srfi 18) extras tcp stack) -;; (use sqlite3 srfi-1 posix regex regex-case srfi-69 csv-xml s11n md5 message-digest base64 format dot-locking z3 typed-records matchable) -;; (import (prefix sqlite3 sqlite3:)) -;; (import (prefix base64 base64:)) -;; -;; (declare (unit db)) -;; (declare (uses common)) -;; (declare (uses keys)) -;; (declare (uses ods)) -;; (declare (uses client)) -;; (declare (uses mt)) -;; -;; (include "common_records.scm") -;; (include "db_records.scm") -;; (include "key_records.scm") -;; (include "run_records.scm") - -(define *number-of-writes* 0) -(define *number-non-write-queries* 0) - -;;====================================================================== -;; R E C O R D S -;;====================================================================== - -;; each db entry is a pair ( db . dbfilepath ) -;; I propose this record evolves into the area record -;; -(defstruct dbr:dbstruct - (tmpdb #f) - (dbstack #f) ;; stack for tmp db handles, do not initialize with a stack - (mtdb #f) - (refndb #f) - (homehost #f) ;; not used yet - (on-homehost #f) ;; not used yet - (read-only #f) - (stmt-cache (make-hash-table)) - (locdbs (make-hash-table)) ;; legacy junk in db_records - ) ;; goal is to converge on one struct for an area but for now it is too confusing - - -;; record for keeping state,status and count for doing roll-ups in -;; iterated tests -;; -(defstruct dbr:counts - (state #f) - (status #f) - (count 0)) - -;;====================================================================== -;; alist-of-alists -;;====================================================================== -;; -;; (define (db:aa-set! dat key1 key2 val) -;; (let loop (( - -;;====================================================================== -;; hash of hashs -;;====================================================================== - - -(define (db:hoh-set! dat key1 key2 val) - (let* ((subhash (hash-table-ref/default dat key1 #f))) - (if subhash - (hash-table-set! subhash key2 val) - (begin - (hash-table-set! dat key1 (make-hash-table)) - (db:hoh-set! dat key1 key2 val))))) - -(define (db:hoh-get dat key1 key2) - (let* ((subhash (hash-table-ref/default dat key1 #f))) - (and subhash - (hash-table-ref/default subhash key2 #f)))) - -(define (db:get-cache-stmth dbstruct db stmt) - (let* ((stmt-cache (dbr:dbstruct-stmt-cache dbstruct)) - (stmth (db:hoh-get stmt-cache db stmt))) - (or stmth - (let* ((newstmth (sqlite3:prepare db stmt))) - (db:hoh-set! stmt-cache db stmt newstmth) - newstmth)))) - ;;====================================================================== ;; SQLITE3 HELPERS ;;====================================================================== (define (db:general-sqlite-error-dump exn stmt . params) Index: dbmod.scm ================================================================== --- dbmod.scm +++ dbmod.scm @@ -27,19 +27,98 @@ chicken.base (prefix sqlite3 sqlite3:) typed-records srfi-18 - + srfi-69 + ) -(define (just-testing) - (print "JUST TESTING")) +;;====================================================================== +;; Database access +;;====================================================================== + +;; dbstruct vector containing all the relevant dbs like main.db, megatest.db, run.db etc + +;; (use (srfi 18) extras tcp stack) +;; (use sqlite3 srfi-1 posix regex regex-case srfi-69 csv-xml s11n md5 message-digest base64 format dot-locking z3 typed-records matchable) +;; (import (prefix sqlite3 sqlite3:)) +;; (import (prefix base64 base64:)) +;; +;; (declare (unit db)) +;; (declare (uses common)) +;; (declare (uses keys)) +;; (declare (uses ods)) +;; (declare (uses client)) +;; (declare (uses mt)) +;; +;; (include "common_records.scm") +;; (include "db_records.scm") +;; (include "key_records.scm") +;; (include "run_records.scm") + +(define *number-of-writes* 0) +(define *number-non-write-queries* 0) + +;;====================================================================== +;; R E C O R D S +;;====================================================================== + +;; each db entry is a pair ( db . dbfilepath ) +;; I propose this record evolves into the area record +;; +(defstruct dbr:dbstruct + (tmpdb #f) + (dbstack #f) ;; stack for tmp db handles, do not initialize with a stack + (mtdb #f) + (refndb #f) + (homehost #f) ;; not used yet + (on-homehost #f) ;; not used yet + (read-only #f) + (stmt-cache (make-hash-table)) + (locdbs (make-hash-table)) ;; legacy junk in db_records + ) ;; goal is to converge on one struct for an area but for now it is too confusing + + +;; record for keeping state,status and count for doing roll-ups in +;; iterated tests +;; +(defstruct dbr:counts + (state #f) + (status #f) + (count 0)) -;; (define (debug:print . params) #f) -;; (define (debug:print-info . params) #f) +;;====================================================================== +;; alist-of-alists +;;====================================================================== ;; -;; (define (set-functions dbgp dbgpinfo) -;; (set! debug:print dbgp) -;; (set! debug:print-info dbgpinfo)) +;; (define (db:aa-set! dat key1 key2 val) +;; (let loop (( + +;;====================================================================== +;; hash of hashs +;;====================================================================== + + +(define (db:hoh-set! dat key1 key2 val) + (let* ((subhash (hash-table-ref/default dat key1 #f))) + (if subhash + (hash-table-set! subhash key2 val) + (begin + (hash-table-set! dat key1 (make-hash-table)) + (db:hoh-set! dat key1 key2 val))))) + +(define (db:hoh-get dat key1 key2) + (let* ((subhash (hash-table-ref/default dat key1 #f))) + (and subhash + (hash-table-ref/default subhash key2 #f)))) + +(define (db:get-cache-stmth dbstruct db stmt) + (let* ((stmt-cache (dbr:dbstruct-stmt-cache dbstruct)) + (stmth (db:hoh-get stmt-cache db stmt))) + (or stmth + (let* ((newstmth (sqlite3:prepare db stmt))) + (db:hoh-set! stmt-cache db stmt newstmth) + newstmth)))) + ) Index: debugprint.scm ================================================================== --- debugprint.scm +++ debugprint.scm @@ -1,7 +1,7 @@ (declare (unit debugprint)) -(declare (uses margsmod)) +(declare (uses mtargs)) (module debugprint * ;;(import scheme chicken data-structures extras files ports) ADDED hostinfo.scm Index: hostinfo.scm ================================================================== --- /dev/null +++ hostinfo.scm @@ -0,0 +1,23 @@ +;;====================================================================== +;; Copyright 2019, Matthew Welland. +;; +;; This file is part of Megatest. +;; +;; Megatest is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. +;; +;; Megatest is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. +;; +;; You should have received a copy of the GNU General Public License +;; along with Megatest. If not, see . + +;;====================================================================== + +(declare (unit hostinfo)) + +(include "hostinfo/hostinfo.scm") Index: hostinfo/hostinfo.scm ================================================================== --- hostinfo/hostinfo.scm +++ hostinfo/hostinfo.scm @@ -56,11 +56,11 @@ (cond-expand [paranoia] [else (declare (no-bound-checks))]) -#> #include "hostinfo.h" <# +#> #include "../hostinfo/hostinfo.h" <# ;; (require-extension srfi-4 lolevel posix) (module hostinfo ;;; Short and sweet lookups Index: http-transport.scm ================================================================== --- http-transport.scm +++ http-transport.scm @@ -224,11 +224,11 @@ (set! *http-requests-in-progress* (- *http-requests-in-progress* 1)) (let loop ((etime (+ (current-seconds) 5))) ;; give up in five seconds (if (> *http-requests-in-progress* 0) (if (> etime (current-seconds)) (begin - (thread-sleep! 0.05) + (thread-sleep! 0.052) (loop etime)) (debug:print-error 0 *default-log-port* "requests still in progress after 5 seconds of waiting. I'm going to pass on cleaning up http connections")) (close-idle-connections!))) (set! *http-connections-next-cleanup* (+ (current-seconds) 10)) (mutex-unlock! *http-mutex*)) @@ -612,11 +612,11 @@ (th3 (make-thread (lambda () (debug:print-info 0 *default-log-port* "Server monitor thread started") (http-transport:keep-running) "Keep running")))) (thread-start! th2) - (thread-sleep! 0.25) ;; give the server time to settle before starting the keep-running monitor. + (thread-sleep! 0.252) ;; give the server time to settle before starting the keep-running monitor. (thread-start! th3) (set! *didsomething* #t) (thread-join! th2) (exit)))) DELETED megatest-version.scm Index: megatest-version.scm ================================================================== --- megatest-version.scm +++ /dev/null @@ -1,23 +0,0 @@ -;; Copyright 2006-2017, Matthew Welland. -;; -;; This file is part of Megatest. -;; -;; Megatest is free software: you can redistribute it and/or modify -;; it under the terms of the GNU General Public License as published by -;; the Free Software Foundation, either version 3 of the License, or -;; (at your option) any later version. -;; -;; Megatest is distributed in the hope that it will be useful, -;; but WITHOUT ANY WARRANTY; without even the implied warranty of -;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -;; GNU General Public License for more details. -;; -;; You should have received a copy of the GNU General Public License -;; along with Megatest. If not, see . - -;; Always use two or four digit decimal -;; 1.01, 1.02...1.10,1.11,1.1101 ... 1.99,2.00.. - -;; (declare (unit megatest-version)) - -(define megatest-version 1.6584) Index: megatest.scm ================================================================== --- megatest.scm +++ megatest.scm @@ -24,105 +24,122 @@ ;; (include "autoload/autoload.scm") ;; (include "dbi/dbi.scm") ;; (include "stml2/cookie.scm") ;; (include "stml2/stml2.scm") ;; (include "pkts/pkts.scm") -(include "csv-xml/csv-xml.scm") +;; (include "csv-xml/csv-xml.scm") ;; (include "ducttape/ducttape-lib.scm") -(include "hostinfo/hostinfo.scm") +;; (include "hostinfo/hostinfo.scm") (include "adjutant.scm") -(declare (uses mutils)) (declare (uses autoload)) (declare (uses pkts)) -(declare (uses ducttape-lib)) (declare (uses stml2)) (declare (uses cookie)) +(declare (uses csv-xml)) +(declare (uses hostinfo)) + +(declare (uses mutils)) +(declare (uses ducttape-lib)) (declare (uses mtargs)) (declare (uses commonmod)) (declare (uses apimod)) (declare (uses dbmod)) (declare (uses rmtmod)) - +(declare (uses servermod)) +(declare (uses mtver)) ;; (include "call-with-environment-variables/call-with-environment-variables.scm") (module megatest-main * - (import scheme - chicken.base - chicken.bitwise - chicken.condition - chicken.file - chicken.file.posix - chicken.format - chicken.io - chicken.irregex - chicken.pathname - chicken.port - chicken.pretty-print - chicken.process - chicken.process-context - chicken.process-context.posix - chicken.process.signal - chicken.random - chicken.repl - chicken.sort - chicken.string - chicken.tcp - chicken.time - chicken.time.posix - - (prefix sqlite3 sqlite3:) - (prefix base64 base64:) - address-info - csv-abnf - directory-utils - fmt - json - matchable - md5 - message-digest - queues - regex - regex-case - sql-de-lite - stack - typed-records - s11n - sparse-vectors - sxml-serializer - sxml-modifications - system-information - z3 - spiffy - uri-common - intarweb - http-client - spiffy-request-vars - intarweb - spiffy-directory-listing - - srfi-1 - srfi-4 - srfi-18 - srfi-13 - srfi-98 - srfi-69 - - ;; local modules - mutils - csv-xml - ducttape-lib - hostinfo - adjutant - ) - -;; (include "common.scm") -(include "megatest-version.scm") - + (import scheme + chicken.base + chicken.bitwise + chicken.condition + chicken.file + chicken.file.posix + chicken.format + chicken.io + chicken.irregex + chicken.pathname + chicken.port + chicken.pretty-print + chicken.process + chicken.process-context + chicken.process-context.posix + chicken.process.signal + chicken.random + chicken.repl + chicken.sort + chicken.string + chicken.tcp + chicken.time + chicken.time.posix + + (prefix sqlite3 sqlite3:) + (prefix base64 base64:) + address-info + csv-abnf + directory-utils + fmt + json + matchable + md5 + message-digest + queues + regex + regex-case + sql-de-lite + stack + typed-records + s11n + sparse-vectors + sxml-serializer + sxml-modifications + system-information + z3 + spiffy + uri-common + intarweb + http-client + spiffy-request-vars + intarweb + spiffy-directory-listing + + srfi-1 + srfi-4 + srfi-18 + srfi-13 + srfi-98 + srfi-69 + + ;; local modules + adjutant + csv-xml + ducttape-lib + hostinfo + mtver + mutils + autoload + cookie + csv-xml + ducttape-lib + mtargs + pkts + stml2 + (prefix dbi dbi:) + + apimod + commonmod + dbmod + rmtmod + servermod + + ) + ;; fake out readline usage of toplevel-command (define (toplevel-command . a) #f) (define setenv set-environment-variable!) (define unsetenv unset-environment-variable!) @@ -145,25 +162,24 @@ ;; (declare (uses tasks)) ;; only used for debugging. ;; (declare (uses env)) ;; (declare (uses diff-report)) ;; (declare (uses ftail)) ;; (import ftail) + +(define (blahblah)(thread-sleep! 1.234)) (define *db* #f) ;; this is only for the repl, do not use in general!!!! (include "common_records.scm") (include "key_records.scm") (include "db_records.scm") (include "run_records.scm") (include "test_records.scm") + +(include "common.scm") (include "megatest-fossil-hash.scm") -(import (prefix dbi dbi:)) -(import stml2) -(import pkts) - -(include "common.scm") (include "configf.scm") (include "margs.scm") (include "process.scm") (include "keys.scm") (include "portlogger.scm") ADDED mtver.scm Index: mtver.scm ================================================================== --- /dev/null +++ mtver.scm @@ -0,0 +1,29 @@ +;; Copyright 2006-2017, Matthew Welland. +;; +;; This file is part of Megatest. +;; +;; Megatest is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. +;; +;; Megatest is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. +;; +;; You should have received a copy of the GNU General Public License +;; along with Megatest. If not, see . + +;; Always use two or four digit decimal +;; 1.01, 1.02...1.10,1.11,1.1101 ... 1.99,2.00.. + +(declare (unit mtver)) + +(module mtver * + +(import scheme chicken.module) + +(define megatest-version 1.6584) + +) Index: rmt.scm ================================================================== --- rmt.scm +++ rmt.scm @@ -68,11 +68,11 @@ (if (> attemptnum 2) (debug:print 0 *default-log-port* "INFO: attemptnum in rmt:send-receive is " attemptnum)) (cond - ((> attemptnum 2) (thread-sleep! 0.05)) + ((> attemptnum 2) (thread-sleep! 0.053)) ((> attemptnum 10) (thread-sleep! 0.5)) ((> attemptnum 20) (thread-sleep! 1))) (if (and (> attemptnum 5) (= 0 (modulo attemptnum 15))) (begin (server:run *toppath*) (thread-sleep! 3))) @@ -614,11 +614,11 @@ (mutex-unlock! multi-run-mutex)) (debug:print-error 0 *default-log-port* "get-tests-for-run-mindata failed for run-id " hed ", testpatt " testpatt ", states " states ", status " status ", not-in " not-in)))) (conc "multi-run-thread for run-id " hed))) (newthreads (cons newthread threads))) (thread-start! newthread) - (thread-sleep! 0.05) ;; give that thread some time to start + (thread-sleep! 0.054) ;; give that thread some time to start (if (null? tal) newthreads (loop (car tal)(cdr tal) newthreads)))))) result)) Index: rmtmod.scm ================================================================== --- rmtmod.scm +++ rmtmod.scm @@ -19,14 +19,10 @@ ;;====================================================================== (declare (unit rmtmod)) (declare (uses commonmod)) (declare (uses apimod)) -;; (declare (uses apimod.import)) -(declare (uses ulex)) - -;; (include "ulex/ulex.scm") (module rmtmod * (import scheme Index: runs.scm ================================================================== --- runs.scm +++ runs.scm @@ -1268,11 +1268,11 @@ ((not have-resources) ;; simply try again after waiting a second (if (runs:lownoise "no resources" 60) (debug:print-info 1 *default-log-port* "no resources to run new tests, waiting ...")) ;; Have gone back and forth on this but db starvation is an issue. ;; wait one second before looking again to run jobs. - (thread-sleep! 0.25) + (thread-sleep! 0.253) ;; could have done hed tal here but doing car/cdr of newtal to rotate tests (list (car newtal)(cdr newtal) reg reruns)) ;; This is the final stage, everything is in place so launch the test ;; Index: server.scm ================================================================== --- server.scm +++ server.scm @@ -35,17 +35,10 @@ ;; ;; (declare (uses daemon)) ;; ;; (include "common_records.scm") ;; (include "db_records.scm") -(define (server:make-server-url hostport) - (if (not hostport) - #f - (conc "http://" (car hostport) ":" (cadr hostport)))) - -(define *server-loop-heart-beat* (current-seconds)) - ;;====================================================================== ;; P K T S S T U F F ;;====================================================================== ;; ??? @@ -206,11 +199,11 @@ (debug:print-info 0 *default-log-port* "Server is in dbprep at " (current-seconds)) (thread-sleep! 25) ) (debug:print-info 0 *default-log-port* "Unable to get server info from " logf " at " (current-seconds)) ) - (list #f #f #f #f))))))))) + (list #f #f #f #f))))))))) ;; get a list of servers with all relevant data ;; ( mod-time host port start-time pid ) ;; (define (server:get-list areapath #!key (limit #f)) @@ -229,12 +222,11 @@ (exn ()(debug:print 0 *default-log-port* "ERROR: Unknown error attemtping to get server list. exn=" exn))) (directory-exists? (conc areapath "/logs"))) '())) ;; Get the list of server logs that do not contain "exiting". Ignore logs for servers that have exited. - (let* ((server-logs-cmd (conc "grep -iL exiting " areapath "/logs/server-*-*.log")) - (server-logs (string-split (string-chomp (call-with-input-pipe server-logs-cmd read-string)))) + (let* ((server-logs (server:get-logs-list areapath)) (num-serv-logs (length server-logs))) (if (or (null? server-logs) (= num-serv-logs 0)) (let () (debug:print 1 *default-log-port* "There are no servers running") '() @@ -382,11 +374,11 @@ (begin (debug:print-info 0 *default-log-port* "Writing " start-flag) (with-output-to-file start-flag (lambda () (print server-key))) - (thread-sleep! 0.25) + (thread-sleep! 0.254) (let ((res (with-input-from-file start-flag (lambda () (read-line))))) (equal? server-key res)))) #t ;; (system (conc "touch " start-flag)) ;; lazy but safe @@ -715,11 +707,11 @@ (if (common:low-noise-print 30) (debug:print-info 0 *default-log-port* "Exiting watchdog timer, *time-to-exit* = " *time-to-exit*" pid="(current-process-id) ))))) (define (server:writable-watchdog-deltasync dbstruct) - (thread-sleep! 0.05) ;; delay for startup + (thread-sleep! 0.054) ;; delay for startup (let ((legacy-sync (common:run-sync?)) (sync-stale-seconds (configf:lookup-number *configdat* "server" "sync-stale-seconds" default: 300)) (debug-mode (debug:debug-mode 1)) (last-time (current-seconds)) (no-sync-db (db:open-no-sync-db)) ADDED servermod.scm Index: servermod.scm ================================================================== --- /dev/null +++ servermod.scm @@ -0,0 +1,53 @@ +;;====================================================================== +;; Copyright 2017, Matthew Welland. +;; +;; This file is part of Megatest. +;; +;; Megatest is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. +;; +;; Megatest is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. +;; +;; You should have received a copy of the GNU General Public License +;; along with Megatest. If not, see . + +;;====================================================================== + +(declare (unit servermod)) + +(module servermod + * + +(import scheme + chicken.base + chicken.string + chicken.process + chicken.io + chicken.time + + (prefix sqlite3 sqlite3:) + + typed-records + srfi-18 + srfi-69 + ) + +(define (server:make-server-url hostport) + (if (not hostport) + #f + (conc "http://" (car hostport) ":" (cadr hostport)))) + +(define *server-loop-heart-beat* (current-seconds)) + +(define (server:get-logs-list area-path) + (let* ((server-logs-cmd (conc "grep -iL exiting " area-path "/logs/server-*-*.log")) + (server-logs (string-split (string-chomp (with-input-from-pipe server-logs-cmd read-string))))) + server-logs)) + + +)