Index: Makefile ================================================================== --- Makefile +++ Makefile @@ -28,14 +28,14 @@ ezsteps.scm lock-queue.scm rmt.scm api.scm \ subrun.scm portlogger.scm archive.scm env.scm \ diff-report.scm cgisetup/models/pgdb.scm # module source files -MSRCFILES = dbmod.scm dbfile.scm debugprint.scm mtargs.scm +MSRCFILES = dbmod.scm dbfile.scm debugprint.scm mtargs.scm commonmod.scm -mofiles/dbfile.o : mofiles/debugprint.o -mofiles/debugprint.o : mofiles/mtargs.o +mofiles/dbfile.o : mofiles/debugprint.o mofiles/commonmod.o commonmod.import.o +mofiles/debugprint.o : mofiles/mtargs.o mofiles/commonmod.o commonmod.import.o # ftail.scm rmtmod.scm commonmod.scm removed # MSRCFILES = ducttape-lib.scm pkts.scm stml2.scm cookie.scm mutils.scm \ # mtargs.scm commonmod.scm dbmod.scm adjutant.scm ulex.scm \ # rmtmod.scm apimod.scm @@ -49,23 +49,28 @@ MOFILES = $(addprefix mofiles/,$(MSRCFILES:%.scm=%.o)) # compiled import files MOIMPFILES = $(MSRCFILES:%.scm=%.import.o) -%.import.o : %.import.scm - csc $(CSCOPTS) -unit $*.import -c $*.import.scm -o $*.import.o +# %.import.o : %.import.scm +# csc $(CSCOPTS) -unit $*.import -c $*.import.scm -o $*.import.o + +# %.import.scm : mofiles/%.o +# sleep 0.1 # I'm not sure the cp is a good idea, changed a lot of things and it may not have been necessary... # mofiles/%.o %.import.scm : %.scm megatest-fossil-hash.scm # @[ -e mofiles ] || mkdir -p mofiles # csc $(CSCOPTS) -I mofiles -I $* -J -c $< -o $*.o # cp $*.o mofiles/$*.o # @touch $*.import.scm # ensure it is touched after the .o is made -mofiles/%.o : %.scm +mofiles/%.o %.import.o : %.scm megatest-fossil-hash.scm mkdir -p mofiles - csc $(CSCOPTS) -J -c $< -o mofiles/$*.o + csc $(CSCOPTS) -J -c $< -o mofiles/$*.o # $(shell ls *.o mofiles/*.o) + csc $(CSCOPTS) -unit $*.import -c $*.import.scm -o $*.import.o # $(shell ls *.o mofiles/*.o) + @touch $*.import.scm # ensure it is younger than the .o ADTLSCR=mt_laststep mt_runstep mt_ezstep HELPERS=$(addprefix $(PREFIX)/bin/,$(ADTLSCR)) DEPLOYHELPERS=$(addprefix deploytarg/,$(ADTLSCR)) MTESTHASH=$(shell fossil info|grep checkout:| awk '{print $$2}') @@ -210,11 +215,11 @@ if ! diff -q megatest-fossil-hash.new megatest-fossil-hash.scm ; then echo copying .new to .scm;cp -f megatest-fossil-hash.new megatest-fossil-hash.scm;fi $(OFILES) $(GOFILES) : common_records.scm %.o : %.scm $(MOFILES) - csc $(CSCOPTS) -c $< $(MOFILES) + csc $(CSCOPTS) -c $< $(MOFILES) $(MOIMPFILES) $(PREFIX)/bin/.$(ARCHSTR)/mtest : mtest utils/mk_wrapper @echo Installing to PREFIX=$(PREFIX) $(INSTALL) mtest $(PREFIX)/bin/.$(ARCHSTR)/mtest utils/mk_wrapper $(PREFIX) mtest $(PREFIX)/bin/megatest Index: api.scm ================================================================== --- api.scm +++ api.scm @@ -255,14 +255,14 @@ ((tasks-add) (apply tasks:add dbstruct params)) ((tasks-set-state-given-param-key) (apply tasks:set-state-given-param-key dbstruct params)) ((tasks-get-last) (apply tasks:get-last dbstruct params)) ;; NO SYNC DB - ((no-sync-set) (apply db:no-sync-set *no-sync-db* params)) - ((no-sync-get/default) (apply db:no-sync-get/default *no-sync-db* params)) - ((no-sync-del!) (apply db:no-sync-del! *no-sync-db* params)) - ((no-sync-get-lock) (apply db:no-sync-get-lock *no-sync-db* params)) + ((no-sync-set) (apply db:no-sync-set (db:no-sync-db *no-sync-db*) params)) + ((no-sync-get/default) (apply db:no-sync-get/default (db:no-sync-db *no-sync-db*) params)) + ((no-sync-del!) (apply db:no-sync-del! (db:no-sync-db *no-sync-db*) params)) + ((no-sync-get-lock) (apply db:no-sync-get-lock (db:no-sync-db *no-sync-db*) params)) ;; ARCHIVES ;; ((archive-get-allocations) ((archive-register-disk) (apply db:archive-register-disk dbstruct params)) ((archive-register-block-name)(apply db:archive-register-block-name dbstruct params)) Index: common.scm ================================================================== --- common.scm +++ common.scm @@ -26,12 +26,12 @@ (prefix sqlite3 sqlite3:) pkts (prefix dbi dbi:) ) (declare (unit common)) -;; (declare (uses commonmod)) -;; (import commonmod) +(declare (uses commonmod)) +(import commonmod) (include "common_records.scm") ;; (require-library margs) @@ -725,55 +725,10 @@ (print-call-chain (current-error-port)) #f) (read (open-input-string (base64:base64-decode instr)))) (read (open-input-string (z3:decode-buffer (base64:base64-decode instr)))))) -;; dot-locking egg seems not to work, using this for now -;; if lock is older than expire-time then remove it and try again -;; to get the lock -;; -(define (common:simple-file-lock fname #!key (expire-time 300)) - (let ((fmod-time (handle-exceptions - ext - (current-seconds) - (file-modification-time fname)))) - (if (common:file-exists? fname) - (if (> (- (current-seconds) fmod-time) expire-time) - (begin - (handle-exceptions exn #f (delete-file* fname)) - (common:simple-file-lock fname expire-time: expire-time)) - #f) - (let ((key-string (conc (get-host-name) "-" (current-process-id)))) - (with-output-to-file fname - (lambda () - (print key-string))) - (thread-sleep! 0.25) - (if (common:file-exists? fname) - (handle-exceptions exn - #f - (with-input-from-file fname - (lambda () - (equal? key-string (read-line))))) - #f))))) - -(define (common:simple-file-lock-and-wait fname #!key (expire-time 300)) - (let ((end-time (+ expire-time (current-seconds)))) - (let loop ((got-lock (common:simple-file-lock fname expire-time: expire-time))) - (if got-lock - #t - (if (> end-time (current-seconds)) - (begin - (thread-sleep! 3) - (loop (common:simple-file-lock fname expire-time: expire-time))) - #f))))) - -(define (common:simple-file-release-lock fname) - (handle-exceptions - exn - #f ;; I don't really care why this failed (at least for now) - (delete-file* fname))) - ;;====================================================================== ;; S T A T E S A N D S T A T U S E S ;;====================================================================== ;; BBnote: *common:std-states* - dashboard filter control and test control state buttons defined here; used in set-fields-panel and dboard:make-controls Index: commonmod.scm ================================================================== --- commonmod.scm +++ commonmod.scm @@ -156,7 +156,97 @@ ;; (define (debug:print-info . params) #f) ;; ;; (define (set-functions dbgp dbgpinfo) ;; (set! debug:print dbgp) ;; (set! debug:print-info dbgpinfo)) + +;; dot-locking egg seems not to work, using this for now +;; if lock is older than expire-time then remove it and try again +;; to get the lock +;; +(define (common:simple-file-lock fname #!key (expire-time 300)) + (let ((fmod-time (handle-exceptions + ext + (current-seconds) + (file-modification-time fname)))) + (if (file-exists? fname) + (if (> (- (current-seconds) fmod-time) expire-time) + (begin + (handle-exceptions exn #f (delete-file* fname)) + (common:simple-file-lock fname expire-time: expire-time)) + #f) + (let ((key-string (conc (get-host-name) "-" (current-process-id)))) + (with-output-to-file fname + (lambda () + (print key-string))) + (thread-sleep! 0.25) + (if (file-exists? fname) + (handle-exceptions exn + #f + (with-input-from-file fname + (lambda () + (equal? key-string (read-line))))) + #f))))) + +(define (common:simple-file-lock-and-wait fname #!key (expire-time 300)) + (let ((end-time (+ expire-time (current-seconds)))) + (let loop ((got-lock (common:simple-file-lock fname expire-time: expire-time))) + (if got-lock + #t + (if (> end-time (current-seconds)) + (begin + (thread-sleep! 3) + (loop (common:simple-file-lock fname expire-time: expire-time))) + #f))))) + +(define (common:simple-file-release-lock fname) + (handle-exceptions + exn + #f ;; I don't really care why this failed (at least for now) + (delete-file* fname))) + +;; dot-locking egg seems not to work, using this for now +;; if lock is older than expire-time then remove it and try again +;; to get the lock +;; +(define (common:simple-file-lock fname #!key (expire-time 300)) + (let ((fmod-time (handle-exceptions + ext + (current-seconds) + (file-modification-time fname)))) + (if (file-exists? fname) + (if (> (- (current-seconds) fmod-time) expire-time) + (begin + (handle-exceptions exn #f (delete-file* fname)) + (common:simple-file-lock fname expire-time: expire-time)) + #f) + (let ((key-string (conc (get-host-name) "-" (current-process-id)))) + (with-output-to-file fname + (lambda () + (print key-string))) + (thread-sleep! 0.25) + (if (file-exists? fname) + (handle-exceptions exn + #f + (with-input-from-file fname + (lambda () + (equal? key-string (read-line))))) + #f))))) + +(define (common:simple-file-lock-and-wait fname #!key (expire-time 300)) + (let ((end-time (+ expire-time (current-seconds)))) + (let loop ((got-lock (common:simple-file-lock fname expire-time: expire-time))) + (if got-lock + #t + (if (> end-time (current-seconds)) + (begin + (thread-sleep! 3) + (loop (common:simple-file-lock fname expire-time: expire-time))) + #f))))) + +(define (common:simple-file-release-lock fname) + (handle-exceptions + exn + #f ;; I don't really care why this failed (at least for now) + (delete-file* fname))) ) Index: db.scm ================================================================== --- db.scm +++ db.scm @@ -161,22 +161,19 @@ ;; if db not open, open inmem, rundb and sync then return inmem ;; inuse gets set automatically for rundb's ;; ;; (define db:get-db db:get-subdb) -;; (define (db:get-db subdb #;dbstruct run-id) ;; RENAME TO db:get-dbh -;; ;; (let* ((subdb (dbfile:get-subdb dbstruct run-id))) -;; (if (stack? (dbr:subdb-dbstack subdb)) -;; (if (stack-empty? (dbr:subdb-dbstack subdb)) -;; (let* ((dbname (db:run-id->dbname run-id)) -;; (newdb (db:open-megatest-db path: (db:dbfile-path) -;; name: dbname))) -;; ;; NOTE: pushing on the stack only happens AFTER the handle has been used -;; ;; (stack-push! (dbr:dbstruct-dbstack dbstruct) newdb) -;; newdb) -;; (stack-pop! (dbr:subdb-dbstack subdb))) -;; (db:open-db subdb run-id))) ;; ) +(define (db:get-db dbstruct run-id) + (let* ((subdb (dbfile:get-subdb dbstruct run-id)) + (dbdat (dbfile:get-dbdat dbstruct run-id))) + (if (dbr:dbdat? dbdat) + dbdat + (dbfile:open-db *dbstruct-dbs* #f db:initialize-main-db) + ) + ) +) (define-inline (db:generic-error-printout exn . message) (print-call-chain (current-error-port)) (apply debug:print-error 0 *default-log-port* message) (debug:print-error 0 *default-log-port* " params: " params @@ -5088,11 +5085,13 @@ (begin (debug:print-info 0 *default-log-port* "Server running, periodic copy-based sync started.") (let loop () ;; run the sync and print out durations - (debug:print-info 0 *default-log-port* "Sync durations: "(db:run-lock-and-sync no-sync-db)) + (let* ((changed (db:run-lock-and-sync no-sync-db))) + (if (not (null? changed)) + (debug:print-info 0 *default-log-port* "Sync durations: "changed))) ;; keep going unless time to exit ;; (if (not *time-to-exit*) (let delay-loop ((count 0)) ;;(debug:print-info 13 *default-log-port* "delay-loop top; count="count" pid="(current-process-id)" this-wd-num="this-wd-num" *time-to-exit*="*time-to-exit*) Index: dbfile.scm ================================================================== --- dbfile.scm +++ dbfile.scm @@ -17,12 +17,14 @@ ;; along with Megatest. If not, see . ;;====================================================================== (declare (unit dbfile)) -;; (declare (uses debugprint)) -;; (declare (uses commonmod)) +(declare (uses debugprint)) +;; (declare (uses debugprint.import)) +(declare (uses commonmod)) +;; (declare (uses commonmod.import)) (module dbfile * (import scheme chicken data-structures extras) @@ -31,11 +33,11 @@ srfi-69 stack files ports - ;; commonmod + commonmod ) ;; (import debugprint) ;;====================================================================== @@ -270,15 +272,12 @@ ;; Open the classic megatest.db file (defaults to open in toppath) ;; ;; NOTE: returns a dbdat not a dbstruct! ;; (define (dbfile:open-sqlite3-db dbpath init-proc) - (let* ((dbexists (file-exists? dbpath)) - (write-access (file-write-access? dbpath)) - (db (dbfile:cautious-open-database dbpath init-proc))) #;(sqlite3:open-database dbpath) - (sqlite3:set-busy-handler! db (sqlite3:make-busy-timeout 10000)) - ;; (init-proc db) + (let* ((write-access (file-write-access? dbpath)) + (db (dbfile:cautious-open-database dbpath init-proc))) (make-dbr:dbdat dbfile: dbpath dbh: db read-only: (not write-access)))) (define (dbfile:print-and-exit . params) (with-output-to-port (current-error-port) @@ -435,74 +434,54 @@ ;;====================================================================== ;; no-sync.db - small bits of data to be shared between servers ;;====================================================================== -;; if we are not a server create a db handle. this is not finalized -;; so watch for problems. I'm still not clear if it is needed to manually -;; finalize sqlite3 dbs with the sqlite3 egg. -;; (define (dbfile:cautious-open-database fname init-proc #!optional (tries-left 10)) - (let* ((lock-file (conc fname".lock")) - (retry (lambda () - (thread-sleep! 1.1) + (let* ((retry (lambda () + (thread-sleep! 0.5) (if (> tries-left 0) (dbfile:cautious-open-database fname init-proc (- tries-left 1)))))) - (assert (>= tries-left 0) (conc "FATAL: Five attempts in dbfile:cautious-open-database of "fname", giving up.")) - (if (and (file-write-access? fname) (not (dbfile:simple-file-lock lock-file))) - (begin - (dbfile:print-err "INFO: lock file "lock-file" exists, trying again in 1 second.") - (thread-sleep! 1) - (dbfile:cautious-open-database fname init-proc (- tries-left 1))) + (condition-case (let* ((db-exists (file-exists? fname)) - (result (condition-case - (let* ((db (sqlite3:open-database fname))) - (if (and init-proc (not db-exists)) - (init-proc db)) - db) - (exn (io-error) - (dbfile:print-err exn "ERROR: i/o error with " fname ". Check permissions, disk space etc. and try again.") - (retry)) - (exn (corrupt) - (dbfile:print-err exn "ERROR: database " fname " is corrupt. Repair it to proceed.") - (retry)) - (exn (busy) - (dbfile:print-err exn "ERROR: database " fname - " is locked. Try copying to another location, remove original and copy back.") - (retry)) - (exn (permission)(dbfile:print-err exn "ERROR: database " fname " has some permissions problem.") - (retry)) - (exn () - (dbfile:print-err exn "ERROR: Unknown error with database " fname " message: " - ((condition-property-accessor 'exn 'message) exn)) - (retry))))) - (if (file-write-access? fname) - (dbfile:simple-file-release-lock lock-file) - ) - result)))) - + (db (sqlite3:open-database fname))) + (if (and init-proc (not db-exists)) + (init-proc db)) + (sqlite3:set-busy-handler! db (sqlite3:make-busy-timeout 136000)) + db) + (exn (io-error) + (dbfile:print-err exn "ERROR: i/o error with " fname ". Check permissions, disk space etc. and try again.") + (retry)) + (exn (corrupt) + (dbfile:print-err exn "ERROR: database " fname " is corrupt. Repair it to proceed.") + (retry)) + (exn (busy) + (dbfile:print-err exn "ERROR: database " fname + " is locked. Try copying to another location, remove original and copy back.") + (retry)) + (exn (permission)(dbfile:print-err exn "ERROR: database " fname " has some permissions problem.") + (retry)) + (exn () + (dbfile:print-err exn "ERROR: Unknown error with database " fname " message: " + ((condition-property-accessor 'exn 'message) exn)) + (retry))))) (define (dbfile:open-no-sync-db dbpath) - (if *no-sync-db* - *no-sync-db* - (begin - (if (not (file-exists? dbpath)) - (create-directory dbpath #t)) - (let* ((dbname (conc dbpath "/no-sync.db")) - (db-exists (file-exists? dbname)) - (init-proc (lambda (db) - (if (not db-exists) - (begin - (sqlite3:execute db "PRAGMA synchronous = 0;") - (sqlite3:execute db "CREATE TABLE IF NOT EXISTS no_sync_metadat (var TEXT,val TEXT, CONSTRAINT no_sync_metadat_constraint UNIQUE (var));")) - ))) - (db (dbfile:cautious-open-database dbname init-proc))) ;; (sqlite3:open-database dbname))) - (sqlite3:set-busy-handler! db (sqlite3:make-busy-timeout 136000)) - ;;(sqlite3:execute db "PRAGMA journal_mode=WAL;") - (set! *no-sync-db* db) - db)))) + (if (not (file-exists? dbpath)) + (create-directory dbpath #t)) + (let* ((dbname (conc dbpath "/no-sync.db")) + (db-exists (file-exists? dbname)) + (db (dbfile:cautious-open-database dbname))) + (if (not db-exists) + (begin + (sqlite3:execute db "PRAGMA synchronous = 0;") + (sqlite3:execute db "CREATE TABLE IF NOT EXISTS no_sync_metadat (var TEXT,val TEXT, CONSTRAINT no_sync_metadat_constraint UNIQUE (var));"))) + (sqlite3:set-busy-handler! db (sqlite3:make-busy-timeout 136000)) + ;;(sqlite3:execute db "PRAGMA journal_mode=WAL;") + (set! *no-sync-db* db) + db)) (define (db:no-sync-set db var val) (sqlite3:execute db "INSERT OR REPLACE INTO no_sync_metadat (var,val) VALUES (?,?);" var val)) (define (db:no-sync-del! db var) @@ -576,62 +555,16 @@ (apply max (map dbfile:lazy-modification-time file-list)))) -;; dot-locking egg seems not to work, using this for now -;; if lock is older than expire-time then remove it and try again -;; to get the lock -;; -(define (dbfile:simple-file-lock fname #!key (expire-time 300)) - (let ((fmod-time (handle-exceptions - ext - (current-seconds) - (file-modification-time fname)))) - (if (file-exists? fname) - (if (> (- (current-seconds) fmod-time) expire-time) - (begin - (handle-exceptions exn #f (delete-file* fname)) - (dbfile:simple-file-lock fname expire-time: expire-time)) - #f) - (let ((key-string (conc (get-host-name) "-" (current-process-id))) - (oup (open-output-file fname))) (with-output-to-port oup (lambda () (print key-string))) (close-output-port oup) #;(with-output-to-file fname ;; bizarre. with-output-to-file does not seem to be cleaning up after itself. - (lambda () - (print key-string))) - (thread-sleep! 0.25) - (if (file-exists? fname) - (handle-exceptions exn - #f - (with-input-from-file fname - (lambda () - (equal? key-string (read-line))))) - #f) - ) - ) - ) -) - -(define (dbfile:simple-file-lock-and-wait fname #!key (expire-time 300)) - (let ((end-time (+ expire-time (current-seconds)))) - (let loop ((got-lock (dbfile:simple-file-lock fname expire-time: expire-time))) - (if got-lock - #t - (if (> end-time (current-seconds)) - (begin - (thread-sleep! 3) - (loop (dbfile:simple-file-lock fname expire-time: expire-time))) - #f))))) - -(define (dbfile:simple-file-release-lock fname) - (handle-exceptions - exn - #f ;; I don't really care why this failed (at least for now) - (delete-file* fname))) - - + ) + ) + ) +) ) Index: megatest.scm ================================================================== --- megatest.scm +++ megatest.scm @@ -42,21 +42,28 @@ (declare (uses tasks)) ;; only used for debugging. (declare (uses env)) (declare (uses diff-report)) (declare (uses dbmod)) (declare (uses dbmod.import)) +(declare (uses commonmod)) +(declare (uses commonmod.import)) +(declare (uses mtargs)) +(declare (uses mtargs.import)) +(declare (uses debugprint)) +(declare (uses debugprint.import)) (declare (uses dbfile)) (declare (uses dbfile.import)) -;; (declare (uses debugprint)) -;; (declare (uses debugprint.import)) ;; (declare (uses mtargs)) ;; (declare (uses mtargs.import)) ;; (declare (uses ftail)) ;; (import ftail) -(import dbmod +(import (prefix mtargs mod:) + commonmod + (prefix debugprint mod:) + dbmod dbfile) (define *db* #f) ;; this is only for the repl, do not use in general!!!! (include "common_records.scm") Index: server.scm ================================================================== --- server.scm +++ server.scm @@ -22,19 +22,21 @@ directory-utils posix-extras matchable utils) (use spiffy uri-common intarweb http-client spiffy-request-vars) (declare (unit server)) - (declare (uses common)) (declare (uses db)) (declare (uses tasks)) ;; tasks are where stuff is maintained about what is running. ;; (declare (uses synchash)) (declare (uses http-transport)) ;;(declare (uses rpc-transport)) (declare (uses launch)) ;; (declare (uses daemon)) + +(declare (uses commonmod)) +(import commonmod) (include "common_records.scm") (include "db_records.scm") (define (server:make-server-url hostport) Index: tests/simplerun/thebeginning.scm ================================================================== --- tests/simplerun/thebeginning.scm +++ tests/simplerun/thebeginning.scm @@ -53,7 +53,7 @@ (define thisdbdat (db:open-db dbstruct #f)) (test #f #t (dbr:dbdat? thisdbdat)) -(test #f #t (dbr:dbdat? (db:get-db dbstruct #f))) -(test #f #t (dbr:dbdat? (db:get-db dbstruct 1))) +(test #f #t (dbr:subdb? (db:get-db dbstruct #f))) +(test #f #t (dbr:subdb? (db:get-db dbstruct 1)))