Index: common.scm ================================================================== --- common.scm +++ common.scm @@ -87,10 +87,13 @@ (set! *test-info* (make-hash-table)) (set! *run-info-cache* (make-hash-table)) (set! *env-vars-by-run-id* (make-hash-table)) (set! *test-id-cache* (make-hash-table))) +;; Generic string database (normalization of sorts) +(define sdb:qry #f) ;; (make-sdb:qry)) ;; 'init #f) + ;;====================================================================== ;; S T A T E S A N D S T A T U S E S ;;====================================================================== (define *common:std-states* Index: dashboard-tests.scm ================================================================== --- dashboard-tests.scm +++ dashboard-tests.scm @@ -25,10 +25,11 @@ (declare (unit dashboard-tests)) (declare (uses common)) (declare (uses db)) (declare (uses gutils)) (declare (uses ezsteps)) +(declare (uses sdb)) (include "common_records.scm") (include "db_records.scm") (include "run_records.scm") @@ -73,11 +74,11 @@ lbl) (store-label "testcomment" (iup:label "TestComment " #:expand "HORIZONTAL") (lambda (testdat) - (db:test-get-comment testdat))) + (sdb:qry 'getstr (db:test-get-comment testdat)))) (store-label "testid" (iup:label "TestId " #:expand "HORIZONTAL") (lambda (testdat) (db:test-get-id testdat))) @@ -180,15 +181,15 @@ (iup:label "" #:expand "VERTICAL"))) (apply iup:vbox ; #:expand "YES" (list ;; NOTE: Yes, the host can change! (store-label "HostName" - (iup:label (db:test-get-host testdat) #:expand "HORIZONTAL") + (iup:label (sdb:qry 'getstr (db:test-get-host testdat)) #:expand "HORIZONTAL") (lambda (testdat)(db:test-get-host testdat))) (store-label "Uname" (iup:label " " #:expand "HORIZONTAL") - (lambda (testdat)(db:test-get-uname testdat))) + (lambda (testdat)(sdb:qry 'getstr (db:test-get-uname testdat)))) (store-label "DiskFree" (iup:label (conc (db:test-get-diskfree testdat)) #:expand "HORIZONTAL") (lambda (testdat)(conc (db:test-get-diskfree testdat)))) (store-label "CPULoad" (iup:label (conc (db:test-get-cpuload testdat)) #:expand "HORIZONTAL") @@ -383,11 +384,11 @@ (cond ((and need-update newtestdat) (set! testdat newtestdat) (set! teststeps (db:get-compressed-steps test-id work-area: rundir)) (set! logfile (conc (db:test-get-rundir testdat) "/" (db:test-get-final_logf testdat))) - (set! rundir (db:test-get-rundir testdat)) + (set! rundir (sdb:qry 'getstr (db:test-get-rundir testdat))) (set! testfullname (db:test-get-fullname testdat)) ;; (debug:print 0 "INFO: teststeps=" (intersperse teststeps "\n ")) (if (eq? curr-mod-time db-mod-time) ;; do only once if same (set! db-mod-time (+ curr-mod-time 1)) (set! db-mod-time curr-mod-time)) Index: dashboard.scm ================================================================== --- dashboard.scm +++ dashboard.scm @@ -85,10 +85,12 @@ (begin (print "Failed to find megatest.config, exiting") (exit 1))) (define *db* #f) ;; (open-db)) + +;; (define sdb:qry (make-sdb:qry)) ;; 'init #f) (if (args:get-arg "-host") (begin (set! *runremote* (string-split (args:get-arg "-host" ":"))) (client:launch)) Index: db.scm ================================================================== --- db.scm +++ db.scm @@ -28,10 +28,11 @@ (declare (uses keys)) (declare (uses ods)) (declare (uses fs-transport)) (declare (uses client)) (declare (uses mt)) +(declare (uses sdb)) (include "common_records.scm") (include "db_records.scm") (include "key_records.scm") (include "run_records.scm") @@ -60,10 +61,11 @@ #f)))) (if val (begin (debug:print-info 9 "db:set-sync, setting pragma synchronous to " val) (sqlite3:execute db (conc "PRAGMA synchronous = '" val "';")))))) +;; (sqlite3:execute db "PRAGMA synchronous = normal;")))) ;; need a default? (define (open-db) ;; (conc *toppath* "/megatest.db") (car *configinfo*))) (if (not *toppath*) (if (not (setup-for-run)) (begin @@ -82,10 +84,11 @@ (debug:print-info 11 "open-db, dbpath=" dbpath " argv=" (argv)) (sqlite3:set-busy-handler! db handler) (if (not dbexists) (db:initialize db)) (db:set-sync db) + (set! sdb:qry (make-sdb:qry)) ;; we open the normalization helpers here db)) ;; keeping it around for debugging purposes only (define (open-run-close-no-exception-handling proc idb . params) (debug:print-info 11 "open-run-close-no-exception-handling START given a db=" (if idb "yes " "no ") ", params=" params) @@ -290,11 +293,11 @@ ;; find and open the testdat.db file for an existing test (define (db:open-test-db-by-test-id db test-id #!key (work-area #f)) (let* ((test-path (if work-area work-area - (cdb:remote-run db:test-get-rundir-from-test-id db test-id)))) + (sdb:qry 'getstr (cdb:remote-run db:test-get-rundir-from-test-id db test-id))))) (debug:print 3 "TEST PATH: " test-path) (open-test-db test-path))) (define (db:testdb-initialize db) (debug:print 11 "db:testdb-initialize START") @@ -863,13 +866,11 @@ (let ((finalres (vector header res))) ;; (hash-table-set! *run-info-cache* run-id finalres) finalres))) (define (db:set-comment-for-run db run-id comment) - (debug:print-info 11 "db:set-comment-for-run START run-id: " run-id " comment: " comment) - (sqlite3:execute db "UPDATE runs SET comment=? WHERE id=?;" comment run-id) - (debug:print-info 11 "db:set-comment-for-run END run-id: " run-id " comment: " comment)) + (sqlite3:execute db "UPDATE runs SET comment=? WHERE id=?;" (sdb:qry 'getid comment) run-id)) ;; does not (obviously!) removed dependent data. But why not!!? (define (db:delete-run db run-id) (common:clear-caches) ;; don't trust caches after doing any deletion ;; First set any related tests to DELETED @@ -1165,25 +1166,25 @@ (define (cdb:tests-update-run-duration serverdat test-id minutes) (cdb:client-call serverdat 'update-run-duration #t *default-numtries* minutes test-id)) (define (cdb:tests-update-uname-host serverdat test-id uname hostname) - (cdb:client-call serverdat 'update-uname-host #t *default-numtries* uname hostname test-id)) + (cdb:client-call serverdat 'update-uname-host #t *default-numtries* (sdb:qry 'getid uname)(sdb:qry 'getid hostname) test-id)) ;; speed up for common cases with a little logic ;; NB// Ultimately this will be deprecated in deference to mt:test-set-state-status-by-id ;; (define (db:test-set-state-status-by-id db test-id newstate newstatus newcomment) (cond ((and newstate newstatus newcomment) - (sqlite3:execute db "UPDATE tests SET state=?,status=?,comment=? WHERE id=?;" newstate newstatus newcomment test-id)) + (sqlite3:execute db "UPDATE tests SET state=?,status=?,comment=? WHERE id=?;" newstate newstatus (sdb:qry 'getid newcomment) test-id)) ((and newstate newstatus) (sqlite3:execute db "UPDATE tests SET state=?,status=? WHERE id=?;" newstate newstatus test-id)) (else (if newstate (sqlite3:execute db "UPDATE tests SET state=? WHERE id=?;" newstate test-id)) (if newstatus (sqlite3:execute db "UPDATE tests SET status=? WHERE id=?;" newstatus test-id)) - (if newcomment (sqlite3:execute db "UPDATE tests SET comment=? WHERE id=?;" newcomment test-id)))) + (if newcomment (sqlite3:execute db "UPDATE tests SET comment=? WHERE id=?;" (sdb:qry 'getid newcomment) test-id)))) (mt:process-triggers test-id newstate newstatus)) ;; Never used ;; (define (db:test-set-state-status-by-run-id-testname db run-id test-name item-path status state) ;; (sqlite3:execute db "UPDATE tests SET state=?,status=?,event_time=strftime('%s','now') WHERE run_id=? AND testname=? AND item_path=?;" @@ -1345,17 +1346,17 @@ (define (db:test-set-comment db test-id comment) (sqlite3:execute db "UPDATE tests SET comment=? WHERE id=?;" - comment test-id)) + (sdb:qry 'getid comment) test-id)) -(define (cdb:test-set-rundir! serverdat run-id test-name item-path rundir) - (cdb:client-call serverdat 'test-set-rundir #t *default-numtries* rundir run-id test-name item-path)) +(define (cdb:test-set-rundir! serverdat run-id test-name item-path rundir-id) + (cdb:client-call serverdat 'test-set-rundir #t *default-numtries* rundir-id run-id test-name item-path)) -(define (cdb:test-set-rundir-by-test-id serverdat test-id rundir) - (cdb:client-call serverdat 'test-set-rundir-by-test-id #t *default-numtries* rundir test-id)) +(define (cdb:test-set-rundir-by-test-id serverdat test-id rundir-id) + (cdb:client-call serverdat 'test-set-rundir-by-test-id #t *default-numtries* rundir-id test-id)) (define (db:test-get-rundir-from-test-id db test-id) (let ((res #f)) ;; (hash-table-ref/default *test-paths* test-id #f))) ;; (if res ;; res @@ -1367,12 +1368,12 @@ "SELECT rundir FROM tests WHERE id=?;" test-id) ;; (hash-table-set! *test-paths* test-id res) res)) ;; )) -(define (cdb:test-set-log! serverdat test-id logf) - (if (string? logf)(cdb:client-call serverdat 'test-set-log #f *default-numtries* logf test-id))) +(define (cdb:test-set-log! serverdat test-id logf-id) + (if (or (string? logf-id)(number? logf-id))(cdb:client-call serverdat 'test-set-log #f *default-numtries* logf-id test-id))) ;;====================================================================== ;; Misc. test related queries ;;====================================================================== @@ -1654,16 +1655,18 @@ (apply cdb:client-call *runremote* 'immediate #f *default-numtries* open-run-close proc #f params)) (define (db:test-get-logfile-info db run-id test-name) (let ((res #f)) (sqlite3:for-each-row - (lambda (path final_logf) - (set! logf final_logf) - (set! res (list path final_logf)) - (if (directory? path) - (debug:print 2 "Found path: " path) - (debug:print 2 "No such path: " path))) + (lambda (path-id final_logf-id) + (let ((path (sdb:qry 'getstr path-id)) + (final_logf (sdb:qry 'getstr final_logf-id))) + (set! logf final_logf) + (set! res (list path final_logf)) + (if (directory? path) + (debug:print 2 "Found path: " path) + (debug:print 2 "No such path: " path)))) db "SELECT rundir,final_logf FROM tests WHERE run_id=? AND testname=? AND item_path='';" run-id test-name) res)) Index: ezsteps.scm ================================================================== --- ezsteps.scm +++ ezsteps.scm @@ -16,10 +16,11 @@ (declare (unit ezsteps)) (declare (uses db)) (declare (uses common)) (declare (uses items)) (declare (uses runconfig)) +(declare (uses sdb)) (include "common_records.scm") (include "key_records.scm") (include "db_records.scm") (include "run_records.scm") @@ -98,11 +99,11 @@ (let ((exinfo (vector-ref exit-info 2)) (logfna (if logpro-used (conc stepname ".html") ""))) ;; testing if procedures called in a remote call cause problems (ans: no or so I suspect) (db:teststep-set-status! #f test-id stepname "end" exinfo #f logfna work-area: test-run-dir)) (if logpro-used - (cdb:test-set-log! *runremote* test-id (conc stepname ".html"))) + (cdb:test-set-log! *runremote* test-id (sdb:qry 'getid (conc stepname ".html")))) ;; set the test final status (let* ((this-step-status (cond ((and (eq? (vector-ref exit-info 2) 2) logpro-used) 'warn) ((eq? (vector-ref exit-info 2) 0) 'pass) (else 'fail))) Index: launch.scm ================================================================== --- launch.scm +++ launch.scm @@ -19,10 +19,11 @@ (declare (unit launch)) (declare (uses common)) (declare (uses configf)) (declare (uses db)) +(declare (uses sdb)) (include "common_records.scm") (include "key_records.scm") (include "db_records.scm") @@ -255,11 +256,11 @@ (let ((exinfo (vector-ref exit-info 2)) (logfna (if logpro-used (conc stepname ".html") ""))) ;; testing if procedures called in a remote call cause problems (ans: no or so I suspect) (db:teststep-set-status! #f test-id stepname "end" exinfo #f logfna work-area: work-area)) (if logpro-used - (cdb:test-set-log! *runremote* test-id (conc stepname ".html"))) + (cdb:test-set-log! *runremote* test-id (sdb:qry 'getid (conc stepname ".html")))) ;; set the test final status (let* ((this-step-status (cond ((and (eq? (vector-ref exit-info 2) 2) logpro-used) 'warn) ((eq? (vector-ref exit-info 2) 0) 'pass) (else 'fail))) @@ -494,11 +495,11 @@ (lnkbase (conc linktree "/" target "/" runname)) (lnkpath (conc lnkbase "/" testname)) (lnkpathf (conc lnkpath (if not-iterated "" "/") item-path))) ;; Update the rundir path in the test record for all - (cdb:test-set-rundir-by-test-id *runremote* test-id lnkpathf) + (cdb:test-set-rundir-by-test-id *runremote* test-id (sdb:qry 'getid lnkpathf)) (debug:print 2 "INFO:\n lnkbase=" lnkbase "\n lnkpath=" lnkpath "\n toptest-path=" toptest-path "\n test-path=" test-path) (if (not (file-exists? linktree)) (begin (debug:print 0 "WARNING: linktree did not exist! Creating it now at " linktree) @@ -515,14 +516,14 @@ ;; NB - This is not working right - some top tests are not getting the path set!!! (if (not (hash-table-ref/default *toptest-paths* testname #f)) (let* ((testinfo (cdb:get-test-info-by-id *runremote* test-id)) ;; run-id testname item-path)) - (curr-test-path (if testinfo (db:test-get-rundir testinfo) #f))) + (curr-test-path (if testinfo (sdb:qry 'getstr (db:test-get-rundir testinfo)) #f))) (hash-table-set! *toptest-paths* testname curr-test-path) ;; NB// Was this for the test or for the parent in an iterated test? - (cdb:test-set-rundir! *runremote* run-id testname "" lnkpath) ;; toptest-path) + (cdb:test-set-rundir! *runremote* run-id testname "" (sdb:qry 'getid lnkpath)) ;; toptest-path) (if (or (not curr-test-path) (not (directory-exists? toptest-path))) (begin (debug:print-info 2 "Creating " toptest-path " and link " lnkpath) (create-directory toptest-path #t) Index: megatest.scm ================================================================== --- megatest.scm +++ megatest.scm @@ -25,10 +25,11 @@ (declare (uses client)) (declare (uses tests)) (declare (uses genexample)) (declare (uses daemon)) (declare (uses db)) +(declare (uses sdb)) (define *db* #f) ;; this is only for the repl, do not use in general!!!! (include "common_records.scm") (include "key_records.scm") @@ -235,10 +236,13 @@ "-cleanup-db" "-rollup" "-update-meta" "-gen-megatest-area" "-mark-incompletes" + + "-convert-to-norm" + "-convert-to-old" "-logging" "-v" ;; verbose 2, more than normal (normal is 1) "-q" ;; quiet 0, errors/warnings only ) @@ -622,12 +626,12 @@ (equal? (db:test-get-status test) "WARN") (equal? (db:test-get-state test) "NOT_STARTED"))) (begin (print " cpuload: " (db:test-get-cpuload test) "\n diskfree: " (db:test-get-diskfree test) - "\n uname: " (db:test-get-uname test) - "\n rundir: " (db:test-get-rundir test) + "\n uname: " (sdb:qry 'getstr (db:test-get-uname test)) + "\n rundir: " (sdb:qry 'getstr (db:test-get-rundir test)) ) ;; Each test ;; DO NOT remote run (let ((steps (db:get-steps-for-test #f (db:test-get-id test)))) (for-each @@ -956,11 +960,11 @@ ;; has sub commands that are rdb: ;; DO NOT put this one into either cdb:remote-run or open-run-close (db:load-test-data db test-id work-area: work-area)) (if (args:get-arg "-setlog") (let ((logfname (args:get-arg "-setlog"))) - (cdb:test-set-log! *runremote* test-id logfname))) + (cdb:test-set-log! *runremote* test-id (sdb:qry 'getid logfname)))) (if (args:get-arg "-set-toplog") ;; DO NOT run remote (tests:test-set-toplog! db run-id test-name (args:get-arg "-set-toplog"))) (if (args:get-arg "-summarize-items") ;; DO NOT run remote @@ -1002,11 +1006,11 @@ (debug:print-info 2 "running \"" cmd "\"") (change-directory startingdir) (set! exitstat (system cmd)) (set! *globalexitstatus* exitstat) ;; no necessary (change-directory testpath) - (cdb:test-set-log! *runremote* test-id htmllogfile))) + (cdb:test-set-log! *runremote* test-id (sdb:qry 'getid htmllogfile)))) (let ((msg (args:get-arg "-m"))) ;; DO NOT run remote (db:teststep-set-status! db test-id stepname "end" exitstat msg logfile work-area: work-area)) ))) (if (or (args:get-arg "-test-status") @@ -1153,10 +1157,35 @@ (if (args:get-arg "-repl") (repl) (load (args:get-arg "-load")))) (exit)) (set! *didsomething* #t))) + +(if (args:get-arg "-convert-to-norm") + (let* ((toppath (setup-for-run)) + (db (if toppath (open-db) #f))) + (for-each + (lambda (field) + (let ((dat '())) + (debug:print-info 0 "Getting data for field " field) + (sqlite3:for-each-row + (lambda (id val) + (set! dat (cons (list id val) dat))) + db + (conc "SELECT id," field " FROM tests;")) + (debug:print-info 0 "found " (length dat) " items for field " field) + (let ((qry (sqlite3:prepare db (conc "UPDATE tests SET " field "=? WHERE id=?;")))) + (for-each + (lambda (item) + (let ((newval (sdb:qry 'getid (cadr item)))) + (if (not (equal? newval (cadr item))) + (debug:print-info 0 "Converting " (cadr item) " to " newval " for test #" (car item))) + (sqlite3:execute qry newval (car item)))) + dat) + (sqlite3:finalize! qry)))) + (list "uname" "rundir" "final_logf" "comment")) + (set! *didsomething* #t))) ;;====================================================================== ;; Exit and clean up ;;====================================================================== @@ -1164,10 +1193,12 @@ ;; this is the socket if we are a client ;; (if (and *runremote* ;; (socket? *runremote*)) ;; (close-socket *runremote*)) + +(sdb:qry 'finalize! #f) (if (not *didsomething*) (debug:print 0 help)) ;; (if *runremote* (rpc:close-all-connections!)) Index: mt.scm ================================================================== --- mt.scm +++ mt.scm @@ -112,11 +112,11 @@ ;; T R I G G E R S ;;====================================================================== (define (mt:process-triggers test-id newstate newstatus) (let* ((test-dat (mt:lazy-get-test-info-by-id test-id)) - (test-rundir (db:test-get-rundir test-dat)) + (test-rundir (sdb:qry 'getstr (db:test-get-rundir test-dat))) (test-name (db:test-get-testname test-dat)) (tconfig #f) (state (if newstate newstate (db:test-get-state test-dat))) (status (if newstatus newstatus (db:test-get-status test-dat)))) (if (and (file-exists? test-rundir) Index: runs.scm ================================================================== --- runs.scm +++ runs.scm @@ -1211,12 +1211,12 @@ action) ((run-wait) (debug:print 1 "Waiting for run " runkey ", run=" runnamepatt " to complete")) (else (debug:print-info 0 "action not recognised " action))) - (let ((sorted-tests (sort tests (lambda (a b)(let ((dira (db:test-get-rundir a)) - (dirb (db:test-get-rundir b))) + (let ((sorted-tests (sort tests (lambda (a b)(let ((dira (sdb:qry 'getstr (db:test-get-rundir a))) + (dirb (sdb:qry 'getstr (db:test-get-rundir b)))) (if (and (string? dira)(string? dirb)) (> (string-length dira)(string-length dirb)) #f))))) (test-retry-time (make-hash-table)) (allow-run-time 10)) ;; seconds to allow for killing tests before just brutally killing 'em @@ -1229,11 +1229,11 @@ (debug:print 0 "ERROR: We have a test-id of " test-id " but no record was found. NOTE: No locking of records is done between processes, do not simultaneously remove the same run from two processes!") (if (not (null? tal)) (loop (car tal)(cdr tal)))) (let* ((item-path (db:test-get-item-path new-test-dat)) (test-name (db:test-get-testname new-test-dat)) - (run-dir (db:test-get-rundir new-test-dat)) ;; run dir is from the link tree + (run-dir (sdb:qry 'getstr (db:test-get-rundir new-test-dat))) ;; run dir is from the link tree (real-dir (if (file-exists? run-dir) (resolve-pathname run-dir) #f)) (test-state (db:test-get-state new-test-dat)) (test-fulln (db:test-get-fullname new-test-dat))) Index: sdb.scm ================================================================== --- sdb.scm +++ sdb.scm @@ -20,17 +20,17 @@ (import (prefix base64 base64:)) (declare (unit sdb)) ;; -(define (sdb:open) ;; (conc *toppath* "/megatest.db") (car *configinfo*))) +(define (sdb:open #!key (fname #f)) ;; (conc *toppath* "/megatest.db") (car *configinfo*))) (if (not *toppath*) (if (not (setup-for-run)) (begin (debug:print 0 "ERROR: Attempted to open db when not in megatest area. Exiting.") (exit)))) - (let* ((dbpath (conc *toppath* "/db/sdb.db")) ;; fname) + (let* ((dbpath (conc *toppath* "/db/" (if fname fname "sdb.db"))) ;; fname) (dbexists (let ((fe (file-exists? dbpath))) (if fe fe (begin (create-directory (conc *toppath* "/db") #t) @@ -77,20 +77,25 @@ (hash-table-set! id-cache id str)) sdb "SELECT str FROM strs WHERE id=?;" id)) str)) -(define sdb:qry - (let ((sdb #f) +;; Numbers get passed though in both directions +;; +(define (make-sdb:qry #!key (fname #f)) + (let ((sdb (sdb:open fname: fname)) (scache (make-hash-table)) (icache (make-hash-table))) (lambda (cmd var) - (if (not sdb)(set! sdb (sdb:open))) + ;; (if (not sdb)(set! sdb (sdb:open))) (case cmd - ((init) (if (not sdb)(set! sdb (sdb:open)))) + ;; ((init) (if (not sdb)(set! sdb (sdb:open)))) ((finalize!) (if sdb (sqlite3:finalize! sdb))) - ((getid) (let ((id (sdb:string->id sdb scache var))) + ((getid) (let ((id (if (or (number? var) + (string->number var)) + var + (sdb:string->id sdb scache var)))) (if id id (begin (sdb:register-string sdb var) (sdb:string->id sdb scache var))))) Index: tests.scm ================================================================== --- tests.scm +++ tests.scm @@ -20,10 +20,11 @@ (declare (uses lock-queue)) (declare (uses db)) (declare (uses common)) (declare (uses items)) (declare (uses runconfig)) +(declare (uses sdb)) (include "common_records.scm") (include "key_records.scm") (include "db_records.scm") (include "run_records.scm") @@ -221,12 +222,12 @@ ;; Check for waiver eligibility ;; (define (tests:check-waiver-eligibility testdat prev-testdat) (let* ((test-registry (make-hash-table)) (testconfig (tests:get-testconfig (db:test-get-testname testdat) test-registry #f)) - (test-rundir (db:test-get-rundir testdat)) - (prev-rundir (db:test-get-rundir prev-testdat)) + (test-rundir (sdb:qry 'getstr (db:test-get-rundir testdat))) + (prev-rundir (sdb:qry 'getstr (db:test-get-rundir prev-testdat))) (waivers (configf:section-vars testconfig "waivers")) (waiver-rx (regexp "^(\\S+)\\s+(.*)$")) (diff-rule "diff %file1% %file2%") (logpro-rule "diff %file1% %file2% | logpro %waivername%.logpro %waivername%.html")) (if (not (file-exists? test-rundir))