Overview
Comment: | Consolidated to single global for dbstruct. Removed *megatest-db* Removed *inmemdb* Removed *write-frequency* Removed *client-non-blocking-mode* Consolidated db:open-local-db-handle in with db:setup Fixed calls which used db to instead use dbstruct Change repl to use db:setup for getting a db handle |
---|---|
Downloads: | Tarball | ZIP archive | SQL archive |
Timelines: | family | ancestors | descendants | both | v1.62-no-rpc |
Files: | files | file ages | folders |
SHA1: |
e335fe582a26acb0893561eb221f8ea3 |
User & Date: | matt on 2016-11-25 11:50:31 |
Other Links: | branch diff | manifest | tags |
Context
2016-11-25
| ||
22:30 | Runs through test4 pretty well. No server enabled yet check-in: 6444c1bdfc user: matt tags: v1.62-no-rpc | |
11:50 | Consolidated to single global for dbstruct. Removed *megatest-db* Removed *inmemdb* Removed *write-frequency* Removed *client-non-blocking-mode* Consolidated db:open-local-db-handle in with db:setup Fixed calls which used db to instead use dbstruct Change repl to use db:setup for getting a db handle check-in: e335fe582a user: matt tags: v1.62-no-rpc | |
2016-11-24
| ||
22:54 | Migrated some (but not all) calls to newer set state status check-in: 5af8fddcf7 user: matt tags: v1.62-no-rpc | |
Changes
Modified common.scm from [9b7d0f2d2f] to [d6de275ae9].
︙ | ︙ | |||
75 76 77 78 79 80 81 | (define *toppath* #f) (define *already-seen-runconfig-info* #f) (define *waiting-queue* (make-hash-table)) (define *test-meta-updated* (make-hash-table)) (define *globalexitstatus* 0) ;; attempt to work around possible thread issues (define *passnum* 0) ;; when running track calls to run-tests or similar | < | > < < < | 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 | (define *toppath* #f) (define *already-seen-runconfig-info* #f) (define *waiting-queue* (make-hash-table)) (define *test-meta-updated* (make-hash-table)) (define *globalexitstatus* 0) ;; attempt to work around possible thread issues (define *passnum* 0) ;; when running track calls to run-tests or similar (define *alt-log-file* #f) ;; used by -log (define *common:denoise* (make-hash-table)) ;; for low noise printing (define *default-log-port* (current-error-port)) (define *time-zero* (current-seconds)) ;; for the watchdog ;; DATABASE (define *dbstruct-db* #f) ;; used when local access is triggered in rmt.scm (define *db-stats* (make-hash-table)) ;; hash of vectors < count duration-total > (define *db-stats-mutex* (make-mutex)) (define *db-sync-mutex* (make-mutex)) (define *db-multi-sync-mutex* (make-mutex)) (define *db-local-sync* (make-hash-table)) ;; used to record last touch of db (define *last-db-access* (current-seconds)) ;; update when db is accessed via server (define *db-write-access* #t) (define *task-db* #f) ;; (vector db path-to-db) (define *db-access-allowed* #t) ;; flag to allow access (define *db-access-mutex* (make-mutex)) (define *db-cache-path* #f) ;; SERVER (define *my-client-signature* #f) (define *transport-type* 'http) (define *transport-type* 'http) ;; override with [server] transport http|rpc|nmsg (define *runremote* (make-hash-table)) ;; if set up for server communication this will hold <host port> (define *max-cache-size* 0) (define *logged-in-clients* (make-hash-table)) (define *server-id* #f) (define *server-info* #f) (define *time-to-exit* #f) (define *received-response* #f) (define *default-numtries* 10) (define *server-run* #t) (define *run-id* #f) |
︙ | ︙ | |||
567 568 569 570 571 572 573 | (begin (set! *time-to-exit* #t) #t)))) (debug:print-info 4 *default-log-port* "starting exit process, finalizing databases.") (if (and no-hurry (debug:debug-mode 18)) (rmt:print-db-stats)) (let ((th1 (make-thread (lambda () ;; thread for cleaning up, give it five seconds | < < < < < < < | < < < < < < < | 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 | (begin (set! *time-to-exit* #t) #t)))) (debug:print-info 4 *default-log-port* "starting exit process, finalizing databases.") (if (and no-hurry (debug:debug-mode 18)) (rmt:print-db-stats)) (let ((th1 (make-thread (lambda () ;; thread for cleaning up, give it five seconds (if *dbstruct-db* (db:close-all *dbstruct-db*)) ;; one second allocated (if *task-db* (let ((db (cdr *task-db*))) (if (sqlite3:database? db) (begin (sqlite3:interrupt! db) (sqlite3:finalize! db #t) (vector-set! *task-db* 0 #f))))) |
︙ | ︙ |
Modified db.scm from [08ad464129] to [8de1aa48cd].
︙ | ︙ | |||
273 274 275 276 277 278 279 | tmpdb)))) ;; Make the dbstruct, setup up auxillary db's and call for main db at least once ;; ;; called in http-transport and replicated in rmt.scm for *local* access. ;; (define (db:setup #!key (areapath #f)) ;; . junk) ;; #!key (run-id #f) (local #f)) | > | | | < < < < < < < | | 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 | tmpdb)))) ;; Make the dbstruct, setup up auxillary db's and call for main db at least once ;; ;; called in http-transport and replicated in rmt.scm for *local* access. ;; (define (db:setup #!key (areapath #f)) ;; . junk) ;; #!key (run-id #f) (local #f)) (or *dbstruct-db* (let* (;; (dbdir (db:dbfile-path)) ;; (conc (configf:lookup *configdat* "setup" "linktree") "/.db")) (dbstruct (make-dbr:dbstruct))) ;; ) ;; path: dbdir local: local))) (db:open-db dbstruct areapath: #f) (set! *dbstruct-db* dbstruct) dbstruct))) ;; Open the classic megatest.db file (defaults to open in toppath) ;; ;; NOTE: returns a dbdat not a dbstruct! ;; (define (db:open-megatest-db #!key (path #f)(name #f)) (let* ((dbpath (conc (or path *toppath*) "/" (or name "megatest.db"))) (dbexists (file-exists? dbpath)) |
︙ | ︙ | |||
318 319 320 321 322 323 324 | (tmpdb (dbr:dbstruct-tmpdb dbstruct)) (mtdb (dbr:dbstruct-mtdb dbstruct)) (refndb (dbr:dbstruct-refndb dbstruct)) ;; (runid (dbr:dbstruct-run-id dbstruct)) ) (debug:print-info 4 *default-log-port* "Syncing for run-id: " run-id) ;; (mutex-lock! *http-mutex*) | | | 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 | (tmpdb (dbr:dbstruct-tmpdb dbstruct)) (mtdb (dbr:dbstruct-mtdb dbstruct)) (refndb (dbr:dbstruct-refndb dbstruct)) ;; (runid (dbr:dbstruct-run-id dbstruct)) ) (debug:print-info 4 *default-log-port* "Syncing for run-id: " run-id) ;; (mutex-lock! *http-mutex*) (db:sync-tables (db:sync-all-tables-list dbstruct) #f tmpdb refndb mtdb))) ;; (if (eq? run-id 0) ;; ;; runid equal to 0 is main.db ;; (if maindb ;; (if (or (not (number? mtime)) ;; (not (number? stime)) ;; (> mtime stime) ;; force-sync) |
︙ | ︙ | |||
3209 3210 3211 3212 3213 3214 3215 | "FAIL" status) statuses) statuses))) *common:std-statuses* >)) (newstate (if (null? all-curr-states) "NOT_STARTED" (car all-curr-states))) (newstatus (if (null? all-curr-statuses) "n/a" (car all-curr-statuses)))) | | | 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 | "FAIL" status) statuses) statuses))) *common:std-statuses* >)) (newstate (if (null? all-curr-states) "NOT_STARTED" (car all-curr-states))) (newstatus (if (null? all-curr-statuses) "n/a" (car all-curr-statuses)))) ;; (print "Setting toplevel to: " newstate "/" newstatus) (db:test-set-state-status-by-id dbstruct run-id tl-test-id newstate newstatus #f))))))) (define db:roll-up-pass-fail-counts db:set-state-status-and-roll-up-items) ;; call with state = #f to roll up with out accounting for state/status of this item ;; ;; (define (db:roll-up-pass-fail-counts dbstruct run-id test-name item-path state status) |
︙ | ︙ | |||
3286 3287 3288 3289 3290 3291 3292 | (set! logf final_logf) (set! res (list path final_logf)) (if (directory? path) (debug:print 2 *default-log-port* "Found path: " path) (debug:print 2 *default-log-port* "No such path: " path))) ;; ) db "SELECT rundir,final_logf FROM tests WHERE testname=? AND item_path='' AND run_id=?;" | | | 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 3294 | (set! logf final_logf) (set! res (list path final_logf)) (if (directory? path) (debug:print 2 *default-log-port* "Found path: " path) (debug:print 2 *default-log-port* "No such path: " path))) ;; ) db "SELECT rundir,final_logf FROM tests WHERE testname=? AND item_path='' AND run_id=?;" test-name run-id) res)))) ;;====================================================================== ;; A G R E G A T E D T R A N S A C T I O N D B W R I T E S ;;====================================================================== (define db:queries |
︙ | ︙ | |||
3515 3516 3517 3518 3519 3520 3521 | ;; can use wildcards. Also can likely be factored in with get test paths? ;; ;; Run this remotely!! ;; (define (db:get-matching-previous-test-run-records dbstruct run-id test-name item-path) (let* ((dbdat (db:get-db dbstruct #f)) (db (db:dbdat-get-db dbdat)) | | | 3509 3510 3511 3512 3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 | ;; can use wildcards. Also can likely be factored in with get test paths? ;; ;; Run this remotely!! ;; (define (db:get-matching-previous-test-run-records dbstruct run-id test-name item-path) (let* ((dbdat (db:get-db dbstruct #f)) (db (db:dbdat-get-db dbdat)) (keys (db:get-keys dbstruct)) (selstr (string-intersperse keys ",")) (qrystr (string-intersperse (map (lambda (x)(conc x "=?")) keys) " AND ")) (keyvals #f) (tests-hash (make-hash-table))) ;; first look up the key values from the run selected by run-id ;; (db:delay-if-busy dbdat) (sqlite3:for-each-row |
︙ | ︙ |
Modified fs-transport.scm from [59920959a9] to [28e812486e].
︙ | ︙ | |||
33 34 35 36 37 38 39 | ;;====================================================================== ;; There is no "server" per se but a convience routine to make it non ;; necessary to be reopening the db over and over again. ;; (define (fs:process-queue-item packet) | | | | | 33 34 35 36 37 38 39 40 41 42 43 44 | ;;====================================================================== ;; There is no "server" per se but a convience routine to make it non ;; necessary to be reopening the db over and over again. ;; (define (fs:process-queue-item packet) (if (not *dbstruct-db*) ;; we will require that (setup-for-run) has already been called (set! *dbstruct-db* (db:setup-db))) (debug:print-info 11 *default-log-port* "fs:process-queue-item called with packet=" packet) (db:process-queue-item *dbstruct-db* packet)) |
Modified http-transport.scm from [a60bbd8be7] to [d70882fd0b].
︙ | ︙ | |||
56 57 58 59 60 61 62 | (ipaddrstr (let ((ipstr (if (string=? "-" hostn) ;; (string-intersperse (map number->string (u8vector->list (hostname->ip hostname))) ".") (server:get-best-guess-address hostname) #f))) (if ipstr ipstr hostn))) ;; hostname))) (start-port (portlogger:open-run-close portlogger:find-port)) (link-tree-path (configf:lookup *configdat* "setup" "linktree"))) | < | 56 57 58 59 60 61 62 63 64 65 66 67 68 69 | (ipaddrstr (let ((ipstr (if (string=? "-" hostn) ;; (string-intersperse (map number->string (u8vector->list (hostname->ip hostname))) ".") (server:get-best-guess-address hostname) #f))) (if ipstr ipstr hostn))) ;; hostname))) (start-port (portlogger:open-run-close portlogger:find-port)) (link-tree-path (configf:lookup *configdat* "setup" "linktree"))) (debug:print-info 0 *default-log-port* "portlogger recommended port: " start-port) (root-path (if link-tree-path link-tree-path (current-directory))) ;; WARNING: SECURITY HOLE. FIX ASAP! (handle-directory spiffy-directory-listing) (handle-exception (lambda (exn chain) (signal (make-composite-condition |
︙ | ︙ | |||
80 81 82 83 84 85 86 | ;; This is were we set up the database connections (let* (($ (request-vars source: 'both)) (dat ($ 'dat)) (res #f)) (cond ((equal? (uri-path (request-uri (current-request))) '(/ "api")) | | | 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 | ;; This is were we set up the database connections (let* (($ (request-vars source: 'both)) (dat ($ 'dat)) (res #f)) (cond ((equal? (uri-path (request-uri (current-request))) '(/ "api")) (send-response body: (api:process-request *dbstruct-db* $) ;; the $ is the request vars proc headers: '((content-type text/plain))) (mutex-lock! *heartbeat-mutex*) (set! *last-db-access* (current-seconds)) (mutex-unlock! *heartbeat-mutex*)) ((equal? (uri-path (request-uri (current-request))) '(/ "")) (send-response body: (http-transport:main-page))) |
︙ | ︙ | |||
391 392 393 394 395 396 397 | (port (cadr server-info)) (last-access 0) (server-timeout (server:get-timeout))) (let loop ((count 0) (server-state 'available) (bad-sync-count 0)) | | | < | | | < < < | 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 | (port (cadr server-info)) (last-access 0) (server-timeout (server:get-timeout))) (let loop ((count 0) (server-state 'available) (bad-sync-count 0)) ;; Use this opportunity to sync the tmp db to megatest.db (if *dbstruct-db* (let ((start-time (current-milliseconds)) (sync-time #f) (rem-time #f)) (condition-case (db:sync-touched *dbstruct-db* *run-id* force-sync: #t) ((sync-failed)(cond ((> bad-sync-count 10) ;; time to give up (http-transport:server-shutdown server-id port)) (else ;; (> bad-sync-count 0) ;; we've had a fail or two, delay and loop (thread-sleep! 5) (loop count server-state (+ bad-sync-count 1))))) ((exn) (debug:print-error 0 *default-log-port* "error from sync code other than 'sync-failed. Attempting to gracefully shutdown the server") (tasks:server-delete-record (db:delay-if-busy tdbdat) server-id " http-transport:keep-running crashed") (exit))) (set! sync-time (- (current-milliseconds) start-time)) (set! rem-time (quotient (- 4000 sync-time) 1000)) (debug:print 4 *default-log-port* "SYNC: time= " sync-time ", rem-time=" rem-time) (if (and (<= rem-time 4) (> rem-time 0)) (thread-sleep! rem-time) (thread-sleep! 4))) ;; fallback for if the math is changed ... ;; ;; no *dbstruct-db* yet, set running after our first pass through and start the db ;; (if (eq? server-state 'available) (let ((new-server-id (tasks:server-am-i-the-server? (db:delay-if-busy tdbdat) run-id))) ;; try to ensure no double registering of servers (if (equal? new-server-id server-id) (begin (tasks:server-set-state! (db:delay-if-busy tdbdat) server-id "dbprep") (thread-sleep! 0.5) ;; give some margin for queries to complete before switching from file based access to server based access (set! *dbstruct-db* (db:setup)) ;; run-id)) (tasks:server-set-state! (db:delay-if-busy tdbdat) server-id "running")) (begin ;; gotta exit nicely (tasks:server-set-state! (db:delay-if-busy tdbdat) server-id "collision") (http-transport:server-shutdown server-id port)))))) (if (< count 1) ;; 3x3 = 9 secs aprox (loop (+ count 1) 'running bad-sync-count)) |
︙ | ︙ | |||
488 489 490 491 492 493 494 | (loop 0 server-state bad-sync-count)) (http-transport:server-shutdown server-id port)))))) (define (http-transport:server-shutdown server-id port) (let ((tdbdat (tasks:open-db))) (debug:print-info 0 *default-log-port* "Starting to shutdown the server.") ;; need to delete only *my* server entry (future use) | > | < | 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 | (loop 0 server-state bad-sync-count)) (http-transport:server-shutdown server-id port)))))) (define (http-transport:server-shutdown server-id port) (let ((tdbdat (tasks:open-db))) (debug:print-info 0 *default-log-port* "Starting to shutdown the server.") ;; need to delete only *my* server entry (future use) (if *dbstruct-db* (db:sync-touched *dbstruct-db* *run-id* force-sync: #t)) (set! *time-to-exit* #t) ;; tell on-exit to be fast as we've already cleaned up ;; ;; start_shutdown ;; (tasks:server-set-state! (db:delay-if-busy tdbdat) server-id "shutting-down") (portlogger:open-run-close portlogger:set-port port "released") (thread-sleep! 5) (debug:print-info 0 *default-log-port* "Max cached queries was " *max-cache-size*) |
︙ | ︙ |
Modified megatest.scm from [784a70461f] to [bb00f9be7d].
︙ | ︙ | |||
762 763 764 765 766 767 768 | (begin ;; (if run-id ;; (client:launch run-id) ;; (client:launch 0) ;; without run-id we'll start a server for "0" #t )))))) | < < < | 762 763 764 765 766 767 768 769 770 771 772 773 774 775 | (begin ;; (if run-id ;; (client:launch run-id) ;; (client:launch 0) ;; without run-id we'll start a server for "0" #t )))))) (if (or (args:get-arg "-list-servers") (args:get-arg "-stop-server") (args:get-arg "-kill-server")) (let ((tl (launch:setup))) (if tl (let* ((tdbdat (tasks:open-db)) (servers (tasks:get-all-servers (db:delay-if-busy tdbdat))) |
︙ | ︙ | |||
1895 1896 1897 1898 1899 1900 1901 | ;; fakeout readline (include "readline-fix.scm") (if (or (getenv "MT_RUNSCRIPT") (args:get-arg "-repl") (args:get-arg "-load")) (let* ((toppath (launch:setup)) | | < | > | 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 | ;; fakeout readline (include "readline-fix.scm") (if (or (getenv "MT_RUNSCRIPT") (args:get-arg "-repl") (args:get-arg "-load")) (let* ((toppath (launch:setup)) (dbstruct (if toppath (db:setup)))) ;; make-dbr:dbstruct path: toppath local: (args:get-arg "-local")) #f))) (if dbstruct (cond ((getenv "MT_RUNSCRIPT") ;; How to run megatest scripts ;; ;; #!/bin/bash ;; ;; export MT_RUNSCRIPT=yes ;; megatest << EOF ;; (print "Hello world") ;; (exit) ;; EOF (repl)) (else (begin (set! *db* dbstruct) (import extras) ;; might not be needed ;; (import csi) (import readline) (import apropos) ;; (import (prefix sqlite3 sqlite3:)) ;; doesn't work ... (if *use-new-readline* (begin (install-history-file (get-environment-variable "HOME") ".megatest_history") ;; [homedir] [filename] [nlines]) (current-input-port (make-readline-port "megatest> "))) (begin (gnu-history-install-file-manager (string-append (or (get-environment-variable "HOME") ".") "/.megatest_history")) (current-input-port (make-gnu-readline-port "megatest> ")))) (if (args:get-arg "-repl") (repl) (load (args:get-arg "-load"))) ;; (db:close-all dbstruct) <= taken care of by on-exit call ) (exit))) (set! *didsomething* #t)))) ;;====================================================================== ;; Wait on a run to complete ;;====================================================================== |
︙ | ︙ |
Modified rmt.scm from [136b52bff0] to [ea574182b1].
︙ | ︙ | |||
217 218 219 220 221 222 223 | res)) (define (rmt:open-qry-close-locally cmd run-id params #!key (remretries 5)) (let* ((qry-is-write (not (member cmd api:read-only-queries))) (db-file-path (db:dbfile-path)) ;; 0)) (dbstruct-local (if *dbstruct-db* *dbstruct-db* | | | | | 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 | res)) (define (rmt:open-qry-close-locally cmd run-id params #!key (remretries 5)) (let* ((qry-is-write (not (member cmd api:read-only-queries))) (db-file-path (db:dbfile-path)) ;; 0)) (dbstruct-local (if *dbstruct-db* *dbstruct-db* (let* ((dbstruct (db:setup))) ;; make-dbr:dbstruct path: dbdir local: #t))) (set! *dbstruct-db* dbstruct) dbstruct))) (read-only (not (file-write-access? db-file-path))) (start (current-milliseconds)) (resdat (if (not (and read-only qry-is-write)) (api:execute-requests dbstruct-local (vector (symbol->string cmd) params)) (vector #t '()))) (success (vector-ref resdat 0)) (res (vector-ref resdat 1)) |
︙ | ︙ |
Modified rpc-transport.scm from [62a65daa58] to [03438a9e32].
︙ | ︙ | |||
94 95 96 97 98 99 100 | (ipaddrstr (if (string=? "-" hostn) (server:get-best-guess-address hostname) ;; (string-intersperse (map number->string (u8vector->list (hostname->ip hostname))) ".") #f)) (portnum (rpc:default-server-port)) (host:port (conc (if ipaddrstr ipaddrstr hostname) ":" portnum)) (tdb (tasks:open-db))) (thread-start! th1) | | | | 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 | (ipaddrstr (if (string=? "-" hostn) (server:get-best-guess-address hostname) ;; (string-intersperse (map number->string (u8vector->list (hostname->ip hostname))) ".") #f)) (portnum (rpc:default-server-port)) (host:port (conc (if ipaddrstr ipaddrstr hostname) ":" portnum)) (tdb (tasks:open-db))) (thread-start! th1) (set! db *dbstruct-db*) (open-run-close tasks:server-set-interface-port tasks:open-db server-id ipaddrstr portnum) (debug:print 0 *default-log-port* "Server started on " host:port) ;; (trace rpc:publish-procedure!) ;; (rpc:publish-procedure! 'server:login server:login) ;; (rpc:publish-procedure! 'testing (lambda () "Just testing")) ;;====================================================================== ;; ;; end of publish-procedure section ;;====================================================================== ;; (on-exit (lambda () (open-run-close tasks:server-set-state! tasks:open-db server-id "stopped"))) (set! *rpc:listener* rpc:listener) (tasks:server-set-state! tdb server-id "running") (set! *dbstruct-db* (db:setup run-id)) ;; if none running or if > 20 seconds since ;; server last used then start shutdown (let loop ((count 0)) (thread-sleep! 5) ;; no need to do this very often (let ((numrunning -1)) ;; (db:get-count-tests-running db))) (if (or (> numrunning 0) (> (+ *last-db-access* 60)(current-seconds))) |
︙ | ︙ |