Overview
Comment: | Merged mtdb change. |
---|---|
Downloads: | Tarball | ZIP archive | SQL archive |
Timelines: | family | ancestors | descendants | both | v1.80 |
Files: | files | file ages | folders |
SHA1: |
a50d34f842e80aedf36281247f3ebc86 |
User & Date: | mrwellan on 2023-03-22 09:05:09 |
Other Links: | branch diff | manifest | tags |
Context
2023-03-22
| ||
13:01 | Merged fork check-in: 9fe68a8fb2 user: mrwellan tags: v1.80, v1.8011 | |
09:05 | Merged mtdb change. check-in: a50d34f842 user: mrwellan tags: v1.80 | |
2023-03-21
| ||
21:09 | Converted .megatest to .mtdb Leaf check-in: f00c7b0bd9 user: matt tags: v1.80-mtdb | |
08:46 | Merged fork check-in: 747321a245 user: mrwellan tags: v1.80 | |
Changes
Modified dashboard.scm from [7828b43e10] to [7b2a364846].
︙ | ︙ | |||
221 222 223 224 225 226 227 | (dboard:commondat-tabdats commondat) tabnum tabdat)) ;; gets and calls updater list based on curr-tab-num ;; (define (dboard:common-run-curr-updaters commondat #!key (tab-num #f)) | | | 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 | (dboard:commondat-tabdats commondat) tabnum tabdat)) ;; gets and calls updater list based on curr-tab-num ;; (define (dboard:common-run-curr-updaters commondat #!key (tab-num #f)) ;; (sync-db-to-tmp (dboard:common-get-tabdat commondat tab-num: tab-num)) ;; no longer applies (if (dboard:common-get-tabdat commondat tab-num: tab-num) ;; only update if there is a tabdat (let* ((tnum (or tab-num (dboard:commondat-curr-tab-num commondat))) (updaters (hash-table-ref/default (dboard:commondat-updaters commondat) tnum '()))) (debug:print 4 *default-log-port* "Found these updaters: " updaters " for tab-num: " tnum) (for-each ;; perform the function calls for the complete updaters list |
︙ | ︙ | |||
682 683 684 685 686 687 688 | ;;(dboard:tabdat-filters-changed tabdat)) 0 (dboard:rundat-last-update run-dat))) (last-db-time (if do-not-use-db-file-timestamps 0 (dboard:rundat-last-db-time run-dat))) (db-path (or (dboard:rundat-db-path run-dat) | | | | 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 | ;;(dboard:tabdat-filters-changed tabdat)) 0 (dboard:rundat-last-update run-dat))) (last-db-time (if do-not-use-db-file-timestamps 0 (dboard:rundat-last-db-time run-dat))) (db-path (or (dboard:rundat-db-path run-dat) (let* ((db-dir (get-environment-variable "MT_RUN_AREA_HOME")) ;; (common:get-db-tmp-area)) (db-pth (conc db-dir "/.mtdb/main.db"))) (dboard:rundat-db-path-set! run-dat db-pth) db-pth))) (db-mod-time (common:lazy-sqlite-db-modification-time db-path)) (db-modified (>= db-mod-time last-db-time)) (multi-get (> (dboard:rundat-run-data-offset run-dat) 0)) ;; multi-get in progress (tmptests (if (or do-not-use-db-file-timestamps (dboard:tabdat-filters-changed tabdat) |
︙ | ︙ | |||
3811 3812 3813 3814 3815 3816 3817 | ;;====================================================================== (stop-the-train) (define (main) ;; (print "Starting dashboard main") | | | 3811 3812 3813 3814 3815 3816 3817 3818 3819 3820 3821 3822 3823 3824 3825 | ;;====================================================================== (stop-the-train) (define (main) ;; (print "Starting dashboard main") (let* ((mtdb-path (conc *toppath* "/.mtdb/main.db")) (target (args:get-arg "-target")) (commondat (dboard:commondat-make))) (if target (begin (args:remove-arg-from-ht "-target") (dboard:commondat-target-set! commondat target) ) |
︙ | ︙ |
Modified db.scm from [f0df6a6b92] to [d85a30ff09].
︙ | ︙ | |||
491 492 493 494 495 496 497 | (file-read-access? f)) (file-modification-time f) 0)))) (max (get-mtime fname) (get-mtime wal-file) (get-mtime shm-file)))) | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | < > | | 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 | (file-read-access? f)) (file-modification-time f) 0)))) (max (get-mtime fname) (get-mtime wal-file) (get-mtime shm-file)))) ;; (define (db:all-db-sync dbstruct) ;; (let* ((dbdat (db:open-db dbstruct #f db:initialize-main-db)) ;; (data-synced 0) ;; count of changed records ;; (tmp-area (common:get-db-tmp-area)) ;; (dbfiles (glob (conc tmp-area"/.megatest/*.db"))) ;; (sync-durations (make-hash-table)) ;; (no-sync-db (db:open-no-sync-db))) ;; (for-each ;; (lambda (file) ;; tmp db file ;; (debug:print-info 3 *default-log-port* "file: " file) ;; (let* ((fname (conc (pathname-file file) ".db")) ;; fname is tmp db file ;; (wal-file (conc fname "-wal")) ;; (shm-file (conc fname "-shm")) ;; (fulln (conc *toppath*"/.megatest/"fname)) ;; fulln is nfs db name ;; (wal-time (if (file-exists? wal-file) ;; (file-modification-time wal-file) ;; 0)) ;; (shm-time (if (file-exists? shm-file) ;; (file-modification-time shm-file) ;; 0)) ;; (time1 (db:get-sqlite3-mod-time file)) ;; ;; (if (file-exists? file) ;; time1 is the max itime of the tmp db, -wal and -shm files. ;; ;; (max (file-modification-time file) wal-time shm-time) ;; ;; (begin ;; ;; (debug:print-info 2 *default-log-port* "Sync - I do not see file "file) ;; ;; 1))) ;; (time2 (db:get-sqlite3-mod-time fulln)) ;; ;; (if (file-exists? fulln) ;; time2 is nfs file time ;; ;; (file-modification-time fulln) ;; ;; (begin ;; ;; (debug:print-info 2 *default-log-port* "Sync - I do not see file "fulln) ;; ;; 0))) ;; (changed (> (- time1 time2) (+ (random 5) 1))) ;; it has been at some few seconds since last synced ;; (changed10 (> (- time1 time2) 10)) ;; it has been at least ten seconds since sync'd ;; (jfile-exists (file-exists? (conc file"-journal"))) ;; i.e. are we busy? ;; (do-cp (cond ;; ((not (file-exists? fulln)) ;; shouldn't happen, but this might recover ;; (cons #t (conc "File "fulln" not found! Copying "fname" to "fulln))) ;; ((and (not jfile-exists) changed) ;; (cons #t "not busy, changed")) ;; not busy and changed ;; ((and jfile-exists changed10) ;; (cons #t "busy but not synced in a while")) ;; busy but not sync'd in over 10 seconds ;; ((and changed *time-to-exit*) ;; (cons #t "Time to exit, forced final sync")) ;; last sync ;; (else ;; (cons #f "No sync needed"))))) ;; (if (car do-cp) ;; (let* ((start-time (current-milliseconds)) ;; (fname (pathname-file file)) ;; (runid (if (string= fname "main") #f (string->number fname)))) ;; (debug:print-info 3 *default-log-port* "db:all-db-sync: fname: " ;; fname", delta: " (- time1 time2) " seconds, reason: "(cdr do-cp)) ;; (db:lock-and-delta-sync no-sync-db dbstruct fname runid (db:get-keys dbstruct) db:initialize-main-db) ;; (hash-table-set! sync-durations (conc fname".db") ;; (- (current-milliseconds) start-time))) ;; (debug:print-info 3 *default-log-port* "skipping sync. " file " is up to date") ;; ))) ;; dbfiles) ;; ;; WHY does the dbdat need to be added back? ;; (if dbdat (dbfile:add-dbdat dbstruct #f dbdat)) ;; ) ;; #t) (define (db:kill-servers) (let* ((servers (server:choose-server *toppath* 'all-valid))) ;; (server:get-list *toppath*)) (for-each (lambda (server) (handle-exceptions exn |
︙ | ︙ | |||
1477 1478 1479 1480 1481 1482 1483 | ;;====================================================================== ;; no-sync.db - small bits of data to be shared between servers ;;====================================================================== (define (db:get-dbsync-path) (case (rmt:transport-mode) ((http)(common:get-db-tmp-area)) | | | | 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 | ;;====================================================================== ;; no-sync.db - small bits of data to be shared between servers ;;====================================================================== (define (db:get-dbsync-path) (case (rmt:transport-mode) ((http)(common:get-db-tmp-area)) ((tcp) (conc *toppath*"/.mtdb")) ((nfs) (conc *toppath*"/.mtdb")) (else "/tmp/dunno-this-gonna-exist"))) ;; This is needed for api.scm (define (db:open-no-sync-db) (dbfile:open-no-sync-db (db:get-dbsync-path))) ;; why get the keys from the db? why not get from the *configdat* |
︙ | ︙ |
Modified dbfile.scm from [49cffc3a66] to [7406bd8270].
︙ | ︙ | |||
203 204 205 206 207 208 209 | ((number? run-id) (modulo run-id (num-run-dbs))) ((not run-id) "main") ;; 0 or main? (else run-id))) ;; POTENTIAL BUG: this implementation could produce a db file if run-id is neither #f or a number (define (dbfile:run-id->dbname run-id) | | | 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 | ((number? run-id) (modulo run-id (num-run-dbs))) ((not run-id) "main") ;; 0 or main? (else run-id))) ;; POTENTIAL BUG: this implementation could produce a db file if run-id is neither #f or a number (define (dbfile:run-id->dbname run-id) (conc ".mtdb/"(dbfile:run-id->dbnum run-id)".db")) ;; Make the dbstruct, setup up auxillary db's and call for main db at least once ;; ;; called in http-transport and replicated in rmt.scm for *local* access. ;; (define (dbfile:setup do-sync areapath tmppath) (cond |
︙ | ︙ |
Modified dbmod.scm from [d1bb9414cd] to [b286c5dacd].
︙ | ︙ | |||
47 48 49 50 51 52 53 | ;; NOTE: This returns only the name "1.db", "main.db", not the path ;; (define (dbmod:run-id->dbfname run-id) (conc (dbfile:run-id->dbnum run-id)".db")) (define (dbmod:get-dbdir dbstruct) (let* ((areapath (dbr:dbstruct-areapath dbstruct)) | | | > > > > | 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 | ;; NOTE: This returns only the name "1.db", "main.db", not the path ;; (define (dbmod:run-id->dbfname run-id) (conc (dbfile:run-id->dbnum run-id)".db")) (define (dbmod:get-dbdir dbstruct) (let* ((areapath (dbr:dbstruct-areapath dbstruct)) (dbdir (conc areapath"/.mtdb"))) (if (and (file-write-access? areapath) (not (file-exists? dbdir))) (create-directory dbdir)) dbdir)) (define (dbmod:run-id->full-dbfname dbstruct run-id) (conc (dbmod:get-dbdir dbstruct run-id )"/"(dbmod:run-id->dbfname run-id))) ;;====================================================================== ;; Read-only inmem cached direct from disk method ;;====================================================================== (define *dbmod:nfs-db-handles* (make-hash-table)) ;; dbfname -> dbstruct |
︙ | ︙ |
Modified megatest-version.scm from [7d35fceb35] to [11989e8c9e].
︙ | ︙ | |||
16 17 18 19 20 21 22 | ;; along with Megatest. If not, see <http://www.gnu.org/licenses/>. ;; Always use two or four digit decimal ;; 1.01, 1.02...1.10,1.11,1.1101 ... 1.99,2.00.. ;; (declare (unit megatest-version)) | | | 16 17 18 19 20 21 22 23 | ;; along with Megatest. If not, see <http://www.gnu.org/licenses/>. ;; Always use two or four digit decimal ;; 1.01, 1.02...1.10,1.11,1.1101 ... 1.99,2.00.. ;; (declare (unit megatest-version)) (define megatest-version 1.8011) |