Overview
Comment: | Patched in the -db2db code and it appears to work fine. |
---|---|
Downloads: | Tarball | ZIP archive | SQL archive |
Timelines: | family | ancestors | descendants | both | v1.80 |
Files: | files | file ages | folders |
SHA1: |
0131a588a0ab4fba7c6e093217fe38e1 |
User & Date: | matt on 2023-05-10 20:33:10 |
Other Links: | branch diff | manifest | tags |
Context
2023-05-11
| ||
05:43 | Use a key for the db lock-down that is unique to the db in .mtdb, this should elminate duplicate, overlapping servers. check-in: 138a40d18e user: matt tags: v1.80 | |
2023-05-10
| ||
20:33 | Patched in the -db2db code and it appears to work fine. check-in: 0131a588a0 user: matt tags: v1.80 | |
2023-05-08
| ||
05:23 | Lock defaults of tmp for cache and none for dashboard check-in: 08314706e4 user: matt tags: v1.80 | |
Changes
Modified dbmod.scm from [08a8b4a0ef] to [a71c3b544a].
︙ | ︙ | |||
27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 | (module dbmod * (import scheme chicken data-structures extras (prefix sqlite3 sqlite3:) posix typed-records srfi-1 srfi-18 srfi-69 commonmod | > > | 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 | (module dbmod * (import scheme chicken data-structures extras files (prefix sqlite3 sqlite3:) matchable posix typed-records srfi-1 srfi-18 srfi-69 commonmod |
︙ | ︙ | |||
206 207 208 209 210 211 212 | (lambda (last-update) (if *sync-in-progress* (debug:print 3 *default-log-port* "WARNING: overlapping calls to sync to disk") (begin (mutex-lock! *db-with-db-mutex*) ;; this mutex is used when overloaded or during a query that modifies the db (set! *sync-in-progress* #t) (dbmod:sync-gasket tables last-update inmem db | | | | | | | 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 | (lambda (last-update) (if *sync-in-progress* (debug:print 3 *default-log-port* "WARNING: overlapping calls to sync to disk") (begin (mutex-lock! *db-with-db-mutex*) ;; this mutex is used when overloaded or during a query that modifies the db (set! *sync-in-progress* #t) (dbmod:sync-gasket tables last-update inmem db dbfullname syncdir keys) (mutex-unlock! *db-with-db-mutex*) (thread-sleep! 0.5) ;; ensure at least 1/2 second down time between sync calls (set! *sync-in-progress* #f))))) ;; (dbmod:sync-tables tables #f db inmem) ;; (if db (dbmod:sync-gasket tables #f inmem db dbfullname 'fromdest keys) ;; ) ;; load into inmem (dbr:dbstruct-last-update-set! dbstruct (current-seconds)) ;; should this be offset back in time by one second? dbstruct)) ;; (if (eq? syncdir 'todisk) ;; sync to disk normally, sync from in dashboard ;; (dbmod:sync-tables tables last-update inmem db) ;; (dbmod:sync-tables tables last-update db inmem)))) ;; direction: 'fromdest 'todest ;; (define (dbmod:sync-gasket tables last-update inmem dbh dbfname direction keys) (assert (sqlite3:database? inmem) "FATAL: sync-gasket: inmem is not a db") (assert (sqlite3:database? inmem) "FATAL: sync-gasket: dbh is not a db") (case (dbfile:sync-method) ((none) #f) ((attach) (dbmod:attach-sync tables inmem dbfname direction)) ((newsync) (dbmod:new-sync tables inmem dbh dbfname direction)) (else (case direction ((todisk) (dbmod:sync-tables tables last-update keys inmem dbh) ) (else (dbmod:sync-tables tables last-update keys dbh inmem)))))) (define (dbmod:close-db dbstruct) ;; do final sync to disk file ;; (do-sync ...) (sqlite3:finalize! (dbr:dbstruct-ondiskdb dbstruct))) ;;====================================================================== |
︙ | ︙ | |||
273 274 275 276 277 278 279 | ;; ;; if last-update specified ("field-name" . time-in-seconds) ;; then sync only records where field-name >= time-in-seconds ;; IFF field-name exists ;; ;; Use (db:sync-all-tables-list keys) to get the tbls input ;; | | | 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 | ;; ;; if last-update specified ("field-name" . time-in-seconds) ;; then sync only records where field-name >= time-in-seconds ;; IFF field-name exists ;; ;; Use (db:sync-all-tables-list keys) to get the tbls input ;; (define (dbmod:sync-tables tbls last-update keys fromdb todb) (debug:print-info 0 *default-log-port* "dbmod:sync-tables called, from: "fromdb", to: "todb) (assert (sqlite3:database? fromdb) "FATAL: dbmod:sync-tables called with fromdb not a database" fromdb) (assert (sqlite3:database? todb) "FATAL: dbmod:sync-tables called with fromdb not a database" todb) (let ((stmts (make-hash-table)) ;; table-field => stmt (all-stmts '()) ;; ( ( stmt1 value1 ) ( stml2 value2 )) (numrecs (make-hash-table)) (start-time (current-milliseconds)) |
︙ | ︙ | |||
750 751 752 753 754 755 756 | (debug:print-info 0 *default-log-port* "Found old test in LAUNCHED state, test-id=" test-id " 1 day since event_time marked") (set! oldlaunched (cons (list test-id run-dir uname testname item-path run-id) oldlaunched))))) stmth3 run-id)))) (list incompleted oldlaunched toplevels))) | > > > | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 | (debug:print-info 0 *default-log-port* "Found old test in LAUNCHED state, test-id=" test-id " 1 day since event_time marked") (set! oldlaunched (cons (list test-id run-dir uname testname item-path run-id) oldlaunched))))) stmth3 run-id)))) (list incompleted oldlaunched toplevels))) ;;====================================================================== ;; db to db sync ;;====================================================================== (define (dbmod:db-to-db-sync src-db dest-db last-update init-proc keys) (if (and (file-exists? src-db) ;; can't proceed without a source (file-read-access? src-db)) (let* ((have-dest (file-exists? dest-db)) (dest-file-wr (and have-dest (file-write-access? dest-db))) ;; exists and writable (dest-dir (or (pathname-directory dest-db) ".")) (dest-dir-wr (and (file-exists? dest-dir) (file-write-access? dest-dir))) (d-wr (or (and have-dest dest-file-wr) dest-dir-wr)) (copied (if (and (not have-dest) dest-dir-wr) (begin (file-copy src-db dest-db) #t) #f))) (if copied (begin (debug:print-info 0 *default-log-port* "db-to-db-sync done with file-copy") #t) (let* ((tables (db:sync-all-tables-list keys)) (sdb (dbmod:safely-open-db src-db init-proc #t)) (ddb (dbmod:safely-open-db dest-db init-proc d-wr)) (res (dbmod:sync-gasket tables last-update sdb ddb dest-db 'todest keys))) (sqlite3:finalize! sdb) (sqlite3:finalize! ddb) res))) #f)) ) |
Modified megatest.scm from [adec828972] to [2473417c38].
︙ | ︙ | |||
282 283 284 285 286 287 288 289 290 291 292 293 294 295 | -list-run-time : list time requered to complete runs. It supports following switches -run-patt <patt> -target-patt <patt> -dumpmode <csv,json,plain-text> -list-test-time : list time requered to complete each test in a run. It following following arguments -runname <patt> -target <patt> -dumpmode <csv,json,plain-text> -syscheck : do some very basic checks; write access and space in tmp, home, runs, links and is $DISPLAY valid -list-waivers : dump waivers for specified target, runname, testpatt to stdout Diff report -diff-rep : generate diff report (must include -src-target, -src-runname, -target, -runname and either -diff-email or -diff-html) -src-target <target> -src-runname <target> -diff-email <emails> : comma separated list of email addresses to send diff report | > | 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 | -list-run-time : list time requered to complete runs. It supports following switches -run-patt <patt> -target-patt <patt> -dumpmode <csv,json,plain-text> -list-test-time : list time requered to complete each test in a run. It following following arguments -runname <patt> -target <patt> -dumpmode <csv,json,plain-text> -syscheck : do some very basic checks; write access and space in tmp, home, runs, links and is $DISPLAY valid -list-waivers : dump waivers for specified target, runname, testpatt to stdout -db2db : sync db to db, use -from, -to for dbs, -period and -timeout for continuous sync Diff report -diff-rep : generate diff report (must include -src-target, -src-runname, -target, -runname and either -diff-email or -diff-html) -src-target <target> -src-runname <target> -diff-email <emails> : comma separated list of email addresses to send diff report |
︙ | ︙ | |||
331 332 333 334 335 336 337 | ":state" "-state" ":status" "-status" "-list-runs" "-testdata-csv" "-testpatt" | | > | 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 | ":state" "-state" ":status" "-status" "-list-runs" "-testdata-csv" "-testpatt" ;; "--modepatt" "-modepatt" "-tagexpr" "-itempatt" "-setlog" "-set-toplog" "-runstep" "-logpro" "-m" "-rerun" "-days" "-rename-run" "-from" "-to" "-dest" "-source" "-time-stamp" ;; values and messages ":category" ":variable" |
︙ | ︙ | |||
376 377 378 379 380 381 382 383 384 385 386 387 388 389 | "-pathmod" "-env2file" "-envcap" "-envdelta" "-setvars" "-set-state-status" "-import-sexpr" ;; move runs stuff here "-remove-keep" "-set-run-status" "-age" ;; archive | > > | 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 | "-pathmod" "-env2file" "-envcap" "-envdelta" "-setvars" "-set-state-status" "-import-sexpr" "-period" ;; sync period in seconds "-timeout" ;; exit sync if timeout in seconds exceeded since last change ;; move runs stuff here "-remove-keep" "-set-run-status" "-age" ;; archive |
︙ | ︙ | |||
493 494 495 496 497 498 499 500 501 502 503 504 505 506 | "-create-megatest-area" "-mark-incompletes" "-convert-to-norm" "-convert-to-old" "-import-megatest.db" "-sync-to-megatest.db" "-sync-brute-force" "-logging" "-v" ;; verbose 2, more than normal (normal is 1) "-q" ;; quiet 0, errors/warnings only "-diff-rep" | > | 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 | "-create-megatest-area" "-mark-incompletes" "-convert-to-norm" "-convert-to-old" "-import-megatest.db" "-sync-to-megatest.db" "-db2db" "-sync-brute-force" "-logging" "-v" ;; verbose 2, more than normal (normal is 1) "-q" ;; quiet 0, errors/warnings only "-diff-rep" |
︙ | ︙ | |||
2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 | (set! *didsomething* #t))) (if (args:get-arg "-sync-to") (let ((toppath (launch:setup))) (tasks:sync-to-postgres *configdat* (args:get-arg "-sync-to")) (set! *didsomething* #t))) (if (args:get-arg "-list-test-time") (let* ((toppath (launch:setup))) (task:get-test-times) (set! *didsomething* #t))) (if (args:get-arg "-list-run-time") (let* ((toppath (launch:setup))) | > > > > > > > > > > > > > > > > > > > > > > > | 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 | (set! *didsomething* #t))) (if (args:get-arg "-sync-to") (let ((toppath (launch:setup))) (tasks:sync-to-postgres *configdat* (args:get-arg "-sync-to")) (set! *didsomething* #t))) ;; use with -from and -to ;; (if (args:get-arg "-db2db") (let* ((duh (launch:setup)) (src-db (args:get-arg "-from")) (dest-db (args:get-arg "-to")) (sync-period (args:get-arg "-period")) ;; NOT IMPLEMENTED YET (sync-timeout (args:get-arg "-timeout")) ;; NOT IMPLEMENTED YET (lockfile (conc dest-db".lock")) (keys (db:get-keys #f)) ) (if (and src-db dest-db) (begin (debug:print-info 0 *default-log-port* "Attempting to sync data from "src-db" to "dest-db"...") (let ((res (dbmod:db-to-db-sync src-db dest-db 0 (dbfile:db-init-proc) keys))) (if res (debug:print 0 *default-log-port* "Synced " res " records from "src-db" to "dest-db) (debug:print 0 *default-log-port* "No sync due to permissions or non-existant source db.")))) (debug:print 0 *default-log-port* "Skipping sync, there is a sync in progress.")) (set! *didsomething* #t)) (debug:print 0 *default-log-port* "Usage for -db2db; -to and -from must be specified")) (if (args:get-arg "-list-test-time") (let* ((toppath (launch:setup))) (task:get-test-times) (set! *didsomething* #t))) (if (args:get-arg "-list-run-time") (let* ((toppath (launch:setup))) |
︙ | ︙ |