Overview
Comment: | added db rotation in .db-snapshot; added some error checking to brute force sync |
---|---|
Downloads: | Tarball | ZIP archive | SQL archive |
Timelines: | family | ancestors | descendants | both | v1.65-dump-for-sync |
Files: | files | file ages | folders |
SHA1: |
78b2c9d57aece8e82b1d13e9fe91fefd |
User & Date: | bjbarcla on 2019-02-08 15:09:03 |
Other Links: | branch diff | manifest | tags |
Context
2019-02-08
| ||
16:48 | fixes check-in: bfc2a0c173 user: bjbarcla tags: v1.65-dump-for-sync | |
15:09 | added db rotation in .db-snapshot; added some error checking to brute force sync check-in: 78b2c9d57a user: bjbarcla tags: v1.65-dump-for-sync | |
14:29 | Added MT_SQLITE3_EXE var check-in: 20b2810c40 user: mrwellan tags: v1.65-dump-for-sync | |
Changes
Modified common.scm from [c75d843fa5] to [b1327a8176].
︙ | ︙ | |||
335 336 337 338 339 340 341 342 343 344 345 346 347 348 | ;; (if full '(dejunk) ;; '()) ) (if (common:api-changed?) (common:set-last-run-version))) ;; Rotate logs, logic: ;; if > 500k and older than 1 week: ;; remove previous compressed log and compress this log ;; WARNING: This proc operates assuming that it is in the directory above the ;; logs directory you wish to log-rotate. ;; (define (common:rotate-logs) | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 | ;; (if full '(dejunk) ;; '()) ) (if (common:api-changed?) (common:set-last-run-version))) (define (common:snapshot-file filepath #!key (subdir ".") ) (if (file-exists? filepath) (let* ((age-sec (lambda (file) (if (file-exists? file) (- (current-seconds) (file-modification-time file)) 1000000000))) ;; return really old value if file doesn't exist. we want to clobber it if old or not exist. (ok-flag #t) (age-mins (lambda (file) (/ (age-sec file) 60))) (age-hrs (lambda (file) (/ (age-min s file) 60))) (age-days (lambda (file) (/ (age-hrs file) 24))) (age-wks (lambda (file) (/ (age-days file) 7))) (docmd (lambda (cmd) (cond (ok-flag (let ((res (system cmd))) (cond ((eq? 0 res) #t) (else (set! ok-flag #f) (debug:print 0 *default-log-port* "ERROR: Command failed with exit code " (if (< res 0) res (/ res 8)) " ["cmd"]" ) #f)))) (else (debug:print 0 *default-log-port* "ERROR: Nor runnining command due to prior error. ["cmd"]") #f)))) (copy (lambda (src dest) (docmd (conc "/bin/cp '"src"' '"dest"'")))) (copy+zip (lambda (src dest) (docmd (conc "gzip -c - < '"src"' > '"dest"'")))) (fullpath (realpath filepath)) (basedir (pathname-directory fullpath)) (basefile (pathname-strip-directory fullpath)) ;;(prevfile (conc filepath ".prev.gz")) (minsfile (conc basedir "/" subdir "/" basefile ".mins.gz")) (hrsfile (conc basedir "/" subdir "/" basefile ".hrs.gz")) (daysfile (conc basedir "/" subdir "/" basefile ".days.gz")) (wksfile (conc basedir "/" subdir "/" basefile ".weeks.gz"))) ;; create subdir it not exists (if (not (directory-exists? (conc basedir "/" subdir))) (docmd (conc "/bin/mkdir -p '"(conc basedir "/" subdir)"'"))) ;; copy&zip <file> to <file>.mins if not exists (if (not (file-exists? minsfile)) (copy+zip filepath minsfile)) ;; copy <file>.mins to <file>.hrs if not exists (if (not (file-exists? hrsfile)) (copy minsfile hrsfile)) ;; copy <file>.hrs to <file>.days if not exists (if (not (file-exists? daysfile)) (copy hrsfile daysfile)) ;; copy <file>.days to <file>.weeks if not exists (if (not (file-exists? wksfile)) (copy daysfile wksfile)) ;; if age(<file>.mins.gz) >= 1h: ;; copy <file>.mins.gz <file>.hrs.gz ;; copy <prev file> <file>.mins.gz (when (>= (age-hrs minsfile) 1) (copy minsfile hrsfile) (copy+zip filepath minsfile)) ;; if age(<file>.hrs.gz) >= 1d: ;; copy <file>.hrs.gz <file>.days.gz ;; copy <file>.mins.gz <file>.hrs.gz (when (>= (age-days hrsfile) 1) (copy hrsfile daysfile) (copy minsfile hrsfile)) ;; if age(<file>.days.gz) >= 1w: ;; copy <file>.days.gz <file>.weeks.gz ;; copy <file>.hrs.gz <file>.days.gz (when (>= (age-weeks daysfile) 1) (copy daysfile wksfile) (copy hrsfile daysfile)) #t) #f)) ;; Rotate logs, logic: ;; if > 500k and older than 1 week: ;; remove previous compressed log and compress this log ;; WARNING: This proc operates assuming that it is in the directory above the ;; logs directory you wish to log-rotate. ;; (define (common:rotate-logs) |
︙ | ︙ |
Modified megatest-version.scm from [58d639f18b] to [f713858d59].
︙ | ︙ | |||
16 17 18 19 20 21 22 | ;; along with Megatest. If not, see <http://www.gnu.org/licenses/>. ;; Always use two or four digit decimal ;; 1.01, 1.02...1.10,1.11,1.1101 ... 1.99,2.00.. (declare (unit megatest-version)) | | | 16 17 18 19 20 21 22 23 | ;; along with Megatest. If not, see <http://www.gnu.org/licenses/>. ;; Always use two or four digit decimal ;; 1.01, 1.02...1.10,1.11,1.1101 ... 1.99,2.00.. (declare (unit megatest-version)) (define megatest-version 1.6524) |
Modified megatest.scm from [6ab2ef604b] to [bd9e9c775d].
︙ | ︙ | |||
341 342 343 344 345 346 347 348 349 350 351 352 353 354 | "-var" "-dumpmode" "-run-id" "-ping" "-refdb2dat" "-o" "-log" "-since" "-fields" "-recover-test" ;; run-id,test-id - used internally to recover a test stuck in RUNNING state "-sort" "-target-db" "-source-db" "-prefix-target" | > | 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 | "-var" "-dumpmode" "-run-id" "-ping" "-refdb2dat" "-o" "-log" "-sync-log" "-since" "-fields" "-recover-test" ;; run-id,test-id - used internally to recover a test stuck in RUNNING state "-sort" "-target-db" "-source-db" "-prefix-target" |
︙ | ︙ |
Modified server.scm from [4e67ff8c04] to [35a6adab78].
︙ | ︙ | |||
501 502 503 504 505 506 507 508 509 510 511 512 | ;; (else #f)))) ;; moving this here as it needs access to db and cannot be in common. ;; (define (server:writable-watchdog dbstruct) (thread-sleep! 10) ;; delay for startup (let* ((legacy-sync (common:run-sync?)) (tmp-area (common:get-db-tmp-area)) (tmp-db (conc tmp-area "/megatest.db")) (staging-file (conc *toppath* "/.megatest.db")) (mtdbfile (conc *toppath* "/megatest.db")) (lockfile (conc tmp-db ".lock")) | > > | < < < > > > | > > | | > > > > | < | 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 | ;; (else #f)))) ;; moving this here as it needs access to db and cannot be in common. ;; (define (server:writable-watchdog dbstruct) (thread-sleep! 10) ;; delay for startup (let* ((legacy-sync (common:run-sync?)) (sqlite-exe (or (get-shell-env-var "MT_SQLITE3_EXE"))) ;; defined in cfg.sh (sync-log (or (args:get-arg "-sync-log") (conc *toppath* "/logs/sync-" (current-process-id) "-" (get-host-name) ".log"))) (tmp-area (common:get-db-tmp-area)) (tmp-db (conc tmp-area "/megatest.db")) (staging-file (conc *toppath* "/.megatest.db")) (mtdbfile (conc *toppath* "/megatest.db")) (lockfile (conc tmp-db ".lock")) (sync-cmd (conc sqlite-exe" " tmp-db " .dump | "sqlite-exe" " staging-file "&>"sync-log)) (min-intersync-delay (configf:lookup-number *configdat* "server" "minimum-intersync-delay" default: 30))) (if (and (not (args:get-arg "-sync-to-megatest.db")) ;; conditions under which we do not run the sync (args:get-arg "-server")) (let loop () (thread-sleep! min-intersync-delay) (if (not (common:file-exists? lockfile)) (begin (if (not (configf:lookup *configdat* "server" "disable-db-snapshot")) (common:snapshot-file mtdbfile subdir: ".db-snapshot")) (delete-file* staging-file) (let ((start-time (current-milliseconds)) (res (system sync-cmd))) (cond ((eq? 0 res) (delete-file* (conc mtdbfile ".backup")) (system (conc "/bin/mv " staging-file " " mtdbfile)) (debug:print 1 *default-log-port* "INFO: SYNC took "(/ (- (current-milliseconds) start-time))" sec") #t) (else (debug:print 0 *default-log-port* "ERROR: Sync failed. See log at "sync-log) (system (conc "mv "mtdbfile ".backup" mtdbfile))))))) ;; keep going unless time to exit ;; (if (not *time-to-exit*) (let delay-loop ((count 0)) ;;(debug:print-info 13 *default-log-port* "delay-loop top; count="count" pid="(current-process-id)" this-wd-num="this-wd-num" *time-to-exit*="*time-to-exit*) |
︙ | ︙ |