Megatest

Check-in [78b2c9d57a]
Login
Overview
Comment:added db rotation in .db-snapshot; added some error checking to brute force sync
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | v1.65-dump-for-sync
Files: files | file ages | folders
SHA1: 78b2c9d57aece8e82b1d13e9fe91fefdd722c907
User & Date: bjbarcla on 2019-02-08 15:09:03
Other Links: branch diff | manifest | tags
Context
2019-02-08
16:48
fixes check-in: bfc2a0c173 user: bjbarcla tags: v1.65-dump-for-sync
15:09
added db rotation in .db-snapshot; added some error checking to brute force sync check-in: 78b2c9d57a user: bjbarcla tags: v1.65-dump-for-sync
14:29
Added MT_SQLITE3_EXE var check-in: 20b2810c40 user: mrwellan tags: v1.65-dump-for-sync
Changes

Modified common.scm from [c75d843fa5] to [b1327a8176].

335
336
337
338
339
340
341


















































































342
343
344
345
346
347
348
   ;; (if full
       '(dejunk)
       ;; '())
       )
  (if (common:api-changed?)
      (common:set-last-run-version)))



















































































;; Rotate logs, logic: 
;;                 if > 500k and older than 1 week:
;;                     remove previous compressed log and compress this log
;; WARNING: This proc operates assuming that it is in the directory above the
;;          logs directory you wish to log-rotate.
;;
(define (common:rotate-logs)







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
   ;; (if full
       '(dejunk)
       ;; '())
       )
  (if (common:api-changed?)
      (common:set-last-run-version)))

(define (common:snapshot-file filepath #!key (subdir  ".") )
  (if (file-exists? filepath)
      (let* ((age-sec  (lambda (file)
                         (if (file-exists? file)
                             (- (current-seconds) (file-modification-time file))
                             1000000000))) ;; return really old value if file doesn't exist.  we want to clobber it if old or not exist.
             (ok-flag  #t)
             (age-mins (lambda (file) (/ (age-sec   file) 60)))
             (age-hrs  (lambda (file) (/ (age-min s file) 60)))
             (age-days (lambda (file) (/ (age-hrs   file) 24)))
             (age-wks  (lambda (file) (/ (age-days  file) 7)))
             (docmd    (lambda (cmd)
                         (cond
                          (ok-flag
                           (let ((res (system cmd)))
                             (cond
                              ((eq? 0 res)
                               #t)
                              (else
                               (set! ok-flag #f)
                               (debug:print 0 *default-log-port* "ERROR: Command failed with exit code "
                                            (if (< res 0)
                                                res
                                                (/ res 8)) " ["cmd"]" )
                               #f))))
                          (else
                           (debug:print 0 *default-log-port* "ERROR: Nor runnining command due to prior error. ["cmd"]")
                           #f))))
             (copy     (lambda (src dest) (docmd (conc "/bin/cp '"src"' '"dest"'"))))
             (copy+zip (lambda (src dest) (docmd (conc "gzip -c - < '"src"' > '"dest"'"))))
             (fullpath (realpath filepath))
             (basedir  (pathname-directory fullpath))
             (basefile (pathname-strip-directory fullpath))
             ;;(prevfile (conc filepath ".prev.gz"))
             (minsfile (conc basedir "/" subdir "/" basefile ".mins.gz"))
             (hrsfile  (conc basedir "/" subdir "/" basefile ".hrs.gz"))
             (daysfile (conc basedir "/" subdir "/" basefile ".days.gz"))
             (wksfile  (conc basedir "/" subdir "/" basefile ".weeks.gz")))

        ;; create subdir it not exists
        (if (not (directory-exists? (conc basedir "/" subdir)))
            (docmd (conc "/bin/mkdir -p '"(conc basedir "/" subdir)"'")))
        
        ;; copy&zip <file> to <file>.mins if not exists
        (if (not (file-exists? minsfile))
            (copy+zip filepath minsfile))
        ;; copy <file>.mins to <file>.hrs if not exists
        (if (not (file-exists? hrsfile))
            (copy minsfile hrsfile))
        ;; copy <file>.hrs to <file>.days if not exists
        (if (not (file-exists? daysfile))
            (copy hrsfile daysfile))
        ;; copy <file>.days to <file>.weeks if not exists
        (if (not (file-exists? wksfile))
            (copy daysfile wksfile))


        ;; if age(<file>.mins.gz) >= 1h:
        ;;     copy <file>.mins.gz <file>.hrs.gz
        ;;     copy <prev file> <file>.mins.gz
        (when (>= (age-hrs minsfile) 1)
          (copy minsfile hrsfile)
          (copy+zip filepath  minsfile))
        
        ;; if age(<file>.hrs.gz) >= 1d:
        ;;     copy <file>.hrs.gz <file>.days.gz
        ;;     copy <file>.mins.gz <file>.hrs.gz
        (when (>= (age-days hrsfile) 1)
          (copy hrsfile daysfile)
          (copy minsfile hrsfile))
        
        ;; if age(<file>.days.gz) >= 1w:
        ;;     copy <file>.days.gz <file>.weeks.gz
        ;;     copy <file>.hrs.gz <file>.days.gz
        (when (>= (age-weeks daysfile) 1)
          (copy daysfile wksfile)
          (copy hrsfile daysfile))
        #t)
      #f))
  
        
        
;; Rotate logs, logic: 
;;                 if > 500k and older than 1 week:
;;                     remove previous compressed log and compress this log
;; WARNING: This proc operates assuming that it is in the directory above the
;;          logs directory you wish to log-rotate.
;;
(define (common:rotate-logs)

Modified megatest-version.scm from [58d639f18b] to [f713858d59].

16
17
18
19
20
21
22
23
;;     along with Megatest.  If not, see <http://www.gnu.org/licenses/>.

;; Always use two or four digit decimal
;; 1.01, 1.02...1.10,1.11,1.1101 ... 1.99,2.00..

(declare (unit megatest-version))

(define megatest-version 1.6523)







|
16
17
18
19
20
21
22
23
;;     along with Megatest.  If not, see <http://www.gnu.org/licenses/>.

;; Always use two or four digit decimal
;; 1.01, 1.02...1.10,1.11,1.1101 ... 1.99,2.00..

(declare (unit megatest-version))

(define megatest-version 1.6524)

Modified megatest.scm from [6ab2ef604b] to [bd9e9c775d].

341
342
343
344
345
346
347

348
349
350
351
352
353
354
			"-var"
			"-dumpmode"
			"-run-id"
			"-ping"
			"-refdb2dat"
			"-o"
			"-log"

			"-since"
			"-fields"
			"-recover-test" ;; run-id,test-id - used internally to recover a test stuck in RUNNING state
			"-sort"
			"-target-db"
			"-source-db"
			"-prefix-target"







>







341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
			"-var"
			"-dumpmode"
			"-run-id"
			"-ping"
			"-refdb2dat"
			"-o"
			"-log"
                        "-sync-log"
			"-since"
			"-fields"
			"-recover-test" ;; run-id,test-id - used internally to recover a test stuck in RUNNING state
			"-sort"
			"-target-db"
			"-source-db"
			"-prefix-target"

Modified server.scm from [4e67ff8c04] to [35a6adab78].

501
502
503
504
505
506
507


508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523


524

525


526
527




528
529
530
531
532
533
534
535
536
;;      (else #f))))

;; moving this here as it needs access to db and cannot be in common.
;;
(define (server:writable-watchdog dbstruct)
  (thread-sleep! 10) ;; delay for startup
  (let* ((legacy-sync  (common:run-sync?))


	 (tmp-area     (common:get-db-tmp-area))
	 (tmp-db       (conc tmp-area "/megatest.db"))
	 (staging-file (conc *toppath* "/.megatest.db"))
	 (mtdbfile     (conc *toppath* "/megatest.db"))
	 (lockfile     (conc tmp-db ".lock"))
	 (cmdline      (conc "megatest -sync-to-megatest.db "
			     (if (args:get-arg "-log")
				 (conc " -log " (args:get-arg "-log"))
				 (conc *toppath* "/logs/sync-" (current-process-id) "-" (get-host-name) ".log"))))
	 (min-intersync-delay (configf:lookup-number *configdat* "server" "minimum-intersync-delay" default: 30)))
    (if (and (not (args:get-arg "-sync-to-megatest.db")) ;; conditions under which we do not run the sync
	     (args:get-arg "-server"))
	(let loop ()
	  (thread-sleep! min-intersync-delay)
	  (if (not (common:file-exists? lockfile))
	      (begin


		(delete-file* staging-file)

		(system (conc "sqlite3 " tmp-db " .dump | sqlite3 " staging-file))


		(delete-file* (conc mtdbfile ".backup"))
		(system (conc "mv " staging-file " " mtdbfile))




		;; (system "megatest -sync-to-megatest.db&"))
		))
	  
	  ;; keep going unless time to exit
	  ;;
	  (if (not *time-to-exit*)
	      (let delay-loop ((count 0))
		;;(debug:print-info 13 *default-log-port* "delay-loop top; count="count" pid="(current-process-id)" this-wd-num="this-wd-num" *time-to-exit*="*time-to-exit*)
		







>
>





|
<
<
<







>
>

>
|
>
>
|
|
>
>
>
>
|
<







501
502
503
504
505
506
507
508
509
510
511
512
513
514
515



516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536

537
538
539
540
541
542
543
;;      (else #f))))

;; moving this here as it needs access to db and cannot be in common.
;;
(define (server:writable-watchdog dbstruct)
  (thread-sleep! 10) ;; delay for startup
  (let* ((legacy-sync  (common:run-sync?))
         (sqlite-exe   (or (get-shell-env-var "MT_SQLITE3_EXE"))) ;; defined in cfg.sh
         (sync-log     (or (args:get-arg "-sync-log") (conc *toppath* "/logs/sync-" (current-process-id) "-" (get-host-name) ".log")))
	 (tmp-area     (common:get-db-tmp-area))
	 (tmp-db       (conc tmp-area "/megatest.db"))
	 (staging-file (conc *toppath* "/.megatest.db"))
	 (mtdbfile     (conc *toppath* "/megatest.db"))
	 (lockfile     (conc tmp-db ".lock"))
         (sync-cmd     (conc sqlite-exe" " tmp-db " .dump | "sqlite-exe" " staging-file "&>"sync-log))



	 (min-intersync-delay (configf:lookup-number *configdat* "server" "minimum-intersync-delay" default: 30)))
    (if (and (not (args:get-arg "-sync-to-megatest.db")) ;; conditions under which we do not run the sync
	     (args:get-arg "-server"))
	(let loop ()
	  (thread-sleep! min-intersync-delay)
	  (if (not (common:file-exists? lockfile))
	      (begin
                (if (not (configf:lookup *configdat* "server" "disable-db-snapshot"))
                    (common:snapshot-file mtdbfile subdir: ".db-snapshot"))
		(delete-file* staging-file)
		(let ((start-time (current-milliseconds))
                      (res (system sync-cmd)))
                  (cond
                   ((eq? 0 res)
		    (delete-file* (conc mtdbfile ".backup"))
		    (system (conc "/bin/mv " staging-file " " mtdbfile))
                    (debug:print 1 *default-log-port* "INFO: SYNC took "(/ (- (current-milliseconds) start-time))" sec")
                    #t)
                   (else
                    (debug:print 0 *default-log-port* "ERROR: Sync failed. See log at "sync-log)
                    (system (conc "mv "mtdbfile ".backup" mtdbfile)))))))

	  
	  ;; keep going unless time to exit
	  ;;
	  (if (not *time-to-exit*)
	      (let delay-loop ((count 0))
		;;(debug:print-info 13 *default-log-port* "delay-loop top; count="count" pid="(current-process-id)" this-wd-num="this-wd-num" *time-to-exit*="*time-to-exit*)