Megatest

Check-in [854b6c8345]
Login
☰Home Timeline Files Branches Tags Tickets Wiki Login
Overview
Comment:streamline sync more
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | v1.60
Files: files | file ages | folders
SHA1: 854b6c8345d955bc8fa3730fcb64e4de3a416526
User & Date: matt on 2014-11-14 20:54:57
Other Links: branch diff | manifest | tags
Context
2014-11-15
03:03
Make sqlite3 sync configurable check-in: cbe91f84d1 user: matt tags: v1.60
2014-11-14
20:54
streamline sync more check-in: 854b6c8345 user: matt tags: v1.60
14:04
Simplified triggering of sync check-in: a1b0d55f23 user: mrwellan tags: v1.60
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Modified db.scm from [b47bb467de] to [bfde4b4e2f].

︙
456
457
458
459
460
461
462



463
464
465
466
467
468
469
470
471
472
473
474
475
476






477
478
479
480

481
482
483
484
485
486
487
488
489
490
491
492
493
494


495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514





















515
516
517
518
519
520
521

456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478

479
480
481
482
483
484
485
486
487

488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504




















505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532








+
+
+













-
+
+
+
+
+
+



-
+














+
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+








		 (field->num (make-hash-table))
		 (num->field (apply vector (map car fields)))
		 (full-sel   (conc "SELECT " (string-intersperse (map car fields) ",") 
				   " FROM " tablename ";"))
		 (full-ins   (conc "INSERT OR REPLACE INTO " tablename " ( " (string-intersperse (map car fields) ",") " ) "
				   " VALUES ( " (string-intersperse (make-list num-fields "?") ",") " );"))
		 (fromdat    '())
		 (fromdats   '())
		 (totrecords 0)
		 (batch-len  (string->number (or (configf:lookup *configdat* "sync" "batchsize") "10")))
		 (todat      (make-hash-table))
		 (count      0))

	    ;; set up the field->num table
	    (for-each
	     (lambda (field)
	       (hash-table-set! field->num field count)
	       (set! count (+ count 1)))
	     fields)

	    ;; read the source table
	    (sqlite3:for-each-row
	     (lambda (a . b)
	       (set! fromdat (cons (apply vector a b) fromdat)))
	       (set! fromdat (cons (apply vector a b) fromdat))
	       (if (> (length fromdat) batch-len)
		   (begin
		     (set! fromdats (cons fromdat fromdats))
		     (set! fromdat  '())
		     (set! totrecords (+ totrecords 1)))))
	     (db:dbdat-get-db fromdb)
	     full-sel)

	    (debug:print-info 2 "found " (length fromdat) " records to sync")
	    (debug:print-info 2 "found " totrecords " records to sync")

	    ;; read the target table
	    (sqlite3:for-each-row
	     (lambda (a . b)
	       (hash-table-set! todat a (apply vector a b)))
	     (db:dbdat-get-db todb)
	     full-sel)

	    ;; first pass implementation, just insert all changed rows
	    (for-each 
	     (lambda (targdb)
	       (let* ((db     (db:dbdat-get-db targdb))
		      (stmth  (sqlite3:prepare db full-ins)))
		 ;; (db:delay-if-busy targdb) ;; NO WAITING
		 (for-each
		  (lambda (fromdat-lst)
		 (sqlite3:with-transaction
		  db
		  (lambda ()
		    (for-each ;; 
		     (lambda (fromrow)
		       (let* ((a    (vector-ref fromrow 0))
			      (curr (hash-table-ref/default todat a #f))
			      (same #t))
			 (let loop ((i 0))
			   (if (or (not curr)
				   (not (equal? (vector-ref fromrow i)(vector-ref curr i))))
			       (set! same #f))
			   (if (and same
				    (< i (- num-fields 1)))
			       (loop (+ i 1))))
			 (if (not same)
			     (begin
			       (apply sqlite3:execute stmth (vector->list fromrow))
			       (hash-table-set! numrecs tablename (+ 1 (hash-table-ref/default numrecs tablename 0)))))))
		     fromdat)))
		    (sqlite3:with-transaction
		     db
		     (lambda ()
		       (for-each ;; 
			(lambda (fromrow)
			  (let* ((a    (vector-ref fromrow 0))
				 (curr (hash-table-ref/default todat a #f))
				 (same #t))
			    (let loop ((i 0))
			      (if (or (not curr)
				      (not (equal? (vector-ref fromrow i)(vector-ref curr i))))
				  (set! same #f))
			      (if (and same
				       (< i (- num-fields 1)))
				  (loop (+ i 1))))
			    (if (not same)
				(begin
				  (apply sqlite3:execute stmth (vector->list fromrow))
				  (hash-table-set! numrecs tablename (+ 1 (hash-table-ref/default numrecs tablename 0)))))))
			fromdat-lst))))
		  fromdats)
		 (sqlite3:finalize! stmth)))
	     (append (list todb) slave-dbs))))
	tbls)
       (let* ((runtime      (- (current-milliseconds) start-time))
	      (should-print (common:low-noise-print 30 "db sync" (> runtime 500)))) ;; low and high sync times treated as separate.
	 (if should-print (debug:print 0 "INFO: db sync, total run time " runtime " ms"))
	 (for-each
︙

Modified megatest.scm from [1795fd261c] to [3ecca6725e].

︙
293
294
295
296
297
298
299
300

301
302
303
304
305
306
307

293
294
295
296
297
298
299

300
301
302
303
304
305
306
307








-
+








       ;; sync for filesystem local db writes
       ;;
       (let ((start-time      (current-seconds))
	     (servers-started (make-hash-table)))
	 (for-each 
	  (lambda (run-id)
	    (mutex-lock! *db-multi-sync-mutex*)
	    (if (hash-table-ref/default *db-local-sync* run-id 0)
	    (if (hash-table-ref/default *db-local-sync* run-id #f)
		;; (if (> (- start-time last-write) 5) ;; every five seconds
		(let ((sync-time (- (current-seconds) start-time)))
		  (db:multi-db-sync (list run-id) 'new2old)
		  (if (common:low-noise-print 30 "sync new to old")
		      (begin
			(debug:print-info 0 "Sync of newdb to olddb for run-id " run-id " completed in " sync-time " seconds")
			(if (and (> sync-time 10) ;; took more than ten seconds, start a server for this run
︙

Modified rmt.scm from [6409627d1a] to [b91ff3594c].

︙
101
102
103
104
105
106
107
108
109



110
111
112
113
114




115
116
117
118
119
120
121

101
102
103
104
105
106
107


108
109
110
111




112
113
114
115
116
117
118
119
120
121
122








-
-
+
+
+

-
-
-
-
+
+
+
+








		;; start with three calls then kill server
		(if (eq? attemptnum 3)(tasks:kill-server-run-id run-id))

		(rmt:send-receive cmd run-id params attemptnum: (+ attemptnum 1)))))
	(let ((max-avg-qry (string->number (or (configf:lookup *configdat* "server" "server-query-threshold") "10"))))
	  (debug:print-info 4 "no server and read-only query, bypassing normal channel")
	  ;; (if (rmt:write-frequency-over-limit? cmd run-id)(server:kind-run run-id))
	  (let ((curr-max (rmt:get-max-query-average run-id)))
	    (if (> (cdr curr-max) max-avg-qry)
	  (let* ((curr-max     (rmt:get-max-query-average run-id))
		 (curr-max-val (cdr curr-max)))
	    (if (> curr-max-val max-avg-qry)
		(if (common:low-noise-print 10 "start server due to max average query too long")
		      (begin
			(debug:print-info 0 "Max average query, " (inexact->exact (round (cdr curr-max))) "ms (" (car curr-max) ") exceeds " max-avg-qry ", try starting server ...")
			(server:kind-run run-id)
			(debug:print-info 3 "Max average query, " (inexact->exact (round (cdr curr-max))) "ms (" (car curr-max) ") below " max-avg-qry ", not starting server...")))))
		    (begin
		      (debug:print-info 0 "Max average query, " (inexact->exact (round curr-max-val)) "ms (" (car curr-max) ") exceeds " max-avg-qry "ms, try starting server ...")
		      (server:kind-run run-id))
		    (debug:print-info 3 "Max average query, " (inexact->exact (round curr-max-val)) "ms (" (car curr-max) ") below " max-avg-qry "ms, not starting server..."))))
	  (rmt:open-qry-close-locally cmd run-id params)))))

(define (rmt:update-db-stats run-id rawcmd params duration)
  (mutex-lock! *db-stats-mutex*)
  (handle-exceptions
   exn
   (begin
︙

Modified synchash.scm from [530875f74d] to [9881f5a738].

︙
115
116
117
118
119
120
121
122

123
124
125
126
127
128

115
116
117
118
119
120
121

122
123
124
125
126
127
128








-
+







		       ;; (debug:print-info 2 "header: " header ", data: " data)
		       (cons (list "header" header)         ;; add the header keyed by the word "header"
			     (map make-indexed data))))        ;; add each element keyed by the keynum'th val
		    (else 
		     ;; (debug:print-info 2 "Non-get runs call")
		     (map make-indexed newdat))))
    ;; (debug:print-info 2 "postdat: " postdat)
    (if (not indb)(sqlite3:finalize! db))
    ;; (if (not indb)(sqlite3:finalize! db))
    (if (not synchash)
	(begin
	  (set! synchash (make-hash-table))
	  (hash-table-set! *synchashes* synckey synchash)))
    (synchash:get-delta postdat synchash)))

Modified tests/fdktestqa/testqa/Makefile from [598f7499e7] to [d3de829000]. 

1
2
3
4

5
6
7
8
9
10
11

1
2
3
4
5
6
7
8
9
10
11
12





+








BINDIR    = $(PWD)/../../../bin
PATH     := $(BINDIR):$(PATH)
MEGATEST  = $(BINDIR)/megatest
DASHBOARD = $(BINDIR)/dashboard
NEWDASHBOARD = $(BINDIR)/newdashboard
RUNNAME   = a


all :
	$(MEGATEST) -remove-runs -target a/b :runname c -testpatt %/%
	$(MEGATEST) -runtests % -target a/b :runname c
︙
22
23
24
25
26
27
28



29
30
31
32
33
34

23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38








+
+
+








bigrun3 :
	$(MEGATEST) -runtests bigrun3 -target a/bigrun3 :runname $(RUNNAME)

dashboard : 
	$(DASHBOARD) -rows 20 &

newdashboard :
	$(NEWDASHBOARD) &

compile :
	(cd ../../..;make -j && make install)

clean :
	rm -rf ../simple*/*/* megatest.db db/* ../simple*/.db/* logs/* monitor.db

This page was generated in about 0.107s by Fossil 2.24 [452f8e1ffc] 2024-03-26 15:36:00