Megatest

Check-in [19861e6399]
Login
Overview
Comment:Use an actual droop
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | v1.80
Files: files | file ages | folders
SHA1: 19861e6399144f886d2d5ff42784450c8c729a8e
User & Date: matt on 2023-02-02 12:54:27
Other Links: branch diff | manifest | tags
Context
2023-02-10
21:06
Merging pretty good branch v1.80-dbperf to v1.80 check-in: 7170e5f43b user: matt tags: v1.80
20:19
Use debugprint module in dbfile module as stepping stone to replacing old debug:print calls with new. Closed-Leaf check-in: 0e8fa15f1d user: matt tags: v1.80-debugprint
2023-02-06
19:35
Squashed v1.80-dbperformance into one commit check-in: 6c7b8be468 user: matt tags: v1.80-dbperf
2023-02-05
11:47
Minor clean up. There were a couple communication errors in sixtyfivek but they looked likely to be host related. Closed-Leaf check-in: e973b1fb77 user: matt tags: v1.80-cleanup
08:36
wip, close idle db connections check-in: 97a3c4ad11 user: matt tags: v1.80-close-idle-connections
2023-02-03
02:16
Reduce load from get-state-status-and-roll-up-run. check-in: 4e634eb46a user: matt tags: v1.80-dbperformance
2023-02-02
12:54
Use an actual droop check-in: 19861e6399 user: matt tags: v1.80
09:27
Change couple queries to use prepared statements check-in: a2e41e0613 user: matt tags: v1.80
Changes

Modified api.scm from [9f9940bc49] to [0676a2f9d1].

188
189
190
191
192
193
194




195

196
197
198
199
200
201
202
188
189
190
191
192
193
194
195
196
197
198

199
200
201
202
203
204
205
206







+
+
+
+
-
+







            (start-t           (current-milliseconds))
            (readonly-mode     (dbr:dbstruct-read-only dbstruct))
            (readonly-command  (member cmd api:read-only-queries))
            (writecmd-in-readonly-mode (and readonly-mode (not readonly-command))))
       (if (not readonly-command)
	   (mutex-lock! write-mutex))
       (let* ((tmppath    (dbr:dbstruct-tmppath  dbstruct))
	      (clean-run-id (cond
			     ((number? run-id)   run-id)
			     ((equal? run-id #f) "main")
			     (else               "other")))
	      (crumbfile  (dbfile:wait-for-qif tmppath run-id (cons cmd params)))
	      (crumbfile  (dbfile:wait-for-qif tmppath clean-run-id (cons cmd params)))
	      (res    
               (if writecmd-in-readonly-mode
                   (conc "attempt to run write command "cmd" on a read-only database")
		   (api:dispatch-request dbstruct cmd run-id params))))
	 (delete-file* crumbfile)
	 (if (not readonly-command)
	     (mutex-unlock! write-mutex))

Modified dbfile.scm from [0def328a84] to [25f8271ef2].

1003
1004
1005
1006
1007
1008
1009

1010
1011
1012
1013
1014
1015
1016
1017


1018
1019
1020
1021
1022
1023
1024
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016


1017
1018
1019
1020
1021
1022
1023
1024
1025







+






-
-
+
+







    (set! *db-write-access* (not (dbr:dbdat-read-only dbdat)))
    ;; (mutex-unlock! *db-open-mutex*)
    dbdat))

(define dbfile:db-init-proc (make-parameter #f))

(define keep-age-param (make-parameter 10))
(define qif-slope      (make-parameter 100))

;; create a dropping near the db file in a qif dir
;; use count of such files to gate queries (queries in flight)
;;
(define (dbfile:wait-for-qif fname run-id params)
  (let* ((thedir  (pathname-directory fname))
	 (destdir (conc thedir"/qif"))
	 (uniqn   (get-area-path-signature (conc run-id params)))
	 (destdir (conc thedir"/qif-"run-id))
	 (uniqn   (get-area-path-signature (conc (or run-id "main") params)))
	 (crumbn  (conc destdir"/"(current-seconds)"-"uniqn"."(current-process-id))))
    (if (not (file-exists? destdir))(create-directory (conc destdir"/attic") #t))
    (let loop ((count 0))
      (let* ((currlks (glob (conc destdir"/*")))
	     (numqrys (length currlks))
	     (delayval (cond ;; do a droopish curve
			((> numqrys 50)
1032
1033
1034
1035
1036
1037
1038



1039

1040
1041
1042
1043
1044
1045
1046







1047
1048
1049
1050
1051
1052
1053
1054
1055

1056
1057
1058
1059
1060
1061
1062
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042

1043
1044






1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059

1060
1061
1062
1063
1064
1065
1066
1067







+
+
+
-
+

-
-
-
-
-
-
+
+
+
+
+
+
+








-
+







					    (file-modification-time f)))
				       (keep-age-param))
				    (let* ((basedir (pathname-directory f))
					   (filen   (pathname-file f))
					   (destf   (conc basedir"/attic/"filen)))
				      (dbfile:print-err "Moving qif file "f" older than 10 seconds to "destf)
				      ;; (delete-file* f)
				      (handle-exceptions
					  exn
					#t
				      (file-move f destf #t))))
					(file-move f destf #t)))))
			      currlks))
			 1)
			((> numqrys 30) 0.50)
			((> numqrys 25) 0.20)
			((> numqrys 20) 0.10)
			((> numqrys 15) 0.05)
			((> numqrys 10) 0.01)
			 1) ;; 50 and above => 1
			((> numqrys 10) (* numqrys (/ 1 (qif-slope)))) ;; slope of 1/100
			;; ((> numqrys 30) 0.50)
			;; ((> numqrys 25) 0.20)
			;; ((> numqrys 20) 0.10)
			;; ((> numqrys 15) 0.05)
			;; ((> numqrys 10) 0.01)
			(else #f))))
	(if (and delayval
		 (< count 5))
	    (begin
	      (thread-sleep! delayval)
	      (loop (+ count 1))))))
    (with-output-to-file crumbn
      (lambda ()
	(print fname" "run-id" "params)
	(print fname" run-id="run-id" params="params)
	))
    crumbn))

(define no-condition-db-with-db (make-parameter #t))

;; (db:with-db dbstruct run-id sqlite3:exec "select blah fgrom blaz;")
;; r/w is a flag to indicate if the db is modified by this query #t = yes, #f = no
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1087
1088
1089
1090
1091
1092
1093

1094
1095
1096
1097
1098
1099
1100







-







			(if use-mutex (mutex-unlock! *db-with-db-mutex*))
			;; (if (vector? dbstruct)(db:done-with dbstruct run-id r/w))
			(if dbdat
			    (dbfile:add-dbdat dbstruct run-id dbdat))
			;; (delete-file* crumbfile)
			res))))

	 ;; (crumbfile (dbfile:wait-for-qif fname run-id params)))
    (assert (sqlite3:database? db) "FATAL: db:with-db, db is not a database, db="db", fname="fname)
    (if (file-exists? jfile)
	(begin
	  (dbfile:print-err "INFO: "jfile" exists, delaying to reduce database load")
	  (thread-sleep! 0.2)))
    (if (and use-mutex
	     (common:low-noise-print 120 "over-50-parallel-api-requests"))

Modified launch.scm from [9881087e2c] to [34af87f4bd].

312
313
314
315
316
317
318
319

320
321
322
323
324
325
326
312
313
314
315
316
317
318

319
320
321
322
323
324
325
326







-
+







		      (tests:test-set-status! run-id test-id "KILLED"  "KILLED" (conc (args:get-arg "-m")" "kill-reason) #f)) ;; BB ADDED kill-reason -- confirm OK with Matt
		    (begin
		      (debug:print-error 0 *default-log-port* "Nothing to kill, pid1=" pid1 ", pid2=" pid2)
		      (tests:test-set-status! run-id test-id "KILLED"  "FAILED TO KILL" (conc (args:get-arg "-m")" "kill-reason) #f) ;; BB ADDED kill-reason -- confirm OK with Matt
		      )))
	      (mutex-unlock! m)
	      ;; no point in sticking around. Exit now. But run end of run before exiting?
        (launch:end-of-run-check run-id)
              (launch:end-of-run-check run-id)
	      (exit)))
	(if (hash-table-ref/default misc-flags 'keep-going #f)
	    (begin
	      (thread-sleep! 3) ;; (+ 3 (random 6))) ;; add some jitter to the call home time to spread out the db accesses
	      (if (hash-table-ref/default misc-flags 'keep-going #f)  ;; keep originals for cpu-load and disk-free unless they change more than the allowed delta
		  (loop (calc-minutes)
                        (or new-cpu-load cpu-load)