Overview
| Comment: | Use an actual droop |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | v1.80 |
| Files: | files | file ages | folders |
| SHA1: |
19861e6399144f886d2d5ff42784450c |
| User & Date: | matt on 2023-02-02 12:54:27 |
| Other Links: | branch diff | manifest | tags |
Context
|
2023-02-10
| ||
| 21:06 | Merging pretty good branch v1.80-dbperf to v1.80 check-in: 7170e5f43b user: matt tags: v1.80 | |
| 20:19 | Use debugprint module in dbfile module as stepping stone to replacing old debug:print calls with new. Closed-Leaf check-in: 0e8fa15f1d user: matt tags: v1.80-debugprint | |
|
2023-02-06
| ||
| 19:35 | Squashed v1.80-dbperformance into one commit check-in: 6c7b8be468 user: matt tags: v1.80-dbperf | |
|
2023-02-05
| ||
| 11:47 | Minor clean up. There were a couple communication errors in sixtyfivek but they looked likely to be host related. Closed-Leaf check-in: e973b1fb77 user: matt tags: v1.80-cleanup | |
| 08:36 | wip, close idle db connections check-in: 97a3c4ad11 user: matt tags: v1.80-close-idle-connections | |
|
2023-02-03
| ||
| 02:16 | Reduce load from get-state-status-and-roll-up-run. check-in: 4e634eb46a user: matt tags: v1.80-dbperformance | |
|
2023-02-02
| ||
| 12:54 | Use an actual droop check-in: 19861e6399 user: matt tags: v1.80 | |
| 09:27 | Change couple queries to use prepared statements check-in: a2e41e0613 user: matt tags: v1.80 | |
Changes
Modified api.scm from [9f9940bc49] to [0676a2f9d1].
| ︙ | ︙ | |||
188 189 190 191 192 193 194 |
(start-t (current-milliseconds))
(readonly-mode (dbr:dbstruct-read-only dbstruct))
(readonly-command (member cmd api:read-only-queries))
(writecmd-in-readonly-mode (and readonly-mode (not readonly-command))))
(if (not readonly-command)
(mutex-lock! write-mutex))
(let* ((tmppath (dbr:dbstruct-tmppath dbstruct))
| > > > > | | 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 |
(start-t (current-milliseconds))
(readonly-mode (dbr:dbstruct-read-only dbstruct))
(readonly-command (member cmd api:read-only-queries))
(writecmd-in-readonly-mode (and readonly-mode (not readonly-command))))
(if (not readonly-command)
(mutex-lock! write-mutex))
(let* ((tmppath (dbr:dbstruct-tmppath dbstruct))
(clean-run-id (cond
((number? run-id) run-id)
((equal? run-id #f) "main")
(else "other")))
(crumbfile (dbfile:wait-for-qif tmppath clean-run-id (cons cmd params)))
(res
(if writecmd-in-readonly-mode
(conc "attempt to run write command "cmd" on a read-only database")
(api:dispatch-request dbstruct cmd run-id params))))
(delete-file* crumbfile)
(if (not readonly-command)
(mutex-unlock! write-mutex))
|
| ︙ | ︙ |
Modified dbfile.scm from [0def328a84] to [25f8271ef2].
| ︙ | ︙ | |||
1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 |
(set! *db-write-access* (not (dbr:dbdat-read-only dbdat)))
;; (mutex-unlock! *db-open-mutex*)
dbdat))
(define dbfile:db-init-proc (make-parameter #f))
(define keep-age-param (make-parameter 10))
;; create a dropping near the db file in a qif dir
;; use count of such files to gate queries (queries in flight)
;;
(define (dbfile:wait-for-qif fname run-id params)
(let* ((thedir (pathname-directory fname))
| > | | | 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 |
(set! *db-write-access* (not (dbr:dbdat-read-only dbdat)))
;; (mutex-unlock! *db-open-mutex*)
dbdat))
(define dbfile:db-init-proc (make-parameter #f))
(define keep-age-param (make-parameter 10))
(define qif-slope (make-parameter 100))
;; create a dropping near the db file in a qif dir
;; use count of such files to gate queries (queries in flight)
;;
(define (dbfile:wait-for-qif fname run-id params)
(let* ((thedir (pathname-directory fname))
(destdir (conc thedir"/qif-"run-id))
(uniqn (get-area-path-signature (conc (or run-id "main") params)))
(crumbn (conc destdir"/"(current-seconds)"-"uniqn"."(current-process-id))))
(if (not (file-exists? destdir))(create-directory (conc destdir"/attic") #t))
(let loop ((count 0))
(let* ((currlks (glob (conc destdir"/*")))
(numqrys (length currlks))
(delayval (cond ;; do a droopish curve
((> numqrys 50)
|
| ︙ | ︙ | |||
1032 1033 1034 1035 1036 1037 1038 | (file-modification-time f))) (keep-age-param)) (let* ((basedir (pathname-directory f)) (filen (pathname-file f)) (destf (conc basedir"/attic/"filen))) (dbfile:print-err "Moving qif file "f" older than 10 seconds to "destf) ;; (delete-file* f) | > > > | | > | | | | | | | 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 |
(file-modification-time f)))
(keep-age-param))
(let* ((basedir (pathname-directory f))
(filen (pathname-file f))
(destf (conc basedir"/attic/"filen)))
(dbfile:print-err "Moving qif file "f" older than 10 seconds to "destf)
;; (delete-file* f)
(handle-exceptions
exn
#t
(file-move f destf #t)))))
currlks))
1) ;; 50 and above => 1
((> numqrys 10) (* numqrys (/ 1 (qif-slope)))) ;; slope of 1/100
;; ((> numqrys 30) 0.50)
;; ((> numqrys 25) 0.20)
;; ((> numqrys 20) 0.10)
;; ((> numqrys 15) 0.05)
;; ((> numqrys 10) 0.01)
(else #f))))
(if (and delayval
(< count 5))
(begin
(thread-sleep! delayval)
(loop (+ count 1))))))
(with-output-to-file crumbn
(lambda ()
(print fname" run-id="run-id" params="params)
))
crumbn))
(define no-condition-db-with-db (make-parameter #t))
;; (db:with-db dbstruct run-id sqlite3:exec "select blah fgrom blaz;")
;; r/w is a flag to indicate if the db is modified by this query #t = yes, #f = no
|
| ︙ | ︙ | |||
1082 1083 1084 1085 1086 1087 1088 | (if use-mutex (mutex-unlock! *db-with-db-mutex*)) ;; (if (vector? dbstruct)(db:done-with dbstruct run-id r/w)) (if dbdat (dbfile:add-dbdat dbstruct run-id dbdat)) ;; (delete-file* crumbfile) res)))) | < | 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 |
(if use-mutex (mutex-unlock! *db-with-db-mutex*))
;; (if (vector? dbstruct)(db:done-with dbstruct run-id r/w))
(if dbdat
(dbfile:add-dbdat dbstruct run-id dbdat))
;; (delete-file* crumbfile)
res))))
(assert (sqlite3:database? db) "FATAL: db:with-db, db is not a database, db="db", fname="fname)
(if (file-exists? jfile)
(begin
(dbfile:print-err "INFO: "jfile" exists, delaying to reduce database load")
(thread-sleep! 0.2)))
(if (and use-mutex
(common:low-noise-print 120 "over-50-parallel-api-requests"))
|
| ︙ | ︙ |
Modified launch.scm from [9881087e2c] to [34af87f4bd].
| ︙ | ︙ | |||
312 313 314 315 316 317 318 | (tests:test-set-status! run-id test-id "KILLED" "KILLED" (conc (args:get-arg "-m")" "kill-reason) #f)) ;; BB ADDED kill-reason -- confirm OK with Matt (begin (debug:print-error 0 *default-log-port* "Nothing to kill, pid1=" pid1 ", pid2=" pid2) (tests:test-set-status! run-id test-id "KILLED" "FAILED TO KILL" (conc (args:get-arg "-m")" "kill-reason) #f) ;; BB ADDED kill-reason -- confirm OK with Matt ))) (mutex-unlock! m) ;; no point in sticking around. Exit now. But run end of run before exiting? | | | 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 |
(tests:test-set-status! run-id test-id "KILLED" "KILLED" (conc (args:get-arg "-m")" "kill-reason) #f)) ;; BB ADDED kill-reason -- confirm OK with Matt
(begin
(debug:print-error 0 *default-log-port* "Nothing to kill, pid1=" pid1 ", pid2=" pid2)
(tests:test-set-status! run-id test-id "KILLED" "FAILED TO KILL" (conc (args:get-arg "-m")" "kill-reason) #f) ;; BB ADDED kill-reason -- confirm OK with Matt
)))
(mutex-unlock! m)
;; no point in sticking around. Exit now. But run end of run before exiting?
(launch:end-of-run-check run-id)
(exit)))
(if (hash-table-ref/default misc-flags 'keep-going #f)
(begin
(thread-sleep! 3) ;; (+ 3 (random 6))) ;; add some jitter to the call home time to spread out the db accesses
(if (hash-table-ref/default misc-flags 'keep-going #f) ;; keep originals for cpu-load and disk-free unless they change more than the allowed delta
(loop (calc-minutes)
(or new-cpu-load cpu-load)
|
| ︙ | ︙ |