Overview
Comment: | Change default number of db's to 10, couple minor fixes. |
---|---|
Downloads: | Tarball | ZIP archive | SQL archive |
Timelines: | family | ancestors | descendants | both | v1.80-dbperformance |
Files: | files | file ages | folders |
SHA1: |
7b86032d25a8b4e522eb4bb5c5b05ec9 |
User & Date: | matt on 2023-02-04 20:00:48 |
Other Links: | branch diff | manifest | tags |
Context
2023-02-06
| ||
19:35 | Squashed v1.80-dbperformance into one commit check-in: 6c7b8be468 user: matt tags: v1.80-dbperf | |
2023-02-04
| ||
20:00 | Change default number of db's to 10, couple minor fixes. Closed-Leaf check-in: 7b86032d25 user: matt tags: v1.80-dbperformance | |
2023-02-03
| ||
21:51 | Reduce zombie cleanup, not clear it helps check-in: c8ec459e79 user: matt tags: v1.80-dbperformance | |
Changes
Modified db.scm from [3db5884da5] to [ffd8a763fc].
︙ | ︙ | |||
466 467 468 469 470 471 472 | fname", delta: " (- time1 time2) " seconds, reason: "(cdr do-cp)) (db:lock-and-delta-sync no-sync-db dbstruct fname runid (db:get-keys dbstruct) db:initialize-main-db) (hash-table-set! sync-durations (conc fname".db") (- (current-milliseconds) start-time))) (debug:print-info 3 *default-log-port* "skipping sync. " file " is up to date") ))) dbfiles) | > | > | 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 | fname", delta: " (- time1 time2) " seconds, reason: "(cdr do-cp)) (db:lock-and-delta-sync no-sync-db dbstruct fname runid (db:get-keys dbstruct) db:initialize-main-db) (hash-table-set! sync-durations (conc fname".db") (- (current-milliseconds) start-time))) (debug:print-info 3 *default-log-port* "skipping sync. " file " is up to date") ))) dbfiles) ;; WHY does the dbdat need to be added back? (if dbdat (dbfile:add-dbdat dbstruct #f dbdat)) ) #t) ;; options: ;; ;; 'killservers - kills all servers ;; 'dejunk - removes junk records ;; 'adj-testids - move test-ids into correct ranges |
︙ | ︙ | |||
596 597 598 599 600 601 602 603 604 605 606 607 608 609 | (lambda (subdb) (let* ((dbname (db:run-id->dbname run-id)) (mtdb (dbr:subdb-mtdb subdb)) (tmpdb (db:get-subdb dbstruct run-id)) (refndb (dbr:subdb-refndb subdb)) (newres (db:sync-tables (db:sync-all-tables-list dbstruct (db:get-keys dbstruct)) last-update tmpdb refndb mtdb))) ;; (stack-push! (dbr:subdb-dbstack subdb) tmpdb) (dbfile:add-dbdat dbstruct run-id tmpdb) (set! res (cons newres res)))) subdbs) res)) ;;;; run-ids ;; if #f use *db-local-sync* : or 'local-sync-flags | > | 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 | (lambda (subdb) (let* ((dbname (db:run-id->dbname run-id)) (mtdb (dbr:subdb-mtdb subdb)) (tmpdb (db:get-subdb dbstruct run-id)) (refndb (dbr:subdb-refndb subdb)) (newres (db:sync-tables (db:sync-all-tables-list dbstruct (db:get-keys dbstruct)) last-update tmpdb refndb mtdb))) ;; (stack-push! (dbr:subdb-dbstack subdb) tmpdb) ;; BUG: verify this is really needed (dbfile:add-dbdat dbstruct run-id tmpdb) (set! res (cons newres res)))) subdbs) res)) ;;;; run-ids ;; if #f use *db-local-sync* : or 'local-sync-flags |
︙ | ︙ | |||
902 903 904 905 906 907 908 909 910 911 912 913 914 915 | db (conc "SELECT d.id,d.archive_area_name,disk_path,last_df,last_df_time FROM archive_disks AS d INNER JOIN archive_blocks AS b ON d.id=b.archive_disk_id WHERE b.id IN (" (string-intersperse (map conc res) ",") ") AND last_df > ?;") dneeded)) (dbfile:add-dbdat dbstruct #f dbdat) blocks)) ;; returns id of the record, register a disk allocated to archiving and record it's last known ;; available space ;; (define (db:archive-register-disk dbstruct bdisk-name bdisk-path df) | > | 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 | db (conc "SELECT d.id,d.archive_area_name,disk_path,last_df,last_df_time FROM archive_disks AS d INNER JOIN archive_blocks AS b ON d.id=b.archive_disk_id WHERE b.id IN (" (string-intersperse (map conc res) ",") ") AND last_df > ?;") dneeded)) ;; BUG: Verfify this is really needed (dbfile:add-dbdat dbstruct #f dbdat) blocks)) ;; returns id of the record, register a disk allocated to archiving and record it's last known ;; available space ;; (define (db:archive-register-disk dbstruct bdisk-name bdisk-path df) |
︙ | ︙ |
Modified dbfile.scm from [94ab27ad7a] to [2ad26c948f].
︙ | ︙ | |||
35 36 37 38 39 40 41 42 43 | posix typed-records srfi-18 srfi-1 srfi-69 stack files ports commonmod ) | > < > > | 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 | posix typed-records srfi-18 srfi-1 srfi-69 stack files ports commonmod ;; debugprint ) (define keep-age-param (make-parameter 10)) ;; qif file age, if over move to attic (define num-run-dbs (make-parameter 10)) ;; number of db's in .megatest ;;====================================================================== ;; R E C O R D S ;;====================================================================== ;; a single Megatest area with it's multiple dbs is ;; managed in a dbstruct |
︙ | ︙ | |||
190 191 192 193 194 195 196 | ;; (define (dbfile:run-id->path apath run-id) (conc apath"/"(dbfile:run-id->dbname run-id))) (define (db:dbname->path apath dbname) (conc apath"/"dbname)) | < < | 192 193 194 195 196 197 198 199 200 201 202 203 204 205 | ;; (define (dbfile:run-id->path apath run-id) (conc apath"/"(dbfile:run-id->dbname run-id))) (define (db:dbname->path apath dbname) (conc apath"/"dbname)) (define (dbfile:run-id->dbnum run-id) (cond ((number? run-id) (modulo run-id (num-run-dbs))) ((not run-id) "main") ;; 0 or main? (else run-id))) |
︙ | ︙ | |||
249 250 251 252 253 254 255 | ;; return a previously opened db handle to the stack of available handles (define (dbfile:add-dbdat dbstruct run-id dbdat) (let* ((subdb (dbfile:get-subdb dbstruct run-id)) (dbstk (dbr:subdb-dbstack subdb)) (count (stack-count dbstk))) (if (> count 15) | | | 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 | ;; return a previously opened db handle to the stack of available handles (define (dbfile:add-dbdat dbstruct run-id dbdat) (let* ((subdb (dbfile:get-subdb dbstruct run-id)) (dbstk (dbr:subdb-dbstack subdb)) (count (stack-count dbstk))) (if (> count 15) (dbfile:print-err "WARNING: stack for "run-id".db is "count".")) (stack-push! dbstk dbdat) dbdat)) ;; set up a subdb ;; (define (dbfile:init-subdb dbstruct run-id init-proc) (let* ((dbname (dbfile:run-id->dbname run-id)) |
︙ | ︙ | |||
1013 1014 1015 1016 1017 1018 1019 | (let* ((dbdat (dbfile:open-db dbstruct run-id dbinit))) (set! *db-write-access* (not (dbr:dbdat-read-only dbdat))) ;; (mutex-unlock! *db-open-mutex*) dbdat)) (define dbfile:db-init-proc (make-parameter #f)) | < < | 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 | (let* ((dbdat (dbfile:open-db dbstruct run-id dbinit))) (set! *db-write-access* (not (dbr:dbdat-read-only dbdat))) ;; (mutex-unlock! *db-open-mutex*) dbdat)) (define dbfile:db-init-proc (make-parameter #f)) ;; in xmaxima this gives a curve close to what I want: ;; plot2d ((exp(x/1.2)-1)/300, [x, 0, 10])$ ;; plot2d ((exp(x/1.5)-1)/40, [x, 0, 10])$ ;; plot2d ((exp(x/5)-1)/40, [x, 0, 20])$ (define (dbfile:droop x) (/ (- (exp (/ x 5)) 1) 40)) ;; (* numqrys (/ 1 (qif-slope)))) |
︙ | ︙ |
Deleted ext-tests/sixtyfivek/run-four.sh version [ac96586fe3].
|
| < < < < < < < < < |
Modified launch.scm from [3243ef2278] to [a705b2e0b2].
︙ | ︙ | |||
27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 | (import (prefix base64 base64:)) (import (prefix sqlite3 sqlite3:)) (declare (unit launch)) (declare (uses subrun)) (declare (uses common)) (declare (uses configf)) (declare (uses db)) (declare (uses ezsteps)) (include "common_records.scm") (include "key_records.scm") (include "db_records.scm") (include "megatest-fossil-hash.scm") ;;====================================================================== ;; ezsteps ;;====================================================================== ;; ezsteps were going to be coded as ;; stepname[,predstep1,predstep2 ...] [{VAR1=first,second,third}] command to execute | > > > | 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 | (import (prefix base64 base64:)) (import (prefix sqlite3 sqlite3:)) (declare (unit launch)) (declare (uses subrun)) (declare (uses common)) (declare (uses commonmod)) (declare (uses configf)) (declare (uses db)) (declare (uses ezsteps)) (include "common_records.scm") (include "key_records.scm") (include "db_records.scm") (include "megatest-fossil-hash.scm") (import commonmod) ;;====================================================================== ;; ezsteps ;;====================================================================== ;; ezsteps were going to be coded as ;; stepname[,predstep1,predstep2 ...] [{VAR1=first,second,third}] command to execute |
︙ | ︙ |
Modified runs.scm from [dd48ee309d] to [e0e242ce63].
︙ | ︙ | |||
1861 1862 1863 1864 1865 1866 1867 | (my-item-path (item-list->path my-itemdat)) (newtestname (db:test-make-full-name hed my-item-path))) ;; test names are unique on testname/item-path (tests:testqueue-set-items! new-test-record #f) (tests:testqueue-set-itemdat! new-test-record my-itemdat) (tests:testqueue-set-item_path! new-test-record my-item-path) (hash-table-set! test-records newtestname new-test-record) | > | | 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 | (my-item-path (item-list->path my-itemdat)) (newtestname (db:test-make-full-name hed my-item-path))) ;; test names are unique on testname/item-path (tests:testqueue-set-items! new-test-record #f) (tests:testqueue-set-itemdat! new-test-record my-itemdat) (tests:testqueue-set-item_path! new-test-record my-item-path) (hash-table-set! test-records newtestname new-test-record) ;; BUG: This next line sucks up a lot of horsepower (set! tal (append tal (list newtestname))))) ;; since these are itemized create new test names testname/itempath items-in-testpatt))) ;; At this point we have possibly added items to tal but all must be handed off to ;; INNER COND logic. I think loop without rotating the queue ;; (loop hed tal reg reruns)) |
︙ | ︙ |