Overview
Comment: | Added brute-force-salvage-db back. It seems to work, at least sometimes. |
---|---|
Downloads: | Tarball | ZIP archive | SQL archive |
Timelines: | family | ancestors | descendants | both | v1.70 |
Files: | files | file ages | folders |
SHA1: |
fb8df287684cd147f6bcfa359e04688b |
User & Date: | matt on 2022-09-18 19:31:27 |
Other Links: | branch diff | manifest | tags |
Context
2022-09-30
| ||
18:13 | Updated megatest version to 1.7007 check-in: 11f871fa5c user: mmgraham tags: v1.70, v1.7007 | |
2022-09-18
| ||
19:31 | Added brute-force-salvage-db back. It seems to work, at least sometimes. check-in: fb8df28768 user: matt tags: v1.70 | |
18:12 | Add back and improve waiting for journal plus few other performance helpers check-in: 76fac9ef30 user: matt tags: v1.70 | |
Changes
Modified api.scm from [3289b67f67] to [8aaec3e750].
︙ | ︙ | |||
233 234 235 236 237 238 239 | ((inc-var) (apply db:inc-var dbstruct params)) ((dec-var) (apply db:dec-var dbstruct params)) ((del-var) (apply db:del-var dbstruct params)) ((add-var) (apply db:add-var dbstruct params)) ;; STEPS ((teststep-set-status!) (apply db:teststep-set-status! dbstruct params)) | | | 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 | ((inc-var) (apply db:inc-var dbstruct params)) ((dec-var) (apply db:dec-var dbstruct params)) ((del-var) (apply db:del-var dbstruct params)) ((add-var) (apply db:add-var dbstruct params)) ;; STEPS ((teststep-set-status!) (apply db:teststep-set-status! dbstruct params)) ((delete-steps-for-test!) (apply db:delete-steps-for-test! dbstruct params)) ;; TEST DATA ((test-data-rollup) (apply db:test-data-rollup dbstruct params)) ((csv->test-data) (apply db:csv->test-data dbstruct params)) ;; MISC ((sync-inmem->db) (let ((run-id (car params))) |
︙ | ︙ |
Modified dbfile.scm from [3ea5a94d80] to [3c40c08f5f].
︙ | ︙ | |||
486 487 488 489 490 491 492 | (if (and (file-write-access? fname) (file-exists? busy-file)) (begin (if (common:low-noise-print 120 busy-file) (dbfile:print-err "INFO: dbfile:cautious-open-database: journal file " busy-file" exists, trying again in few seconds.")) (thread-sleep! 1) | | | | | | 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 | (if (and (file-write-access? fname) (file-exists? busy-file)) (begin (if (common:low-noise-print 120 busy-file) (dbfile:print-err "INFO: dbfile:cautious-open-database: journal file " busy-file" exists, trying again in few seconds.")) (thread-sleep! 1) (if (eq? tries-left 2) (begin (dbfile:print-err "INFO: forcing journal rollup "busy-file) (dbfile:brute-force-salvage-db fname))) (dbfile:cautious-open-database fname init-proc (- tries-left 1))) (let* ((result (condition-case (if dir-access (dbfile:with-simple-file-lock (conc fname ".lock") (lambda () |
︙ | ︙ | |||
1216 1217 1218 1219 1220 1221 1222 | (exn (busy) (db:generic-error-printout exn "ERROR: database " fname " is locked. Try copying to another location, remove original and copy back.")) (exn (permission)(db:generic-error-printout exn "ERROR: database " fname " has some permissions problem.")) (exn () (db:generic-error-printout exn "ERROR: Unknown error with database " fname " message: " ((condition-property-accessor 'exn 'message) exn)))))) | | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 | (exn (busy) (db:generic-error-printout exn "ERROR: database " fname " is locked. Try copying to another location, remove original and copy back.")) (exn (permission)(db:generic-error-printout exn "ERROR: database " fname " has some permissions problem.")) (exn () (db:generic-error-printout exn "ERROR: Unknown error with database " fname " message: " ((condition-property-accessor 'exn 'message) exn)))))) ;;====================================================================== ;; another attempt at a transactionized queue ;;====================================================================== ;; ;; ;; (define *transaction-queues* (make-hash-table)) ;; ;; ;; ;; ;; ;; (define (db:get-queue run-id) ;; ;; ;; (let* ((res (hash-table-ref/default *transaction-queues* run-id #f))) ;; ;; ;; (if res ;; ;; ;; res ;; ;; ;; (let* ((newq (make-queue))) ;; ;; ;; (hash-table-set! *transaction-queues* run-id newq) ;; ;; ;; newq)))) ;; ;; ;; ;; ;; ;; (define (db:add-to-transaction-queue dbstruct proc params) ;; ;; ;; (let* ((mbox (make-mailbox)) ;; ;; ;; (q (db:get-queue run-id))) ;; ;; ;; (queue-add! *transaction-queue* (list dbstruct proc mbox)) ;; ;; ;; (mailbox-receive mbox))) ;; ;; ;; ;; ;; ;; (define (db:process-transaction-queue *dbstruct-dbs*) ;; ;; ;; (for-each ;; ;; ;; (lambda (run-id) ;; ;; ;; (let* ((q (hash-table-ref *transaction-queue* run-id))) ;; ;; ;; ;; with-transaction ;; ;; ;; ;; dbstruct ;; ;; ;; ;; pop items from queue and execute them, return results via mailbox ;; ;; ;; q ;; ;; ;; ;; pop ;; ;; ;; )) ;; ;; ;; (hash-table-keys *transaction-queues*))) ;;====================================================================== ;; file utils ;;====================================================================== ;;====================================================================== ;; lazy-safe get file mod time. on any error (file not existing etc.) return 0 |
︙ | ︙ |
Modified http-transport.scm from [8048b5ff50] to [0337db9fa9].
︙ | ︙ | |||
481 482 483 484 485 486 487 | ;; (thread-start! *watchdog*) ) (if (and no-sync-db (common:low-noise-print 5 "sync-all")) ;; cheesy way to reduce frequency of running sync :) (begin (debug:print 0 *default-log-port* "keep-running calling db:all-db-sync at " (time->string (seconds->local-time) "%H:%M:%S")) | > > > > | > | 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 | ;; (thread-start! *watchdog*) ) (if (and no-sync-db (common:low-noise-print 5 "sync-all")) ;; cheesy way to reduce frequency of running sync :) (begin (debug:print 0 *default-log-port* "keep-running calling db:all-db-sync at " (time->string (seconds->local-time) "%H:%M:%S")) ;; This is tougher than it seems - have to deal with multiple dbs ;; (db:process-transaction-queue *dbstruct-dbs*) (db:all-db-sync *dbstruct-dbs*) ;; (db:do-sync no-sync-db) ;; (db:run-lock-and-sync *no-sync-db*) ) ) ) ;; when things go wrong we don't want to be doing the various queries too often |
︙ | ︙ |