Megatest

Check-in [eca566d9f9]
Login
Overview
Comment:Added asserts, improved message/assert on lock failure
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | v1.80
Files: files | file ages | folders
SHA1: eca566d9f94cd052a9827d80c129c98c810fc3e1
User & Date: mmgraham on 2023-04-06 16:25:26
Other Links: branch diff | manifest | tags
Context
2023-04-06
16:37
merge-fork check-in: c574c7b21b user: matt tags: v1.80
16:25
Added asserts, improved message/assert on lock failure check-in: eca566d9f9 user: mmgraham tags: v1.80
09:23
Fixed performance issue with append in runs.scm check-in: 0dc6c83d6e user: matt tags: v1.80
Changes

Modified db.scm from [315e0db07f] to [00e228e110].

2876
2877
2878
2879
2880
2881
2882
2883

2884
2885
2886
2887
2888
2889
2890
    (if res
	res
	(if (> tries 0)
	    (begin
	      (thread-sleep! 1)
	      (db:keep-trying-until-true proc params (- tries 1)))
	    (begin
	      (debug:print-info 0 *default-log-port* "proc never returned true, params="params)

	      #f)))))
  
(define (db:get-test-info dbstruct run-id test-name item-path)
  (db:with-db
   dbstruct
   run-id
   #f







|
>







2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
    (if res
	res
	(if (> tries 0)
	    (begin
	      (thread-sleep! 1)
	      (db:keep-trying-until-true proc params (- tries 1)))
	    (begin
	      ;; (debug:print-info 0 *default-log-port* "proc never returned true, params="params)
	      (print"db:keep-trying-until-true proc never returned true, proc = " proc " params =" params " tries = " tries)
	      #f)))))
  
(define (db:get-test-info dbstruct run-id test-name item-path)
  (db:with-db
   dbstruct
   run-id
   #f

Modified dbfile.scm from [d5febb23fb] to [8e990891b1].

483
484
485
486
487
488
489

490
491
492
493
494
495
496
(define (db:no-sync-set db var val)
  (sqlite3:execute db "INSERT OR REPLACE INTO no_sync_metadat (var,val) VALUES (?,?);" var val))

(define (db:no-sync-del! db var)
  (sqlite3:execute db "DELETE FROM no_sync_metadat WHERE var=?;" var))

(define (db:no-sync-get/default db var default)

  (let ((res default))
    (sqlite3:for-each-row
     (lambda (val)
       (set! res val))
     db
     "SELECT val FROM no_sync_metadat WHERE var=?;"
     var)







>







483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
(define (db:no-sync-set db var val)
  (sqlite3:execute db "INSERT OR REPLACE INTO no_sync_metadat (var,val) VALUES (?,?);" var val))

(define (db:no-sync-del! db var)
  (sqlite3:execute db "DELETE FROM no_sync_metadat WHERE var=?;" var))

(define (db:no-sync-get/default db var default)
  (assert (sqlite3:database? db) "FATAL: db:no-sync-get/default called with a bad db handle:" db)
  (let ((res default))
    (sqlite3:for-each-row
     (lambda (val)
       (set! res val))
     db
     "SELECT val FROM no_sync_metadat WHERE var=?;"
     var)
1344
1345
1346
1347
1348
1349
1350










1351
1352
1353
1354
1355
1356
1357
1358

(define (dbfile:with-simple-file-lock fname proc #!key (expire-time 300))
  (let ((gotlock (dbfile:simple-file-lock-and-wait fname expire-time: expire-time)))
    (if gotlock
	(let ((res (proc)))
	  (dbfile:simple-file-release-lock fname)
	  res)










	(assert #t "FATAL: simple file lock never got a lock."))))

(define *get-cache-stmth-mutex* (make-mutex))

(define (db:get-cache-stmth dbdat db stmt)
  (mutex-lock! *get-cache-stmth-mutex*)
  (let* (;; (dbdat       (dbfile:get-dbdat dbstruct run-id))
	 (stmt-cache  (dbr:dbdat-stmt-cache dbdat))







>
>
>
>
>
>
>
>
>
>
|







1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369

(define (dbfile:with-simple-file-lock fname proc #!key (expire-time 300))
  (let ((gotlock (dbfile:simple-file-lock-and-wait fname expire-time: expire-time)))
    (if gotlock
	(let ((res (proc)))
	  (dbfile:simple-file-release-lock fname)
	  res)
        (begin
          (dbfile:print-err "dbfile:with-simple-file-lock: " fname " is locked by " )
          (with-input-from-file fname
             (lambda ()
                (dbfile:print-err (read-line))))
	  (assert #f (conc "ERROR: simple file lock could not get a lock for " fname " in " expire-time " seconds"))
        )
    )
  )
)


(define *get-cache-stmth-mutex* (make-mutex))

(define (db:get-cache-stmth dbdat db stmt)
  (mutex-lock! *get-cache-stmth-mutex*)
  (let* (;; (dbdat       (dbfile:get-dbdat dbstruct run-id))
	 (stmt-cache  (dbr:dbdat-stmt-cache dbdat))

Modified dbmod.scm from [51b0a8133b] to [08e196df1c].

145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165


166
167
168
169
170
171
172
  (let* ((dbstruct     (or dbstruct-in (make-dbr:dbstruct areapath: areapath)))
	 (dbfname      (or dbfname-in (dbmod:run-id->dbfname run-id)))
	 (dbpath       (dbmod:get-dbdir dbstruct))             ;; directory where all the .db files are kept
	 (dbfullname   (conc dbpath"/"dbfname)) ;; (dbmod:run-id->full-dbfname dbstruct run-id))
	 (dbexists     (file-exists? dbfullname))
	 (inmem        (dbmod:open-inmem-db init-proc))
	 (write-access (file-write-access? dbpath))
	 (open-the-db  (lambda ()
			 (dbfile:with-simple-file-lock
			  (conc dbfullname".lock")
			  (lambda ()
			    (let* ((db      (sqlite3:open-database dbfullname))
				   (handler (sqlite3:make-busy-timeout 136000)))
			      (sqlite3:set-busy-handler! db handler)
			      (if write-access
				  (init-proc db))
			      db)))))
	 (db            ;; (if (dbmod:need-on-disk-db-handle)
			    (open-the-db))
;;			    #f))
	 (tables       (db:sync-all-tables-list keys)))


    (dbr:dbstruct-inmem-set!     dbstruct inmem)
    (dbr:dbstruct-ondiskdb-set!  dbstruct db)
    (dbr:dbstruct-dbfile-set!    dbstruct dbfullname)
    (dbr:dbstruct-dbfname-set!   dbstruct dbfname)
    (dbr:dbstruct-sync-proc-set! dbstruct
				 (lambda (last-update)
				   ;; (if db







<
|







|
<
<
<

>
>







145
146
147
148
149
150
151

152
153
154
155
156
157
158
159
160



161
162
163
164
165
166
167
168
169
170
  (let* ((dbstruct     (or dbstruct-in (make-dbr:dbstruct areapath: areapath)))
	 (dbfname      (or dbfname-in (dbmod:run-id->dbfname run-id)))
	 (dbpath       (dbmod:get-dbdir dbstruct))             ;; directory where all the .db files are kept
	 (dbfullname   (conc dbpath"/"dbfname)) ;; (dbmod:run-id->full-dbfname dbstruct run-id))
	 (dbexists     (file-exists? dbfullname))
	 (inmem        (dbmod:open-inmem-db init-proc))
	 (write-access (file-write-access? dbpath))

	 (db           (dbfile:with-simple-file-lock
			  (conc dbfullname".lock")
			  (lambda ()
			    (let* ((db      (sqlite3:open-database dbfullname))
				   (handler (sqlite3:make-busy-timeout 136000)))
			      (sqlite3:set-busy-handler! db handler)
			      (if write-access
				  (init-proc db))
			      db))))



	 (tables       (db:sync-all-tables-list keys)))
    (assert (sqlite3:database? inmem) "FATAL: open-dbmoddb: inmem is not a db")
    (assert (sqlite3:database? db) "FATAL:  open-dbmoddb: db is not a db")
    (dbr:dbstruct-inmem-set!     dbstruct inmem)
    (dbr:dbstruct-ondiskdb-set!  dbstruct db)
    (dbr:dbstruct-dbfile-set!    dbstruct dbfullname)
    (dbr:dbstruct-dbfname-set!   dbstruct dbfname)
    (dbr:dbstruct-sync-proc-set! dbstruct
				 (lambda (last-update)
				   ;; (if db
181
182
183
184
185
186
187


188
189
190
191
192
193
194
;;    (if (eq? syncdir 'todisk) ;; sync to disk normally, sync from in dashboard
;;        (dbmod:sync-tables tables last-update inmem db)
;;        (dbmod:sync-tables tables last-update db inmem))))

;; direction: 'fromdest 'todest
;;
(define (dbmod:sync-gasket tables last-update inmem dbh dbfname direction)


  (case (dbfile:sync-method)
    ((none) #f)
    ((attach)
     (dbmod:attach-sync tables inmem dbfname direction))
    ((newsync)
     (dbmod:new-sync tables inmem dbh dbfname direction))
    (else







>
>







179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
;;    (if (eq? syncdir 'todisk) ;; sync to disk normally, sync from in dashboard
;;        (dbmod:sync-tables tables last-update inmem db)
;;        (dbmod:sync-tables tables last-update db inmem))))

;; direction: 'fromdest 'todest
;;
(define (dbmod:sync-gasket tables last-update inmem dbh dbfname direction)
  (assert (sqlite3:database? inmem) "FATAL: sync-gasket: inmem is not a db")
  (assert (sqlite3:database? inmem) "FATAL: sync-gasket: dbh is not a db")
  (case (dbfile:sync-method)
    ((none) #f)
    ((attach)
     (dbmod:attach-sync tables inmem dbfname direction))
    ((newsync)
     (dbmod:new-sync tables inmem dbh dbfname direction))
    (else