Megatest

Check-in [b34c691e5f]
Login
Overview
Comment:avoided locking db when read-only, changed db:get-keys to use keys:config-get-fields, corrected formatting in list-servers
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | v1.7001-multi-db-rb01
Files: files | file ages | folders
SHA1: b34c691e5f4ee21ca6d6c8f8f68e092011cc2a08
User & Date: mmgraham on 2022-04-18 12:48:42
Other Links: branch diff | manifest | tags
Context
2022-05-07
20:14
unknown changes Closed-Leaf check-in: 7e90cd0862 user: matt tags: v1.7001-multi-db-unknown, v1.7001-multi-db-rb01
2022-05-02
11:29
WIP Fixed db:sync-touched and db:sync-tables. Disabled watchdog threads. Will use only keep-running thread. check-in: c597d4e8cf user: mmgraham tags: v1.7001-multi-db-rb01
2022-04-18
12:48
avoided locking db when read-only, changed db:get-keys to use keys:config-get-fields, corrected formatting in list-servers check-in: b34c691e5f user: mmgraham tags: v1.7001-multi-db-rb01
2022-04-12
19:06
minor reformatting check-in: 98e76b1aca user: mrwellan tags: v1.7001-multi-db-rb01
Changes

Modified db.scm from [5c6fcf9a46] to [33f582feca].

2136
2137
2138
2139
2140
2141
2142


2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
;; re-read the db over and over again for the keys since they never
;; change

;; why get the keys from the db? why not get from the *configdat*
;; using keys:config-get-fields?

(define (db:get-keys dbstruct)


  (if *db-keys* *db-keys* 
      (let ((res '()))
	(db:with-db dbstruct #f #f
		    (lambda (dbdat db)
		      (sqlite3:for-each-row 
		       (lambda (key)
			 (set! res (cons key res)))
		       db
		       "SELECT fieldname FROM keys ORDER BY id DESC;")))
	(set! *db-keys* res)
	res)))

;; extract index number given a header/data structure
(define (db:get-index-by-header header field)
  (list-index (lambda (x)(equal? x field)) header))

;; look up values in a header/data structure
(define (db:get-value-by-header row header field)







>
>
|
|
|
|
|
|
|
|
|
|
|







2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
;; re-read the db over and over again for the keys since they never
;; change

;; why get the keys from the db? why not get from the *configdat*
;; using keys:config-get-fields?

(define (db:get-keys dbstruct)
  (keys:config-get-fields *configdat*)
)
;;  (if *db-keys* *db-keys* 
;;      (let ((res '()))
;;	(db:with-db dbstruct #f #f
;;		    (lambda (dbdat db)
;;		      (sqlite3:for-each-row 
;;		       (lambda (key)
;;			 (set! res (cons key res)))
;;		       db
;;		       "SELECT fieldname FROM keys ORDER BY id DESC;")))
;;	(set! *db-keys* res)
;;	res)))

;; extract index number given a header/data structure
(define (db:get-index-by-header header field)
  (list-index (lambda (x)(equal? x field)) header))

;; look up values in a header/data structure
(define (db:get-value-by-header row header field)

Modified dbfile.scm from [57bcb686f3] to [d652e662dd].

445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
(define (dbfile:cautious-open-database fname init-proc #!optional (tries-left 10))
  (let* ((lock-file (conc fname".lock"))
	 (retry (lambda ()
		  (thread-sleep! 1.1)
		  (if (> tries-left 0)
		      (dbfile:cautious-open-database fname init-proc (- tries-left 1))))))
    (assert (>= tries-left 0) (conc "FATAL: Five attempts in dbfile:cautious-open-database of "fname", giving up."))
    (if (not (dbfile:simple-file-lock lock-file))
	(begin
	  (dbfile:print-err "INFO: lock file "lock-file" exists, trying again in 1 second.")
	  (thread-sleep! 1)
	  (dbfile:cautious-open-database fname init-proc (- tries-left 1)))
	(let* ((db-exists (file-exists? fname))
	       (result (condition-case
			   (let* ((db (sqlite3:open-database fname)))







|







445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
(define (dbfile:cautious-open-database fname init-proc #!optional (tries-left 10))
  (let* ((lock-file (conc fname".lock"))
	 (retry (lambda ()
		  (thread-sleep! 1.1)
		  (if (> tries-left 0)
		      (dbfile:cautious-open-database fname init-proc (- tries-left 1))))))
    (assert (>= tries-left 0) (conc "FATAL: Five attempts in dbfile:cautious-open-database of "fname", giving up."))
    (if (and (file-write-access? fname) (not (dbfile:simple-file-lock lock-file)))
	(begin
	  (dbfile:print-err "INFO: lock file "lock-file" exists, trying again in 1 second.")
	  (thread-sleep! 1)
	  (dbfile:cautious-open-database fname init-proc (- tries-left 1)))
	(let* ((db-exists (file-exists? fname))
	       (result (condition-case
			   (let* ((db (sqlite3:open-database fname)))
472
473
474
475
476
477
478

479

480
481
482
483
484
485
486
			     (retry))
			(exn (permission)(dbfile:print-err exn "ERROR: database " fname " has some permissions problem.")
			     (retry))
			(exn ()
			     (dbfile:print-err exn "ERROR: Unknown error with database " fname " message: "
					       ((condition-property-accessor 'exn 'message) exn))
			     (retry)))))

	  (dbfile:simple-file-release-lock lock-file)

	  result))))


(define (dbfile:open-no-sync-db dbpath)
  (if *no-sync-db*
      *no-sync-db*
      (begin







>
|
>







472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
			     (retry))
			(exn (permission)(dbfile:print-err exn "ERROR: database " fname " has some permissions problem.")
			     (retry))
			(exn ()
			     (dbfile:print-err exn "ERROR: Unknown error with database " fname " message: "
					       ((condition-property-accessor 'exn 'message) exn))
			     (retry)))))
          (if (file-write-access? fname)
	    (dbfile:simple-file-release-lock lock-file)
          )
	  result))))


(define (dbfile:open-no-sync-db dbpath)
  (if *no-sync-db*
      *no-sync-db*
      (begin
604
605
606
607
608
609
610
611




612
613
614
615
616
617
618
	  (thread-sleep! 0.25)
	  (if (file-exists? fname)
	      (handle-exceptions exn
                #f 
                (with-input-from-file fname
	  	  (lambda ()
		    (equal? key-string (read-line)))))
	      #f)))))





(define (dbfile:simple-file-lock-and-wait fname #!key (expire-time 300))
  (let ((end-time (+ expire-time (current-seconds))))
    (let loop ((got-lock (dbfile:simple-file-lock fname expire-time: expire-time)))
      (if got-lock
	  #t
	  (if (> end-time (current-seconds))







|
>
>
>
>







606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
	  (thread-sleep! 0.25)
	  (if (file-exists? fname)
	      (handle-exceptions exn
                #f 
                (with-input-from-file fname
	  	  (lambda ()
		    (equal? key-string (read-line)))))
	      #f)
       )
    )
  )
)

(define (dbfile:simple-file-lock-and-wait fname #!key (expire-time 300))
  (let ((end-time (+ expire-time (current-seconds))))
    (let loop ((got-lock (dbfile:simple-file-lock fname expire-time: expire-time)))
      (if got-lock
	  #t
	  (if (> end-time (current-seconds))

Modified megatest.scm from [718f8c5f41] to [fd40d13cfe].

942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
      (set! *didsomething* #t)))

(if (or (args:get-arg "-list-servers")
        (args:get-arg "-kill-servers"))
    (let ((tl (launch:setup)))
      (if tl ;; all roads from here exit
	  (let* ((servers (server:get-list *toppath*))
		 (fmtstr  "~8a~22a~20a~20a~8a\n"))
	    (format #t fmtstr "pid" "Interface:port" "age (hms)" "Last mod" "State")
	    (format #t fmtstr "===" "==============" "=========" "========" "=====")
	    (for-each ;;  ( mod-time host port start-time pid )
	     (lambda (server)
	       (let* ((mtm (any->number (car server)))
		      (mod (if mtm (- (current-seconds) mtm) "unk"))
		      (age (- (current-seconds)(or (any->number (list-ref server 3)) (current-seconds))))
		      (url (conc (cadr server) ":" (caddr server)))
		      (pid (list-ref server 4))







|
|
|







942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
      (set! *didsomething* #t)))

(if (or (args:get-arg "-list-servers")
        (args:get-arg "-kill-servers"))
    (let ((tl (launch:setup)))
      (if tl ;; all roads from here exit
	  (let* ((servers (server:get-list *toppath*))
		 (fmtstr  "~33a~22a~20a~20a~8a\n"))
	    (format #t fmtstr "ID" "host:port" "age (hms)" "Last mod" "State")
	    (format #t fmtstr "==" "=========" "=========" "========" "=====")
	    (for-each ;;  ( mod-time host port start-time pid )
	     (lambda (server)
	       (let* ((mtm (any->number (car server)))
		      (mod (if mtm (- (current-seconds) mtm) "unk"))
		      (age (- (current-seconds)(or (any->number (list-ref server 3)) (current-seconds))))
		      (url (conc (cadr server) ":" (caddr server)))
		      (pid (list-ref server 4))

Modified tests/simplerun/thebeginning.scm from [1a8187c724] to [50a75d414c].

50
51
52
53
54
55
56





;; *************** db.scm tests ****************


(define thisdbdat (db:open-db dbstruct #f))
(test #f #t (dbr:dbdat? thisdbdat))










>
>
>
50
51
52
53
54
55
56
57
58
59


;; *************** db.scm tests ****************


(define thisdbdat (db:open-db dbstruct #f))
(test #f #t (dbr:dbdat? thisdbdat))

(test #f #t (dbr:dbdat? (db:get-db dbstruct #f)))
(test #f #t (dbr:dbdat? (db:get-db dbstruct 1)))