Megatest

Check-in [2ce6e734b4]
Login
Overview
Comment:Added back auto fallback to direct fs writes/reads
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | multi-transport
Files: files | file ages | folders
SHA1: 2ce6e734b464ee69146bd7de30205706a3081cb0
User & Date: mrwellan on 2014-12-01 19:39:35
Other Links: branch diff | manifest | tags
Context
2014-12-02
05:02
Add jitter to start time on servers and when starting bunch of servers rapidly do so in bunches of no more than five. check-in: 7e1b097420 user: matt tags: multi-transport
2014-12-01
19:39
Added back auto fallback to direct fs writes/reads check-in: 2ce6e734b4 user: mrwellan tags: multi-transport
13:54
Merged v1.60 into multi-transport check-in: 1888a270be user: mrwellan tags: multi-transport
Changes

Modified http-transport.scm from [2164d5e761] to [9b98c664db].

74
75
76
77
78
79
80






81
82
83
84
85
86
87
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93







+
+
+
+
+
+







	 (link-tree-path  (configf:lookup *configdat* "setup" "linktree")))
    ;; (set! db *inmemdb*)
    (debug:print-info 0 "portlogger recommended port: " start-port)
    (root-path     (if link-tree-path 
		       link-tree-path
		       (current-directory))) ;; WARNING: SECURITY HOLE. FIX ASAP!
    (handle-directory spiffy-directory-listing)
    (handle-exception (lambda (exn chain)
			(signal (make-composite-condition
				 (make-property-condition 
				  'server
				  'message "server error")))))

    ;; http-transport:handle-directory) ;; simple-directory-handler)
    ;; Setup the web server and a /ctrl interface
    ;;
    (vhost-map `(((* any) . ,(lambda (continue)
			       ;; open the db on the first call 
				 ;; This is were we set up the database connections
			       (let* (($   (request-vars source: 'both))

Modified rmt.scm from [d0047679a0] to [6f19056a06].

139
140
141
142
143
144
145
146

147
148
149


150
151
152
153
154
155
156
139
140
141
142
143
144
145

146
147


148
149
150
151
152
153
154
155
156







-
+

-
-
+
+







	      (hash-table-delete! *runremote* run-id)
	      ;; (mutex-unlock! *send-receive-mutex*)
	      (tasks:start-and-wait-for-server (db:delay-if-busy (tasks:open-db)) run-id 10)
	      (client:setup run-id)
	      (thread-sleep! (random 5)) ;; give some time to settle and minimize collison?
	      (rmt:send-receive cmd rid params attemptnum: (+ attemptnum 1)))
	    (begin
	      (debug:print 0 "ERROR: Communication failed!")
	      ;; (debug:print 0 "ERROR: Communication failed!")
	      ;; (mutex-unlock! *send-receive-mutex*)
	      (exit)
	      ;; (rmt:open-qry-close-locally cmd run-id params))))
	      ;; (exit)
	      (rmt:open-qry-close-locally cmd run-id params)
	      )))))

(define (rmt:update-db-stats run-id rawcmd params duration)
  (mutex-lock! *db-stats-mutex*)
  (handle-exceptions
   exn
   (begin
212
213
214
215
216
217
218
219

220
221
222
223
224
225
226
212
213
214
215
216
217
218

219
220
221
222
223
224
225
226







-
+







			     (let* ((dbdir (conc    (configf:lookup *configdat* "setup" "linktree") "/.db"))
				    (db (make-dbr:dbstruct path:  dbdir local: #t)))
			       (set! *dbstruct-db* db)
			       db)))
	 (db-file-path   (db:dbfile-path 0)))
    ;; (read-only      (not (file-read-access? db-file-path)))
    (let* ((start         (current-milliseconds))
	   (resdat        (api:execute-requests dbstruct-local (symbol->string cmd) params))
	   (resdat        (api:execute-requests dbstruct-local (vector (symbol->string cmd) params)))
	   (res           (vector-ref resdat 1))
	   (duration      (- (current-milliseconds) start)))
      (rmt:update-db-stats run-id cmd params duration)
      ;; mark this run as dirty if this was a write
      (if (not (member cmd api:read-only-queries))
	  (let ((start-time (current-seconds)))
	    (mutex-lock! *db-multi-sync-mutex*)

Modified tasks.scm from [299efaed94] to [b000052392].

364
365
366
367
368
369
370
371

372
373
374
375
376
377
378
364
365
366
367
368
369
370

371
372
373
374
375
376
377
378







-
+







    (cond
     (forced 
      (if (common:low-noise-print 60 run-id "server required is set")
	  (debug:print-info 0 "Server required is set, starting server for run-id " run-id "."))
      #t)
     ((> maxqry threshold)
      (if (common:low-noise-print 60 run-id "Max query time execeeded")
	  (debug:print-info 0 "Max avg query time of " maxqry "ms exceeds limit of " threshold "ms, starting server."))
	  (debug:print-info 0 "Max avg query time of " maxqry "ms exceeds limit of " threshold "ms, server needed for run-id " run-id "."))
      #t)
     (else
      #f))))

;; try to start a server and wait for it to be available
;;
(define (tasks:start-and-wait-for-server tdbdat run-id delay-max-tries)

Modified tests/fullrun/megatest.config from [54ee5ec660] to [0e8e9bc7a7].

137
138
139
140
141
142
143
144

145
146
147
148

149
150
151
152
153
154
155
137
138
139
140
141
142
143

144
145
146
147

148
149
150
151
152
153
154
155







-
+



-
+








# This server will keep running this number of hours after last access. 
# Three minutes is 0.05 hours
# timeout 0.025
timeout 0.061

# Server is required - slower but more resistant to Sqlite issues.
required yes
# required yes

# Start server when average query takes longer than this
# server-query-threshold 55500
server-query-threshold -1
server-query-threshold 100

# daemonize yes
# hostname #{scheme (get-host-name)}

## disks are:
## name host:/path/to/area
## -or-