Modified api.scm
from [fafaf8a322]
to [debe6a9315].
| ︙ | | |
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
|
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
|
-
-
+
+
|
(print-call-chain (current-error-port))
(debug:print 0 " message: " ((condition-property-accessor 'exn 'message) exn))
(vector #f (vector exn call-chain dat))) ;; return some stuff for debug if an exception happens
(if (not (vector? dat)) ;; it is an error to not receive a vector
(vector #f #f "remote must be called with a vector")
(vector ;; return a vector + the returned data structure
#t
(let ((cmd (vector-ref dat 0))
(params (vector-ref dat 1)))
(let ((cmd (safe-vector-ref dat 0))
(params (safe-vector-ref dat 1)))
(case (if (symbol? cmd)
cmd
(string->symbol cmd))
;;===============================================
;; READ/WRITE QUERIES
;;===============================================
|
| ︙ | | |
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
|
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
|
-
+
|
;; NB// Runs on the server as part of the server loop
;;
(define (api:process-request dbstruct $) ;; the $ is the request vars proc
(let* ((cmd ($ 'cmd))
(paramsj ($ 'params))
(params (db:string->obj paramsj transport: 'http)) ;; (rmt:json-str->dat paramsj))
(resdat (api:execute-requests dbstruct (vector cmd params))) ;; #( flag result )
(res (vector-ref resdat 1)))
(res (safe-vector-ref resdat 1)))
;; This can be here but needs controls to ensure it doesn't run more than every 4 seconds
;; (rmt:dat->json-str
;; (if (or (string? res)
;; (list? res)
;; (number? res)
;; (boolean? res))
;; res
;; (list "ERROR, not string, list, number or boolean" 1 cmd params res)))))
(db:obj->string res transport: 'http)))
|
Modified archive.scm
from [399e9c79b1]
to [26003e84d9].
| ︙ | | |
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
|
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
|
-
-
+
+
|
(define (archive:get-archive testname itempath dused)
;; look up in archive_allocations if there is a pre-used archive
;; with adequate diskspace
;;
(let* ((existing-blocks (rmt:archive-get-allocations testname itempath dused))
(candidate-disks (map (lambda (block)
(list
(vector-ref block 1) ;; archive-area-name
(vector-ref block 2))) ;; disk-path
(safe-vector-ref block 1) ;; archive-area-name
(safe-vector-ref block 2))) ;; disk-path
existing-blocks)))
(or (common:get-disk-with-most-free-space candidate-disks dused)
(archive:allocate-new-archive-block testname itempath))))
;; allocate a new archive area
;;
(define (archvie:allocate-new-archive-block testname itempath dneeded)
|
| ︙ | | |
Modified client.scm
from [a61d4e6d81]
to [6ee60fdb9d].
| ︙ | | |
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
|
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
|
-
+
|
(start-res (case *transport-type*
((http)(http-transport:client-connect iface port))
((nmsg)(nmsg-transport:client-connect hostname port))))
(ping-res (case *transport-type*
((http)(rmt:login-no-auto-client-setup start-res run-id))
((nmsg)(let ((logininfo (rmt:login-no-auto-client-setup start-res run-id)))
(if logininfo
(car (vector-ref logininfo 1))
(car (safe-vector-ref logininfo 1))
#f))))))
(if (and start-res
ping-res)
(begin
(hash-table-set! *runremote* run-id start-res)
(debug:print-info 2 "connected to " (http-transport:server-dat-make-url start-res))
start-res)
|
| ︙ | | |
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
|
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
|
-
-
-
-
-
-
+
+
+
+
+
+
+
-
+
|
(thread-sleep! (+ 1 (random 5))) ;; spread out the starts a little
(thread-sleep! (+ 15 (random 20)))) ;; it isn't going well. give it plenty of time
(server:try-running run-id)
(thread-sleep! 5) ;; give server a little time to start up
(client:setup run-id remaining-tries: (- remaining-tries 1))
)))
(begin ;; no server registered
(let ((num-available (tasks:num-in-available-state (db:dbdat-get-db tdbdat) run-id)))
(debug:print-info 0 "client:setup, no server registered, remaining-tries=" remaining-tries " num-available=" num-available)
(if (< num-available 2)
(server:try-running run-id))
(thread-sleep! (+ 5 (random (- 20 remaining-tries)))) ;; give server a little time to start up, randomize a little to avoid start storms.
(client:setup run-id remaining-tries: (- remaining-tries 1)))))))))
;; (let ((num-available (tasks:num-in-available-state (db:dbdat-get-db tdbdat) run-id)))
;; (debug:print-info 0 "client:setup, no server registered, remaining-tries=" remaining-tries " num-available=" num-available)
;; (if (< num-available 2)
;; (server:try-running run-id))
(tasks:start-and-wait-for-server tdbdat run-id delay-max-tries)
;; (thread-sleep! (+ 2 (random (- 20 remaining-tries)))) ;; give server a little time to start up, randomize a little to avoid start storms.
(client:setup run-id remaining-tries: (- remaining-tries 1))))))))
;; (let ((host-info (hash-table-ref/default *runremote* run-id #f)))
;; (if host-info ;; this is a bit circular. the host-info *is* the start-res FIXME
;; (let* ((iface (http-transport:server-dat-get-iface host-info))
;; (port (http-transport:server-dat-get-port host-info))
;; (start-res (case *transport-type*
;; ((http)(http-transport:client-connect iface port))
;; ((nmsg)(nmsg-transport:client-connect iface port)) ;; (http-transport:server-dat-get-socket host-info))
;; (else #f)))
;; (ping-res (case *transport-type*
;; ((http)(rmt:login-no-auto-client-setup start-res run-id))
;; ((nmsg)(let ((logininfo (rmt:login-no-auto-client-setup start-res run-id)))
;; (if logininfo
;; (vector-ref (vector-ref logininfo 1) 1)
;; (safe-vector-ref (safe-vector-ref logininfo 1) 1)
;; #f)))
;; (else #f))))
;; (if ping-res ;; sucessful login?
;; (begin
;; (debug:print-info 2 "client:setup, ping is good using host-info=" host-info ", remaining-tries=" remaining-tries)
;; start-res) ;; return the server info
;; ;; have host info but no ping. shutdown the current connection and try again
|
| ︙ | | |
Modified common_records.scm
from [08c9f6257d]
to [e1ead0c092].
| ︙ | | |
25
26
27
28
29
30
31
32
33
34
35
36
37
38
|
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
|
+
+
+
+
+
+
+
+
|
(syntax-rules ()
((_ (name arg ...) body ...)
(define-syntax name (syntax-rules () ((name arg ...) (begin body ...)))))))
(define-syntax common:handle-exceptions
(syntax-rules ()
((_ exn-in errstmt ...)(handle-exceptions exn-in errstmt ...))))
(define (safe-vector-ref vec indx)
(if vec
(vector-ref vec indx)
(begin
(debug:print 0 "vector-ref called with #f")
(print-call-chain (current-error-port))
(exit))))
(define (debug:calc-verbosity vstr)
(cond
((number? vstr) vstr)
((not (string? vstr)) 1)
;; ((string-match "^\\s*$" vstr) 1)
(vstr (let ((debugvals (filter number? (map string->number (string-split vstr ",")))))
|
| ︙ | | |
Modified dashboard-guimonitor.scm
from [10390e6373]
to [86c3ecd4b7].
| ︙ | | |
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
|
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
|
-
+
-
+
|
(key-params (make-hash-table))
(monitordat '()) ;; list of monitor records
(keyentries (iup:frame
#:title "Keys"
(apply
iup:vbox
(map (lambda (key)
(iup:hbox (iup:label (vector-ref key 0) #:size "60x15") ; #:expand "HORIZONTAL")
(iup:hbox (iup:label (safe-vector-ref key 0) #:size "60x15") ; #:expand "HORIZONTAL")
(iup:textbox #:expand "HORIZONTAL"
#:action (lambda (obj a val)
(hash-table-set! key-params (vector-ref key 0) val)))))
(hash-table-set! key-params (safe-vector-ref key 0) val)))))
keys))))
(othervars (iup:frame
#:title "Run Vars"
(apply
iup:vbox
(map (lambda (var)
(iup:hbox (iup:label var #:size "60x15")
|
| ︙ | | |
Modified dashboard-tests.scm
from [224dddeb50]
to [018ef98bdc].
| ︙ | | |
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
|
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
|
-
-
+
+
|
;; use a global for setting the buttons colors
;; state status teststeps
(define *state-status* (vector #f #f #f))
(define (update-state-status-buttons testdat)
(let* ((state (db:test-get-state testdat))
(status (db:test-get-status testdat))
(color (car (gutils:get-color-for-state-status state status))))
((vector-ref *state-status* 0) state color)
((vector-ref *state-status* 1) status color)))
((safe-vector-ref *state-status* 0) state color)
((safe-vector-ref *state-status* 1) status color)))
(define *dashboard-test-db* #t)
(define *dashboard-comment-share-slot* #f)
;;======================================================================
;; Set fields
;;======================================================================
|
| ︙ | | |
Modified dashboard.scm
from [6d6a8350b9]
to [8b5244f3e8].
| ︙ | | |
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
|
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
|
-
+
-
-
-
-
+
+
+
+
|
(define *tests-sort-reverse*
(let ((t-sort (assoc (configf:lookup *configdat* "dashboard" "testsort") *tests-sort-type-index*)))
(if t-sort
(cadr t-sort)
3)))
(define (get-curr-sort)
(vector-ref *tests-sort-options* *tests-sort-reverse*))
(safe-vector-ref *tests-sort-options* *tests-sort-reverse*))
(define *hide-empty-runs* #f)
(define *hide-not-hide* #t) ;; toggle for hide/not hide
(define *hide-not-hide-button* #f)
(define *hide-not-hide-tabs* #f)
(define *current-tab-number* 0)
(define *updaters* (make-hash-table))
(debug:setup)
(define uidat #f)
(define-inline (dboard:uidat-get-keycol vec)(vector-ref vec 0))
(define-inline (dboard:uidat-get-lftcol vec)(vector-ref vec 1))
(define-inline (dboard:uidat-get-header vec)(vector-ref vec 2))
(define-inline (dboard:uidat-get-runsvec vec)(vector-ref vec 3))
(define-inline (dboard:uidat-get-keycol vec)(safe-vector-ref vec 0))
(define-inline (dboard:uidat-get-lftcol vec)(safe-vector-ref vec 1))
(define-inline (dboard:uidat-get-header vec)(safe-vector-ref vec 2))
(define-inline (dboard:uidat-get-runsvec vec)(safe-vector-ref vec 3))
(if (get-environment-variable "MT_RUN_AREA_HOME")(change-directory (get-environment-variable "MT_RUN_AREA_HOME")))
(define (message-window msg)
(iup:show
(iup:dialog
(iup:vbox
|
| ︙ | | |
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
|
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
|
-
-
+
+
|
(header (db:get-header allruns))
(runs (db:get-rows allruns))
(result '())
(maxtests 0)
(states (hash-table-keys *state-ignore-hash*))
(statuses (hash-table-keys *status-ignore-hash*))
(sort-info (get-curr-sort))
(sort-by (vector-ref sort-info 1))
(sort-order (vector-ref sort-info 2))
(sort-by (safe-vector-ref sort-info 1))
(sort-order (safe-vector-ref sort-info 2))
(bubble-type (if (member sort-order '(testname))
'testname
'itempath)))
;;
;; trim runs to only those that are changing often here
;;
(for-each (lambda (run)
|
| ︙ | | |
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
|
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
|
-
+
|
(debug:print-info 6 "*allruns* has " (length *allruns*) " runs")
maxtests))
(define *collapsed* (make-hash-table))
; (define *row-lookup* (make-hash-table)) ;; testname => (rownum lableobj)
(define (toggle-hide lnum) ; fulltestname)
(let* ((btn (vector-ref (dboard:uidat-get-lftcol uidat) lnum))
(let* ((btn (safe-vector-ref (dboard:uidat-get-lftcol uidat) lnum))
(fulltestname (iup:attribute btn "TITLE"))
(parts (string-split fulltestname "("))
(basetestname (if (null? parts) "" (car parts))))
;(print "Toggling " basetestname " currently " (hash-table-ref/default *collapsed* basetestname #f))
(if (hash-table-ref/default *collapsed* basetestname #f)
(begin
;(iup:attribute-set! btn "FGCOLOR" "0 0 0")
|
| ︙ | | |
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
|
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
|
-
-
+
+
-
-
-
+
+
+
-
-
+
+
-
+
-
-
+
+
-
-
+
+
|
(if (> (length splst) 1)
(vector-set! res 1 (car (string-split (cadr splst) ")"))))
res))
lst))
(define (collapse-rows inlst)
(let* ((sort-info (get-curr-sort))
(sort-by (vector-ref sort-info 1))
(sort-order (vector-ref sort-info 2))
(sort-by (safe-vector-ref sort-info 1))
(sort-order (safe-vector-ref sort-info 2))
(bubble-type (if (member sort-order '(testname))
'testname
'itempath))
(newlst (filter (lambda (x)
(let* ((tparts (string-split x "("))
(basetname (if (null? tparts) x (car tparts))))
;(print "x " x " tparts: " tparts " basetname: " basetname)
(cond
((string-match blank-line-rx x) #f)
((equal? x basetname) #t)
((hash-table-ref/default *collapsed* basetname #f)
;(print "Removing " basetname " from items")
#f)
(else #t))))
inlst))
(vlst (run-item-name->vectors newlst))
(vlst2 (bubble-up vlst priority: bubble-type)))
(map (lambda (x)
(if (equal? (vector-ref x 1) "")
(vector-ref x 0)
(conc (vector-ref x 0) "(" (vector-ref x 1) ")")))
(if (equal? (safe-vector-ref x 1) "")
(safe-vector-ref x 0)
(conc (safe-vector-ref x 0) "(" (safe-vector-ref x 1) ")")))
vlst2)))
(define (update-labels uidat)
(let* ((rown 0)
(keycol (dboard:uidat-get-keycol uidat))
(lftcol (dboard:uidat-get-lftcol uidat))
(numcols (vector-length lftcol))
(maxn (- numcols 1))
(allvals (make-vector numcols "")))
(for-each (lambda (name)
(if (<= rown maxn)
(vector-set! allvals rown name)) ;)
(set! rown (+ 1 rown)))
*alltestnamelst*)
(let loop ((i 0))
(let* ((lbl (vector-ref lftcol i))
(keyval (vector-ref keycol i))
(let* ((lbl (safe-vector-ref lftcol i))
(keyval (safe-vector-ref keycol i))
(oldval (iup:attribute lbl "TITLE"))
(newval (vector-ref allvals i)))
(newval (safe-vector-ref allvals i)))
(if (not (equal? oldval newval))
(let ((munged-val (let ((parts (string-split newval "(")))
(if (> (length parts) 1)(conc " " (car (string-split (cadr parts) ")"))) newval))))
(vector-set! keycol i newval)
(iup:attribute-set! lbl "TITLE" munged-val)))
(iup:attribute-set! lbl "FGCOLOR" (if (hash-table-ref/default *collapsed* newval #f) "0 112 112" "0 0 0"))
(if (< i maxn)
(loop (+ i 1)))))))
;;
(define (get-itemized-tests test-dats)
(let ((tnames '()))
(for-each (lambda (tdat)
(let ((tname (vector-ref tdat 0)) ;; (db:test-get-testname tdat))
(ipath (vector-ref tdat 1))) ;; (db:test-get-item-path tdat)))
(let ((tname (safe-vector-ref tdat 0)) ;; (db:test-get-testname tdat))
(ipath (safe-vector-ref tdat 1))) ;; (db:test-get-item-path tdat)))
(if (not (equal? ipath ""))
(if (and (list? tnames)
(string? tname)
(not (member tname tnames)))
(set! tnames (append tnames (list tname)))))))
test-dats)
tnames))
;; Bubble up the top tests to above the items, collect the items underneath
;; all while preserving the sort order from the SQL query as best as possible.
;;
(define (bubble-up test-dats #!key (priority 'itempath))
(if (null? test-dats)
test-dats
(begin
(let* ((tnames '()) ;; list of names used to reserve order
(tests (make-hash-table)) ;; hash of lists, used to build as we go
(itemized (get-itemized-tests test-dats)))
(for-each
(lambda (testdat)
(let* ((tname (vector-ref testdat 0)) ;; db:test-get-testname testdat))
(ipath (vector-ref testdat 1))) ;; db:test-get-item-path testdat)))
(let* ((tname (safe-vector-ref testdat 0)) ;; db:test-get-testname testdat))
(ipath (safe-vector-ref testdat 1))) ;; db:test-get-item-path testdat)))
;; (seen (hash-table-ref/default tests tname #f)))
(if (not (member tname tnames))
(if (or (and (eq? priority 'itempath)
(not (equal? ipath "")))
(and (eq? priority 'testname)
(equal? ipath ""))
(not (member tname itemized)))
|
| ︙ | | |
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
|
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
|
-
+
|
(table (dboard:uidat-get-runsvec uidat))
(coln 0))
(set! *alltestnamelst* '())
;; create a concise list of test names
(for-each
(lambda (rundat)
(if (vector? rundat)
(let* ((testdat (vector-ref rundat 1))
(let* ((testdat (safe-vector-ref rundat 1))
(testnames (map test:test-get-fullname testdat)))
(if (not (and *hide-empty-runs*
(null? testnames)))
(for-each (lambda (testname)
(if (not (member testname *alltestnamelst*))
(begin
(set! *alltestnamelst* (append *alltestnamelst* (list testname))))))
|
| ︙ | | |
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
|
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
|
-
-
-
+
+
+
-
+
-
+
-
+
-
+
|
(append xl (make-list (- *num-tests* (length xl)) ""))))
(update-labels uidat)
(for-each
(lambda (rundat)
(if (not rundat) ;; handle padded runs
;; ;; id run-id testname state status event-time host cpuload diskfree uname rundir item-path run-duration
(set! rundat (vector (make-vector 20 #f) '() (map (lambda (x) "") *keys*))));; 3)))
(let* ((run (vector-ref rundat 0))
(testsdat (vector-ref rundat 1))
(key-val-dat (vector-ref rundat 2))
(let* ((run (safe-vector-ref rundat 0))
(testsdat (safe-vector-ref rundat 1))
(key-val-dat (safe-vector-ref rundat 2))
(run-id (db:get-value-by-header run *header* "id"))
(key-vals (append key-val-dat
(list (let ((x (db:get-value-by-header run *header* "runname")))
(if x x "")))))
(run-key (string-intersperse key-vals "\n")))
;; fill in the run header key values
(let ((rown 0)
(headercol (vector-ref tableheader coln)))
(headercol (safe-vector-ref tableheader coln)))
(for-each (lambda (kval)
(let* ((labl (vector-ref headercol rown)))
(let* ((labl (safe-vector-ref headercol rown)))
(if (not (equal? kval (iup:attribute labl "TITLE")))
(iup:attribute-set! (vector-ref headercol rown) "TITLE" kval))
(iup:attribute-set! (safe-vector-ref headercol rown) "TITLE" kval))
(set! rown (+ rown 1))))
key-vals))
;; For this run now fill in the buttons for each test
(let ((rown 0)
(columndat (vector-ref table coln)))
(columndat (safe-vector-ref table coln)))
(for-each
(lambda (testname)
(let ((buttondat (hash-table-ref/default *buttondat* (mkstr coln rown) #f)))
(if buttondat
(let* ((test (let ((matching (filter
(lambda (x)(equal? (test:test-get-fullname x) testname))
testsdat)))
|
| ︙ | | |
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
|
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
|
-
+
-
-
+
+
|
(buttontxt (cond
((equal? teststate "COMPLETED") teststatus)
((and (equal? teststate "NOT_STARTED")
(member teststatus '("ZERO_ITEMS" "BLOCKED" "PREQ_FAIL" "PREQ_DISCARDED" "TIMED_OUT" "KEEP_TRYING" "TEN_STRIKES")))
teststatus)
(else
teststate)))
(button (vector-ref columndat rown))
(button (safe-vector-ref columndat rown))
(color (car (gutils:get-color-for-state-status teststate teststatus)))
(curr-color (vector-ref buttondat 1)) ;; (iup:attribute button "BGCOLOR"))
(curr-title (vector-ref buttondat 2))) ;; (iup:attribute button "TITLE")))
(curr-color (safe-vector-ref buttondat 1)) ;; (iup:attribute button "BGCOLOR"))
(curr-title (safe-vector-ref buttondat 2))) ;; (iup:attribute button "TITLE")))
(if (not (equal? curr-color color))
(iup:attribute-set! button "BGCOLOR" color))
(if (not (equal? curr-title buttontxt))
(iup:attribute-set! button "TITLE" buttontxt))
(vector-set! buttondat 0 run-id)
(vector-set! buttondat 1 color)
(vector-set! buttondat 2 buttontxt)
|
| ︙ | | |
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
|
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
|
-
-
+
+
|
(let ((newval (car values)))
(iup:attribute-set! lb "VALUE" newval)
newval))))))
(define (dashboard:update-target-selector key-lbs #!key (action-proc #f))
(let* ((runconf-targs (common:get-runconfig-targets))
(db-target-dat (db:get-targets *dbstruct-local*))
(header (vector-ref db-target-dat 0))
(db-targets (vector-ref db-target-dat 1))
(header (safe-vector-ref db-target-dat 0))
(db-targets (safe-vector-ref db-target-dat 1))
(all-targets (append db-targets
(map (lambda (x)
(list->vector
(take (append (string-split x "/")
(make-list (length header) "na"))
(length header))))
runconf-targs)))
|
| ︙ | | |
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
|
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
|
-
-
+
+
|
#:action (lambda (obj val index lbstate)
(iup:attribute-set! tb "VALUE" val)
(dboard:data-set-run-name! *data* val)
(dashboard:update-run-command))))
(refresh-runs-list (lambda ()
(let* ((target (dboard:data-get-target-string *data*))
(runs-for-targ (db:get-runs-by-patt *dbstruct-local* *keys* "%" target #f #f))
(runs-header (vector-ref runs-for-targ 0))
(runs-dat (vector-ref runs-for-targ 1))
(runs-header (safe-vector-ref runs-for-targ 0))
(runs-dat (safe-vector-ref runs-for-targ 1))
(run-names (cons default-run-name
(map (lambda (x)
(db:get-value-by-header x runs-header "runname"))
runs-dat))))
(iup:attribute-set! lb "REMOVEITEM" "ALL")
(iuplistbox-fill-list lb run-names selected-item: default-run-name)))))
(set! updater-for-runs refresh-runs-list)
|
| ︙ | | |
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
|
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
|
-
+
-
-
+
+
-
+
|
(let* ((toolpath (car (argv)))
(key (conc lin ":" col))
(test-id (hash-table-ref/default cell-lookup key -1))
(cmd (conc toolpath " -test " (dboard:data-get-curr-run-id *data*) "," test-id "&")))
(system cmd)))))
(updater (lambda ()
(let* ((runs-dat (db:get-runs-by-patt db *keys* "%" #f #f #f))
(runs-header (vector-ref runs-dat 0)) ;; 0 is header, 1 is list of records
(runs-header (safe-vector-ref runs-dat 0)) ;; 0 is header, 1 is list of records
(run-id (dboard:data-get-curr-run-id *data*))
(tests-dat (let ((tdat (db:get-tests-for-run db run-id
(hash-table-ref/default *searchpatts* "test-name" "%/%")
(hash-table-keys *state-ignore-hash*) ;; '()
(hash-table-keys *status-ignore-hash*) ;; '()
#f #f
*hide-not-hide*
#f #f
"id,testname,item_path,state,status"))) ;; get 'em all
(sort tdat (lambda (a b)
(let* ((aval (vector-ref a 2))
(bval (vector-ref b 2))
(let* ((aval (safe-vector-ref a 2))
(bval (safe-vector-ref b 2))
(anum (string->number aval))
(bnum (string->number bval)))
(if (and anum bnum)
(< anum bnum)
(string<= aval bval)))))))
(tests-mindat (dcommon:minimize-test-data tests-dat))
(indices (common:sparse-list-generate-index tests-mindat)) ;; proc: set-cell))
(row-indices (cadr indices))
(col-indices (car indices))
(max-row (if (null? row-indices) 1 (common:max (map cadr row-indices))))
(max-col (if (null? col-indices) 1 (common:max (map cadr col-indices))))
(max-visible (max (- *num-tests* 15) 3)) ;; *num-tests* is proportional to the size of the window
(numrows 1)
(numcols 1)
(changed #f)
(runs-hash (let ((ht (make-hash-table)))
(for-each (lambda (run)
(hash-table-set! ht (db:get-value-by-header run runs-header "id") run))
(vector-ref runs-dat 1))
(safe-vector-ref runs-dat 1))
ht))
(run-ids (sort (filter number? (hash-table-keys runs-hash))
(lambda (a b)
(let* ((record-a (hash-table-ref runs-hash a))
(record-b (hash-table-ref runs-hash b))
(time-a (db:get-value-by-header record-a runs-header "event_time"))
(time-b (db:get-value-by-header record-b runs-header "event_time")))
|
| ︙ | | |
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
|
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
|
-
+
|
(default-cmd (car (list-ref *tests-sort-type-index* *tests-sort-reverse*))))
(iuplistbox-fill-list lb cmds-list selected-item: default-cmd)
(mark-for-update)
;; (set! *tests-sort-reverse* *tests-sort-reverse*0)
lb)
;; (iup:button "Sort -t" #:action (lambda (obj)
;; (next-sort-option)
;; (iup:attribute-set! obj "TITLE" (vector-ref (vector-ref *tests-sort-options* *tests-sort-reverse*) 0))
;; (iup:attribute-set! obj "TITLE" (safe-vector-ref (safe-vector-ref *tests-sort-options* *tests-sort-reverse*) 0))
;; (mark-for-update)))
(iup:button "HideEmpty" #:action (lambda (obj)
(set! *hide-empty-runs* (not *hide-empty-runs*))
(iup:attribute-set! obj "TITLE" (if *hide-empty-runs* "+HideE" "-HideE"))
(mark-for-update)))
(let ((hideit (iup:button "HideTests" #:action (lambda (obj)
(set! *hide-not-hide* (not *hide-not-hide*))
|
| ︙ | | |
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
|
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
|
-
-
+
+
|
(butn (iup:button "" ;; button-key
#:size "60x15"
#:expand "HORIZONTAL"
#:fontsize "10"
#:action (lambda (x)
(let* ((toolpath (car (argv)))
(buttndat (hash-table-ref *buttondat* button-key))
(test-id (db:test-get-id (vector-ref buttndat 3)))
(run-id (db:test-get-run_id (vector-ref buttndat 3)))
(test-id (db:test-get-id (safe-vector-ref buttndat 3)))
(run-id (db:test-get-run_id (safe-vector-ref buttndat 3)))
(cmd (conc toolpath " -test " run-id "," test-id "&")))
;(print "Launching " cmd)
(system cmd))))))
(hash-table-set! *buttondat* button-key (vector 0 "100 100 100" button-key #f #f))
(vector-set! testvec testnum butn)
(loop runnum (+ testnum 1) testvec (cons butn res))))))
;; now assemble the hdrlst and bdylst and kick off the dialog
|
| ︙ | | |
Modified datashare.scm
from [2abd8aec1c]
to [d9f97e0b97].
| ︙ | | |
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
|
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
|
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
|
;;======================================================================
;; RECORDS
;;======================================================================
;; make-vector-record "testing" datastore pkg id area version_name store_type copied source_path iteration submitter datetime storegrp datavol quality disk_id comment
;; testing
(define (make-datashare:pkg)(make-vector 15))
(define-inline (datashare:pkg-get-id vec) (vector-ref vec 0))
(define-inline (datashare:pkg-get-area vec) (vector-ref vec 1))
(define-inline (datashare:pkg-get-version_name vec) (vector-ref vec 2))
(define-inline (datashare:pkg-get-store_type vec) (vector-ref vec 3))
(define-inline (datashare:pkg-get-copied vec) (vector-ref vec 4))
(define-inline (datashare:pkg-get-source_path vec) (vector-ref vec 5))
(define-inline (datashare:pkg-get-iteration vec) (vector-ref vec 6))
(define-inline (datashare:pkg-get-submitter vec) (vector-ref vec 7))
(define-inline (datashare:pkg-get-datetime vec) (vector-ref vec 8))
(define-inline (datashare:pkg-get-storegrp vec) (vector-ref vec 9))
(define-inline (datashare:pkg-get-datavol vec) (vector-ref vec 10))
(define-inline (datashare:pkg-get-quality vec) (vector-ref vec 11))
(define-inline (datashare:pkg-get-disk_id vec) (vector-ref vec 12))
(define-inline (datashare:pkg-get-comment vec) (vector-ref vec 13))
(define-inline (datashare:pkg-get-stored_path vec) (vector-ref vec 14))
(define-inline (datashare:pkg-get-id vec) (safe-vector-ref vec 0))
(define-inline (datashare:pkg-get-area vec) (safe-vector-ref vec 1))
(define-inline (datashare:pkg-get-version_name vec) (safe-vector-ref vec 2))
(define-inline (datashare:pkg-get-store_type vec) (safe-vector-ref vec 3))
(define-inline (datashare:pkg-get-copied vec) (safe-vector-ref vec 4))
(define-inline (datashare:pkg-get-source_path vec) (safe-vector-ref vec 5))
(define-inline (datashare:pkg-get-iteration vec) (safe-vector-ref vec 6))
(define-inline (datashare:pkg-get-submitter vec) (safe-vector-ref vec 7))
(define-inline (datashare:pkg-get-datetime vec) (safe-vector-ref vec 8))
(define-inline (datashare:pkg-get-storegrp vec) (safe-vector-ref vec 9))
(define-inline (datashare:pkg-get-datavol vec) (safe-vector-ref vec 10))
(define-inline (datashare:pkg-get-quality vec) (safe-vector-ref vec 11))
(define-inline (datashare:pkg-get-disk_id vec) (safe-vector-ref vec 12))
(define-inline (datashare:pkg-get-comment vec) (safe-vector-ref vec 13))
(define-inline (datashare:pkg-get-stored_path vec) (safe-vector-ref vec 14))
(define-inline (datashare:pkg-set-id! vec val)(vector-set! vec 0 val))
(define-inline (datashare:pkg-set-area! vec val)(vector-set! vec 1 val))
(define-inline (datashare:pkg-set-version_name! vec val)(vector-set! vec 2 val))
(define-inline (datashare:pkg-set-store_type! vec val)(vector-set! vec 3 val))
(define-inline (datashare:pkg-set-copied! vec val)(vector-set! vec 4 val))
(define-inline (datashare:pkg-set-source_path! vec val)(vector-set! vec 5 val))
(define-inline (datashare:pkg-set-iteration! vec val)(vector-set! vec 6 val))
|
| ︙ | | |
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
|
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
|
-
-
-
-
-
-
+
+
+
+
+
+
|
(db (datashare:open-db configdat))
(versions (datashare:get-versions-for-area db (car args) version-patt: (args:get-arg "-vpatt"))))
;; (print "area-name=" area-name " args=" args " *args-hash*=" (hash-table->alist *args-hash*))
(map (lambda (x)
(if (args:get-arg "-full")
(format #t
"~10a~10a~4a~27a~30a\n"
(vector-ref x 0)
(vector-ref x 1)
(vector-ref x 2)
(conc "\"" (time->string (seconds->local-time (vector-ref x 3))) "\"")
(conc "\"" (vector-ref x 4) "\""))
(print (vector-ref x 0))))
(safe-vector-ref x 0)
(safe-vector-ref x 1)
(safe-vector-ref x 2)
(conc "\"" (time->string (seconds->local-time (safe-vector-ref x 3))) "\"")
(conc "\"" (safe-vector-ref x 4) "\""))
(print (safe-vector-ref x 0))))
versions)
(sqlite3:finalize! db)))))
;; ease debugging by loading ~/.dashboardrc - REMOVE FROM PRODUCTION!
(let ((debugcontrolf (conc (get-environment-variable "HOME") "/.datasharerc")))
(if (file-exists? debugcontrolf)
(load debugcontrolf)))
|
| ︙ | | |
Modified db.scm
from [8121f8fcc1]
to [1371806ee5].
| ︙ | | |
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
|
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
|
-
+
|
res))))
;;======================================================================
;; K E E P F I L E D B I N dbstruct
;;======================================================================
;; (define (db:get-filedb dbstruct run-id)
;; (let ((db (vector-ref dbstruct 2)))
;; (let ((db (safe-vector-ref dbstruct 2)))
;; (if db
;; db
;; (let ((fdb (filedb:open-db (conc *toplevel* "/db/files.db"))))
;; (vector-set! dbstruct 2 fdb)
;; fdb))))
;;
;; ;; Can also be used to save arbitrary strings
|
| ︙ | | |
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
|
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
|
-
+
-
+
|
(for-each
(lambda (fromdat-lst)
(sqlite3:with-transaction
db
(lambda ()
(for-each ;;
(lambda (fromrow)
(let* ((a (vector-ref fromrow 0))
(let* ((a (safe-vector-ref fromrow 0))
(curr (hash-table-ref/default todat a #f))
(same #t))
(let loop ((i 0))
(if (or (not curr)
(not (equal? (vector-ref fromrow i)(vector-ref curr i))))
(not (equal? (safe-vector-ref fromrow i)(safe-vector-ref curr i))))
(set! same #f))
(if (and same
(< i (- num-fields 1)))
(loop (+ i 1))))
(if (not same)
(begin
(apply sqlite3:execute stmth (vector->list fromrow))
|
| ︙ | | |
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
|
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
|
-
-
+
+
|
(tdbdat (tasks:open-db))
(servers (tasks:get-all-servers (db:delay-if-busy tdbdat))))
;; kill servers
(if (member 'killservers options)
(for-each
(lambda (server)
(tasks:server-delete-record (db:delay-if-busy tdbdat) (vector-ref server 0) "dbmigration")
(tasks:kill-server (vector-ref server 2)(vector-ref server 1)))
(tasks:server-delete-record (db:delay-if-busy tdbdat) (safe-vector-ref server 0) "dbmigration")
(tasks:kill-server (safe-vector-ref server 2)(safe-vector-ref server 1)))
servers))
;; clear out junk records
;;
(if (member 'dejunk options)
(begin
(db:delay-if-busy mtdb)
|
| ︙ | | |
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
|
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
|
-
+
-
-
+
+
|
;; look up values in a header/data structure
(define (db:get-value-by-header row header field)
(if (null? header) #f
(let loop ((hed (car header))
(tal (cdr header))
(n 0))
(if (equal? hed field)
(vector-ref row n)
(safe-vector-ref row n)
(if (null? tal) #f (loop (car tal)(cdr tal)(+ n 1)))))))
;; Accessors for the header/data structure
;; get rows and header from
(define (db:get-header vec)(vector-ref vec 0))
(define (db:get-rows vec)(vector-ref vec 1))
(define (db:get-header vec)(safe-vector-ref vec 0))
(define (db:get-rows vec)(safe-vector-ref vec 1))
;;======================================================================
;; R U N S
;;======================================================================
(define (db:get-run-name-from-id dbstruct run-id)
(db:with-db
|
| ︙ | | |
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
|
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
|
-
-
-
-
-
+
+
+
+
+
-
+
|
((shortlist)(map db:test-short-record->norm res))
((#f) res)
(else res)))))
(define (db:test-short-record->norm inrec)
;; "id,run_id,testname,item_path,state,status"
;; "id,run_id,testname,state,status,event_time,host,cpuload,diskfree,uname,rundir,item_path,run_duration,final_logf,comment
(vector (vector-ref inrec 0) ;; id
(vector-ref inrec 1) ;; run_id
(vector-ref inrec 2) ;; testname
(vector-ref inrec 4) ;; state
(vector-ref inrec 5) ;; status
(vector (safe-vector-ref inrec 0) ;; id
(safe-vector-ref inrec 1) ;; run_id
(safe-vector-ref inrec 2) ;; testname
(safe-vector-ref inrec 4) ;; state
(safe-vector-ref inrec 5) ;; status
-1 "" -1 -1 "" "-"
(vector-ref inrec 3) ;; item-path
(safe-vector-ref inrec 3) ;; item-path
-1 "-" "-"))
(define (db:get-tests-for-run-state-status dbstruct run-id testpatt)
(let* ((res '())
(tests-match-qry (tests:match->sqlqry testpatt))
(qry (conc "SELECT id,testname,item_path,state,status FROM tests WHERE run_id=? "
|
| ︙ | | |
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
|
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
|
-
+
|
;; move test ids into the 30k * run_id range
;;
(define (db:prep-megatest.db-adj-test-ids mtdb run-id testrecs)
(debug:print-info 0 "Adjusting test ids in megatest.db for run " run-id)
(let ((min-test-id (* run-id 30000)))
(for-each
(lambda (testrec)
(let* ((test-id (vector-ref testrec (db:field->number "id" db:test-record-fields))))
(let* ((test-id (safe-vector-ref testrec (db:field->number "id" db:test-record-fields))))
(db:adj-test-id (db:dbdat-get-db mtdb) min-test-id test-id)))
testrecs)))
;; 1. move test ids into the 30k * run_id range
;; 2. move step ids into the 30k * run_id range
;;
(define (db:prep-megatest.db-for-migration mtdb)
|
| ︙ | | |
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
|
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
|
-
-
+
+
|
;;
;; Run this remotely!!
;;
(define (db:get-matching-previous-test-run-records dbstruct run-id test-name item-path)
(let* ((dbdat (db:get-db dbstruct #f))
(db (db:dbdat-get-db dbdat))
(keys (db:get-keys db))
(selstr (string-intersperse (map (lambda (x)(vector-ref x 0)) keys) ","))
(qrystr (string-intersperse (map (lambda (x)(conc (vector-ref x 0) "=?")) keys) " AND "))
(selstr (string-intersperse (map (lambda (x)(safe-vector-ref x 0)) keys) ","))
(qrystr (string-intersperse (map (lambda (x)(conc (safe-vector-ref x 0) "=?")) keys) " AND "))
(keyvals #f)
(tests-hash (make-hash-table)))
;; first look up the key values from the run selected by run-id
(db:delay-if-busy dbdat)
(sqlite3:for-each-row
(lambda (a . b)
(set! keyvals (cons a b)))
|
| ︙ | | |
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
|
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
|
-
-
-
-
-
-
+
+
+
+
+
+
|
(if pathmod
(let* ((vb (apply vector b))
(keyvals (let loop ((i 0)
(res '()))
(if (>= i numkeys)
res
(loop (+ i 1)
(append res (list (vector-ref vb (+ i 2))))))))
(runname (vector-ref vb 1))
(testname (vector-ref vb (+ 2 numkeys)))
(item-path (vector-ref vb (+ 3 numkeys)))
(final-log (vector-ref vb (+ 7 numkeys)))
(run-dir (vector-ref vb (+ 18 numkeys)))
(append res (list (safe-vector-ref vb (+ i 2))))))))
(runname (safe-vector-ref vb 1))
(testname (safe-vector-ref vb (+ 2 numkeys)))
(item-path (safe-vector-ref vb (+ 3 numkeys)))
(final-log (safe-vector-ref vb (+ 7 numkeys)))
(run-dir (safe-vector-ref vb (+ 18 numkeys)))
(log-fpath (conc run-dir "/" final-log))) ;; (string-intersperse keyvals "/") "/" testname "/" item-path "/"
(debug:print 4 "log: " log-fpath " exists: " (file-exists? log-fpath))
(vector-set! vb (+ 7 numkeys) (if (file-exists? log-fpath)
(let ((newpath (conc pathmod "/"
(string-intersperse keyvals "/")
"/" runname "/" testname "/"
(if (string=? item-path "") "" (conc "/" item-path))
|
| ︙ | | |
Modified db_records.scm
from [858bdddce0]
to [fd3df95a8c].
| ︙ | | |
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
|
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
|
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
|
;; |-1.db
;; |-<N>.db
;;
;;
;; Accessors for a dbstruct
;;
(define-inline (dbr:dbstruct-get-main vec) (vector-ref vec 0)) ;; ( db path )
(define-inline (dbr:dbstruct-get-strdb vec) (vector-ref vec 1)) ;; ( db path )
(define-inline (dbr:dbstruct-get-path vec) (vector-ref vec 2))
(define-inline (dbr:dbstruct-get-local vec) (vector-ref vec 3))
(define-inline (dbr:dbstruct-get-rundb vec) (vector-ref vec 4)) ;; ( db path )
(define-inline (dbr:dbstruct-get-inmem vec) (vector-ref vec 5)) ;; ( db #f )
(define-inline (dbr:dbstruct-get-mtime vec) (vector-ref vec 6))
(define-inline (dbr:dbstruct-get-rtime vec) (vector-ref vec 7))
(define-inline (dbr:dbstruct-get-stime vec) (vector-ref vec 8))
(define-inline (dbr:dbstruct-get-inuse vec) (vector-ref vec 9))
(define-inline (dbr:dbstruct-get-refdb vec) (vector-ref vec 10)) ;; ( db path )
(define-inline (dbr:dbstruct-get-locdbs vec) (vector-ref vec 11))
(define-inline (dbr:dbstruct-get-olddb vec) (vector-ref vec 12)) ;; ( db path )
;; (define-inline (dbr:dbstruct-get-main-path vec) (vector-ref vec 13))
;; (define-inline (dbr:dbstruct-get-rundb-path vec) (vector-ref vec 14))
;; (define-inline (dbr:dbstruct-get-run-id vec) (vector-ref vec 13))
(define-inline (dbr:dbstruct-get-main vec) (safe-vector-ref vec 0)) ;; ( db path )
(define-inline (dbr:dbstruct-get-strdb vec) (safe-vector-ref vec 1)) ;; ( db path )
(define-inline (dbr:dbstruct-get-path vec) (safe-vector-ref vec 2))
(define-inline (dbr:dbstruct-get-local vec) (safe-vector-ref vec 3))
(define-inline (dbr:dbstruct-get-rundb vec) (safe-vector-ref vec 4)) ;; ( db path )
(define-inline (dbr:dbstruct-get-inmem vec) (safe-vector-ref vec 5)) ;; ( db #f )
(define-inline (dbr:dbstruct-get-mtime vec) (safe-vector-ref vec 6))
(define-inline (dbr:dbstruct-get-rtime vec) (safe-vector-ref vec 7))
(define-inline (dbr:dbstruct-get-stime vec) (safe-vector-ref vec 8))
(define-inline (dbr:dbstruct-get-inuse vec) (safe-vector-ref vec 9))
(define-inline (dbr:dbstruct-get-refdb vec) (safe-vector-ref vec 10)) ;; ( db path )
(define-inline (dbr:dbstruct-get-locdbs vec) (safe-vector-ref vec 11))
(define-inline (dbr:dbstruct-get-olddb vec) (safe-vector-ref vec 12)) ;; ( db path )
;; (define-inline (dbr:dbstruct-get-main-path vec) (safe-vector-ref vec 13))
;; (define-inline (dbr:dbstruct-get-rundb-path vec) (safe-vector-ref vec 14))
;; (define-inline (dbr:dbstruct-get-run-id vec) (safe-vector-ref vec 13))
(define-inline (dbr:dbstruct-set-main! vec val)(vector-set! vec 0 val))
(define-inline (dbr:dbstruct-set-strdb! vec val)(vector-set! vec 1 val))
(define-inline (dbr:dbstruct-set-path! vec val)(vector-set! vec 2 val))
(define-inline (dbr:dbstruct-set-local! vec val)(vector-set! vec 3 val))
(define-inline (dbr:dbstruct-set-rundb! vec val)(vector-set! vec 4 val))
(define-inline (dbr:dbstruct-set-inmem! vec val)(vector-set! vec 5 val))
|
| ︙ | | |
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
|
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
|
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-
+
+
-
-
-
-
-
-
-
+
+
+
+
+
+
+
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
-
-
-
-
-
-
-
+
+
+
+
+
+
+
-
-
-
-
-
+
+
+
+
+
-
-
-
-
-
-
+
+
+
+
+
+
|
(hash-table-ref/default (dbr:dbstruct-get-locdbs v) run-id #f))
(define (dbr:dbstruct-set-localdb! v run-id db)
(hash-table-set! (dbr:dbstruct-get-locdbs v) run-id db))
(define (make-db:test)(make-vector 20))
(define-inline (db:test-get-id vec) (vector-ref vec 0))
(define-inline (db:test-get-run_id vec) (vector-ref vec 1))
(define-inline (db:test-get-testname vec) (vector-ref vec 2))
(define-inline (db:test-get-state vec) (vector-ref vec 3))
(define-inline (db:test-get-status vec) (vector-ref vec 4))
(define-inline (db:test-get-event_time vec) (vector-ref vec 5))
(define-inline (db:test-get-host vec) (vector-ref vec 6))
(define-inline (db:test-get-cpuload vec) (vector-ref vec 7))
(define-inline (db:test-get-diskfree vec) (vector-ref vec 8))
(define-inline (db:test-get-uname vec) (vector-ref vec 9))
;; (define-inline (db:test-get-rundir vec) (sdb:qry 'getstr (vector-ref vec 10)))
(define-inline (db:test-get-rundir vec) (vector-ref vec 10))
(define-inline (db:test-get-item-path vec) (vector-ref vec 11))
(define-inline (db:test-get-run_duration vec) (vector-ref vec 12))
(define-inline (db:test-get-final_logf vec) (vector-ref vec 13))
(define-inline (db:test-get-comment vec) (vector-ref vec 14))
(define-inline (db:test-get-process_id vec) (vector-ref vec 16))
;; (define-inline (db:test-get-pass_count vec) (vector-ref vec 15))
;; (define-inline (db:test-get-fail_count vec) (vector-ref vec 16))
(define-inline (db:test-get-id vec) (safe-vector-ref vec 0))
(define-inline (db:test-get-run_id vec) (safe-vector-ref vec 1))
(define-inline (db:test-get-testname vec) (safe-vector-ref vec 2))
(define-inline (db:test-get-state vec) (safe-vector-ref vec 3))
(define-inline (db:test-get-status vec) (safe-vector-ref vec 4))
(define-inline (db:test-get-event_time vec) (safe-vector-ref vec 5))
(define-inline (db:test-get-host vec) (safe-vector-ref vec 6))
(define-inline (db:test-get-cpuload vec) (safe-vector-ref vec 7))
(define-inline (db:test-get-diskfree vec) (safe-vector-ref vec 8))
(define-inline (db:test-get-uname vec) (safe-vector-ref vec 9))
;; (define-inline (db:test-get-rundir vec) (sdb:qry 'getstr (safe-vector-ref vec 10)))
(define-inline (db:test-get-rundir vec) (safe-vector-ref vec 10))
(define-inline (db:test-get-item-path vec) (safe-vector-ref vec 11))
(define-inline (db:test-get-run_duration vec) (safe-vector-ref vec 12))
(define-inline (db:test-get-final_logf vec) (safe-vector-ref vec 13))
(define-inline (db:test-get-comment vec) (safe-vector-ref vec 14))
(define-inline (db:test-get-process_id vec) (safe-vector-ref vec 16))
;; (define-inline (db:test-get-pass_count vec) (safe-vector-ref vec 15))
;; (define-inline (db:test-get-fail_count vec) (safe-vector-ref vec 16))
(define-inline (db:test-get-fullname vec)
(conc (db:test-get-testname vec) "/" (db:test-get-item-path vec)))
(define-inline (db:test-get-first_err vec) (printable (vector-ref vec 15)))
(define-inline (db:test-get-first_warn vec) (printable (vector-ref vec 16)))
(define-inline (db:test-get-first_err vec) (printable (safe-vector-ref vec 15)))
(define-inline (db:test-get-first_warn vec) (printable (safe-vector-ref vec 16)))
(define-inline (db:test-set-cpuload! vec val)(vector-set! vec 7 val))
(define-inline (db:test-set-diskfree! vec val)(vector-set! vec 8 val))
(define-inline (db:test-set-testname! vec val)(vector-set! vec 2 val))
(define-inline (db:test-set-state! vec val)(vector-set! vec 3 val))
(define-inline (db:test-set-status! vec val)(vector-set! vec 4 val))
(define-inline (db:test-set-run_duration! vec val)(vector-set! vec 12 val))
(define-inline (db:test-set-final_logf! vec val)(vector-set! vec 13 val))
;; Test record utility functions
;; Is a test a toplevel?
;;
(define (db:test-get-is-toplevel vec)
(and (equal? (db:test-get-item-path vec) "") ;; test is not an item
(equal? (db:test-get-uname vec) "n/a"))) ;; test has never been run
;; make-vector-record "" db mintest id run_id testname state status event_time item_path
;;
(define (make-db:mintest)(make-vector 7))
(define-inline (db:mintest-get-id vec) (vector-ref vec 0))
(define-inline (db:mintest-get-run_id vec) (vector-ref vec 1))
(define-inline (db:mintest-get-testname vec) (vector-ref vec 2))
(define-inline (db:mintest-get-state vec) (vector-ref vec 3))
(define-inline (db:mintest-get-status vec) (vector-ref vec 4))
(define-inline (db:mintest-get-event_time vec) (vector-ref vec 5))
(define-inline (db:mintest-get-item_path vec) (vector-ref vec 6))
(define-inline (db:mintest-get-id vec) (safe-vector-ref vec 0))
(define-inline (db:mintest-get-run_id vec) (safe-vector-ref vec 1))
(define-inline (db:mintest-get-testname vec) (safe-vector-ref vec 2))
(define-inline (db:mintest-get-state vec) (safe-vector-ref vec 3))
(define-inline (db:mintest-get-status vec) (safe-vector-ref vec 4))
(define-inline (db:mintest-get-event_time vec) (safe-vector-ref vec 5))
(define-inline (db:mintest-get-item_path vec) (safe-vector-ref vec 6))
;; make-vector-record db testmeta id testname author owner description reviewed iterated avg_runtime avg_disk
(define (make-db:testmeta)(make-vector 10 ""))
(define-inline (db:testmeta-get-id vec) (vector-ref vec 0))
(define-inline (db:testmeta-get-testname vec) (vector-ref vec 1))
(define-inline (db:testmeta-get-author vec) (vector-ref vec 2))
(define-inline (db:testmeta-get-owner vec) (vector-ref vec 3))
(define-inline (db:testmeta-get-description vec) (vector-ref vec 4))
(define-inline (db:testmeta-get-reviewed vec) (vector-ref vec 5))
(define-inline (db:testmeta-get-iterated vec) (vector-ref vec 6))
(define-inline (db:testmeta-get-avg_runtime vec) (vector-ref vec 7))
(define-inline (db:testmeta-get-avg_disk vec) (vector-ref vec 8))
(define-inline (db:testmeta-get-tags vec) (vector-ref vec 9))
(define-inline (db:testmeta-get-id vec) (safe-vector-ref vec 0))
(define-inline (db:testmeta-get-testname vec) (safe-vector-ref vec 1))
(define-inline (db:testmeta-get-author vec) (safe-vector-ref vec 2))
(define-inline (db:testmeta-get-owner vec) (safe-vector-ref vec 3))
(define-inline (db:testmeta-get-description vec) (safe-vector-ref vec 4))
(define-inline (db:testmeta-get-reviewed vec) (safe-vector-ref vec 5))
(define-inline (db:testmeta-get-iterated vec) (safe-vector-ref vec 6))
(define-inline (db:testmeta-get-avg_runtime vec) (safe-vector-ref vec 7))
(define-inline (db:testmeta-get-avg_disk vec) (safe-vector-ref vec 8))
(define-inline (db:testmeta-get-tags vec) (safe-vector-ref vec 9))
(define-inline (db:testmeta-set-id! vec val)(vector-set! vec 0 val))
(define-inline (db:testmeta-set-testname! vec val)(vector-set! vec 1 val))
(define-inline (db:testmeta-set-author! vec val)(vector-set! vec 2 val))
(define-inline (db:testmeta-set-owner! vec val)(vector-set! vec 3 val))
(define-inline (db:testmeta-set-description! vec val)(vector-set! vec 4 val))
(define-inline (db:testmeta-set-reviewed! vec val)(vector-set! vec 5 val))
(define-inline (db:testmeta-set-iterated! vec val)(vector-set! vec 6 val))
(define-inline (db:testmeta-set-avg_runtime! vec val)(vector-set! vec 7 val))
(define-inline (db:testmeta-set-avg_disk! vec val)(vector-set! vec 8 val))
;;======================================================================
;; T E S T D A T A
;;======================================================================
(define (make-db:test-data)(make-vector 10))
(define-inline (db:test-data-get-id vec) (vector-ref vec 0))
(define-inline (db:test-data-get-test_id vec) (vector-ref vec 1))
(define-inline (db:test-data-get-category vec) (vector-ref vec 2))
(define-inline (db:test-data-get-variable vec) (vector-ref vec 3))
(define-inline (db:test-data-get-value vec) (vector-ref vec 4))
(define-inline (db:test-data-get-expected vec) (vector-ref vec 5))
(define-inline (db:test-data-get-tol vec) (vector-ref vec 6))
(define-inline (db:test-data-get-units vec) (vector-ref vec 7))
(define-inline (db:test-data-get-comment vec) (vector-ref vec 8))
(define-inline (db:test-data-get-status vec) (vector-ref vec 9))
(define-inline (db:test-data-get-type vec) (vector-ref vec 10))
(define-inline (db:test-data-get-id vec) (safe-vector-ref vec 0))
(define-inline (db:test-data-get-test_id vec) (safe-vector-ref vec 1))
(define-inline (db:test-data-get-category vec) (safe-vector-ref vec 2))
(define-inline (db:test-data-get-variable vec) (safe-vector-ref vec 3))
(define-inline (db:test-data-get-value vec) (safe-vector-ref vec 4))
(define-inline (db:test-data-get-expected vec) (safe-vector-ref vec 5))
(define-inline (db:test-data-get-tol vec) (safe-vector-ref vec 6))
(define-inline (db:test-data-get-units vec) (safe-vector-ref vec 7))
(define-inline (db:test-data-get-comment vec) (safe-vector-ref vec 8))
(define-inline (db:test-data-get-status vec) (safe-vector-ref vec 9))
(define-inline (db:test-data-get-type vec) (safe-vector-ref vec 10))
(define-inline (db:test-data-set-id! vec val)(vector-set! vec 0 val))
(define-inline (db:test-data-set-test_id! vec val)(vector-set! vec 1 val))
(define-inline (db:test-data-set-category! vec val)(vector-set! vec 2 val))
(define-inline (db:test-data-set-variable! vec val)(vector-set! vec 3 val))
(define-inline (db:test-data-set-value! vec val)(vector-set! vec 4 val))
(define-inline (db:test-data-set-expected! vec val)(vector-set! vec 5 val))
(define-inline (db:test-data-set-tol! vec val)(vector-set! vec 6 val))
(define-inline (db:test-data-set-units! vec val)(vector-set! vec 7 val))
(define-inline (db:test-data-set-comment! vec val)(vector-set! vec 8 val))
(define-inline (db:test-data-set-status! vec val)(vector-set! vec 9 val))
(define-inline (db:test-data-set-type! vec val)(vector-set! vec 10 val))
;;======================================================================
;; S T E P S
;;======================================================================
;; Run steps
;; make-vector-record "Run steps" db step id test_id stepname step_complete step_pass event_time
(define (make-db:step)(make-vector 7))
(define-inline (tdb:step-get-id vec) (vector-ref vec 0))
(define-inline (tdb:step-get-test_id vec) (vector-ref vec 1))
(define-inline (tdb:step-get-stepname vec) (vector-ref vec 2))
(define-inline (tdb:step-get-state vec) (vector-ref vec 3))
(define-inline (tdb:step-get-status vec) (vector-ref vec 4))
(define-inline (tdb:step-get-event_time vec) (vector-ref vec 5))
(define-inline (tdb:step-get-logfile vec) (vector-ref vec 6))
(define-inline (tdb:step-get-id vec) (safe-vector-ref vec 0))
(define-inline (tdb:step-get-test_id vec) (safe-vector-ref vec 1))
(define-inline (tdb:step-get-stepname vec) (safe-vector-ref vec 2))
(define-inline (tdb:step-get-state vec) (safe-vector-ref vec 3))
(define-inline (tdb:step-get-status vec) (safe-vector-ref vec 4))
(define-inline (tdb:step-get-event_time vec) (safe-vector-ref vec 5))
(define-inline (tdb:step-get-logfile vec) (safe-vector-ref vec 6))
(define-inline (tdb:step-set-id! vec val)(vector-set! vec 0 val))
(define-inline (tdb:step-set-test_id! vec val)(vector-set! vec 1 val))
(define-inline (tdb:step-set-stepname! vec val)(vector-set! vec 2 val))
(define-inline (tdb:step-set-state! vec val)(vector-set! vec 3 val))
(define-inline (tdb:step-set-status! vec val)(vector-set! vec 4 val))
(define-inline (tdb:step-set-event_time! vec val)(vector-set! vec 5 val))
(define-inline (tdb:step-set-logfile! vec val)(vector-set! vec 6 val))
;; The steps table
(define (make-db:steps-table)(make-vector 5))
(define-inline (tdb:steps-table-get-stepname vec) (vector-ref vec 0))
(define-inline (tdb:steps-table-get-start vec) (vector-ref vec 1))
(define-inline (tdb:steps-table-get-end vec) (vector-ref vec 2))
(define-inline (tdb:steps-table-get-status vec) (vector-ref vec 3))
(define-inline (tdb:steps-table-get-runtime vec) (vector-ref vec 4))
(define-inline (tdb:steps-table-get-stepname vec) (safe-vector-ref vec 0))
(define-inline (tdb:steps-table-get-start vec) (safe-vector-ref vec 1))
(define-inline (tdb:steps-table-get-end vec) (safe-vector-ref vec 2))
(define-inline (tdb:steps-table-get-status vec) (safe-vector-ref vec 3))
(define-inline (tdb:steps-table-get-runtime vec) (safe-vector-ref vec 4))
(define-inline (tdb:step-stable-set-stepname! vec val)(vector-set! vec 0 val))
(define-inline (tdb:step-stable-set-start! vec val)(vector-set! vec 1 val))
(define-inline (tdb:step-stable-set-end! vec val)(vector-set! vec 2 val))
(define-inline (tdb:step-stable-set-status! vec val)(vector-set! vec 3 val))
(define-inline (tdb:step-stable-set-runtime! vec val)(vector-set! vec 4 val))
;; The data structure for handing off requests via wire
(define (make-cdb:packet)(make-vector 6))
(define-inline (cdb:packet-get-client-sig vec) (vector-ref vec 0))
(define-inline (cdb:packet-get-qtype vec) (vector-ref vec 1))
(define-inline (cdb:packet-get-immediate vec) (vector-ref vec 2))
(define-inline (cdb:packet-get-query-sig vec) (vector-ref vec 3))
(define-inline (cdb:packet-get-params vec) (vector-ref vec 4))
(define-inline (cdb:packet-get-qtime vec) (vector-ref vec 5))
(define-inline (cdb:packet-get-client-sig vec) (safe-vector-ref vec 0))
(define-inline (cdb:packet-get-qtype vec) (safe-vector-ref vec 1))
(define-inline (cdb:packet-get-immediate vec) (safe-vector-ref vec 2))
(define-inline (cdb:packet-get-query-sig vec) (safe-vector-ref vec 3))
(define-inline (cdb:packet-get-params vec) (safe-vector-ref vec 4))
(define-inline (cdb:packet-get-qtime vec) (safe-vector-ref vec 5))
(define-inline (cdb:packet-set-client-sig! vec val)(vector-set! vec 0 val))
(define-inline (cdb:packet-set-qtype! vec val)(vector-set! vec 1 val))
(define-inline (cdb:packet-set-immediate! vec val)(vector-set! vec 2 val))
(define-inline (cdb:packet-set-query-sig! vec val)(vector-set! vec 3 val))
(define-inline (cdb:packet-set-params! vec val)(vector-set! vec 4 val))
(define-inline (cdb:packet-set-qtime! vec val)(vector-set! vec 5 val))
|
Modified dcommon.scm
from [31fd59f2b3]
to [5c9e81b0f3].
| ︙ | | |
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
|
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
|
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
-
-
-
-
-
-
-
+
+
+
+
+
+
+
-
-
+
+
|
;;======================================================================
;;
;; A single data structure for all the data used in a dashboard.
;; Share this structure between newdashboard and dashboard with the
;; intent of converging on a single app.
;;
(define *data* (make-vector 25 #f))
(define (dboard:data-get-runs vec) (vector-ref vec 0))
(define (dboard:data-get-tests vec) (vector-ref vec 1))
(define (dboard:data-get-runs-matrix vec) (vector-ref vec 2))
(define (dboard:data-get-tests-tree vec) (vector-ref vec 3))
(define (dboard:data-get-run-keys vec) (vector-ref vec 4))
(define (dboard:data-get-curr-test-ids vec) (vector-ref vec 5))
;; (define (dboard:data-get-test-details vec) (vector-ref vec 6))
(define (dboard:data-get-path-test-ids vec) (vector-ref vec 7))
(define (dboard:data-get-updaters vec) (vector-ref vec 8))
(define (dboard:data-get-path-run-ids vec) (vector-ref vec 9))
(define (dboard:data-get-curr-run-id vec) (vector-ref vec 10))
(define (dboard:data-get-runs-tree vec) (vector-ref vec 11))
(define (dboard:data-get-runs vec) (safe-vector-ref vec 0))
(define (dboard:data-get-tests vec) (safe-vector-ref vec 1))
(define (dboard:data-get-runs-matrix vec) (safe-vector-ref vec 2))
(define (dboard:data-get-tests-tree vec) (safe-vector-ref vec 3))
(define (dboard:data-get-run-keys vec) (safe-vector-ref vec 4))
(define (dboard:data-get-curr-test-ids vec) (safe-vector-ref vec 5))
;; (define (dboard:data-get-test-details vec) (safe-vector-ref vec 6))
(define (dboard:data-get-path-test-ids vec) (safe-vector-ref vec 7))
(define (dboard:data-get-updaters vec) (safe-vector-ref vec 8))
(define (dboard:data-get-path-run-ids vec) (safe-vector-ref vec 9))
(define (dboard:data-get-curr-run-id vec) (safe-vector-ref vec 10))
(define (dboard:data-get-runs-tree vec) (safe-vector-ref vec 11))
;; For test-patts convert #f to ""
(define (dboard:data-get-test-patts vec)
(let ((val (vector-ref vec 12)))(if val val "")))
(define (dboard:data-get-states vec) (vector-ref vec 13))
(define (dboard:data-get-statuses vec) (vector-ref vec 14))
(define (dboard:data-get-logs-textbox vec val)(vector-ref vec 15))
(define (dboard:data-get-command vec) (vector-ref vec 16))
(define (dboard:data-get-command-tb vec) (vector-ref vec 17))
(define (dboard:data-get-target vec) (vector-ref vec 18))
(let ((val (safe-vector-ref vec 12)))(if val val "")))
(define (dboard:data-get-states vec) (safe-vector-ref vec 13))
(define (dboard:data-get-statuses vec) (safe-vector-ref vec 14))
(define (dboard:data-get-logs-textbox vec val)(safe-vector-ref vec 15))
(define (dboard:data-get-command vec) (safe-vector-ref vec 16))
(define (dboard:data-get-command-tb vec) (safe-vector-ref vec 17))
(define (dboard:data-get-target vec) (safe-vector-ref vec 18))
(define (dboard:data-get-target-string vec)
(let ((targ (dboard:data-get-target vec)))
(if (list? targ)(string-intersperse targ "/") "no-target-specified")))
(define (dboard:data-get-run-name vec) (vector-ref vec 19))
(define (dboard:data-get-runs-listbox vec) (vector-ref vec 20))
(define (dboard:data-get-run-name vec) (safe-vector-ref vec 19))
(define (dboard:data-get-runs-listbox vec) (safe-vector-ref vec 20))
(define (dboard:data-set-runs! vec val)(vector-set! vec 0 val))
(define (dboard:data-set-tests! vec val)(vector-set! vec 1 val))
(define (dboard:data-set-runs-matrix! vec val)(vector-set! vec 2 val))
(define (dboard:data-set-tests-tree! vec val)(vector-set! vec 3 val))
(define (dboard:data-set-run-keys! vec val)(vector-set! vec 4 val))
(define (dboard:data-set-curr-test-ids! vec val)(vector-set! vec 5 val))
|
| ︙ | | |
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
|
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
|
-
-
-
-
-
+
+
+
+
+
|
;;
(define (dcommon:minimize-test-data tests-dat)
(if (null? tests-dat)
'()
(let loop ((hed (car tests-dat))
(tal (cdr tests-dat))
(res '()))
(let* ((test-id (vector-ref hed 0)) ;; look at the tests-dat spec for locations
(test-name (vector-ref hed 1))
(item-path (vector-ref hed 2))
(state (vector-ref hed 3))
(status (vector-ref hed 4))
(let* ((test-id (safe-vector-ref hed 0)) ;; look at the tests-dat spec for locations
(test-name (safe-vector-ref hed 1))
(item-path (safe-vector-ref hed 2))
(state (safe-vector-ref hed 3))
(status (safe-vector-ref hed 4))
(newitem (list test-name item-path (list test-id state status))))
(if (null? tal)
(reverse (cons newitem res))
(loop (car tal)(cdr tal)(cons newitem res)))))))
;;======================================================================
|
| ︙ | | |
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
|
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
|
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
|
;; (iup:attribute-set! servers-matrix (conc "0:" colnum) colname)
;; (set! colnum (+ 1 colnum)))
;; colnames)
(set! rownum 1)
(for-each
(lambda (server)
(set! colnum 0)
(let* ((vals (list (vector-ref server 0) ;; Id
(vector-ref server 9) ;; MT-Ver
(vector-ref server 1) ;; Pid
(vector-ref server 2) ;; Hostname
(conc (vector-ref server 3) ":" (vector-ref server 4)) ;; IP:Port
(seconds->hr-min-sec (- (current-seconds)(vector-ref server 6)))
;; (vector-ref server 5) ;; Pubport
;; (vector-ref server 10) ;; Last beat
;; (vector-ref server 6) ;; Start time
;; (vector-ref server 7) ;; Priority
;; (vector-ref server 8) ;; State
(vector-ref server 8) ;; State
(vector-ref server 12) ;; RunId
(let* ((vals (list (safe-vector-ref server 0) ;; Id
(safe-vector-ref server 9) ;; MT-Ver
(safe-vector-ref server 1) ;; Pid
(safe-vector-ref server 2) ;; Hostname
(conc (safe-vector-ref server 3) ":" (safe-vector-ref server 4)) ;; IP:Port
(seconds->hr-min-sec (- (current-seconds)(safe-vector-ref server 6)))
;; (safe-vector-ref server 5) ;; Pubport
;; (safe-vector-ref server 10) ;; Last beat
;; (safe-vector-ref server 6) ;; Start time
;; (safe-vector-ref server 7) ;; Priority
;; (safe-vector-ref server 8) ;; State
(safe-vector-ref server 8) ;; State
(safe-vector-ref server 12) ;; RunId
)))
(for-each (lambda (val)
(let* ((row-col (conc rownum ":" colnum))
(curr-val (iup:attribute servers-matrix row-col)))
(if (not (equal? (conc val) curr-val))
(begin
(iup:attribute-set! servers-matrix row-col val)
|
| ︙ | | |
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
|
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
|
-
+
-
-
-
+
+
+
|
"\nid: " (tdb:step-get-id step)
"\nstepname: " (tdb:step-get-stepname step)
"\nstate: " (tdb:step-get-state step)
"\nstatus: " (tdb:step-get-status step)
"\ntime: " (tdb:step-get-event_time step))
(case (string->symbol (tdb:step-get-state step))
((start)(vector-set! record 1 (tdb:step-get-event_time step))
(vector-set! record 3 (if (equal? (vector-ref record 3) "")
(vector-set! record 3 (if (equal? (safe-vector-ref record 3) "")
(tdb:step-get-status step)))
(if (> (string-length (tdb:step-get-logfile step))
0)
(vector-set! record 5 (tdb:step-get-logfile step))))
((end)
(vector-set! record 2 (any->number (tdb:step-get-event_time step)))
(vector-set! record 3 (tdb:step-get-status step))
(vector-set! record 4 (let ((startt (any->number (vector-ref record 1)))
(endt (any->number (vector-ref record 2))))
(debug:print 4 "record[1]=" (vector-ref record 1)
(vector-set! record 4 (let ((startt (any->number (safe-vector-ref record 1)))
(endt (any->number (safe-vector-ref record 2))))
(debug:print 4 "record[1]=" (safe-vector-ref record 1)
", startt=" startt ", endt=" endt
", get-status: " (tdb:step-get-status step))
(if (and (number? startt)(number? endt))
(seconds->hr-min-sec (- endt startt)) "-1")))
(if (> (string-length (tdb:step-get-logfile step))
0)
(vector-set! record 5 (tdb:step-get-logfile step))))
|
| ︙ | | |
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
|
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
|
-
-
+
+
-
+
-
-
-
+
+
+
-
-
+
+
-
-
+
+
-
+
|
(define (dcommon:get-compressed-steps dbstruct run-id test-id)
(let* ((steps-data (db:get-steps-for-test dbstruct run-id test-id))
(comprsteps (dcommon:process-steps-table steps-data))) ;; (open-run-close db:get-steps-table #f test-id work-area: work-area)))
(map (lambda (x)
;; take advantage of the \n on time->string
(vector
(vector-ref x 0)
(let ((s (vector-ref x 1)))
(safe-vector-ref x 0)
(let ((s (safe-vector-ref x 1)))
(if (number? s)(seconds->time-string s) s))
(let ((s (vector-ref x 2)))
(let ((s (safe-vector-ref x 2)))
(if (number? s)(seconds->time-string s) s))
(vector-ref x 3) ;; status
(vector-ref x 4)
(vector-ref x 5))) ;; time delta
(safe-vector-ref x 3) ;; status
(safe-vector-ref x 4)
(safe-vector-ref x 5))) ;; time delta
(sort (hash-table-values comprsteps)
(lambda (a b)
(let ((time-a (vector-ref a 1))
(time-b (vector-ref b 1)))
(let ((time-a (safe-vector-ref a 1))
(time-b (safe-vector-ref b 1)))
(if (and (number? time-a)(number? time-b))
(if (< time-a time-b)
#t
(if (eq? time-a time-b)
(string<? (conc (vector-ref a 2))
(conc (vector-ref b 2)))
(string<? (conc (safe-vector-ref a 2))
(conc (safe-vector-ref b 2)))
#f))
(string<? (conc time-a)(conc time-b)))))))))
(define (dcommon:populate-steps teststeps steps-matrix)
(let ((max-row 0))
(if (null? teststeps)
(iup:attribute-set! steps-matrix "CLEARVALUE" "CONTENTS")
(let loop ((hed (car teststeps))
(tal (cdr teststeps))
(rownum 1)
(colnum 1))
(if (> rownum max-row)(set! max-row rownum))
(let ((val (vector-ref hed (- colnum 1)))
(let ((val (safe-vector-ref hed (- colnum 1)))
(mtrx-rc (conc rownum ":" colnum)))
(iup:attribute-set! steps-matrix mtrx-rc (if val (conc val) ""))
(if (< colnum 6)
(loop hed tal rownum (+ colnum 1))
(if (not (null? tal))
(loop (car tal)(cdr tal)(+ rownum 1) 1))))))
(if (> max-row 0)
|
| ︙ | | |
Modified ezsteps.scm
from [18ab86f9c8]
to [6e0bd5f400].
| ︙ | | |
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
|
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
|
-
+
|
(if (not (> (length ezstepslst) 0))
(message-window "ERROR: You can only re-run steps defined via ezsteps")
(begin
(let loop ((ezstep (car ezstepslst))
(tal (cdr ezstepslst))
(prevstep #f)
(runflag #f)) ;; flag used to skip steps when not starting at the beginning
(if (vector-ref exit-info 1)
(if (safe-vector-ref exit-info 1)
(let* ((stepname (car ezstep)) ;; do stuff to run the step
(stepinfo (cadr ezstep))
(stepparts (string-match (regexp "^(\\{([^\\}]*)\\}\\s*|)(.*)$") stepinfo))
(stepparms (list-ref stepparts 2)) ;; for future use, {VAR=1,2,3}, run step for each
(stepcmd (list-ref stepparts 3))
(script "") ; "#!/bin/bash\n") ;; yep, we depend on bin/bash FIXME!!!
(logpro-used #f))
|
| ︙ | | |
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
|
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
|
-
+
-
-
+
+
-
+
-
+
-
+
-
+
|
(vector-set! exit-info 2 exit-code)
(mutex-unlock! run-mutex)
(if (eq? pid-val 0)
(begin
(thread-sleep! 1)
(processloop (+ i 1))))
))
(let ((exinfo (vector-ref exit-info 2))
(let ((exinfo (safe-vector-ref exit-info 2))
(logfna (if logpro-used (conc stepname ".html") "")))
(rmt:teststep-set-status! run-id test-id stepname "end" exinfo #f logfna))
(if logpro-used
(rmt:test-set-log! test-id (conc stepname ".html")))
;; set the test final status
(let* ((this-step-status (cond
((and (eq? (vector-ref exit-info 2) 2) logpro-used) 'warn)
((eq? (vector-ref exit-info 2) 0) 'pass)
((and (eq? (safe-vector-ref exit-info 2) 2) logpro-used) 'warn)
((eq? (safe-vector-ref exit-info 2) 0) 'pass)
(else 'fail)))
(overall-status (cond
((eq? rollup-status 2) 'warn)
((eq? rollup-status 0) 'pass)
(else 'fail)))
(next-status (cond
((eq? overall-status 'pass) this-step-status)
((eq? overall-status 'warn)
(if (eq? this-step-status 'fail) 'fail 'warn))
(else 'fail))))
(debug:print 4 "Exit value received: " (vector-ref exit-info 2) " logpro-used: " logpro-used
(debug:print 4 "Exit value received: " (safe-vector-ref exit-info 2) " logpro-used: " logpro-used
" this-step-status: " this-step-status " overall-status: " overall-status
" next-status: " next-status " rollup-status: " rollup-status)
(case next-status
((warn)
(set! rollup-status 2)
;; NB// test-set-status! does rdb calls under the hood
(tests:test-set-status! test-id "RUNNING" "WARN"
(if (eq? this-step-status 'warn) "Logpro warning found" #f)
#f))
((pass)
(tests:test-set-status! test-id "RUNNING" "PASS" #f #f))
(else ;; 'fail
(set! rollup-status 1) ;; force fail
(tests:test-set-status! test-id "RUNNING" "FAIL" (conc "Failed at step " stepname) #f)
))))
(if (and (steprun-good? logpro-used (vector-ref exit-info 2))
(if (and (steprun-good? logpro-used (safe-vector-ref exit-info 2))
(not (null? tal)))
(if (not run-one) ;; if we got here we completed the step, if run-one is true, stop
(loop (car tal) (cdr tal) stepname runflag))))
(debug:print 4 "WARNING: a prior step failed, stopping at " ezstep)))
;; Once done with step/steps update the test record
;;
(let* ((item-path (db:test-get-item-path testdat)) ;; (item-list->path itemdat))
(testinfo (rmt:get-testinfo-by-id run-id test-id))) ;; refresh the testdat, call it iteminfo in case need prev/curr
;; Am I completed?
(if (equal? (db:test-get-state testinfo) "RUNNING") ;; (not (equal? (db:test-get-state testinfo) "COMPLETED"))
(let ((new-state (if kill-job "KILLED" "COMPLETED") ;; (if (eq? (vector-ref exit-info 2) 0) ;; exited with "good" status
(let ((new-state (if kill-job "KILLED" "COMPLETED") ;; (if (eq? (safe-vector-ref exit-info 2) 0) ;; exited with "good" status
;; "COMPLETED"
;; (db:test-get-state testinfo))) ;; else preseve the state as set within the test
)
(new-status (cond
((not (vector-ref exit-info 1)) "FAIL") ;; job failed to run
((not (safe-vector-ref exit-info 1)) "FAIL") ;; job failed to run
((eq? rollup-status 0)
;; if the current status is AUTO the defer to the calculated value (i.e. leave this AUTO)
(if (equal? (db:test-get-status testinfo) "AUTO") "AUTO" "PASS"))
((eq? rollup-status 1) "FAIL")
((eq? rollup-status 2)
;; if the current status is AUTO the defer to the calculated value but qualify (i.e. make this AUTO-WARN)
(if (equal? (db:test-get-status testinfo) "AUTO") "AUTO-WARN" "WARN"))
|
| ︙ | | |
Modified fdb_records.scm
from [bbb0371221]
to [9cfbefb51b].
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
|
-
-
-
-
-
+
+
+
+
+
-
-
-
+
+
+
|
;; Single record for managing a filedb
;; make-vector-record "Filedb record" filedb fdb db dbpath pathcache idcache partcache
;; Filedb record
(define (make-filedb:fdb)(make-vector 5))
(define-inline (filedb:fdb-get-db vec) (vector-ref vec 0))
(define-inline (filedb:fdb-get-dbpath vec) (vector-ref vec 1))
(define-inline (filedb:fdb-get-pathcache vec) (vector-ref vec 2))
(define-inline (filedb:fdb-get-idcache vec) (vector-ref vec 3))
(define-inline (filedb:fdb-get-partcache vec) (vector-ref vec 4))
(define-inline (filedb:fdb-get-db vec) (safe-vector-ref vec 0))
(define-inline (filedb:fdb-get-dbpath vec) (safe-vector-ref vec 1))
(define-inline (filedb:fdb-get-pathcache vec) (safe-vector-ref vec 2))
(define-inline (filedb:fdb-get-idcache vec) (safe-vector-ref vec 3))
(define-inline (filedb:fdb-get-partcache vec) (safe-vector-ref vec 4))
(define-inline (filedb:fdb-set-db! vec val)(vector-set! vec 0 val))
(define-inline (filedb:fdb-set-dbpath! vec val)(vector-set! vec 1 val))
(define-inline (filedb:fdb-set-pathcache! vec val)(vector-set! vec 2 val))
(define-inline (filedb:fdb-set-idcache! vec val)(vector-set! vec 3 val))
(define-inline (filedb:fdb-set-partcache! vec val)(vector-set! vec 4 val))
;; children records, should have use something other than "child"
(define-inline (filedb:child-get-id vec) (vector-ref vec 0))
(define-inline (filedb:child-get-path vec) (vector-ref vec 1))
(define-inline (filedb:child-get-parent_id vec)(vector-ref vec 2))
(define-inline (filedb:child-get-id vec) (safe-vector-ref vec 0))
(define-inline (filedb:child-get-path vec) (safe-vector-ref vec 1))
(define-inline (filedb:child-get-parent_id vec)(safe-vector-ref vec 2))
|
Modified filedb.scm
from [91e90bcdc7]
to [e0ba84ec3a].
| ︙ | | |
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
|
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
|
-
-
-
-
-
+
+
+
+
+
|
;; 12 number of blocks allocated st_blocks
(define (filedb:add-path-stat db path parent statinfo)
(let ((stmt (sqlite3:prepare db "INSERT INTO paths (path,parent_id,mode,uid,gid,size,mtime) VALUES (?,?,?,?,?,?,?);")))
(sqlite3:execute stmt
path
parent
(vector-ref statinfo 1) ;; mode
(vector-ref statinfo 3) ;; uid
(vector-ref statinfo 4) ;; gid
(vector-ref statinfo 5) ;; size
(vector-ref statinfo 8) ;; mtime
(safe-vector-ref statinfo 1) ;; mode
(safe-vector-ref statinfo 3) ;; uid
(safe-vector-ref statinfo 4) ;; gid
(safe-vector-ref statinfo 5) ;; size
(safe-vector-ref statinfo 8) ;; mtime
)
(sqlite3:finalize! stmt))) ;; (filedb:get-current-time-string))))
(define (filedb:add-path db path parent)
(let ((stmt (sqlite3:prepare db "INSERT INTO paths (path,parent_id) VALUES (?,?);")))
(sqlite3:execute stmt path parent)
(sqlite3:finalize! stmt)))
|
| ︙ | | |
Modified http-transport.scm
from [30e3275ed4]
to [a767e14558].
| ︙ | | |
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
|
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
|
-
+
|
exn
(begin
(set! success #f)
(debug:print 0 "WARNING: failure in with-input-from-request to " fullurl ".")
(debug:print 0 " message: " ((condition-property-accessor 'exn 'message) exn))
(hash-table-delete! *runremote* run-id)
;; Killing associated server to allow clean retry.")
(tasks:kill-server-run-id run-id) ;; better to kill the server in the logic that called this routine?
;; (tasks:kill-server-run-id run-id) ;; better to kill the server in the logic that called this routine?
(mutex-unlock! *http-mutex*)
(signal (make-composite-condition
(make-property-condition 'commfail 'message "failed to connect to server")))
"communications failed")
(with-input-from-request ;; was dat
fullurl
(list (cons 'key "thekey")
|
| ︙ | | |
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
|
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
|
-
-
+
+
-
+
-
-
+
+
-
-
-
-
-
-
-
+
+
+
+
+
+
+
|
(th1 (make-thread send-recieve "with-input-from-request"))
(th2 (make-thread time-out "time out")))
(thread-start! th1)
(thread-start! th2)
(thread-join! th1)
(thread-terminate! th2)
(debug:print-info 11 "got res=" res)
(if (vector? res)
(if (vector-ref res 0)
(if (and res (vector? res))
(if (safe-vector-ref res 0)
res
(begin ;; note: this code also called in nmsg-transport - consider consolidating it
(debug:print 0 "ERROR: error occured at server, info=" (vector-ref res 2))
(debug:print 0 "ERROR: error occured at server, info=" (safe-vector-ref res 2))
(debug:print 0 " client call chain:")
(print-call-chain (current-error-port))
(debug:print 0 " server call chain:")
(pp (vector-ref res 1) (current-error-port))
(signal (vector-ref result 0))))
(pp (safe-vector-ref res 1) (current-error-port))
(signal (safe-vector-ref result 0))))
(signal (make-composite-condition
(make-property-condition
'timeout
'message "nmsg-transport:client-api-send-receive-raw timed out talking to server")))))))
;; careful closing of connections stored in *runremote*
;;
(define (http-transport:close-connections run-id)
(let* ((server-dat (hash-table-ref/default *runremote* run-id #f)))
(if (vector? server-dat)
(let ((api-dat (http-transport:server-dat-get-api-uri server-dat)))
(close-connection! api-dat)
#t)
#f)))
(define (make-http-transport:server-dat)(make-vector 6))
(define (http-transport:server-dat-get-iface vec) (vector-ref vec 0))
(define (http-transport:server-dat-get-port vec) (vector-ref vec 1))
(define (http-transport:server-dat-get-api-uri vec) (vector-ref vec 2))
(define (http-transport:server-dat-get-api-url vec) (vector-ref vec 3))
(define (http-transport:server-dat-get-api-req vec) (vector-ref vec 4))
(define (http-transport:server-dat-get-last-access vec) (vector-ref vec 5))
(define (http-transport:server-dat-get-socket vec) (vector-ref vec 6))
(define (http-transport:server-dat-get-iface vec) (safe-vector-ref vec 0))
(define (http-transport:server-dat-get-port vec) (safe-vector-ref vec 1))
(define (http-transport:server-dat-get-api-uri vec) (safe-vector-ref vec 2))
(define (http-transport:server-dat-get-api-url vec) (safe-vector-ref vec 3))
(define (http-transport:server-dat-get-api-req vec) (safe-vector-ref vec 4))
(define (http-transport:server-dat-get-last-access vec) (safe-vector-ref vec 5))
(define (http-transport:server-dat-get-socket vec) (safe-vector-ref vec 6))
(define (http-transport:server-dat-make-url vec)
(if (and (http-transport:server-dat-get-iface vec)
(http-transport:server-dat-get-port vec))
(conc "http://"
(http-transport:server-dat-get-iface vec)
":"
|
| ︙ | | |
435
436
437
438
439
440
441
442
443
444
445
446
447
448
|
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
|
+
|
(if (eq? server-state 'available)
(let ((new-server-id (tasks:server-am-i-the-server? (db:delay-if-busy tdbdat) run-id))) ;; try to ensure no double registering of servers
(if (equal? new-server-id server-id)
(begin
(tasks:server-set-state! (db:delay-if-busy tdbdat) server-id "dbprep")
(thread-sleep! 0.5) ;; give some margin for queries to complete before switching from file based access to server based access
(set! *inmemdb* (db:setup run-id))
(thread-sleep! 0.1)
(tasks:server-set-state! (db:delay-if-busy tdbdat) server-id "running"))
(begin ;; gotta exit nicely
(tasks:server-set-state! (db:delay-if-busy tdbdat) server-id "collision")
(http-transport:server-shutdown server-id port))))))
(if (< count 1) ;; 3x3 = 9 secs aprox
(loop (+ count 1) 'running bad-sync-count))
|
| ︙ | | |
Modified launch.scm
from [7676bcea71]
to [1dc0071816].
| ︙ | | |
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
|
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
|
-
-
+
+
|
(if (not (file-exists? ".ezsteps"))(create-directory ".ezsteps"))
;; if ezsteps was defined then we are sure to have at least one step but check anyway
(if (not (> (length ezstepslst) 0))
(debug:print 0 "ERROR: ezsteps defined but ezstepslst is zero length")
(let loop ((ezstep (car ezstepslst))
(tal (cdr ezstepslst))
(prevstep #f))
;; check exit-info (vector-ref exit-info 1)
(if (vector-ref exit-info 1)
;; check exit-info (safe-vector-ref exit-info 1)
(if (safe-vector-ref exit-info 1)
(let* ((stepname (car ezstep)) ;; do stuff to run the step
(stepinfo (cadr ezstep))
(stepparts (string-match (regexp "^(\\{([^\\}]*)\\}\\s*|)(.*)$") stepinfo))
(stepparms (list-ref stepparts 2)) ;; for future use, {VAR=1,2,3}, run step for each
(stepcmd (list-ref stepparts 3))
(script "") ; "#!/bin/bash\n") ;; yep, we depend on bin/bash FIXME!!!
(logpro-used #f))
|
| ︙ | | |
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
|
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
|
-
+
-
-
+
+
-
+
-
+
|
(vector-set! exit-info 2 exit-code)
(mutex-unlock! m)
(if (eq? pid-val 0)
(begin
(thread-sleep! 2)
(processloop (+ i 1))))
))
(let ((exinfo (vector-ref exit-info 2))
(let ((exinfo (safe-vector-ref exit-info 2))
(logfna (if logpro-used (conc stepname ".html") "")))
(rmt:teststep-set-status! run-id test-id stepname "end" exinfo #f logfna))
(if logpro-used
(rmt:test-set-log! run-id test-id (conc stepname ".html")))
;; set the test final status
(let* ((this-step-status (cond
((and (eq? (vector-ref exit-info 2) 2) logpro-used) 'warn)
((eq? (vector-ref exit-info 2) 0) 'pass)
((and (eq? (safe-vector-ref exit-info 2) 2) logpro-used) 'warn)
((eq? (safe-vector-ref exit-info 2) 0) 'pass)
(else 'fail)))
(overall-status (cond
((eq? rollup-status 2) 'warn)
((eq? rollup-status 0) 'pass)
(else 'fail)))
(next-status (cond
((eq? overall-status 'pass) this-step-status)
((eq? overall-status 'warn)
(if (eq? this-step-status 'fail) 'fail 'warn))
(else 'fail)))
(next-state ;; "RUNNING") ;; WHY WAS THIS CHANGED TO NOT USE (null? tal) ??
(cond
((null? tal) ;; more to run?
"COMPLETED")
(else "RUNNING")))
)
(debug:print 4 "Exit value received: " (vector-ref exit-info 2) " logpro-used: " logpro-used
(debug:print 4 "Exit value received: " (safe-vector-ref exit-info 2) " logpro-used: " logpro-used
" this-step-status: " this-step-status " overall-status: " overall-status
" next-status: " next-status " rollup-status: " rollup-status)
(case next-status
((warn)
(set! rollup-status 2)
;; NB// test-set-status! does rdb calls under the hood
(tests:test-set-status! run-id test-id next-state "WARN"
(if (eq? this-step-status 'warn) "Logpro warning found" #f)
#f))
((pass)
(tests:test-set-status! run-id test-id next-state "PASS" #f #f))
(else ;; 'fail
(set! rollup-status 1) ;; force fail, this used to be next-state but that doesn't make sense. should always be "COMPLETED"
(tests:test-set-status! run-id test-id "COMPLETED" "FAIL" (conc "Failed at step " stepname) #f)
))))
(if (and (steprun-good? logpro-used (vector-ref exit-info 2))
(if (and (steprun-good? logpro-used (safe-vector-ref exit-info 2))
(not (null? tal)))
(loop (car tal) (cdr tal) stepname)))
(debug:print 4 "WARNING: a prior step failed, stopping at " ezstep))))))))
(monitorjob (lambda ()
(let* ((start-seconds (current-seconds))
(calc-minutes (lambda ()
(inexact->exact
|
| ︙ | | |
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
|
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
|
-
+
|
(tests:update-central-meta-info run-id test-id (get-cpu-load) (get-df (current-directory))(calc-minutes) #f #f)
(if kill-job?
(begin
(mutex-lock! m)
;; NOTE: The pid can change as different steps are run. Do we need handshaking between this
;; section and the runit section? Or add a loop that tries three times with a 1/4 second
;; between tries?
(let* ((pid1 (vector-ref exit-info 0))
(let* ((pid1 (safe-vector-ref exit-info 0))
(pid2 (rmt:test-get-top-process-pid run-id test-id))
(pids (delete-duplicates (filter number? (list pid1 pid2)))))
(if (not (null? pids))
(begin
(for-each
(lambda (pid)
(handle-exceptions
|
| ︙ | | |
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
|
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
|
-
+
-
+
-
+
-
-
-
+
+
+
|
(thread-sleep! 1) ;; give thread th1 a chance to be done TODO: Verify this is needed. At 0.1 I was getting fail to stop, increased to total of 1.1 sec.
(mutex-lock! m)
(let* ((item-path (item-list->path itemdat))
;; only state and status needed - use lazy routine
(testinfo (rmt:get-testinfo-state-status run-id test-id)))
;; Am I completed?
(if (member (db:test-get-state testinfo) '("REMOTEHOSTSTART" "RUNNING")) ;; NOTE: It should *not* be REMOTEHOSTSTART but for reasons I don't yet understand it sometimes gets stuck in that state ;; (not (equal? (db:test-get-state testinfo) "COMPLETED"))
(let ((new-state (if kill-job? "KILLED" "COMPLETED") ;; (if (eq? (vector-ref exit-info 2) 0) ;; exited with "good" status
(let ((new-state (if kill-job? "KILLED" "COMPLETED") ;; (if (eq? (safe-vector-ref exit-info 2) 0) ;; exited with "good" status
;; "COMPLETED"
;; (db:test-get-state testinfo))) ;; else preseve the state as set within the test
)
(new-status (cond
((not (vector-ref exit-info 1)) "FAIL") ;; job failed to run
((not (safe-vector-ref exit-info 1)) "FAIL") ;; job failed to run
((eq? rollup-status 0)
;; if the current status is AUTO then defer to the calculated value (i.e. leave this AUTO)
(if (equal? (db:test-get-status testinfo) "AUTO") "AUTO" "PASS"))
((eq? rollup-status 1) "FAIL")
((eq? rollup-status 2)
;; if the current status is AUTO the defer to the calculated value but qualify (i.e. make this AUTO-WARN)
(if (equal? (db:test-get-status testinfo) "AUTO") "AUTO-WARN" "WARN"))
(else "FAIL")))) ;; (db:test-get-status testinfo)))
(debug:print-info 1 "Test exited in state=" (db:test-get-state testinfo) ", setting state/status based on exit code of " (vector-ref exit-info 1) " and rollup-status of " rollup-status)
(debug:print-info 1 "Test exited in state=" (db:test-get-state testinfo) ", setting state/status based on exit code of " (safe-vector-ref exit-info 1) " and rollup-status of " rollup-status)
(tests:test-set-status! run-id
test-id
new-state
new-status
(args:get-arg "-m") #f)
;; need to update the top test record if PASS or FAIL and this is a subtest
;; NO NEED TO CALL roll-up-pass-fail-counts HERE, THIS IS DONE IN roll-up-pass-fail-counts called by tests:test-set-status!
))
;; for automated creation of the rollup html file this is a good place...
(if (not (equal? item-path ""))
(tests:summarize-items run-id test-id test-name #f))) ;; don't force - just update if no
(mutex-unlock! m)
(debug:print 2 "Output from running " fullrunscript ", pid " (vector-ref exit-info 0) " in work area "
work-area ":\n====\n exit code " (vector-ref exit-info 2) "\n" "====\n")
(if (not (vector-ref exit-info 1))
(debug:print 2 "Output from running " fullrunscript ", pid " (safe-vector-ref exit-info 0) " in work area "
work-area ":\n====\n exit code " (safe-vector-ref exit-info 2) "\n" "====\n")
(if (not (safe-vector-ref exit-info 1))
(exit 4)))))))
;; set up the very basics needed for doing anything here.
(define (launch:setup-for-run #!key (force #f))
;; would set values for KEYS in the environment here for better support of env-override but
;; have chicken/egg scenario. need to read megatest.config then read it again. Going to
;; pass on that idea for now
|
| ︙ | | |
Modified lock-queue.scm
from [fb7e24faf1]
to [26af3a3a46].
| ︙ | | |
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
|
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
|
-
-
+
+
|
;;======================================================================
;;======================================================================
;; db record, <vector db path-to-db>
;;======================================================================
(define (make-lock-queue:db-dat)(make-vector 3))
(define-inline (lock-queue:db-dat-get-db vec) (vector-ref vec 0))
(define-inline (lock-queue:db-dat-get-path vec) (vector-ref vec 1))
(define-inline (lock-queue:db-dat-get-db vec) (safe-vector-ref vec 0))
(define-inline (lock-queue:db-dat-get-path vec) (safe-vector-ref vec 1))
(define-inline (lock-queue:db-dat-set-db! vec val)(vector-set! vec 0 val))
(define-inline (lock-queue:db-dat-set-path! vec val)(vector-set! vec 1 val))
(define (lock-queue:open-db fname #!key (count 10))
(let* ((actualfname (conc fname ".lockdb"))
(dbexists (file-exists? actualfname))
(db (sqlite3:open-database actualfname))
|
| ︙ | | |
Modified megatest.scm
from [f33b053b58]
to [4f78592855].
| ︙ | | |
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
|
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
|
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
|
(killinfo (args:get-arg "-stop-server"))
(khost-port (if killinfo (if (substring-index ":" killinfo)(string-split ":") #f) #f))
(sid (if killinfo (if (substring-index ":" killinfo) #f (string->number killinfo)) #f)))
(format #t fmtstr "Id" "MTver" "Pid" "Host" "Interface:OutPort" "InPort" "LastBeat" "State" "Transport")
(format #t fmtstr "==" "=====" "===" "====" "=================" "======" "========" "=====" "=========")
(for-each
(lambda (server)
(let* ((id (vector-ref server 0))
(pid (vector-ref server 1))
(hostname (vector-ref server 2))
(interface (vector-ref server 3))
(pullport (vector-ref server 4))
(pubport (vector-ref server 5))
(start-time (vector-ref server 6))
(priority (vector-ref server 7))
(state (vector-ref server 8))
(mt-ver (vector-ref server 9))
(last-update (vector-ref server 10))
(transport (vector-ref server 11))
(let* ((id (safe-vector-ref server 0))
(pid (safe-vector-ref server 1))
(hostname (safe-vector-ref server 2))
(interface (safe-vector-ref server 3))
(pullport (safe-vector-ref server 4))
(pubport (safe-vector-ref server 5))
(start-time (safe-vector-ref server 6))
(priority (safe-vector-ref server 7))
(state (safe-vector-ref server 8))
(mt-ver (safe-vector-ref server 9))
(last-update (safe-vector-ref server 10))
(transport (safe-vector-ref server 11))
(killed #f)
(status (< last-update 20)))
;; (zmq-sockets (if status (server:client-connect hostname port) #f)))
;; no need to login as status of #t indicates we are connecting to correct
;; server
(if (equal? state "dead")
(if (> last-update (* 25 60 60)) ;; keep records around for slighly over a day.
|
| ︙ | | |
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
|
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
|
-
-
+
+
-
+
|
(general-run-call
"-set-run-status"
"set run status"
(lambda (target runname keys keyvals)
(let* ((runsdat (rmt:get-runs-by-patt keys runname
(common:args-get-target)
#f #f))
(header (vector-ref runsdat 0))
(rows (vector-ref runsdat 1)))
(header (safe-vector-ref runsdat 0))
(rows (safe-vector-ref runsdat 1)))
(if (null? rows)
(begin
(debug:print-info 0 "No matching run found.")
(exit 1))
(let* ((row (car (vector-ref runsdat 1)))
(let* ((row (car (safe-vector-ref runsdat 1)))
(run-id (db:get-value-by-header row header "id")))
(if (args:get-arg "-set-run-status")
(rmt:set-run-status run-id (args:get-arg "-set-run-status") msg: (args:get-arg "-m"))
(print (rmt:get-run-status run-id))
)))))))
;;======================================================================
|
| ︙ | | |
Modified mt.scm
from [15956fcc00]
to [bfe855850e].
| ︙ | | |
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
|
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
|
-
-
+
+
|
;;
(define (mt:get-runs-by-patt keys runnamepatt targpatt)
(let loop ((runsdat (rmt:get-runs-by-patt keys runnamepatt targpatt 0 500))
(res '())
(offset 0)
(limit 500))
;; (print "runsdat: " runsdat)
(let* ((header (vector-ref runsdat 0))
(runslst (vector-ref runsdat 1))
(let* ((header (safe-vector-ref runsdat 0))
(runslst (safe-vector-ref runsdat 1))
(full-list (append res runslst))
(have-more (eq? (length runslst) limit)))
;; (debug:print 0 "header: " header " runslst: " runslst " have-more: " have-more)
(if have-more
(let ((new-offset (+ offset limit))
(next-batch (rmt:get-runs-by-patt keys runnamepatt targpatt offset limit)))
(debug:print-info 4 "More than " limit " runs, have " (length full-list) " runs so far.")
|
| ︙ | | |
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
|
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
|
-
+
-
+
-
+
|
new-offset
limit))
full-list))))
(define (mt:lazy-get-prereqs-not-met run-id waitons ref-item-path #!key (mode '(normal))(itemmap #f) )
(let* ((key (list run-id waitons ref-item-path mode))
(res (hash-table-ref/default *pre-reqs-met-cache* key #f))
(useres (let ((last-time (if (vector? res) (vector-ref res 0) #f)))
(useres (let ((last-time (if (vector? res) (safe-vector-ref res 0) #f)))
(if last-time
(< (current-seconds)(+ last-time 5))
#f))))
(if useres
(let ((result (vector-ref res 1)))
(let ((result (safe-vector-ref res 1)))
(debug:print 4 "Using lazy value res: " result)
result)
(let ((newres (rmt:get-prereqs-not-met run-id waitons ref-item-path mode: mode itemmap: itemmap)))
;; (let ((newres (db:get-prereqs-not-met run-id waitons ref-item-path mode: mode itemmap: itemmap)))
(hash-table-set! *pre-reqs-met-cache* key (vector (current-seconds) newres))
newres))))
(define (mt:get-run-stats dbstruct run-id)
;; Get run stats from local access, move this ... but where?
(db:get-run-stats dbstruct run-id))
(define (mt:discard-blocked-tests run-id failed-test tests test-records)
(if (null? tests)
tests
(begin
(debug:print-info 1 "Discarding tests from " tests " that are waiting on " failed-test)
(let loop ((testn (car tests))
(remt (cdr tests))
(res '()))
(let* ((test-dat (hash-table-ref/default test-records testn (vector #f #f '())))
(waitons (vector-ref test-dat 2)))
(waitons (safe-vector-ref test-dat 2)))
;; (print "mt:discard-blocked-tests run-id: " run-id " failed-test: " failed-test " testn: " testn " with waitons: " waitons)
(if (null? remt)
(let ((new-res (reverse res)))
;; (print " new-res: " new-res)
new-res)
(loop (car remt)
(cdr remt)
|
| ︙ | | |
Modified nmsg-transport.scm
from [2023441101]
to [5b3387bd23].
| ︙ | | |
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
|
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
|
-
+
|
soc)))
(success #t)
(dat (vector "ping" our-key))
(result (condition-case
(nmsg-transport:client-api-send-receive-raw req dat timeout: timeout)
((timeout)(set! success #f) #f)))
(key (if success
(vector-ref result 1)
(safe-vector-ref result 1)
#f)))
(debug:print 0 "success=" success ", key=" key ", expected-key=" expected-key ", equal? " (equal? key expected-key))
(if (and success
(or (not expected-key) ;; just getting a reply is good enough then
(equal? key expected-key)))
(if return-socket
req
|
| ︙ | | |
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
|
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
|
-
+
-
+
-
-
+
+
|
(thread-start! timeout)
(thread-start! send-recv)
(thread-join! send-recv)
(if success (thread-terminate! timeout)))
;; raise timeout error if timed out
(if success
(if (and (vector? result)
(vector-ref result 0)) ;; did it fail at the server?
(safe-vector-ref result 0)) ;; did it fail at the server?
result ;; nope, all good
(begin
(debug:print 0 "ERROR: error occured at server, info=" (vector-ref result 2))
(debug:print 0 "ERROR: error occured at server, info=" (safe-vector-ref result 2))
(debug:print 0 " client call chain:")
(print-call-chain (current-error-port))
(debug:print 0 " server call chain:")
(pp (vector-ref result 1) (current-error-port))
(signal (vector-ref result 0))))
(pp (safe-vector-ref result 1) (current-error-port))
(signal (safe-vector-ref result 0))))
(signal (make-composite-condition
(make-property-condition 'timeout 'message "nmsg-transport:client-api-send-receive-raw timed out talking to server"))))))
;; run nmsg-transport:keep-running in a parallel thread to monitor that the db is being
;; used and to shutdown after sometime if it is not.
;;
(define (nmsg-transport:keep-running server-id run-id)
|
| ︙ | | |
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
|
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
|
-
-
+
+
|
(define (nmsg-transport:client-api-send-receive run-id connection-info cmd param #!key (remtries 5))
;; NB// In the html version of this routine there is a call to
;; tasks:kill-server-run-id when there is an exception
(mutex-lock! *http-mutex*)
(let* ((packet (vector cmd param))
(reqsoc (http-transport:server-dat-get-socket connection-info))
(res (nmsg-transport:client-api-send-receive-raw reqsoc packet)))
;; (status (vector-ref rawres 0))
;; (result (vector-ref rawres 1)))
;; (status (safe-vector-ref rawres 0))
;; (result (safe-vector-ref rawres 1)))
(mutex-unlock! *http-mutex*)
res)) ;; (vector status (if status (db:string->obj result transport: 'nmsg) result))))
;;======================================================================
;; J U N K
;;======================================================================
|
| ︙ | | |
Modified rmt.scm
from [c2992f6eb2]
to [340929d17a].
| ︙ | | |
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
|
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
|
-
-
+
+
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
|
(define (rmt:write-frequency-over-limit? cmd run-id)
(and (not (member cmd api:read-only-queries))
(let* ((tmprec (hash-table-ref/default *write-frequency* run-id #f))
(record (if tmprec tmprec
(let ((v (vector (current-seconds) 0)))
(hash-table-set! *write-frequency* run-id v)
v)))
(count (+ 1 (vector-ref record 1)))
(start (vector-ref record 0))
(count (+ 1 (safe-vector-ref record 1)))
(start (safe-vector-ref record 0))
(queries-per-second (/ (* count 1.0)
(max (- (current-seconds) start) 1))))
(vector-set! record 1 count)
(if (and (> count 10)
(> queries-per-second 10))
(begin
(debug:print-info 1 "db write rate too high, starting a server, count=" count " start=" start " run-id=" run-id " queries-per-second=" queries-per-second)
#t)
#f))))
(define (rmt:get-connection-info run-id)
(let ((cinfo (hash-table-ref/default *runremote* run-id #f)))
(if cinfo
cinfo
;; NB// can cache the answer for server running for 10 seconds ...
;; ;; (and (not (rmt:write-frequency-over-limit? cmd run-id))
(if (tasks:server-running-or-starting? (db:delay-if-busy (tasks:open-db)) run-id)
(client:setup run-id)
#f))))
;; (define (rmt:get-connection-info run-id)
;; (let ((cinfo (hash-table-ref/default *runremote* run-id #f)))
;; (if cinfo
;; cinfo
;; ;; NB// can cache the answer for server running for 10 seconds ...
;; ;; ;; (and (not (rmt:write-frequency-over-limit? cmd run-id))
;; ;; (if (tasks:server-running-or-starting? (db:delay-if-busy (tasks:open-db)) run-id)
;; ;; (begin
;; ;; (tasks:start-and-wait-for-server (db:delay-if-busy (tasks:open-db)) run-id)
;; (client:setup run-id))))
;; ;; #f))))
(define *send-receive-mutex* (make-mutex)) ;; should have separate mutex per run-id
(define (rmt:send-receive cmd rid params #!key (attemptnum 1)) ;; start attemptnum at 1 so the modulo below works as expected
;; clean out old connections
(mutex-lock! *db-multi-sync-mutex*)
(let ((expire-time (- (current-seconds) (server:get-timeout) 10))) ;; don't forget the 10 second margin
(for-each
|
| ︙ | | |
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
|
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
|
-
+
+
+
-
-
+
+
+
+
-
-
-
+
+
+
+
-
+
+
-
-
-
-
-
-
-
-
-
-
-
-
-
+
-
-
-
-
-
-
+
+
-
+
-
-
+
+
-
-
+
+
|
((nmsg)(nn-close (http-transport:server-dat-get-socket
(hash-table-ref *runremote* run-id)))))
(hash-table-delete! *runremote* run-id)))))
(hash-table-keys *runremote*)))
(mutex-unlock! *db-multi-sync-mutex*)
;; (mutex-lock! *send-receive-mutex*)
(let* ((run-id (if rid rid 0))
(connection-info (rmt:get-connection-info run-id)))
(connection-info (hash-table-ref/default *runremote* run-id #f))) ;; (rmt:get-connection-info run-id)))
;; the nmsg method does the encoding under the hood (the http method should be changed to do this also)
(if connection-info
;; use the server if have connection info
(let* ((dat (case *transport-type*
((http)(condition-case
(http-transport:client-api-send-receive run-id connection-info cmd params)
((commfail)
(tasks:kill-server-run-id run-id)
((commfail)(vector #f "communications fail"))
((exn)(vector #f "other fail"))))
(vector #f "communications fail"))
((exn)
(tasks:kill-server-run-id run-id)
(vector #f "other fail"))))
((nmsg)(condition-case
(nmsg-transport:client-api-send-receive run-id connection-info cmd params)
((timeout)(vector #f "timeout talking to server"))))
(else (exit))))
(success (if (vector? dat) (vector-ref dat 0) #f))
(res (if (vector? dat) (vector-ref dat 1) #f)))
(if (vector? connection-info)(http-transport:server-dat-update-last-access connection-info))
(success (if (vector? dat) (safe-vector-ref dat 0) #f))
(res (if (vector? dat) (safe-vector-ref dat 1) #f)))
(if (and connection-info (vector? connection-info))(http-transport:server-dat-update-last-access connection-info))
(if success
(begin
;; (mutex-unlock! *send-receive-mutex*)
;; all is well, return the result!
(case *transport-type*
((http) res) ;; (db:string->obj res))
((nmsg) res))) ;; (vector-ref res 1)))
((nmsg) res))) ;; (safe-vector-ref res 1)))
;; we had a connection but it is borked. clean up and reconnect
(begin ;; let ((new-connection-info (client:setup run-id)))
(debug:print 0 "WARNING: Communication failed, trying call to rmt:send-receive again.")
;; (case *transport-type*
;; ((nmsg)(nn-close (http-transport:server-dat-get-socket connection-info))))
(hash-table-delete! *runremote* run-id) ;; don't keep using the same connection
;; NOTE: killing server causes this process to block forever. No idea why. Dec 2.
;; (if (eq? (modulo attemptnum 5) 0)
;; (tasks:kill-server-run-id run-id tag: "api-send-receive-failed"))
;; (mutex-unlock! *send-receive-mutex*) ;; close the mutex here to allow other threads access to communications
(tasks:start-and-wait-for-server (tasks:open-db) run-id 15)
;; (nmsg-transport:client-api-send-receive run-id connection-info cmd param remtries: (- remtries 1))))))
;; no longer killing the server in http-transport:client-api-send-receive
;; may kill it here but what are the criteria?
;; start with three calls then kill server
;; (if (eq? attemptnum 3)(tasks:kill-server-run-id run-id))
;; (thread-sleep! 2)
;; (client:setup run-id) ;; client setup happens in rmt:get-connection-info
(client:setup run-id)
(thread-sleep! (random 5)) ;; give some time to settle and minimize collison?
;; (debug:print 0 "ERROR: Communication failed!")
;; (mutex-unlock! *send-receive-mutex*)
;; (exit)
(vector-set! stat-vec 0 (+ (vector-ref stat-vec 0) 1))
(vector-set! stat-vec 1 (+ (vector-ref stat-vec 1) duration))))
(vector-set! stat-vec 0 (+ (safe-vector-ref stat-vec 0) 1))
(vector-set! stat-vec 1 (+ (safe-vector-ref stat-vec 1) duration))))
(mutex-unlock! *db-stats-mutex*))
(define (rmt:print-db-stats)
(let ((fmtstr "~40a~7-d~9-d~20,2-f")) ;; "~20,2-f"
(debug:print 18 "DB Stats\n========")
(debug:print 18 (format #f "~40a~8a~10a~10a" "Cmd" "Count" "TotTime" "Avg"))
(for-each (lambda (cmd)
(let ((cmd-dat (hash-table-ref *db-stats* cmd)))
(debug:print 18 (format #f fmtstr cmd (vector-ref cmd-dat 0) (vector-ref cmd-dat 1) (/ (vector-ref cmd-dat 1)(vector-ref cmd-dat 0))))))
(debug:print 18 (format #f fmtstr cmd (safe-vector-ref cmd-dat 0) (safe-vector-ref cmd-dat 1) (/ (safe-vector-ref cmd-dat 1)(safe-vector-ref cmd-dat 0))))))
(sort (hash-table-keys *db-stats*)
(lambda (a b)
(> (vector-ref (hash-table-ref *db-stats* a) 0)
(vector-ref (hash-table-ref *db-stats* b) 0)))))))
(> (safe-vector-ref (hash-table-ref *db-stats* a) 0)
(safe-vector-ref (hash-table-ref *db-stats* b) 0)))))))
(define (rmt:get-max-query-average run-id)
(mutex-lock! *db-stats-mutex*)
(let* ((runkey (conc "run-id=" run-id " "))
(cmds (filter (lambda (x)
(substring-index runkey x))
(hash-table-keys *db-stats*)))
(res (if (null? cmds)
(cons 'none 0)
(let loop ((cmd (car cmds))
(tal (cdr cmds))
(max-cmd (car cmds))
(res 0))
(let* ((cmd-dat (hash-table-ref *db-stats* cmd))
(tot (vector-ref cmd-dat 0))
(curravg (/ (vector-ref cmd-dat 1) (vector-ref cmd-dat 0))) ;; count is never zero by construction
(tot (safe-vector-ref cmd-dat 0))
(curravg (/ (safe-vector-ref cmd-dat 1) (safe-vector-ref cmd-dat 0))) ;; count is never zero by construction
(currmax (max res curravg))
(newmax-cmd (if (> curravg res) cmd max-cmd)))
(if (null? tal)
(if (> tot 10)
(cons newmax-cmd currmax)
(cons 'none 0))
(loop (car tal)(cdr tal) newmax-cmd currmax)))))))
|
| ︙ | | |
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
|
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
|
-
+
-
+
-
+
|
(db (make-dbr:dbstruct path: dbdir local: #t)))
(set! *dbstruct-db* db)
db)))
(db-file-path (db:dbfile-path 0)))
;; (read-only (not (file-read-access? db-file-path)))
(let* ((start (current-milliseconds))
(resdat (api:execute-requests dbstruct-local (vector (symbol->string cmd) params)))
(res (vector-ref resdat 1))
(res (safe-vector-ref resdat 1))
(duration (- (current-milliseconds) start)))
(rmt:update-db-stats run-id cmd params duration)
;; mark this run as dirty if this was a write
(if (not (member cmd api:read-only-queries))
(let ((start-time (current-seconds)))
(mutex-lock! *db-multi-sync-mutex*)
;; (if (not (hash-table-ref/default *db-local-sync* run-id #f))
;; just set it every time. Is a write more expensive than a read and does it matter?
(hash-table-set! *db-local-sync* (or run-id 0) start-time) ;; the oldest "write"
(mutex-unlock! *db-multi-sync-mutex*)))
res)))
(define (rmt:send-receive-no-auto-client-setup connection-info cmd run-id params)
(let* ((run-id (if run-id run-id 0))
;; (jparams (db:obj->string params)) ;; (rmt:dat->json-str params))
(res (handle-exceptions
exn
#f
(http-transport:client-api-send-receive run-id connection-info cmd params))))
;; ((commfail) (vector #f "communications fail")))))
(if (and res (vector-ref res 0))
(if (and res (safe-vector-ref res 0))
res
#f)))
;; (db:string->obj (vector-ref dat 1))
;; (db:string->obj (safe-vector-ref dat 1))
;; (begin
;; (debug:print 0 "ERROR: rmt:send-receive-no-auto-client-setup failed, attempting to continue. Got " dat)
;; dat))))
;; Wrap json library for strings (why the ports crap in the first place?)
(define (rmt:dat->json-str dat)
(with-output-to-string
|
| ︙ | | |
Modified run_records.scm
from [1580836de1]
to [5ad78642d1].
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
|
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-
-
-
-
-
+
+
+
+
+
+
|
;;======================================================================
;; Copyright 2006-2012, Matthew Welland.
;;
;; This program is made available under the GNU GPL version 2.0 or
;; greater. See the accompanying file COPYING for details.
;;
;; This program is distributed WITHOUT ANY WARRANTY; without even the
;; implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
;; PURPOSE.
;;======================================================================
(define-inline (runs:runrec-make-record) (make-vector 13))
(define-inline (runs:runrec-get-target vec)(vector-ref vec 0)) ;; a/b/c
(define-inline (runs:runrec-get-runname vec)(vector-ref vec 1)) ;; string
(define-inline (runs:runrec-testpatt vec)(vector-ref vec 2)) ;; a,b/c,d%
(define-inline (runs:runrec-keys vec)(vector-ref vec 3)) ;; (key1 key2 ...)
(define-inline (runs:runrec-keyvals vec)(vector-ref vec 4)) ;; ((key1 val1)(key2 val2) ...)
(define-inline (runs:runrec-environment vec)(vector-ref vec 5)) ;; environment, alist key val
(define-inline (runs:runrec-mconfig vec)(vector-ref vec 6)) ;; megatest.config
(define-inline (runs:runrec-runconfig vec)(vector-ref vec 7)) ;; runconfigs.config
(define-inline (runs:runrec-serverdat vec)(vector-ref vec 8)) ;; (host port)
(define-inline (runs:runrec-transport vec)(vector-ref vec 9)) ;; 'http
(define-inline (runs:runrec-db vec)(vector-ref vec 10)) ;; <sqlite3db> (if 'fs)
(define-inline (runs:runrec-top-path vec)(vector-ref vec 11)) ;; *toppath*
(define-inline (runs:runrec-run_id vec)(vector-ref vec 12)) ;; run-id
(define-inline (runs:runrec-get-target vec)(safe-vector-ref vec 0)) ;; a/b/c
(define-inline (runs:runrec-get-runname vec)(safe-vector-ref vec 1)) ;; string
(define-inline (runs:runrec-testpatt vec)(safe-vector-ref vec 2)) ;; a,b/c,d%
(define-inline (runs:runrec-keys vec)(safe-vector-ref vec 3)) ;; (key1 key2 ...)
(define-inline (runs:runrec-keyvals vec)(safe-vector-ref vec 4)) ;; ((key1 val1)(key2 val2) ...)
(define-inline (runs:runrec-environment vec)(safe-vector-ref vec 5)) ;; environment, alist key val
(define-inline (runs:runrec-mconfig vec)(safe-vector-ref vec 6)) ;; megatest.config
(define-inline (runs:runrec-runconfig vec)(safe-vector-ref vec 7)) ;; runconfigs.config
(define-inline (runs:runrec-serverdat vec)(safe-vector-ref vec 8)) ;; (host port)
(define-inline (runs:runrec-transport vec)(safe-vector-ref vec 9)) ;; 'http
(define-inline (runs:runrec-db vec)(safe-vector-ref vec 10)) ;; <sqlite3db> (if 'fs)
(define-inline (runs:runrec-top-path vec)(safe-vector-ref vec 11)) ;; *toppath*
(define-inline (runs:runrec-run_id vec)(safe-vector-ref vec 12)) ;; run-id
(define-inline (test:get-id vec) (vector-ref vec 0))
(define-inline (test:get-run_id vec) (vector-ref vec 1))
(define-inline (test:get-test-name vec)(vector-ref vec 2))
(define-inline (test:get-state vec) (vector-ref vec 3))
(define-inline (test:get-status vec) (vector-ref vec 4))
(define-inline (test:get-item-path vec)(vector-ref vec 5))
(define-inline (test:get-id vec) (safe-vector-ref vec 0))
(define-inline (test:get-run_id vec) (safe-vector-ref vec 1))
(define-inline (test:get-test-name vec)(safe-vector-ref vec 2))
(define-inline (test:get-state vec) (safe-vector-ref vec 3))
(define-inline (test:get-status vec) (safe-vector-ref vec 4))
(define-inline (test:get-item-path vec)(safe-vector-ref vec 5))
(define-inline (test:test-get-fullname test)
(conc (db:test-get-testname test)
(if (equal? (db:test-get-item-path test) "")
""
(conc "(" (db:test-get-item-path test) ")"))))
|
Modified runs.scm
from [15eacb131a]
to [59984075fd].
| ︙ | | |
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
|
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
|
-
-
+
+
|
;;
(define (runs:operate-on action target runnamepatt testpatt #!key (state #f)(status #f)(new-state-status #f)(remove-data-only #f))
(common:clear-caches) ;; clear all caches
(let* ((db #f)
(tdbdat (tasks:open-db))
(keys (rmt:get-keys))
(rundat (mt:get-runs-by-patt keys runnamepatt target))
(header (vector-ref rundat 0))
(runs (vector-ref rundat 1))
(header (safe-vector-ref rundat 0))
(runs (safe-vector-ref rundat 1))
(states (if state (string-split state ",") '()))
(statuses (if status (string-split status ",") '()))
(state-status (if (string? new-state-status) (string-split new-state-status ",") '(#f #f))))
(debug:print-info 4 "runs:operate-on => Header: " header " action: " action " new-state-status: " new-state-status)
(if (> 2 (length state-status))
(begin
(debug:print 0 "ERROR: the parameter to -set-state-status is a comma delimited string. E.g. COMPLETED,FAIL")
|
| ︙ | | |
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
|
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
|
-
-
+
+
|
;;======================================================================
;; Lock/unlock runs
;;======================================================================
(define (runs:handle-locking target keys runname lock unlock user)
(let* ((db #f)
(rundat (mt:get-runs-by-patt keys runname target))
(header (vector-ref rundat 0))
(runs (vector-ref rundat 1)))
(header (safe-vector-ref rundat 0))
(runs (safe-vector-ref rundat 1)))
(for-each (lambda (run)
(let ((run-id (db:get-value-by-header run header "id")))
(if (or lock
(and unlock
(begin
(print "Do you really wish to unlock run " run-id "?\n y/n: ")
(equal? "y" (read-line)))))
|
| ︙ | | |
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
|
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
|
-
+
|
(rmt:testmeta-add-record test-name)))
(for-each
(lambda (key)
(let* ((idx (cadr key))
(fld (car key))
(val (config-lookup test-conf "test_meta" fld)))
;; (debug:print 5 "idx: " idx " fld: " fld " val: " val)
(if (and val (not (equal? (vector-ref currrecord idx) val)))
(if (and val (not (equal? (safe-vector-ref currrecord idx) val)))
(begin
(print "Updating " test-name " " fld " to " val)
(rmt:testmeta-update-field test-name fld val)))))
'(("author" 2)("owner" 3)("description" 4)("reviewed" 5)("tags" 9)("jobgroup" 10)))))
;; Update test_meta for all tests
(define (runs:update-all-test_meta db)
|
| ︙ | | |
Modified server.scm
from [d9e8792ebb]
to [c1492c153b].
| ︙ | | |
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
|
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
|
-
+
|
(debug:print-info 11 "server:reply return-addr=" return-addr ", result=" result)
;; (send-message pubsock target send-more: #t)
;; (send-message pubsock
(case (server:get-transport)
((rpc) (db:obj->string (vector success/fail query-sig result)))
((http) (db:obj->string (vector success/fail query-sig result)))
((zmq)
(let ((pub-socket (vector-ref *runremote* 1)))
(let ((pub-socket (safe-vector-ref *runremote* 1)))
(send-message pub-socket return-addr send-more: #t)
(send-message pub-socket (db:obj->string (vector success/fail query-sig result)))))
((fs) result)
(else
(debug:print 0 "ERROR: unrecognised transport type: " *transport-type*)
result)))
|
| ︙ | | |
Modified synchash.scm
from [9881f5a738]
to [655a49680e].
| ︙ | | |
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
|
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
|
-
+
-
-
+
+
|
((db:get-test-info-by-ids) db:get-test-info-by-ids)
(else
(print "ERROR: sync for hash " proc " not setup! Edits needed in synchash.scm")
print))
db params))
(postdat #f)
(make-indexed (lambda (x)
(list (vector-ref x keynum) x))))
(list (safe-vector-ref x keynum) x))))
;; Now process newdat based on the query type
(set! postdat (case proc
((db:get-runs)
;; (debug:print-info 2 "Get runs call")
(let ((header (vector-ref newdat 0))
(data (vector-ref newdat 1)))
(let ((header (safe-vector-ref newdat 0))
(data (safe-vector-ref newdat 1)))
;; (debug:print-info 2 "header: " header ", data: " data)
(cons (list "header" header) ;; add the header keyed by the word "header"
(map make-indexed data)))) ;; add each element keyed by the keynum'th val
(else
;; (debug:print-info 2 "Non-get runs call")
(map make-indexed newdat))))
;; (debug:print-info 2 "postdat: " postdat)
|
| ︙ | | |
Modified task_records.scm
from [8f450896f9]
to [73558fec1c].
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
|
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
-
-
-
-
-
-
+
+
+
+
+
+
|
;;======================================================================
;; Copyright 2006-2012, Matthew Welland.
;;
;; This program is made available under the GNU GPL version 2.0 or
;; greater. See the accompanying file COPYING for details.
;;
;; This program is distributed WITHOUT ANY WARRANTY; without even the
;; implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
;; PURPOSE.
;;======================================================================
;; make-vector-record tasks task id action owner state target name test item params creation_time execution_time
(define (make-tasks:task)(make-vector 11))
(define-inline (tasks:task-get-id vec) (vector-ref vec 0))
(define-inline (tasks:task-get-action vec) (vector-ref vec 1))
(define-inline (tasks:task-get-owner vec) (vector-ref vec 2))
(define-inline (tasks:task-get-state vec) (vector-ref vec 3))
(define-inline (tasks:task-get-target vec) (vector-ref vec 4))
(define-inline (tasks:task-get-name vec) (vector-ref vec 5))
(define-inline (tasks:task-get-test vec) (vector-ref vec 6))
(define-inline (tasks:task-get-item vec) (vector-ref vec 7))
(define-inline (tasks:task-get-params vec) (vector-ref vec 8))
(define-inline (tasks:task-get-creation_time vec) (vector-ref vec 9))
(define-inline (tasks:task-get-execution_time vec) (vector-ref vec 10))
(define-inline (tasks:task-get-id vec) (safe-vector-ref vec 0))
(define-inline (tasks:task-get-action vec) (safe-vector-ref vec 1))
(define-inline (tasks:task-get-owner vec) (safe-vector-ref vec 2))
(define-inline (tasks:task-get-state vec) (safe-vector-ref vec 3))
(define-inline (tasks:task-get-target vec) (safe-vector-ref vec 4))
(define-inline (tasks:task-get-name vec) (safe-vector-ref vec 5))
(define-inline (tasks:task-get-test vec) (safe-vector-ref vec 6))
(define-inline (tasks:task-get-item vec) (safe-vector-ref vec 7))
(define-inline (tasks:task-get-params vec) (safe-vector-ref vec 8))
(define-inline (tasks:task-get-creation_time vec) (safe-vector-ref vec 9))
(define-inline (tasks:task-get-execution_time vec) (safe-vector-ref vec 10))
(define-inline (tasks:task-set-state! vec val)(vector-set! vec 3 val))
;; make-vector-record tasks monitor id pid start_time last_update hostname username
(define (make-tasks:monitor)(make-vector 5))
(define-inline (tasks:monitor-get-id vec) (vector-ref vec 0))
(define-inline (tasks:monitor-get-pid vec) (vector-ref vec 1))
(define-inline (tasks:monitor-get-start_time vec) (vector-ref vec 2))
(define-inline (tasks:monitor-get-last_update vec) (vector-ref vec 3))
(define-inline (tasks:monitor-get-hostname vec) (vector-ref vec 4))
(define-inline (tasks:monitor-get-username vec) (vector-ref vec 5))
(define-inline (tasks:monitor-get-id vec) (safe-vector-ref vec 0))
(define-inline (tasks:monitor-get-pid vec) (safe-vector-ref vec 1))
(define-inline (tasks:monitor-get-start_time vec) (safe-vector-ref vec 2))
(define-inline (tasks:monitor-get-last_update vec) (safe-vector-ref vec 3))
(define-inline (tasks:monitor-get-hostname vec) (safe-vector-ref vec 4))
(define-inline (tasks:monitor-get-username vec) (safe-vector-ref vec 5))
|
Modified tasks.scm
from [edd9ff6647]
to [a0cd34a8aa].
| ︙ | | |
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
|
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
|
-
-
-
-
-
-
-
+
+
+
+
+
+
+
|
*task-db*))))
;;======================================================================
;; Server and client management
;;======================================================================
;; make-vector-record tasks hostinfo id interface port pubport transport pid hostname
(define (tasks:hostinfo-get-id vec) (vector-ref vec 0))
(define (tasks:hostinfo-get-interface vec) (vector-ref vec 1))
(define (tasks:hostinfo-get-port vec) (vector-ref vec 2))
(define (tasks:hostinfo-get-pubport vec) (vector-ref vec 3))
(define (tasks:hostinfo-get-transport vec) (vector-ref vec 4))
(define (tasks:hostinfo-get-pid vec) (vector-ref vec 5))
(define (tasks:hostinfo-get-hostname vec) (vector-ref vec 6))
(define (tasks:hostinfo-get-id vec) (safe-vector-ref vec 0))
(define (tasks:hostinfo-get-interface vec) (safe-vector-ref vec 1))
(define (tasks:hostinfo-get-port vec) (safe-vector-ref vec 2))
(define (tasks:hostinfo-get-pubport vec) (safe-vector-ref vec 3))
(define (tasks:hostinfo-get-transport vec) (safe-vector-ref vec 4))
(define (tasks:hostinfo-get-pid vec) (safe-vector-ref vec 5))
(define (tasks:hostinfo-get-hostname vec) (safe-vector-ref vec 6))
(define (tasks:server-lock-slot mdb run-id)
(tasks:server-clean-out-old-records-for-run-id mdb run-id " tasks:server-lock-slot")
(if (< (tasks:num-in-available-state mdb run-id) 4)
(begin
(tasks:server-set-available mdb run-id)
(thread-sleep! (/ (random 1500) 1000)) ;; (thread-sleep! 2) ;; Try removing this. It may not be needed.
|
| ︙ | | |
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
|
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
|
-
+
-
+
-
+
|
(else
#f))))
;; try to start a server and wait for it to be available
;;
(define (tasks:start-and-wait-for-server tdbdat run-id delay-max-tries)
;; ensure a server is running for this run
(let loop ((server-dat (tasks:get-server (db:delay-if-busy tdbdat) run-id))
(let loop ((server-running (tasks:server-running? (db:delay-if-busy tdbdat) run-id))
(delay-time 0))
(if (and (not server-dat)
(if (and (not server-running)
(< delay-time delay-max-tries))
(begin
(if (common:low-noise-print 60 "tasks:start-and-wait-for-server" run-id)
(debug:print 0 "Try starting server for run-id " run-id))
(thread-sleep! (/ (random 2000) 1000))
(server:kind-run run-id)
(thread-sleep! (min delay-time 1))
(loop (tasks:get-server (db:delay-if-busy tdbdat) run-id)(+ delay-time 1))))))
(loop (tasks:server-running? (db:delay-if-busy tdbdat) run-id)(+ delay-time 1))))))
(define (tasks:get-all-servers mdb)
(let ((res '()))
(sqlite3:for-each-row
(lambda (id pid hostname interface port pubport start-time priority state mt-version last-update transport run-id)
;; 0 1 2 3 4 5 6 7 8 9 10 11 12
(set! res (cons (vector id pid hostname interface port pubport start-time priority state mt-version last-update transport run-id) res)))
|
| ︙ | | |
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
|
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
|
-
-
-
+
+
+
|
;; look up a server by run-id and send it a kill, also delete the record for that server
;;
(define (tasks:kill-server-run-id run-id #!key (tag "default"))
(let* ((tdbdat (tasks:open-db))
(sdat (tasks:get-server (db:delay-if-busy tdbdat) run-id)))
(if sdat
(let ((hostname (vector-ref sdat 6))
(pid (vector-ref sdat 5))
(server-id (vector-ref sdat 0)))
(let ((hostname (safe-vector-ref sdat 6))
(pid (safe-vector-ref sdat 5))
(server-id (safe-vector-ref sdat 0)))
(tasks:server-set-state! (db:delay-if-busy tdbdat) server-id "killed")
(debug:print-info 0 "Killing server " server-id " for run-id " run-id " on host " hostname " with pid " pid)
(tasks:kill-server hostname pid)
(tasks:server-delete-record (db:delay-if-busy tdbdat) server-id tag) )
(debug:print-info 0 "No server found for run-id " run-id ", nothing to kill"))
;; (sqlite3:finalize! tdb)
))
|
| ︙ | | |
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
|
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
|
-
+
-
+
|
owner
target
runname
testpatt
(if params params "")))))
(define (keys:key-vals-hash->target keys key-params)
(let ((tmp (hash-table-ref/default key-params (vector-ref (car keys) 0) "")))
(let ((tmp (hash-table-ref/default key-params (safe-vector-ref (car keys) 0) "")))
(if (> (length keys) 1)
(for-each (lambda (key)
(set! tmp (conc tmp "/" (hash-table-ref/default key-params (vector-ref key 0) ""))))
(set! tmp (conc tmp "/" (hash-table-ref/default key-params (safe-vector-ref key 0) ""))))
(cdr keys)))
tmp))
;; for use from the gui, not ported
;;
;; (define (tasks:add-from-params mdb action keys key-params var-params)
;; (let ((target (keys:key-vals-hash->target keys key-params))
|
| ︙ | | |
Modified tdb.scm
from [575d5c7ba8]
to [f95da50e05].
| ︙ | | |
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
|
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
|
-
+
-
-
-
+
+
+
|
"\nid: " (tdb:step-get-id step)
"\nstepname: " (tdb:step-get-stepname step)
"\nstate: " (tdb:step-get-state step)
"\nstatus: " (tdb:step-get-status step)
"\ntime: " (tdb:step-get-event_time step))
(case (string->symbol (tdb:step-get-state step))
((start)(vector-set! record 1 (tdb:step-get-event_time step))
(vector-set! record 3 (if (equal? (vector-ref record 3) "")
(vector-set! record 3 (if (equal? (safe-vector-ref record 3) "")
(tdb:step-get-status step)))
(if (> (string-length (tdb:step-get-logfile step))
0)
(vector-set! record 5 (tdb:step-get-logfile step))))
((end)
(vector-set! record 2 (any->number (tdb:step-get-event_time step)))
(vector-set! record 3 (tdb:step-get-status step))
(vector-set! record 4 (let ((startt (any->number (vector-ref record 1)))
(endt (any->number (vector-ref record 2))))
(debug:print 4 "record[1]=" (vector-ref record 1)
(vector-set! record 4 (let ((startt (any->number (safe-vector-ref record 1)))
(endt (any->number (safe-vector-ref record 2))))
(debug:print 4 "record[1]=" (safe-vector-ref record 1)
", startt=" startt ", endt=" endt
", get-status: " (tdb:step-get-status step))
(if (and (number? startt)(number? endt))
(seconds->hr-min-sec (- endt startt)) "-1")))
(if (> (string-length (tdb:step-get-logfile step))
0)
(vector-set! record 5 (tdb:step-get-logfile step))))
|
| ︙ | | |
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
|
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
|
-
+
-
-
-
+
+
+
|
"\nid: " (tdb:step-get-id step)
"\nstepname: " (tdb:step-get-stepname step)
"\nstate: " (tdb:step-get-state step)
"\nstatus: " (tdb:step-get-status step)
"\ntime: " (tdb:step-get-event_time step))
(case (string->symbol (tdb:step-get-state step))
((start)(vector-set! record 1 (tdb:step-get-event_time step))
(vector-set! record 3 (if (equal? (vector-ref record 3) "")
(vector-set! record 3 (if (equal? (safe-vector-ref record 3) "")
(tdb:step-get-status step)))
(if (> (string-length (tdb:step-get-logfile step))
0)
(vector-set! record 5 (tdb:step-get-logfile step))))
((end)
(vector-set! record 2 (any->number (tdb:step-get-event_time step)))
(vector-set! record 3 (tdb:step-get-status step))
(vector-set! record 4 (let ((startt (any->number (vector-ref record 1)))
(endt (any->number (vector-ref record 2))))
(debug:print 4 "record[1]=" (vector-ref record 1)
(vector-set! record 4 (let ((startt (any->number (safe-vector-ref record 1)))
(endt (any->number (safe-vector-ref record 2))))
(debug:print 4 "record[1]=" (safe-vector-ref record 1)
", startt=" startt ", endt=" endt
", get-status: " (tdb:step-get-status step))
(if (and (number? startt)(number? endt))
(seconds->hr-min-sec (- endt startt)) "-1")))
(if (> (string-length (tdb:step-get-logfile step))
0)
(vector-set! record 5 (tdb:step-get-logfile step))))
|
| ︙ | | |
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
|
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
|
-
-
+
+
-
+
-
-
-
+
+
+
-
-
+
+
-
-
+
+
|
;;
;; Move to steps.scm
;;
(define (tdb:get-compressed-steps comprsteps) ;; from tdb:get-steps-table
(map (lambda (x)
;; take advantage of the \n on time->string
(vector
(vector-ref x 0)
(let ((s (vector-ref x 1)))
(safe-vector-ref x 0)
(let ((s (safe-vector-ref x 1)))
(if (number? s)(seconds->time-string s) s))
(let ((s (vector-ref x 2)))
(let ((s (safe-vector-ref x 2)))
(if (number? s)(seconds->time-string s) s))
(vector-ref x 3) ;; status
(vector-ref x 4)
(vector-ref x 5))) ;; time delta
(safe-vector-ref x 3) ;; status
(safe-vector-ref x 4)
(safe-vector-ref x 5))) ;; time delta
(sort (hash-table-values comprsteps)
(lambda (a b)
(let ((time-a (vector-ref a 1))
(time-b (vector-ref b 1)))
(let ((time-a (safe-vector-ref a 1))
(time-b (safe-vector-ref b 1)))
(if (and (number? time-a)(number? time-b))
(if (< time-a time-b)
#t
(if (eq? time-a time-b)
(string<? (conc (vector-ref a 2))
(conc (vector-ref b 2)))
(string<? (conc (safe-vector-ref a 2))
(conc (safe-vector-ref b 2)))
#f))
(string<? (conc time-a)(conc time-b))))))))
;;
(define (tdb:remote-update-testdat-meta-info run-id test-id work-area cpuload diskfree minutes)
(let ((tdb (rmt:open-test-db-by-test-id run-id test-id work-area: work-area)))
(if (sqlite3:database? tdb)
(begin
(sqlite3:execute tdb "INSERT INTO test_rundat (update_time,cpuload,diskfree,run_duration) VALUES (strftime('%s','now'),?,?,?);"
cpuload diskfree minutes)
(sqlite3:finalize! tdb))
(debug:print 2 "Can't update testdat.db for test " test-id " read-only or non-existant"))))
|
Modified test_records.scm
from [9245906f33]
to [373e59ec86].
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
|
-
-
-
-
+
+
+
+
-
-
-
+
+
+
|
;; make-vector-record tests testqueue testname testconfig waitons priority items
(define (make-tests:testqueue)(make-vector 7 #f))
(define-inline (tests:testqueue-get-testname vec) (vector-ref vec 0))
(define-inline (tests:testqueue-get-testconfig vec) (vector-ref vec 1))
(define-inline (tests:testqueue-get-waitons vec) (vector-ref vec 2))
(define-inline (tests:testqueue-get-priority vec) (vector-ref vec 3))
(define-inline (tests:testqueue-get-testname vec) (safe-vector-ref vec 0))
(define-inline (tests:testqueue-get-testconfig vec) (safe-vector-ref vec 1))
(define-inline (tests:testqueue-get-waitons vec) (safe-vector-ref vec 2))
(define-inline (tests:testqueue-get-priority vec) (safe-vector-ref vec 3))
;; items: #f=no items, list=list of items remaining, proc=need to call to get items
(define-inline (tests:testqueue-get-items vec) (vector-ref vec 4))
(define-inline (tests:testqueue-get-itemdat vec) (vector-ref vec 5))
(define-inline (tests:testqueue-get-item_path vec) (vector-ref vec 6))
(define-inline (tests:testqueue-get-items vec) (safe-vector-ref vec 4))
(define-inline (tests:testqueue-get-itemdat vec) (safe-vector-ref vec 5))
(define-inline (tests:testqueue-get-item_path vec) (safe-vector-ref vec 6))
(define-inline (tests:testqueue-set-testname! vec val)(vector-set! vec 0 val))
(define-inline (tests:testqueue-set-testconfig! vec val)(vector-set! vec 1 val))
(define-inline (tests:testqueue-set-waitons! vec val)(vector-set! vec 2 val))
(define-inline (tests:testqueue-set-priority! vec val)(vector-set! vec 3 val))
(define-inline (tests:testqueue-set-items! vec val)(vector-set! vec 4 val))
(define-inline (tests:testqueue-set-itemdat! vec val)(vector-set! vec 5 val))
|
| ︙ | | |
Modified tests.scm
from [2a580a2e0e]
to [9e78be79cc].
| ︙ | | |
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
|
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
|
-
-
-
-
-
-
-
+
+
+
+
+
+
+
|
(with-output-to-port
oup
(lambda ()
(set! outtxt (conc outtxt "<html><title>Summary: " test-name
"</title><body><h2>Summary for " test-name "</h2>"))
(for-each
(lambda (testrecord)
(let ((id (vector-ref testrecord 0))
(itempath (vector-ref testrecord 1))
(state (vector-ref testrecord 2))
(status (vector-ref testrecord 3))
(run_duration (vector-ref testrecord 4))
(logf (vector-ref testrecord 5))
(comment (vector-ref testrecord 6)))
(let ((id (safe-vector-ref testrecord 0))
(itempath (safe-vector-ref testrecord 1))
(state (safe-vector-ref testrecord 2))
(status (safe-vector-ref testrecord 3))
(run_duration (safe-vector-ref testrecord 4))
(logf (safe-vector-ref testrecord 5))
(comment (safe-vector-ref testrecord 6)))
(hash-table-set! counts status (+ 1 (hash-table-ref/default counts status 0)))
(hash-table-set! statecounts state (+ 1 (hash-table-ref/default statecounts state 0)))
(set! outtxt (conc outtxt "<tr>"
"<td><a href=\"" itempath "/" logf "\"> " itempath "</a></td>"
"<td>" state "</td>"
"<td><font color=" (common:get-color-from-status status)
">" status "</font></td>"
|
| ︙ | | |
Modified zmq-transport.scm
from [e1f3152a02]
to [a4b47f5fb1].
| ︙ | | |
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
|
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
|
-
-
+
+
|
(define *server-loop-heart-beat* (current-seconds))
(define *heartbeat-mutex* (make-mutex))
;;======================================================================
;; S E R V E R
;;======================================================================
(define-inline (zmqsock:get-pub dat)(vector-ref dat 0))
(define-inline (zmqsock:get-pull dat)(vector-ref dat 1))
(define-inline (zmqsock:get-pub dat)(safe-vector-ref dat 0))
(define-inline (zmqsock:get-pull dat)(safe-vector-ref dat 1))
(define-inline (zmqsock:set-pub! dat s)(vector-set! dat s 0))
(define-inline (zmqsock:set-pull! dat s)(vector-set! dat s 0))
(define (zmq-transport:run hostn)
(debug:print 2 "Attempting to start the server ...")
(if (not *toppath*)
(if (not (setup-for-run))
|
| ︙ | | |