849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
|
(lambda (id) ;; run-id testname state status event-time host cpuload diskfree uname rundir item-path run_duration final_logf comment )
(set! res id)) ;; (vector id run-id testname state status event-time host cpuload diskfree uname rundir item-path run_duration final_logf comment )))
db
"SELECT id FROM tests WHERE run_id=? AND testname=? AND item_path=?;"
run-id testname item-path)
res))
(define db:get-test-id db:get-test-id-cached)
;; given a test-info record, patch in the latest data from the testdat.db file
;; found in the test run directory
(define (db:patch-tdb-data-into-test-info db test-id res)
(let ((tdb (db:open-test-db-by-test-id db test-id)))
;; get state and status from megatest.db in real time
;; other fields that perhaps should be updated:
|
|
|
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
|
(lambda (id) ;; run-id testname state status event-time host cpuload diskfree uname rundir item-path run_duration final_logf comment )
(set! res id)) ;; (vector id run-id testname state status event-time host cpuload diskfree uname rundir item-path run_duration final_logf comment )))
db
"SELECT id FROM tests WHERE run_id=? AND testname=? AND item_path=?;"
run-id testname item-path)
res))
(define db:get-test-id db:get-test-id-not-cached)
;; given a test-info record, patch in the latest data from the testdat.db file
;; found in the test run directory
(define (db:patch-tdb-data-into-test-info db test-id res)
(let ((tdb (db:open-test-db-by-test-id db test-id)))
;; get state and status from megatest.db in real time
;; other fields that perhaps should be updated:
|
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
|
(let ((calling-path (car remparam)))
(if (equal? calling-path *toppath*)
#t ;; path matches - pass! Should vet the caller at this time ...
#f)))) ;; else fail to login
((flush)
(db:write-cached-data)
#t)
(else
(mutex-lock! *incoming-mutex*)
(set! *last-db-access* (current-seconds))
(set! *incoming-data* (cons
(vector qry-name
(current-milliseconds)
remparam)
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
|
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
|
(let ((calling-path (car remparam)))
(if (equal? calling-path *toppath*)
#t ;; path matches - pass! Should vet the caller at this time ...
#f)))) ;; else fail to login
((flush)
(db:write-cached-data)
#t)
((immediate)
(db:write-cached-data)
(if (not (null? remparam))
(apply (car remparam) (cdr remparam))
"ERROR"))
((killserver)
(db:write-cached-data)
(debug:print-info 0 "Remotely killed server on host " (get-host-name) " pid " (current-process-id))
(set! *time-to-exit* #t)
#t)
((set-verbosity)
(set! *verbosity* (caddr params))
*verbosity*)
((get-verbosity)
*verbosity*)
(else
(mutex-lock! *incoming-mutex*)
(set! *last-db-access* (current-seconds))
(set! *incoming-data* (cons
(vector qry-name
(current-milliseconds)
remparam)
|
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
|
(let ((zdat (db:obj->string params)) ;; (with-output-to-string (lambda ()(serialize params))))
(res #f))
(send-message zmq-socket zdat)
(set! res (db:string->obj (receive-message zmq-socket zdat)))
(debug:print-info 11 "zmq-socket " (car params) " res=" res)
res))
(define (cdb:test-set-status-state zmqsocket test-id status state msg)
(if msg
(cdb:client-call zmqsocket 'state-status-msg #t state status msg test-id)
(cdb:client-call zmqsocket 'state-status #t state status test-id))) ;; run-id test-name item-path minutes cpuload diskfree tmpfree)
(define (cdb:test-rollup-test_data-pass-fail zmqsocket test-id)
(cdb:client-call zmqsocket 'test_data-pf-rollup #t test-id test-id test-id test-id))
(define (cdb:pass-fail-counts zmqsocket test-id fail-count pass-count)
(cdb:client-call zmqsocket 'pass-fail-counts #t fail-count pass-count test-id))
(define (cdb:tests-register-test zmqsocket run-id test-name item-path)
(let ((item-paths (if (equal? item-path "")
(list item-path)
(list item-path ""))))
(cdb:client-call zmqsocket 'register-test #t run-id test-name item-path)))
(define (cdb:flush-queue zmqsocket)
(cdb:client-call zmqsocket 'flush #f))
(define db:queries
'((register-test "INSERT OR IGNORE INTO tests (run_id,testname,event_time,item_path,state,status) VALUES (?,?,strftime('%s','now'),?,'NOT_STARTED','n/a');")
(state-status "UPDATE tests SET state=?,status=? WHERE id=?;")
(state-status-msg "UPDATE tests SET state=?,status=?,comment=? WHERE id=?;")
(pass-fail-counts "UPDATE tests SET fail_count=?,pass_count=? WHERE id=?;")
(test_data-pf-rollup "UPDATE tests
SET status=CASE WHEN (SELECT fail_count FROM tests WHERE id=?) > 0
THEN 'FAIL'
WHEN (SELECT pass_count FROM tests WHERE id=?) > 0 AND
(SELECT status FROM tests WHERE id=?) NOT IN ('WARN','FAIL')
THEN 'PASS'
ELSE status
END WHERE id=?;")
(rollup-tests-pass-fail "UPDATE tests
SET fail_count=(SELECT count(id) FROM tests WHERE
run_id=? AND testname=? AND item_path != '' AND status='FAIL'),
pass_count=(SELECT count(id) FROM tests WHERE
run_id=? AND testname=? AND item_path != '' AND (status='PASS' OR status='WARN' OR status='WAIVED'))
WHERE run_id=? AND testname=? AND item_path='';")
(test-set-log "UPDATE tests SET final_logf=? WHERE id=?;")
(test-set-rundir-by-test-id "UPDATE tests SET rundir=? WHERE id=?")))
(define db:special-queries '(rollup-tests-pass-fail))
(define db:run-local-queries '(rollup-tests-pass-fail))
;; The queue is a list of vectors where the zeroth slot indicates the type of query to
;; apply and the second slot is the time of the query and the third entry is a list of
;; values to be applied
;;
(define (db:write-cached-data)
(open-run-close
(lambda (db . junkparams)
(let ((queries (make-hash-table))
(data #f))
(mutex-lock! *incoming-mutex*)
(set! data (sort *incoming-data* (lambda (a b)(< (vector-ref a 1)(vector-ref b 1)))))
(set! *incoming-data* '())
(mutex-unlock! *incoming-mutex*)
(if (> (length data) 0)
(debug:print-info 4 "Writing cached data " data))
;; prepare the needed statements
(for-each (lambda (request-item)
(let ((stmt-key (vector-ref request-item 0)))
(if (not (hash-table-ref/default queries stmt-key #f))
(let ((stmt (alist-ref stmt-key db:queries)))
(if stmt
(hash-table-set! queries stmt-key (sqlite3:prepare db (car stmt)))
(debug:print 0 "ERROR: Missing query spec for " stmt-key "!"))))))
data)
(let outerloop ((special-qry #f)
(stmts data))
(if special-qry
;; handle a query that cannot be part of the grouped queries
(let* ((stmt-key (vector-ref special-qry 0))
(qry (hash-table-ref queries stmt-key))
(params (vector-ref special-qry 2)))
(apply sqlite3:execute db qry params)
(if (not (null? stmts))
(outerloop #f stmts)))
;; handle normal queries
(sqlite3:with-transaction
db
(lambda ()
(debug:print-info 11 "flushing " stmts " to db")
(if (not (null? stmts))
(let innerloop ((hed (car stmts))
(tal (cdr stmts)))
(let ((params (vector-ref hed 2))
(stmt-key (vector-ref hed 0)))
(if (not (member stmt-key db:special-queries))
(begin
(debug:print-info 11 "Executing " stmt-key " for " params)
(apply sqlite3:execute (hash-table-ref queries stmt-key) params)
(if (not (null? tal))
(innerloop (car tal)(cdr tal))))
(outerloop hed tal)))))))))
(for-each (lambda (stmt-key)
(sqlite3:finalize! (hash-table-ref queries stmt-key)))
(hash-table-keys queries))
(let ((cache-size (length data)))
(if (> cache-size *max-cache-size*)
(set! *max-cache-size* cache-size)))
))
#f))
(define (db:roll-up-pass-fail-counts db run-id test-name item-path status)
(cdb:flush-queue *runremote*)
(if (and (not (equal? item-path ""))
(or (equal? status "PASS")
(equal? status "WARN")
(equal? status "FAIL")
(equal? status "WAIVED")
(equal? status "RUNNING")))
(begin
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
|
|
|
|
>
|
|
|
|
|
|
|
|
<
<
<
<
<
<
|
|
>
>
|
>
>
>
|
>
|
>
>
|
>
>
>
>
>
|
>
>
>
>
>
>
>
|
|
|
|
|
>
|
|
|
|
>
|
|
>
>
>
|
|
|
|
>
|
>
>
>
|
|
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
|
(let ((zdat (db:obj->string params)) ;; (with-output-to-string (lambda ()(serialize params))))
(res #f))
(send-message zmq-socket zdat)
(set! res (db:string->obj (receive-message zmq-socket zdat)))
(debug:print-info 11 "zmq-socket " (car params) " res=" res)
res))
(define (cdb:set-verbosity zmqsocket val)
(cdb:client-call zmqsocket 'set-verbosity #f val))
(define (cdb:test-set-status-state zmqsocket test-id status state msg)
(if msg
(cdb:client-call zmqsocket 'state-status-msg #t state status msg test-id)
(cdb:client-call zmqsocket 'state-status #t state status test-id))) ;; run-id test-name item-path minutes cpuload diskfree tmpfree)
(define (cdb:test-rollup-test_data-pass-fail zmqsocket test-id)
(cdb:client-call zmqsocket 'test_data-pf-rollup #t test-id test-id test-id test-id))
(define (cdb:pass-fail-counts zmqsocket test-id fail-count pass-count)
(cdb:client-call zmqsocket 'pass-fail-counts #t fail-count pass-count test-id))
(define (cdb:tests-register-test zmqsocket run-id test-name item-path)
(let ((item-paths (if (equal? item-path "")
(list item-path)
(list item-path ""))))
(cdb:client-call zmqsocket 'register-test #t run-id test-name item-path)))
(define (cdb:flush-queue zmqsocket)
(cdb:client-call zmqsocket 'flush #f))
(define (cdb:kill-server zmqsocket)
(cdb:client-call zmqsocket 'killserver #f))
(define (cdb:roll-up-pass-fail-counts zmqsocket run-id test-name item-path status)
(cdb:client-call zmqsocket 'immediate #f open-run-close db:roll-up-pass-fail-counts #f run-id test-name item-path status))
(define (cdb:get-test-info zmqsocket run-id test-name item-path)
(cdb:client-call zmqsocket 'immediate #f open-run-close db:get-test-info #f run-id test-name item-path))
;; db should be db open proc or #f
(define (cdb:remote-run proc db . params)
(apply cdb:client-call *runremote* 'immediate #f open-run-close proc #f params))
(define db:queries
(list '(register-test "INSERT OR IGNORE INTO tests (run_id,testname,event_time,item_path,state,status) VALUES (?,?,strftime('%s','now'),?,'NOT_STARTED','n/a');")
'(state-status "UPDATE tests SET state=?,status=? WHERE id=?;")
'(state-status-msg "UPDATE tests SET state=?,status=?,comment=? WHERE id=?;")
'(pass-fail-counts "UPDATE tests SET fail_count=?,pass_count=? WHERE id=?;")
;; test_data-pf-rollup is used to set a tests PASS/FAIL based on the pass/fail info from the steps
'(test_data-pf-rollup "UPDATE tests
SET status=CASE WHEN (SELECT fail_count FROM tests WHERE id=?) > 0
THEN 'FAIL'
WHEN (SELECT pass_count FROM tests WHERE id=?) > 0 AND
(SELECT status FROM tests WHERE id=?) NOT IN ('WARN','FAIL')
THEN 'PASS'
ELSE status
END WHERE id=?;")
'(test-set-log "UPDATE tests SET final_logf=? WHERE id=?;")
'(test-set-rundir-by-test-id "UPDATE tests SET rundir=? WHERE id=?")
))
;; do not run these as part of the transaction
(define db:special-queries '(rollup-tests-pass-fail
db:roll-up-pass-fail-counts))
;; not used, intended to indicate to run in calling process
(define db:run-local-queries '()) ;; rollup-tests-pass-fail))
;; The queue is a list of vectors where the zeroth slot indicates the type of query to
;; apply and the second slot is the time of the query and the third entry is a list of
;; values to be applied
;;
(define (db:write-cached-data)
(open-run-close
(lambda (db . junkparams)
(let ((queries (make-hash-table))
(data #f))
(mutex-lock! *incoming-mutex*)
(set! data (sort *incoming-data* (lambda (a b)(< (vector-ref a 1)(vector-ref b 1)))))
(set! *incoming-data* '())
(mutex-unlock! *incoming-mutex*)
(if (> (length data) 0)
(debug:print-info 4 "Writing cached data " data))
;; prepare the needed statements, do each only once
(for-each (lambda (request-item)
(let ((stmt-key (vector-ref request-item 0)))
(if (not (hash-table-ref/default queries stmt-key #f))
(let ((stmt (alist-ref stmt-key db:queries)))
(if stmt
(hash-table-set! queries stmt-key (sqlite3:prepare db (car stmt)))
(if (procedure? stmt-key)
(hash-table-set! queries stmt-key #f)
(debug:print 0 "ERROR: Missing query spec for " stmt-key "!")))))))
data)
;; outer loop to handle special queries that cannot be handled in the
;; transaction.
(let outerloop ((special-qry #f)
(stmts data))
(if special-qry
;; handle a query that cannot be part of the grouped queries
(let* ((stmt-key (vector-ref special-qry 0))
(qry (hash-table-ref queries stmt-key))
(params (vector-ref special-qry 2)))
(if (string? qry)
(apply sqlite3:execute db qry params)
(if (procedure? stmt-key)
(begin
;; we are being handed a procedure so call it
(debug:print-info 11 "Running (apply " stmt-key " " db " " params ")")
(apply stmt-key db params))
(debug:print 0 "ERROR: Unrecognised queued call " qry " " params)))
(if (not (null? stmts))
(outerloop #f stmts)))
;; handle normal queries
(let ((rem (sqlite3:with-transaction
db
(lambda ()
(debug:print-info 11 "flushing " stmts " to db")
(if (null? stmts)
stmts
(let innerloop ((hed (car stmts))
(tal (cdr stmts)))
(let ((params (vector-ref hed 2))
(stmt-key (vector-ref hed 0)))
(if (or (procedure? stmt-key)
(member stmt-key db:special-queries))
(begin
(debug:print-info 11 "Handling special statement " stmt-key)
(cons hed tal))
(begin
(debug:print-info 11 "Executing " stmt-key " for " params)
(apply sqlite3:execute (hash-table-ref queries stmt-key) params)
(if (not (null? tal))
(innerloop (car tal)(cdr tal))
'()))
))))))))
(if (not (null? rem))
(outerloop (car rem)(cdr rem))))))
(for-each (lambda (stmt-key)
(sqlite3:finalize! (hash-table-ref queries stmt-key)))
(hash-table-keys queries))
(let ((cache-size (length data)))
(if (> cache-size *max-cache-size*)
(set! *max-cache-size* cache-size)))
))
#f))
;; Rollup the pass/fail counts from itemized tests into fail_count and pass_count
(define (db:roll-up-pass-fail-counts db run-id test-name item-path status)
;; (cdb:flush-queue *runremote*)
(if (and (not (equal? item-path ""))
(or (equal? status "PASS")
(equal? status "WARN")
(equal? status "FAIL")
(equal? status "WAIVED")
(equal? status "RUNNING")))
(begin
|
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
|
SET state=CASE WHEN (SELECT count(id) FROM tests WHERE run_id=? AND testname=? AND item_path != '' AND state in ('RUNNING','NOT_STARTED')) > 0 THEN
'RUNNING'
ELSE 'COMPLETED' END,
status=CASE WHEN fail_count > 0 THEN 'FAIL' WHEN pass_count > 0 AND fail_count=0 THEN 'PASS' ELSE 'UNKNOWN' END
WHERE run_id=? AND testname=? AND item_path='';"
run-id test-name run-id test-name))
#f)
#f))
;;======================================================================
;; Tests meta data
;;======================================================================
;; read the record given a testname
|
<
|
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
|
SET state=CASE WHEN (SELECT count(id) FROM tests WHERE run_id=? AND testname=? AND item_path != '' AND state in ('RUNNING','NOT_STARTED')) > 0 THEN
'RUNNING'
ELSE 'COMPLETED' END,
status=CASE WHEN fail_count > 0 THEN 'FAIL' WHEN pass_count > 0 AND fail_count=0 THEN 'PASS' ELSE 'UNKNOWN' END
WHERE run_id=? AND testname=? AND item_path='';"
run-id test-name run-id test-name))
#f)
#f))
;;======================================================================
;; Tests meta data
;;======================================================================
;; read the record given a testname
|