Megatest

Check-in [b2c735197b]
Login
☰Home Timeline Files Branches Tags Tickets Wiki Login
Overview
Comment:wip
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | v1.6584-nanomsg
Files: files | file ages | folders
SHA1: b2c735197b1485dde357fafc78bfe873018537d3
User & Date: matt on 2021-11-15 19:11:36
Other Links: branch diff | manifest | tags
Context
2021-11-25
18:25
Simplerun now passes check-in: c8d4b40864 user: matt tags: v1.6584-nanomsg
2021-11-15
19:11
wip check-in: b2c735197b user: matt tags: v1.6584-nanomsg
15:05
Merged fork check-in: 18439ca550 user: matt tags: v1.6584-nanomsg
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Modified dashboard.scm from [eb6f650afa] to [5354c7dd14].

︙
3743
3744
3745
3746
3747
3748
3749

3750

3751
3752
3753

3743
3744
3745
3746
3747
3748
3749
3750

3751
3752
3753
3754








+
-
+





)

(import dashboard)

;; ease debugging by loading ~/.dashboardrc
(let ((debugcontrolf (get-debugcontrolf)))
  (if debugcontrolf
  (load debugcontrolf))
      (load debugcontrolf)))

(main)

Modified dbmod.scm from [ab3d898579] to [f04c36c6ab].

︙
295
296
297
298
299
300
301

302
303
304
305
306
307
308

295
296
297
298
299
300
301
302
303
304
305
306
307
308
309








+








	chicken.time
	chicken.time.posix
	system-information
	
	(prefix base64 base64:)
	csv-xml
	directory-utils
	format
	matchable
	regex
	s11n
	srfi-1
	srfi-13
	srfi-18
	srfi-69
︙
1052
1053
1054
1055
1056
1057
1058
1059

1060
1061
1062
1063
1064
1065
1066

1053
1054
1055
1056
1057
1058
1059

1060
1061
1062
1063
1064
1065
1066
1067








-
+








      (if should-print (debug:print 3 *default-log-port* "INFO: db sync, total run time " runtime " ms"))
      (for-each 
       (lambda (dat)
	 (let ((tblname (car dat))
	       (count   (cdr dat)))
	   (set! tot-count (+ tot-count count))
	   (if (> count 0)
	       (if should-print (debug:print 0 *default-log-port* "   "tblname" "count))))) ;; (format #f "    ~10a ~5a" tblname count))))))
	       (if should-print (debug:print 0 *default-log-port* (format #f "    ~10a ~5a" tblname count))))))
       (sort (hash-table->alist numrecs)(lambda (a b)(> (cdr a)(cdr b))))))
    tot-count))

(define (db:patch-schema-rundb frundb)
  ;;
  ;; remove this some time after September 2016 (added in version v1.6031
  ;;
︙
4166
4167
4168
4169
4170
4171
4172
4173
4174
4175



4176
4177
4178
4179
4180
4181
4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
4192
4193
4194


4195
4196
4197
4198
4199
4200
4201
4202








4203
4204
4205
4206
4207
4208





4209
4210
4211
4212
4213
4214
4215
4216
4217
4218
4219
4220
4221
4222
4223
4224
4225
4226
4227
4228
4229
4230
4231
4232
4233
4234
4235
4236
4237
4238
4239
4240
4241
4242
4243
4244
4245




































4246
4247
4248


4249
4250
4251
4252
4253
4254
4255
4256
4257
4258
4259
4260
4261
4262
4263
4264






4265
4266
4267
4268
4269
4270
4271
4272
4273
4274
4275
4276
4277
4278








4279
4280
4281
4282
4283
4284
4285

4167
4168
4169
4170
4171
4172
4173



4174
4175
4176
4177
4178
4179
4180
4181
4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
4192
4193
4194
4195
4196
4197








4198
4199
4200
4201
4202
4203
4204
4205
4206





4207
4208
4209
4210
4211
4212




































4213
4214
4215
4216
4217
4218
4219
4220
4221
4222
4223
4224
4225
4226
4227
4228
4229
4230
4231
4232
4233
4234
4235
4236
4237
4238
4239
4240
4241
4242
4243
4244
4245
4246
4247
4248
4249


4250
4251
4252
4253
4254
4255
4256
4257
4258
4259
4260
4261






4262
4263
4264
4265
4266
4267
4268
4269
4270
4271
4272
4273








4274
4275
4276
4277
4278
4279
4280
4281
4282
4283
4284
4285
4286
4287
4288








-
-
-
+
+
+



















+
+
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+

-
-
-
-
-
+
+
+
+
+

-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

-
-
+
+










-
-
-
-
-
-
+
+
+
+
+
+






-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+








               db
               (lambda ()
                 ;; NB// Pass the db so it is part fo the transaction
		  ;; this call sets the item state/status
                 (db:test-set-state-status db run-id test-id state status comment)
                 (if (not (equal? item-path "")) ;; only roll up IF incoming test is an item
                     (let* ((state-status-counts  (db:get-all-state-status-counts-for-test dbstruct run-id test-name item-path state status)) ;; item-path is used to exclude current state/status of THIS test
			      						  (state-stauses (db:roll-up-rules state-status-counts state status))
                          (newstate (car state-stauses))
                          (newstatus (cadr state-stauses)))
			    (state-statuses        (db:roll-up-rules state-status-counts state status))
                            (newstate             (car state-statuses))
                            (newstatus            (cadr state-statuses)))
                       (debug:print 4 *default-log-port* "BB> tl-test-id="tl-test-id" ; "test-name":"item-path" newstate="newstate" newstatus="newstatus" len(sscs)="(length state-status-counts)  " state-status-counts: "
							(apply conc
                  (map (lambda (x)
                     (conc
		      (with-output-to-string (lambda () (pp (dbr:counts->alist x)))) " | "))
		       state-status-counts))); end debug:print
		       
                       (if tl-test-id
			   (db:test-set-state-status db run-id tl-test-id newstate newstatus #f)) ;; we are still in the transaction - must access the db and not the dbstruct
		       ))))))
         ;; (mutex-unlock! *db-transaction-mutex*)
         (if (and test-id state status (equal? status "AUTO")) 
             (db:test-data-rollup dbstruct run-id test-id status))
         tr-res)))
    ;; this was moved out of test-set-state-status
    (mt:process-triggers dbstruct run-id test-id state status)))


(define (db:roll-up-rules state-status-counts state status)
  (if (null? state-status-counts)
      '(#f #f)
		(let* ((running     (length (filter (lambda (x)
                          (member (dbr:counts-state x) *common:running-states*))
                                 state-status-counts)))
           (bad-not-started      (length (filter (lambda (x)
                                      (and (equal? (dbr:counts-state x) "NOT_STARTED") 
                                        (not (member (dbr:counts-status x)  *common:not-started-ok-statuses*))))
																	state-status-counts)))
           (all-curr-states      (common:special-sort  ;; worst -> best (sort of)
      (let* ((running              (length (filter (lambda (x)
						     (member (dbr:counts-state x) *common:running-states*))
						   state-status-counts)))
             (bad-not-started      (length (filter (lambda (x)
						     (and (equal? (dbr:counts-state x) "NOT_STARTED") 
							  (not (member (dbr:counts-status x)  *common:not-started-ok-statuses*))))
						   state-status-counts)))
             (all-curr-states      (common:special-sort  ;; worst -> best (sort of)
                                    (delete-duplicates
                                      (if (and state (not (member state *common:dont-roll-up-states*)))
                                          (cons state (map dbr:counts-state state-status-counts))
                                          (map dbr:counts-state state-status-counts)))
                                                  *common:std-states* >))
           (all-curr-statuses    (common:special-sort  ;; worst -> best
                                     (if (and state (not (member state *common:dont-roll-up-states*)))
					 (cons state (map dbr:counts-state state-status-counts))
					 (map dbr:counts-state state-status-counts)))
                                    *common:std-states* >))
             (all-curr-statuses    (common:special-sort  ;; worst -> best
                                    (delete-duplicates
                                      (if (and state status (not (member state *common:dont-roll-up-states*)))
                                          (cons status (map dbr:counts-status state-status-counts))
                                          (map dbr:counts-status state-status-counts)))
                                                   *common:std-statuses* >))
           (non-completes        (filter (lambda (x)
							 										 (not (member x (cons "COMPLETED" *common:dont-roll-up-states*))))
						       									all-curr-states))
			     (preq-fails        (filter (lambda (x)
							 								(equal? x "PREQ_FAIL"))
						       							all-curr-statuses))
           (num-non-completes (length non-completes))
 					 (newstate          (cond
															((> running 0)           "RUNNING")            ;; anything running, call the situation running
                              ((> (length preq-fails) 0) "NOT_STARTED")
															((> bad-not-started 0)   "COMPLETED")          ;; we have an ugly situation, it is completed in the sense we cannot do more.
															((> num-non-completes 0) (car non-completes))  ;;  (remove (lambda (x)(equal? "COMPLETED" x)) all-curr-states))) ;; only rollup DELETED if all DELETED
															(else                    (car all-curr-states))))
           (newstatus         (cond
                              ((> (length preq-fails) 0)  "PREQ_FAIL")
                              ((or (> bad-not-started 0)
                                   (and (equal? newstate "NOT_STARTED")
                                      (> num-non-completes 0)))
                                            "STARTED")
                              (else (car all-curr-statuses)))))
 					(debug:print-info 2 *default-log-port*
                                         "\n--> probe db:set-state-status-and-roll-up-items: "
                                         "\n--> state-status-counts: "(map dbr:counts->alist state-status-counts)
                                         "\n--> running:             "running
                                         "\n--> bad-not-started:     "bad-not-started
                                         "\n--> non-non-completes:   "num-non-completes
                                         "\n--> non-completes:       "non-completes
                                         "\n--> all-curr-states:     "all-curr-states
                                         "\n--> all-curr-statuses:     "all-curr-statuses
                                         "\n--> newstate              "newstate
                                         "\n--> newstatus            "newstatus
                                         "\n\n")
                                     (if (and state status (not (member state *common:dont-roll-up-states*)))
					 (cons status (map dbr:counts-status state-status-counts))
					 (map dbr:counts-status state-status-counts)))
                                    *common:std-statuses* >))
             (non-completes        (filter (lambda (x)
					     (not (member x (cons "COMPLETED" *common:dont-roll-up-states*))))
					   all-curr-states))
	     (preq-fails        (filter (lambda (x)
					  (equal? x "PREQ_FAIL"))
					all-curr-statuses))
             (num-non-completes (length non-completes))
 	     (newstate          (cond
				 ((> running 0)           "RUNNING")            ;; anything running, call the situation running
				 ((> (length preq-fails) 0) "NOT_STARTED")
				 ((> bad-not-started 0)   "COMPLETED")          ;; we have an ugly situation, it is completed in the sense we cannot do more.
				 ((> num-non-completes 0) (car non-completes))  ;;  (remove (lambda (x)(equal? "COMPLETED" x)) all-curr-states))) ;; only rollup DELETED if all DELETED
				 (else                    (car all-curr-states))))
             (newstatus         (cond
				 ((> (length preq-fails) 0)  "PREQ_FAIL")
				 ((or (> bad-not-started 0)
                                      (and (equal? newstate "NOT_STARTED")
					   (> num-non-completes 0)))
				  "STARTED")
				 (else (car all-curr-statuses)))))
	(debug:print-info 2 *default-log-port*
			  "\n--> probe db:set-state-status-and-roll-up-items: "
			  "\n--> state-status-counts: "(map dbr:counts->alist state-status-counts)
			  "\n--> running:             "running
			  "\n--> bad-not-started:     "bad-not-started
			  "\n--> non-non-completes:   "num-non-completes
			  "\n--> non-completes:       "non-completes
			  "\n--> all-curr-states:     "all-curr-states
			  "\n--> all-curr-statuses:     "all-curr-statuses
			  "\n--> newstate              "newstate
			  "\n--> newstatus            "newstatus
			  "\n\n")

                        ;; NB// Pass the db so it is part of the transaction
         (list newstate newstatus)))
	;; NB// Pass the db so it is part of the transaction
	(list newstate newstatus))))

(define (db:set-state-status-and-roll-up-run dbstruct run-id curr-state curr-status)
  ;; (mutex-lock! *db-transaction-mutex*)
    (db:with-db
     dbstruct #f #f
     (lambda (db)
       (let ((tr-res
              (sqlite3:with-transaction
               db
               (lambda ()
                   (let* ((state-status-counts  (db:get-all-state-status-counts-for-run dbstruct run-id))
													(state-stauses (db:roll-up-rules state-status-counts #f #f ))
                          (newstate (car state-stauses))
                          (newstatus (cadr state-stauses))) 
                    (if (or (not (eq? newstate curr-state)) (not (eq?  newstatus curr-status)))
                   (db:set-run-state-status dbstruct run-id newstate newstatus )))))))
                 (let* ((state-status-counts  (db:get-all-state-status-counts-for-run dbstruct run-id))
			(state-statuses       (db:roll-up-rules state-status-counts #f #f ))
                        (newstate             (car state-statuses))
                        (newstatus            (cadr state-statuses))) 
                   (if (or (not (eq? newstate curr-state)) (not (eq?  newstatus curr-status)))
                       (db:set-run-state-status dbstruct run-id newstate newstatus )))))))
         ;; (mutex-unlock! *db-transaction-mutex*)
         tr-res))))


(define (db:get-all-state-status-counts-for-run dbstruct run-id)
 (let* ((test-count-recs (db:with-db
                                  dbstruct #f #f
                                  (lambda (db)
                                    (sqlite3:map-row
                                     (lambda (state status count)
                                        (make-dbr:counts state: state status: status count: count))
                                     db
                                     "SELECT state,status,count(id) FROM tests WHERE run_id=?  GROUP BY state,status;"
                                     run-id )))))
                          dbstruct #f #f
                          (lambda (db)
                            (sqlite3:map-row
                             (lambda (state status count)
                               (make-dbr:counts state: state status: status count: count))
                             db
                             "SELECT state,status,count(id) FROM tests WHERE run_id=?  GROUP BY state,status;"
                             run-id )))))
   test-count-recs))


;; BBnote: db:get-all-state-status-counts-for-test returns dbr:counts object aggregating state and status of items of a given test, *not including rollup state/status*
;;
;; NOTE: This is called within a transaction
;;
︙

Modified launchmod.scm from [38950f6d5c] to [26d43d79a0].

︙
1899
1900
1901
1902
1903
1904
1905
1906

1907
1908
1909
1910
1911
1912
1913

1899
1900
1901
1902
1903
1904
1905

1906
1907
1908
1909
1910
1911
1912
1913








-
+








    (runs:update-junit-test-reporter-xml run-id) 
    (cond 
     ((and all-test-launched (eq? not-completed-cnt 0) (equal? all-test-launched "yes" ))
      (if (and (equal? (rmt:get-var run-id (conc "end-of-run-" run-id)) "no") (common:simple-lock (conc "endOfRun" run-id)))
	  (begin
	    (debug:print 4 *default-log-port* "look for  post hook. currseconds: " (current-seconds) " EOR " (rmt:get-var run-id (conc "end-of-run-" run-id)))
	    (debug:print 0 *default-log-port* "End of Run Detected.")
	    (rmt:set-var (conc "end-of-run-" run-id) "yes")
	    (rmt:set-var run-id (conc "end-of-run-" run-id) "yes")
					;(thread-sleep! 10)
	    (runs:run-post-hook run-id)
	    (debug:print 4 *default-log-port* "currseconds: " (current-seconds)" eor: " (rmt:get-var run-id (conc "end-of-run-" run-id)))
	    (common:simple-unlock (conc "endOfRun" run-id)))
	  (debug:print 0 *default-log-port* "End of Run Detected but not running post hook. This should happen when eor is set to yes. This will happen only when 2 tests exit at smae time. eor= " (rmt:get-var run-id (conc "end-of-run-" run-id)))))
     ((> running-cnt 3) 
      (debug:print 0 *default-log-port* "There are " running-cnt " tests running." ))
︙

Modified megatest.scm from [e3b480ce07] to [bf964b0d3e].

︙
90
91
92
93
94
95
96

97
98
99
100
101
102
103

90
91
92
93
94
95
96
97
98
99
100
101
102
103
104








+








	  (prefix base64 base64:)
	  (prefix sqlite3 sqlite3:)
	  (prefix sxml-modifications sxml-)
	  address-info
	  csv-abnf
	  directory-utils
	  fmt
	  format
	  http-client
	  intarweb
	  json
	  linenoise
	  matchable
	  md5
	  message-digest
︙

Modified rmtmod.scm from [b3af46403d] to [70b915874b].

︙
38
39
40
41
42
43
44
45

46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61

62
63
64
65
66
67
68

38
39
40
41
42
43
44

45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69








-
+
















+








	
(import scheme
		
	chicken.base
	chicken.condition
	chicken.file
	chicken.file.posix
	chicken.format
	;; chicken.format
	chicken.io
	chicken.pathname
	chicken.port
	chicken.pretty-print
	chicken.process
	chicken.process-context
	chicken.process-context.posix
	chicken.sort
	chicken.string
	;; chicken.tcp
	chicken.random
	chicken.time
	chicken.time.posix
	(prefix sqlite3 sqlite3:)
	
	directory-utils
	format
	;; http-client
	;; intarweb
	matchable
	md5
	message-digest
	nng ;; nanomsg
	(prefix base64 base64:)
︙

Modified runsmod.scm from [c5ee22b35e] to [155dc51460].

︙
63
64
65
66
67
68
69

70
71
72
73
74
75
76

63
64
65
66
67
68
69
70
71
72
73
74
75
76
77








+








	chicken.time.posix
	chicken.random
	chicken.process.signal
	
	(prefix base64 base64:)
	csv-xml
	directory-utils
	format
	matchable
	regex
	s11n
	srfi-1
	srfi-13
	srfi-18
	srfi-69
︙
1299
1300
1301
1302
1303
1304
1305


1306

1307
1308
1309
1310
1311
1312
1313

1300
1301
1302
1303
1304
1305
1306
1307
1308

1309
1310
1311
1312
1313
1314
1315
1316








+
+
-
+








	       (if (and (not (member state '("DELETED" "REMOTEHOSTSTART" "RUNNING" "LAUNCHED""NOT_STARTED")))
			(not (and prevdat
				  (equal? state  (db:test-get-state  prevdat))
				  (equal? status (db:test-get-status prevdat)))))
		   (let ((fmt   (runs:gendat-inc-results-fmt runs-data))
			 (dtime (seconds->year-work-week/day-time event-time))) 
		     (if (runs:lownoise "inc-print" 600)
			 (begin
			   (print "fmt=" fmt)
			 (format #t fmt "State" "Status" "Start Time" "Duration" "Test path"))
			   (format #t fmt "State" "Status" "Start Time" "Duration" "Test path")))
		     ;; (debug:print 0 *default-log-port* "fmt: " fmt " state: " state " status: " status " test-name: " test-name " item-path: " item-path " dtime: " dtime)
		     ;; (debug:print 0 #f "event-time: " event-time " duration: " duration)
		     (format #t fmt
			     state
			     status
			     dtime
			     (seconds->hr-min-sec duration)
︙

Modified tasksmod.scm from [a2fcddb156] to [0b1b410563].

︙
36
37
38
39
40
41
42
43

44
45
46
47
48
49
50
51
52
53
54
55
56
57
58

59
60
61
62
63
64
65

36
37
38
39
40
41
42

43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66








-
+















+








	
(import scheme
	(prefix sqlite3 sqlite3:)
	chicken.base
	chicken.condition
	chicken.file
	chicken.file.posix
	chicken.format
	;; chicken.format
	chicken.io
	chicken.pathname
	chicken.port
	chicken.pretty-print
	chicken.process
	chicken.process-context
	chicken.process-context.posix
	chicken.sort
	chicken.string
	chicken.time
	chicken.time.posix

	(prefix base64 base64:)
	;; csv-xml
	directory-utils
	format
	matchable
	regex
	s11n
	srfi-1
	srfi-13
	srfi-18
	srfi-69
︙

Modified testsmod.scm from [46b455d991] to [13d6172d0b].

︙
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212




1213
1214
1215
1216
1217
1218
1219

1220
1221
1222

1223
1224
1225
1226
1227
1228
1229

1202
1203
1204
1205
1206
1207
1208




1209
1210
1211
1212
1213
1214
1215
1216
1217
1218

1219
1220
1221

1222
1223
1224
1225
1226
1227
1228
1229








-
-
-
-
+
+
+
+






-
+


-
+








	;; ))
	(sort all-tests sort-fn1)))) ;; avoid dealing with deleted tests, look at the hash table

(define (tests:easy-dot test-records outtype)
  (let-values (((fd temp-path) (file-mkstemp (conc "/tmp/" (current-user-name) ".XXXXXX"))))
    (let ((all-testnames (hash-table-keys test-records))
	  (temp-port     (open-output-file* fd)))
      ;; (format temp-port "This file is ~A.~%" temp-path)
      (format temp-port "digraph tests {\n")
      (format temp-port "  size=4,8\n")
      ;; (format temp-port "   splines=none\n")
      ;; (chicken.format#format temp-port "This file is ~A.~%" temp-path)
      (chicken.format#format temp-port "digraph tests {\n")
      (chicken.format#format temp-port "  size=4,8\n")
      ;; (chicken.format#format temp-port "   splines=none\n")
      (for-each
       (lambda (testname)
	 (let* ((testrec (hash-table-ref test-records testname))
		(waitons (or (tests:testqueue-get-waitons testrec) '())))
	   (for-each
	    (lambda (waiton)
	      (format temp-port (conc "   " waiton " -> " testname " [splines=ortho]\n")))
	      (chicken.format#format temp-port (conc "   " waiton " -> " testname " [splines=ortho]\n")))
	    waitons)))
       all-testnames)
      (format temp-port "}\n")
      (chicken.format#format temp-port "}\n")
      (close-output-port temp-port)
      (with-input-from-pipe
       (conc "env -i PATH=$PATH dot -T" outtype " < " temp-path)
       (lambda ()
	 (let ((res (read-lines)))
	   ;; (delete-file temp-path)
	   res))))))
︙

This page was generated in about 0.134s by Fossil 2.24 [452f8e1ffc] 2024-03-26 15:36:00