Megatest

Check-in [7993b83cc3]
Login
☰Home Timeline Files Branches Tags Tickets Wiki Login
Overview
Comment:Converted calls to rdb: back to db:
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | test-specific-db | v1.4602
Files: files | file ages | folders
SHA1: 7993b83cc3f3fc28a69204afd4b10f3c9c68c0ed
User & Date: matt on 2012-09-27 23:11:41
Other Links: branch diff | manifest | tags
Context
2012-09-27
23:30
Switched to OFF for sync and using exception handling on db access check-in: d805adeb13 user: mrwellan tags: test-specific-db, v.14603
23:11
Converted calls to rdb: back to db: check-in: 7993b83cc3 user: matt tags: test-specific-db, v1.4602
22:22
Added fossil signature to help check-in: 44cc049083 user: matt tags: test-specific-db, v1.4601
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Modified db.scm from [76b911ed38] to [8860c23826].

︙
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
























89
90
91
92
93
94
95

65
66
67
68
69
70
71

















72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102








-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+








    (sqlite3:set-busy-handler! db handler)
    (if (not dbexists)
	(db:initialize db))
    (db:set-sync db)
    db))

(define (open-run-close  proc idb . params)
  (let ((runner (lambda ()
		  (let* ((db   (if idb idb (open-db)))
			 (res #f))
		    (db:set-sync db)
		    (set! res (apply proc db params))
		    (if (not idb)(sqlite3:finalize! db))
		    res))))
    (handle-exceptions
     exn
     (begin
       (debug:print 0 "EXCEPTION: database probably overloaded?")
       (debug:print 0 "  " exn)
       (print-call-chain)
       (thread-sleep! (random 120))
       (debug:print 0 "trying db call one more time....")
       (runner))
     (runner))))
 (let* ((db   (if idb idb (open-db)))
	(res #f))
   (db:set-sync db)
   (set! res (apply proc db params))
   (if (not idb)(sqlite3:finalize! db))
   res))

;;   (let ((runner (lambda ()
;; 		  (let* ((db   (if idb idb (open-db)))
;; 			 (res #f))
;; 		    (db:set-sync db)
;; 		    (set! res (apply proc db params))
;; 		    (if (not idb)(sqlite3:finalize! db))
;; 		    res))))
;;     (handle-exceptions
;;      exn
;;      (begin
;;        (debug:print 0 "EXCEPTION: database probably overloaded?")
;;        (debug:print 0 "  " exn)
;;        (print-call-chain)
;;        (thread-sleep! (random 120))
;;        (debug:print 0 "trying db call one more time....")
;;        (runner))
;;      (runner))))

(define *global-delta* 0)
(define *last-global-delta-printed* 0)

(define (open-run-close-measure  proc idb . params)
  (let* ((start-ms (current-milliseconds))
	 (db       (if idb idb (open-db)))
︙

Modified megatest.scm from [d4ef4fb90f] to [3ee9ed6771].

︙
277
278
279
280
281
282
283
284

285
286
287

288
289
290
291
292
293
294
295
296
297
298
299
300
301
302

303
304
305
306
307
308
309

277
278
279
280
281
282
283

284
285
286

287
288
289
290
291
292
293
294
295
296
297
298
299
300
301

302
303
304
305
306
307
308
309








-
+


-
+














-
+








(if (args:get-arg "-list-runs")
    (let* ((db       (begin
		       (setup-for-run)
		       (open-db)))
	   (runpatt  (args:get-arg "-list-runs"))
	   (testpatt (args:get-arg "-testpatt"))
	   (itempatt (args:get-arg "-itempatt"))
	   (runsdat  (rdb:get-runs db runpatt #f #f '()))
	   (runsdat  (db:get-runs db runpatt #f #f '()))
	   (runs     (db:get-rows runsdat))
	   (header   (db:get-header runsdat))
	   (keys     (rdb:get-keys db))
	   (keys     (db:get-keys db))
	   (keynames (map key:get-fieldname keys)))
      (if (not (args:get-arg "-server"))
	  (server:client-setup db))
      ;; Each run
      (for-each 
       (lambda (run)
	 (debug:print 1 "Run: "
		(string-intersperse (map (lambda (x)
					   (db:get-value-by-header run header x))
					 keynames) "/")
		"/"
		(db:get-value-by-header run header "runname")
		" status: " (db:get-value-by-header run header "state"))
	 (let ((run-id (db:get-value-by-header run header "id")))
	   (let ((tests (rdb:get-tests-for-run db run-id testpatt itempatt '() '())))
	   (let ((tests (db:get-tests-for-run db run-id testpatt itempatt '() '())))
	     ;; Each test
	     (for-each 
	      (lambda (test)
		(format #t
			"  Test: ~25a State: ~15a Status: ~15a Runtime: ~5@as Time: ~22a Host: ~10a\n"
			(conc (db:test-get-testname test)
			      (if (equal? (db:test-get-item-path test) "")
︙
482
483
484
485
486
487
488
489

490
491

492
493
494
495
496
497
498
499
500
501
502

503
504
505
506
507
508
509

482
483
484
485
486
487
488

489
490

491
492
493
494
495
496
497
498
499
500
501

502
503
504
505
506
507
508
509








-
+

-
+










-
+








	      (begin
		(debug:print 0 "Failed to setup, giving up on -test-paths or -test-files, exiting")
		(exit 1)))
	  (set! db (open-db))    
	  (if (not (args:get-arg "-server"))
	      (server:client-setup db))
	  (let* ((itempatt (args:get-arg "-itempatt"))
		 (keys     (rdb:get-keys db))
		 (keys     (db:get-keys db))
		 (keynames (map key:get-fieldname keys))
		 (paths    (rdb:test-get-paths-matching db keynames target (args:get-arg "-test-files"))))
		 (paths    (db:test-get-paths-matching db keynames target (args:get-arg "-test-files"))))
	    (set! *didsomething* #t)
	    (for-each (lambda (path)
			(print path))
		      paths)))
	;; else do a general-run-call
	(general-run-call 
	 "-test-files"
	 "Get paths to test"
	 (lambda (db target runname keys keynames keyvallst)
	   (let* ((itempatt (args:get-arg "-itempatt"))
		  (paths    (rdb:test-get-paths-matching db keynames target (args:get-arg "-test-files"))))
		  (paths    (db:test-get-paths-matching db keynames target (args:get-arg "-test-files"))))
	     (for-each (lambda (path)
			 (print path))
		       paths))))))

;;======================================================================
;; Archive tests
;;======================================================================
︙
532
533
534
535
536
537
538
539

540
541
542
543
544
545
546

532
533
534
535
536
537
538

539
540
541
542
543
544
545
546








-
+








	      (begin
		(debug:print 0 "Failed to setup, giving up on -archive, exiting")
		(exit 1)))
	  (set! db (open-db))   
	  (if (not (args:get-arg "-server"))
	      (server:client-setup db))
	  (let* ((itempatt (args:get-arg "-itempatt"))
		 (keys     (rdb:get-keys db))
		 (keys     (db:get-keys db))
		 (keynames (map key:get-fieldname keys))
		 (paths    (db:test-get-paths-matching db keynames target)))
	    (set! *didsomething* #t)
	    (for-each (lambda (path)
			(print path))
		      paths)))
	;; else do a general-run-call
︙
610
611
612
613
614
615
616
617

618
619
620
621
622
623
624

610
611
612
613
614
615
616

617
618
619
620
621
622
623
624








-
+








	      (begin
		(debug:print 0 "Failed to setup, exiting")
		(exit 1)))
	  (set! db (open-db))
	  (if (not (args:get-arg "-server"))
	      (server:client-setup db))
	  (if (and state status)
	      (rdb:teststep-set-status! db test-id step state status itemdat (args:get-arg "-m") logfile)
	      (db:teststep-set-status! db test-id step state status itemdat (args:get-arg "-m") logfile)
	      (begin
		(debug:print 0 "ERROR: You must specify :state and :status with every call to -step")
		(exit 6)))
	  (sqlite3:finalize! db)
	  (set! *didsomething* #t))))

(if (or (args:get-arg "-setlog")       ;; since setting up is so costly lets piggyback on -test-status
︙
653
654
655
656
657
658
659
660

661
662
663
664
665
666
667

653
654
655
656
657
658
659

660
661
662
663
664
665
666
667








-
+








	  (if (not (args:get-arg "-server"))
	      (server:client-setup db))
	  (if (args:get-arg "-load-test-data")
	      ;; has sub commands that are rdb:
	      (db:load-test-data db test-id))
	  (if (args:get-arg "-setlog")
	      (let ((logfname (args:get-arg "-setlog")))
		(rdb:test-set-log! db test-id logfname)))
		(db:test-set-log! db test-id logfname)))
	  (if (args:get-arg "-set-toplog")
	      (rtests:test-set-toplog! db run-id test-name (args:get-arg "-set-toplog")))
	  (if (args:get-arg "-summarize-items")
	      (tests:summarize-items db run-id test-name #t)) ;; do force here
	  (if (args:get-arg "-runstep")
	      (if (null? remargs)
		  (begin
︙
679
680
681
682
683
684
685
686

687
688
689
690
691
692
693

679
680
681
682
683
684
685

686
687
688
689
690
691
692
693








-
+








				       ((tcsh csh ksh)    ">&")
				       ((zsh bash sh ash) "2>&1 >")
				       (else ">&")))
			 (fullcmd    (conc "(" (string-intersperse 
						(cons cmd params) " ")
					   ") " redir " " logfile)))
		    ;; mark the start of the test
		    (rdb:teststep-set-status! db test-id stepname "start" "n/a" itemdat (args:get-arg "-m") logfile)
		    (db:teststep-set-status! db test-id stepname "start" "n/a" itemdat (args:get-arg "-m") logfile)
		    ;; close the db
		    ;; (sqlite3:finalize! db)
		    ;; run the test step
		    (debug:print 2 "INFO: Running \"" fullcmd "\"")
		    (change-directory startingdir)
		    (set! exitstat (system fullcmd)) ;; cmd params))
		    (set! *globalexitstatus* exitstat)
︙
702
703
704
705
706
707
708
709

710
711

712
713
714
715
716
717
718

702
703
704
705
706
707
708

709
710

711
712
713
714
715
716
717
718








-
+

-
+








			       (oldexitstat exitstat)
			       (cmd         (string-intersperse (list "logpro" logprofile htmllogfile "<" logfile ">" (conc stepname "_logpro.log")) " ")))
			  (debug:print 2 "INFO: running \"" cmd "\"")
			  (change-directory startingdir)
			  (set! exitstat (system cmd))
			  (set! *globalexitstatus* exitstat) ;; no necessary
			  (change-directory testpath)
			  (rdb:test-set-log! db test-id htmllogfile)))
			  (db:test-set-log! db test-id htmllogfile)))
		    (let ((msg (args:get-arg "-m")))
		      (rdb:teststep-set-status! db test-id stepname "end" exitstat itemdat msg logfile))
		      (db:teststep-set-status! db test-id stepname "end" exitstat itemdat msg logfile))
		    ;; (sqlite3:finalize! db)
		    ;;(if (not (eq? exitstat 0))
		    ;;	(exit 254)) ;; (exit exitstat) doesn't work?!?
		  ;; open the db
		  ;; mark the end of the test
		  )))
	  (if (or (args:get-arg "-test-status")
︙
752
753
754
755
756
757
758
759

760
761
762
763
764
765
766

752
753
754
755
756
757
758

759
760
761
762
763
764
765
766








-
+








      (if (not (setup-for-run))
	  (begin
	    (debug:print 0 "Failed to setup, exiting")
	    (exit 1)))
      (set! db (open-db))
      (if (not (args:get-arg "-server"))
	  (server:client-setup db))
      (set! keys (rdb:get-keys db))
      (set! keys (db:get-keys db))
      (debug:print 1 "Keys: " (string-intersperse (map key:get-fieldname keys) ", "))
      (sqlite3:finalize! db)
      (set! *didsomething* #t)))

(if (args:get-arg "-gui")
    (begin
      (debug:print 0 "Look at the dashboard for now")
︙

Modified runconfig.scm from [c41d686cb1] to [2fcfa8c039].

︙
9
10
11
12
13
14
15
16
17


18
19
20
21
22
23
24

9
10
11
12
13
14
15


16
17
18
19
20
21
22
23
24








-
-
+
+








(declare (uses common))

(include "common_records.scm")



(define (setup-env-defaults db fname run-id already-seen #!key (environ-patt #f))
  (let* ((keys    (rdb:get-keys db))
	 (keyvals (rdb:get-key-vals db run-id))
  (let* ((keys    (db:get-keys db))
	 (keyvals (db:get-key-vals db run-id))
	 (thekey  (string-intersperse (map (lambda (x)(if x x "-na-")) keyvals) "/"))
	 ;; Why was system disallowed in the reading of the runconfigs file?
	 ;; NOTE: Should be setting env vars based on (target|default)
	 (confdat (read-config fname #f #t environ-patt: environ-patt sections: (list "default" thekey)))
	 (whatfound (make-hash-table))
	 (sections (list "default" thekey)))
    (if (not *target*)(set! *target* thekey)) ;; may save a db access or two but repeats db:get-target code
︙

Modified runs.scm from [ab03495e0c] to [32036156e7].

︙
180
181
182
183
184
185
186
187

188
189
190
191
192
193
194

180
181
182
183
184
185
186

187
188
189
190
191
192
193
194








-
+








	(begin
	  (debug:print 0 "ERROR: Called without all necessary keys")
	  #f))))

;; This is a duplicate of run-tests (which has been deprecated). Use this one instead of run tests.
;; keyvals
(define (runs:run-tests db target runname test-patts user flags)
  (let* ((keys        (rdb:get-keys db))
  (let* ((keys        (db:get-keys db))
	 (keyvallst   (keys:target->keyval keys target))
	 (run-id      (runs:register-run db keys keyvallst runname "new" "n/a" user))  ;;  test-name)))
	 (deferred    '()) ;; delay running these since they have a waiton clause
	 ;; keepgoing is the defacto modality now, will add hit-n-run a bit later
	 ;; (keepgoing   (hash-table-ref/default flags "-keepgoing" #f))
	 (test-names  '())
	 (runconfigf   (conc  *toppath* "/runconfigs.config"))
︙
675
676
677
678
679
680
681
682

683
684
685
686
687
688
689

675
676
677
678
679
680
681

682
683
684
685
686
687
688
689








-
+








;; action:
;;    'remove-runs
;;    'set-state-status
;;
;; NB// should pass in keys?
;;
(define (runs:operate-on db action runnamepatt testpatt itempatt #!key (state #f)(status #f)(new-state-status #f))
  (let* ((keys         (rdb:get-keys db))
  (let* ((keys         (db:get-keys db))
	 (rundat       (runs:get-runs-by-patt db keys runnamepatt))
	 (header       (vector-ref rundat 0))
	 (runs         (vector-ref rundat 1))
	 (states       (if state  (string-split state  ",") '()))
	 (statuses     (if status (string-split status ",") '()))
	 (state-status (if (string? new-state-status) (string-split new-state-status ",") '(#f #f))))
    (debug:print 2 "Header: " header " action: " action " new-state-status: " new-state-status)
︙
799
800
801
802
803
804
805
806

807
808
809
810
811
812
813

799
800
801
802
803
804
805

806
807
808
809
810
811
812
813








-
+








	      (exit 1)))
	(set! db   (open-db))
	(if (args:get-arg "-server")
	    (server:start db (args:get-arg "-server"))
	    (if (not (or (args:get-arg "-runall")
			  (args:get-arg "-runtests")))
		(server:client-setup db)))
	(set! keys (rdb:get-keys db))
	(set! keys (db:get-keys db))
	;; have enough to process -target or -reqtarg here
	(if (args:get-arg "-reqtarg")
	    (let* ((runconfigf (conc  *toppath* "/runconfigs.config")) ;; DO NOT EVALUATE ALL 
		   (runconfig  (read-config runconfigf #f #f environ-patt: #f))) 
	      (if (hash-table-ref/default runconfig (args:get-arg "-reqtarg") #f)
		  (keys:target-set-args keys (args:get-arg "-reqtarg") args:arg-hash)
		  (begin
︙
885
886
887
888
889
890
891
892

893
894
895
896
897
898
899

885
886
887
888
889
890
891

892
893
894
895
896
897
898
899








-
+









;; This could probably be refactored into one complex query ...
(define (runs:rollup-run db keys keyvallst runname user) ;; was target, now keyvallst
  (debug:print 4 "runs:rollup-run, keys: " keys " keyvallst: " keyvallst " :runname " runname " user: " user)
  (let* (; (keyvalllst      (keys:target->keyval keys target))
	 (new-run-id      (runs:register-run db keys keyvallst runname "new" "n/a" user))
	 (prev-tests      (test:get-matching-previous-test-run-records db new-run-id "%" "%"))
	 (curr-tests      (rdb:get-tests-for-run db new-run-id "%" "%" '() '()))
	 (curr-tests      (db:get-tests-for-run db new-run-id "%" "%" '() '()))
	 (curr-tests-hash (make-hash-table)))
    (db:update-run-event_time db new-run-id)
    ;; index the already saved tests by testname and itemdat in curr-tests-hash
    (for-each
     (lambda (testdat)
       (let* ((testname  (db:test-get-testname testdat))
	      (item-path (db:test-get-item-path testdat))
︙
913
914
915
916
917
918
919
920

921
922
923
924
925
926
927

913
914
915
916
917
918
919

920
921
922
923
924
925
926
927








-
+








	      (new-test-record #f))
	 ;; replace these with insert ... select
	 (apply sqlite3:execute 
		db 
		(conc "INSERT OR REPLACE INTO tests (run_id,testname,state,status,event_time,host,cpuload,diskfree,uname,rundir,item_path,run_duration,final_logf,comment) "
		      "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?);")
		new-run-id (cddr (vector->list testdat)))
	 (set! new-testdat (car (rdb:get-tests-for-run db new-run-id testname item-path '() '())))
	 (set! new-testdat (car (db:get-tests-for-run db new-run-id testname item-path '() '())))
	 (hash-table-set! curr-tests-hash full-name new-testdat) ;; this could be confusing, which record should go into the lookup table?
	 ;; Now duplicate the test steps
	 (debug:print 4 "Copying records in test_steps from test_id=" (db:test-get-id testdat) " to " (db:test-get-id new-testdat))
	 (sqlite3:execute 
	  db 
	  (conc "INSERT OR REPLACE INTO test_steps (test_id,stepname,state,status,event_time,comment) "
		"SELECT " (db:test-get-id new-testdat) ",stepname,state,status,event_time,comment FROM test_steps WHERE test_id=?;")
︙

This page was generated in about 0.147s by Fossil 2.24 [452f8e1ffc] 2024-03-26 15:36:00