Megatest

Check-in [c32ae5e108]
Login
☰Home Timeline Files Branches Tags Tickets Wiki Login
Overview
Comment:Fixed some remote run issues in tests:update-central-meta. Corrected issue with redirecting output to mt_launch.log
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | v1.54
Files: files | file ages | folders
SHA1: c32ae5e10895f3184a029e67841b8f03371ac115
User & Date: matt on 2013-04-27 14:54:20
Other Links: branch diff | manifest | tags
Context
2013-04-27
16:52
Converted some open-run-close calls to cdb:remote-run check-in: 13cd0c805d user: matt tags: v1.54
15:15
Merged in v1.54 changes Closed-Leaf check-in: c196e891d9 user: matt tags: threaded-db-calls
14:54
Fixed some remote run issues in tests:update-central-meta. Corrected issue with redirecting output to mt_launch.log check-in: c32ae5e108 user: matt tags: v1.54
11:36
Added (but not yet used) cdb: calls for remaining test table updates. Remotified and renamed register-run check-in: eec815277b user: matt tags: v1.54
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Modified db.scm from [9fdbb19be4] to [14db6902e7].

︙
905
906
907
908
909
910
911
912
913


914
915
916
917
918
919

920
921
922
923
924
925
926

905
906
907
908
909
910
911


912
913
914
915
916
917
918

919
920
921
922
923
924
925
926








-
-
+
+





-
+








		;;(debug:print 0 "QRY: " qry)
		(sqlite3:execute db qry run-id newstate newstatus testname testname)))
	    testnames))

(define (cdb:delete-tests-in-state serverdat run-id state)
  (cdb:client-call serverdat 'delete-tests-in-state #t *default-numtries* run-id state))

(define (cbd:tests-update-cpuload-diskfree serverdat test-id cpuload diskfree)
  (cbd:client-call serverdat 'update-cpuload-diskfree #t *default-numtries cpuload diskfree test-id))
(define (cdb:tests-update-cpuload-diskfree serverdat test-id cpuload diskfree)
  (cdb:client-call serverdat 'update-cpuload-diskfree #t *default-numtries* cpuload diskfree test-id))

(define (cdb:tests-update-run-duration serverdat test-id minutes)
  (cdb:client-call serverdat 'update-run-duration #t *default-numtries* minutes test-id))

(define (cdb:tests-update-uname-host serverdat test-id uname hostname)
  (cdb:client-call serverdat 'update-uname-host #t *default-numtries* uname hostname))
  (cdb:client-call serverdat 'update-uname-host #t *default-numtries* test-id uname hostname))

;; speed up for common cases with a little logic
(define (db:test-set-state-status-by-id db test-id newstate newstatus newcomment)
  (cond
   ((and newstate newstatus newcomment)
    (sqlite3:exectute db "UPDATE tests SET state=?,status=?,comment=? WHERE id=?;" newstate newstatus test-id))
   ((and newstate newstatus)
︙

Modified launch.scm from [87b2ca50e6] to [9aca7f57e1].

︙
274
275
276
277
278
279
280
281

282
283
284
285
286
287
288

274
275
276
277
278
279
280

281
282
283
284
285
286
287
288








-
+








							    (current-seconds) 
							    start-seconds)))))
					(kill-tries 0))
				   (let loop ((minutes   (calc-minutes)))
				     (begin
				       (set! kill-job? (test-get-kill-request test-id)) ;; run-id test-name itemdat))
				       ;; open-run-close not needed for test-set-meta-info
				       (test-set-meta-info #f test-id run-id test-name itemdat minutes)
				       (tests:set-meta-info #f test-id run-id test-name itemdat minutes)
				       (if kill-job? 
					   (begin
					     (mutex-lock! m)
					     (let* ((pid (vector-ref exit-info 0)))
					       (if (number? pid)
						   (begin
						     (debug:print 0 "WARNING: Request received to kill job (attempt # " kill-tries ")")
︙
535
536
537
538
539
540
541
542

543
544
545
546


547
548
549
550
551
552
553

535
536
537
538
539
540
541

542
543
544
545

546
547
548
549
550
551
552
553
554








-
+



-
+
+








    ;;   (system  (conc "ln -sf " test-path " " testlink)))
    (if (directory? test-path)
	(begin
	  (let* ((ovrcmd (let ((cmd (config-lookup *configdat* "setup" "testcopycmd")))
			   (if cmd
			       ;; substitute the TEST_SRC_PATH and TEST_TARG_PATH
			       (string-substitute "TEST_TARG_PATH" test-path
						  (string-substitute "TEST_SRC_PATH" test-src-path cmd))
						  (string-substitute "TEST_SRC_PATH" test-src-path cmd #t) #t)
			       #f)))
		 (cmd    (if ovrcmd 
			     ovrcmd
			     (conc "rsync -av" (if (debug:debug-mode 1) "" "q") " " test-src-path "/ " test-path "/")))
			     (conc "rsync -av" (if (debug:debug-mode 1) "" "q") " " test-src-path "/ " test-path "/"
				   " >> " test-path "/mt_launch.log >>2 " test-path "/mt_launch.log")))
		 (status (system cmd)))
	    (if (not (eq? status 0))
		(debug:print 2 "ERROR: problem with running \"" cmd "\"")))
	  (list lnkpathf lnkpath ))
	(list #f #f))))

;; 1. look though disks list for disk with most space
︙
676
677
678
679
680
681
682
683


684
685
686
687
688
689
690

677
678
679
680
681
682
683

684
685
686
687
688
689
690
691
692








-
+
+








				      (car fullcmd))
				  ;; conc
				  (if useshell
				      '()
				      (cdr fullcmd))))) ;;  launcher fullcmd)));; (apply cmd-run-proc-each-line launcher print fullcmd))) ;; (cmd-run->list fullcmd))
      (with-output-to-file "mt_launch.log"
	(lambda ()
	  (apply print launch-results)))
	  (apply print launch-results))
	#:append)
      (debug:print 2 "Launching completed, updating db")
      (debug:print 2 "Launch results: " launch-results)
      (if (not launch-results)
	  (begin
	    (print "ERROR: Failed to run " (string-intersperse fullcmd " ") ", exiting now")
	    ;; (sqlite3:finalize! db)
	    ;; good ole "exit" seems not to work
︙

Modified megatest.scm from [c9058ed68c] to [ab8687b3c9].

︙
31
32
33
34
35
36
37



38
39



40
41
42
43
44
45
46

31
32
33
34
35
36
37
38
39
40


41
42
43
44
45
46
47
48
49
50








+
+
+
-
-
+
+
+








(include "common_records.scm")
(include "key_records.scm")
(include "db_records.scm")
(include "megatest-fossil-hash.scm")

;; (use trace dot-locking)
;; (trace
;;  cdb:tests-update-cpuload-diskfree
;;  cdb:tests-update-uname-host
;;  cdb:tests-update-run-duration
;;  cdb:client-call
;;  cdb:remote-run
;;  ;;  cdb:client-call
;;  ;; cdb:remote-run
;; )
;;  cdb:test-set-status-state
;;  change-directory
;;  db:process-queue-item
;;  db:test-get-logfile-info
;;  db:teststep-set-status!
;;  nice-path
;;  obtain-dot-lock
︙

Modified tests.scm from [5affe7cafb] to [db49b893d4].

︙
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565















566
567



568
569
570
571
572
573
574
575
576
577

578
579
580
581
582
583
584

551
552
553
554
555
556
557








558
559
560
561
562
563
564
565
566
567
568
569
570
571
572


573
574
575
576
577
578
579
580
581
582
583
584

585
586
587
588
589
590
591
592








-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-
+
+
+









-
+








	 (lambda (count)
	   (set! res count))
	 tdb
	 "SELECT count(id) FROM test_rundat;")
	res))
  0)

(define (db:update-central-meta-info db test-id cpuload diskfree minutes num-records uname hostname)
  (sqlite3:execute db "UPDATE tests SET cpuload=?,diskfree=? WHERE id=?;"
		   cpuload
		   diskfree
		   test-id)
  (if minutes (sqlite3:execute db "UPDATE tests SET run_duration=? WHERE id=?;" minutes test-id))
  (if (eq? num-records 0)
      (sqlite3:execute db "UPDATE tests SET uname=?,host=? WHERE id=?;"
(define (tests:update-central-meta-info test-id cpuload diskfree minutes num-records uname hostname)
  ;; This is a good candidate for threading the requests to enable
  ;; transactionized write at the server
  (cdb:tests-update-cpuload-diskfree *runremote* test-id cpuload diskfree)
  ;; (let ((db (open-db)))
    ;; (sqlite3:execute db "UPDATE tests SET cpuload=?,diskfree=? WHERE id=?;"
    ;;     	     cpuload
    ;;     	     diskfree
    ;;     	     test-id)
    (if minutes 
	(cdb:tests-update-run-duration *runremote* test-id minutes))
	;; (sqlite3:execute db "UPDATE tests SET run_duration=? WHERE id=?;" minutes test-id))
    (if (eq? num-records 0)
	(cdb:tests-update-uname-host *runremote* test-id uname hostname))
	;;(sqlite3:execute db "UPDATE tests SET uname=?,host=? WHERE id=?;" uname hostname test-id))
		       uname hostname test-id)))

    ;;(sqlite3:finalize! db))
    )
  
(define (tests:set-meta-info db test-id run-id testname itemdat minutes)
  ;; DOES cdb:remote-run under the hood!
  (let* ((tdb         (db:open-test-db-by-test-id db test-id))
	 (num-records (test:tdb-get-rundat-count tdb))
	 (cpuload  (get-cpu-load))
	 (diskfree (get-df (current-directory))))
    (if (eq? (modulo num-records 10) 0) ;; every ten records update central
	(let ((uname    (get-uname "-srvpio"))
	      (hostname (get-host-name)))
	  (cdb:remote-run db:update-central-meta-info db test-id cpuload diskfree minutes num-records uname hostname)))
	  (tests:update-central-meta-info test-id cpuload diskfree minutes num-records uname hostname)))
    (sqlite3:execute tdb "INSERT INTO test_rundat (update_time,cpuload,diskfree,run_duration) VALUES (strftime('%s','now'),?,?,?);"
		     cpuload diskfree minutes)))
	  
;;======================================================================
;; A R C H I V I N G
;;======================================================================
︙

Modified tests/Makefile from [60f7cab260] to [7cc2ac6e26].

︙
43
44
45
46
47
48
49
50

51
52
53
54
55
56
57

43
44
45
46
47
48
49

50
51
52
53
54
55
56
57








-
+








	sleep 40;cd fullrun;megatest -target ubuntu/nfs/none :runname $(RUNNAME) -set-state-status COMPLETED,FORCED :state COMPLETED :status PASS -testpatt ez_p%s,runfirst/ -debug $(DEBUG) $(LOGGING)


test3 : fullprep
	cd fullrun;$(MEGATEST) -runtests runfirst -reqtarg ubuntu/nfs/none :runname $(RUNNAME)_b -debug 10

test4 : fullprep
	cd fullrun;$(MEGATEST) -debug $(DEBUG) -runall -reqtarg ubuntu/nfs/none :runname $(RUNNAME)_b -m "This is a comment specific to a run" -v $(LOGGING)
	cd fullrun;time $(MEGATEST) -debug $(DEBUG) -runall -reqtarg ubuntu/nfs/none :runname $(RUNNAME)_b -m "This is a comment specific to a run" -v $(LOGGING)

# NOTE: Only one instance can be a server
test5 : fullprep
	cd fullrun;sleep 0;$(MEGATEST) -runtests % -target $(TARGET) :runname $(RUNNAME)_aa -debug $(DEBUG) $(LOGGING) > aa.log 2> aa.log &
	cd fullrun;sleep 0;$(MEGATEST) -runtests % -target $(TARGET) :runname $(RUNNAME)_ab -debug $(DEBUG) $(LOGGING) > ab.log 2> ab.log &
	cd fullrun;sleep 0;$(MEGATEST) -runtests % -target $(TARGET) :runname $(RUNNAME)_ac -debug $(DEBUG) $(LOGGING) > ac.log 2> ac.log &
	cd fullrun;sleep 0;$(MEGATEST) -runtests % -target $(TARGET) :runname $(RUNNAME)_ad -debug $(DEBUG) $(LOGGING) > ad.log 2> ad.log &
︙

Modified tests/fullrun/megatest.config from [d25787fc32] to [b9e3240f7d].

︙
12
13
14
15
16
17
18
19

20
21
22
23
24
25
26

12
13
14
15
16
17
18

19
20
21
22
23
24
25
26








-
+









[setup]

# It is possible (but not recommended) to override the rsync command used
# to populate the test directories. For test development the following 
# example can be useful
#
testcopycmd cp --remove-destination -rsv TEST_SRC_PATH/. TEST_TARG_PATH/.
testcopycmd cp --remove-destination -rsv TEST_SRC_PATH/. TEST_TARG_PATH/. >> TEST_TARG_PATH/mt_launch.log 2>> TEST_TARG_PATH/mt_launch.log

# or for hard links

# testcopycmd cp --remove-destination -rlv TEST_SRC_PATH/. TEST_TARG_PATH/.

# FULL or 2, NORMAL or 1, OFF or 0
synchronous OFF
︙

This page was generated in about 0.105s by Fossil 2.24 [452f8e1ffc] 2024-03-26 15:36:00