Megatest

Changes On Branch 03ef168ab1775755
Login
☰Home Timeline Files Branches Tags Tickets Wiki Login
Hide Diff Patch Side-by-Side Diff

Changes In Branch v1.65-modularization Through [03ef168ab1] Excluding Merge-Ins

This is equivalent to a diff from b9f1218ee5 to 03ef168ab1

2018-03-12
14:08
Bumped version to v1.6511 check-in: 1dd24dcdda user: mrwellan tags: v1.65, v1.6511
2018-03-08
16:25
Broken. Compiles but has some issues with referencing other compilation units check-in: 90e6a7e419 user: mrwellan tags: v1.65-modularization
14:34
First serious steps for coverting to modules. db and common now have module alternatives in src. NOTE: these cannot be mixed with the current common and db compilation unit based files. check-in: 03ef168ab1 user: mrwellan tags: v1.65-modularization
2018-03-07
20:17
Added placeholder for gather area db's check-in: b9f1218ee5 user: mrwellan tags: v1.65
2018-03-06
09:36
Added tcmt objects to make clean check-in: f5f300b27d user: jmoon18 tags: v1.65

Modified Makefile from [786bb282c9] to [9ee8079b86].

︙
26
27
28
29
30
31
32



33

34
35
36
37
38
39
40
41
42
43
44
45
46

47
48

49
50
51
52
53
54
55

26
27
28
29
30
31
32
33
34
35

36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51

52
53
54
55
56
57
58
59








+
+
+
-
+













+

-
+








   http-transport.scm filedb.scm tdb.scm \
   client.scm mt.scm \
   ezsteps.scm lock-queue.scm sdb.scm \
   rmt.scm api.scm subrun.scm \
   portlogger.scm archive.scm env.scm diff-report.scm cgisetup/models/pgdb.scm

# module source files
MSRCFILES = ftail.scm db.scm common.scm

# mtest module source files actually used by mtest building
MSRCFILES = ftail.scm
MTMSRCFILES = ftail.scm

# Eggs to install (straightforward ones)
EGGS=matchable readline apropos base64 regex-literals format regex-case test coops trace csv \
dot-locking posix-utils posix-extras directory-utils hostinfo tcp-server rpc csv-xml fmt \
json md5 awful http-client spiffy uri-common intarweb spiffy-request-vars \
spiffy-directory-listing ssax sxml-serializer sxml-modifications iup canvas-draw sqlite3

GUISRCF  = dashboard-context-menu.scm dashboard-tests.scm dashboard-guimonitor.scm gutils.scm dcommon.scm tree.scm vg.scm

OFILES   = $(SRCFILES:%.scm=%.o)
GOFILES  = $(GUISRCF:%.scm=%.o)

MOFILES = $(addprefix mofiles/,$(MSRCFILES:%.scm=%.o))
MTMOFILES =  $(addprefix mofiles/,$(MTMSRCFILES:%.scm=%.o))

mofiles/%.o : %.scm
mofiles/%.o : src/%.scm
	mkdir -p mofiles
	csc $(CSCOPTS) -J -c $< -o mofiles/$*.o

ADTLSCR=mt_laststep mt_runstep mt_ezstep
HELPERS=$(addprefix $(PREFIX)/bin/,$(ADTLSCR))
DEPLOYHELPERS=$(addprefix deploytarg/,$(ADTLSCR))
MTESTHASH=$(shell fossil info|grep checkout:| awk '{print $$2}')
︙
63
64
65
66
67
68
69
70
71


72
73
74
75


76
77
78
79
80

81
82
83
84
85
86
87

67
68
69
70
71
72
73


74
75
76
77


78
79
80
81
82
83

84
85
86
87
88
89
90
91








-
-
+
+


-
-
+
+




-
+








# ARCHSTR=$(shell bash -c "echo \$$MACHTYPE")

PNGFILES = $(shell cd docs/manual;ls *png)

#all : $(PREFIX)/bin/.$(ARCHSTR) mtest dboard mtut ndboard
all : $(PREFIX)/bin/.$(ARCHSTR) mtest dboard mtut

mtest: $(OFILES) readline-fix.scm megatest.o $(MOFILES) mofiles/ftail.o
	csc $(CSCOPTS) $(OFILES) $(MOFILES) megatest.o -o mtest
mtest: $(OFILES) readline-fix.scm megatest.o $(MTMOFILES)
	csc $(CSCOPTS) $(OFILES) $(MTMOFILES) megatest.o -o mtest


dboard : $(OFILES) $(GOFILES) dashboard.scm $(MOFILES)
	csc $(CSCOPTS) $(OFILES) dashboard.scm $(GOFILES) $(MOFILES) -o dboard
dboard : $(OFILES) $(GOFILES) dashboard.scm $(MTMOFILES)
	csc $(CSCOPTS) $(OFILES) dashboard.scm $(GOFILES) $(MTMOFILES) -o dboard

ndboard : newdashboard.scm $(OFILES) $(GOFILES)
	csc $(CSCOPTS) $(OFILES) $(GOFILES) newdashboard.scm -o ndboard

mtut: $(OFILES) megatest-fossil-hash.scm mtut.scm
mtut: $(OFILES) megatest-fossil-hash.scm mtut.scm $(MOFILES)
	csc $(CSCOPTS) $(OFILES) mtut.scm -o mtut

TCMTOBJS = \
	api.o \
	archive.o \
	cgisetup/models/pgdb.o \
	client.o \
︙
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111

101
102
103
104
105
106
107

108
109
110
111
112
113
114








-








	margs.o \
	mt.o \
	megatest-version.o \
	ods.o \
	portlogger.o \
	process.o \
	rmt.o \
	rpc-transport.o \
	runconfig.o \
	runs.o \
	server.o \
	tasks.o \
	tdb.o \
	tests.o \
	subrun.o \
︙
145
146
147
148
149
150
151


152
153
154
155
156
157
158

148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163








+
+








tests.o tasks.o dashboard-tasks.o : task_records.scm
runs.o : test_records.scm
megatest.o : megatest-fossil-hash.scm
rmt.scm client.scm common.scm configf.scm dashboard-guimonitor.scm dashboard-tests.scm dashboard.scm db.scm dcommon.scm ezsteps.scm fs-transport.scm http-transport.scm index-tree.scm items.scm keys.scm launch.scm megatest.scm monitor.scm mt.scm newdashboard.scm runconfig.scm runs.scm server.scm tdb.scm tests.scm tree.scm : common_records.scm
common_records.scm : altdb.scm
vg.o dashboard.o : vg_records.scm
dcommon.o : run_records.scm
mofiles/db.o : mofiles/common.o

# Temporary while transitioning to new routine
# runs.o : run-tests-queue-classic.scm  run-tests-queue-new.scm

megatest-fossil-hash.scm : $(SRCFILES) megatest.scm *_records.scm
	echo "(define megatest-fossil-hash \"$(MTESTHASH)\")" > megatest-fossil-hash.new
	if ! diff -q megatest-fossil-hash.new megatest-fossil-hash.scm ; then echo copying .new to .scm;cp -f megatest-fossil-hash.new megatest-fossil-hash.scm;fi
︙

Modified common.scm from [7382d07655] to [7ec508d3f4].

︙
706
707
708
709
710
711
712


713
714
715
716
717
718
719
720

721
722
723
724
725
726
727

706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721

722
723
724
725
726
727
728
729








+
+







-
+








      (getenv "MT_TESTSUITE_NAME")
      (if (string? *toppath* )
          (pathname-file *toppath*)
          #f))) ;; (pathname-file (current-directory)))))

(define common:get-area-name common:get-testsuite-name)

;; get-db-tmp-area is improved/replicated src/db.scm
;;
(define (common:get-db-tmp-area . junk)
  (if *db-cache-path*
      *db-cache-path*
      (if *toppath* ;; common:get-create-writeable-dir
	  (handle-exceptions
	      exn
	      (begin
		(debug:print-error 0 *default-log-port* "Couldn't create path to " dbdir)
		(debug:print-error 0 *default-log-port* "Couldn't create path to /tmp/ area")
		(exit 1))
	    (let ((dbpath (common:get-create-writeable-dir
			   (list (conc "/tmp/" (current-user-name)
				       "/megatest_localdb/"
				       (common:get-testsuite-name) "/"
				       (string-translate *toppath* "/" ".")))))) ;;  #t))))
	      (set! *db-cache-path* dbpath)
︙
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422

1394
1395
1396
1397
1398
1399
1400

















1401
1402
1403
1404
1405
1406
1407








-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-









;; if it looks like a number -> convert it to a number, else return it
;;
(define (common:lazy-convert inval)
  (let* ((as-num (if (string? inval)(string->number inval) #f)))
    (or as-num inval)))

;; convert string a=1; b=2; c=a silly thing; d=
;; to '((a . 1)(b . 2)(c . "a silly thing")(d . ""))
;;
(define (common:val->alist val #!key (convert #f))
  (let ((val-list (string-split-fields ";\\s*" val #:infix)))
    (if val-list
	(map (lambda (x)
	       (let ((f (string-split-fields "\\s*=\\s*" x #:infix)))
		 (case (length f)
		   ((0) `(,#f))  ;; null string case
		   ((1) `(,(string->symbol (car f))))
		   ((2) `(,(string->symbol (car f)) . ,(let ((inval (cadr f)))
							 (if convert (common:lazy-convert inval) inval))))
		   (else f))))
	     val-list)
	'())))

;;======================================================================
;; S Y S T E M   S T U F F
;;======================================================================

;; lazy-safe get file mod time. on any error (file not existing etc.) return 0
;;
(define (common:lazy-modification-time fpath)
︙

Modified configf.scm from [af30e1aa5d] to [3ed3fb9db8].

︙
240
241
242
243
244
245
246



































247
248
249
250
251
252
253

240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288








+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+








			      (val  (cadr bundle))
			      (meta (if (> (length bundle) 2)(caddr bundle) #f)))
			  (hash-table-set! ht section (config:assoc-safe-add (hash-table-ref ht section) key val metadata: meta))))
		      vars)))))
         (hash-table-keys ht))))
  ht)

;;======================================================================
;; Extended config lines, allows storing more hierarchial data in the config lines
;;   ABC a=1; b=hello world; c=a
;;
;; NOTE: implementation is quite limited. You currently cannot have
;;       semicolons in your string values.
;;======================================================================

;; convert string a=1; b=2; c=a silly thing; d=
;; to '((a . 1)(b . 2)(c . "a silly thing")(d . ""))
;;
(define (configf:val->alist val #!key (convert #f))
  (let ((val-list (string-split-fields ";\\s*" val #:infix)))
    (if val-list
	(map (lambda (x)
	       (let ((f (string-split-fields "\\s*=\\s*" x #:infix)))
		 (case (length f)
		   ((0) `(,#f))  ;; null string case
		   ((1) `(,(string->symbol (car f))))
		   ((2) `(,(string->symbol (car f)) . ,(let ((inval (cadr f)))
							 (if convert (common:lazy-convert inval) inval))))
		   (else f))))
	     val-list)
	'())))

;; I don't want configf to turn into a weak yaml format but this extention is really useful
;;
(define (configf:section->val-alist cfgdat section-name #!key (convert #f))
  (let ((section (configf:get-section cfgdat section-name)))
    (map (lambda (item)
           (let ((key (car item))
                 (val (cadr item))) ;; BUG IN WAIT. sections are not returned as proper alists, should fix this.
             (cons key (configf:val->alist val convert: convert))))
         section)))
  
;; read a config file, returns hash table of alists

;; read a config file, returns hash table of alists
;; adds to ht if given (must be #f otherwise)
;; allow-system:
;;    #f - do not evaluate [system
;;    #t - immediately evaluate [system and store result as string
︙

Modified mtut.scm from [e2f911af30] to [40d2029578].

︙
27
28
29
30
31
32
33
34

35
36
37
38


39
40
41
42
43
44
45

27
28
29
30
31
32
33

34
35
36
37
38
39
40
41
42
43
44
45
46
47








-
+




+
+








     (prefix dbi dbi:)
     nanomsg)

(declare (uses common))
(declare (uses megatest-version))
(declare (uses margs))
(declare (uses configf))
;; (declare (uses rmt))
(declare (uses db)) ;; WARNING: This is NOT the db from megatest/db.scm, is it src/db.scm

(include "megatest-fossil-hash.scm")

(require-library stml)

(import (prefix db db:))

;; stuff for the mapper and checker functions
;;
(define *target-mappers*  (make-hash-table)) 
(define *runname-mappers* (make-hash-table)) 
(define *area-checkers*   (make-hash-table))
︙
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317

304
305
306
307
308
309
310


311
312
313
314
315
316
317








-
-








;; given a mtutil param, return the old megatest equivalent
;;
(define (megatest-param->mtutil-param param)
  (let* ((mapping-alist (common:get-param-mapping flavor: 'switch-symbol)))
    (alist-ref (string->symbol param) mapping-alist eq? param)
    param))

(define val->alist common:val->alist)

(define (push-run-spec torun contour runkey spec)
  (configf:section-var-set! torun contour runkey
			    (cons spec
				  (or (configf:lookup torun contour runkey)
				      '()))))

(define (fossil:clone-or-sync url name dest-dir)
︙
664
665
666
667
668
669
670
671

672
673
674
675
676
677
678

664
665
666
667
668
669
670

671
672
673
674
675
676
677
678








-
+








;;
;; Override the run start time record with sched. Usually #f is fine.
;; 
(define (create-run-pkt mtconf action area runkey target runname mode-patt 
                        tag-expr pktsdir reason contour sched dbdest append-conf
                        runtrans)
  (let* ((good-val   (lambda (inval)(and inval (string? inval)(not (string-null? inval)))))
	 (area-dat   (common:val->alist (or (configf:lookup mtconf "areas" area) "")))
	 (area-dat   (configf:val->alist (or (configf:lookup mtconf "areas" area) "")))
	 (area-path  (alist-ref 'path      area-dat))
	 ;; (area-xlatr (alist-ref 'targtrans area-dat))
         ;; (xlatr-key  (if area-xlatr (string->symbol area-xlatr) #f))
         (new-runname (let* ((callname (if (string? runtrans)(string->symbol runtrans) #f))
			     (mapper   (if callname (hash-table-ref/default *runname-mappers* callname #f) #f)))
			;; (print "callname=" callname " runtrans=" runtrans " mapper=" mapper)
			(if (and callname
︙
763
764
765
766
767
768
769
770

771
772
773
774
775
776
777

763
764
765
766
767
768
769

770
771
772
773
774
775
776
777








-
+








			(keyparts   (string-split key ":")) ;; contour:ruletype:action:optional
			(contour    (car keyparts))
			(len-key    (length keyparts))
			(ruletype   (if (> len-key 1)(cadr keyparts) #f))
			(action     (if (> len-key 2)(caddr keyparts) #f))
			(optional   (if (> len-key 3)(cadddr keyparts) #f))
			;; (val-list   (string-split-fields ";\\s*" val #:infix)) ;; (string-split val)) ;; runname-rule params
			(val-alist  (common:val->alist val))
			(val-alist  (configf:val->alist val))
			(runname    (make-runname "" ""))
			(runtrans   (alist-ref 'runtrans val-alist))

			;; these may or may not be defined and not all are used in each handler type in the case below
			(run-name   (alist-ref 'run-name val-alist))
			(target     (alist-ref 'target   val-alist))
			(crontab    (alist-ref 'cron     val-alist))
︙
1008
1009
1010
1011
1012
1013
1014
1015

1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033

1034
1035
1036
1037
1038
1039
1040

1008
1009
1010
1011
1012
1013
1014

1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032

1033
1034
1035
1036
1037
1038
1039
1040








-
+

















-
+








	       keydats))) ;; sense rules
	  (hash-table-keys rgconf))
	 
	 ;; now have to run populated
	 (for-each
	  (lambda (contour)
	    (let* ((cval       (or (configf:lookup mtconf "contours" contour) ""))
		   (cval-alist (common:val->alist cval))                     ;; BEWARE ... NOT the same val-alist as above!
		   (cval-alist (configf:val->alist cval))                     ;; BEWARE ... NOT the same val-alist as above!
		   (areas      (val-alist->areas cval-alist))
		   (selector   (alist-ref 'selector cval-alist))
		   (mode-tag   (and selector (string-split-fields "/" selector #:infix)))
		   (mode-patt  (and mode-tag (if (eq? (length mode-tag) 2)(cadr mode-tag) #f)))
		   (tag-expr   (and mode-tag (if (null? mode-tag) #f (car mode-tag)))))
	      (print "contour: " contour " areas=" areas " cval=" cval)
	      (for-each
	       (lambda (runkeydatset) 
		 ;; (print "runkeydatset: ")(pp runkeydatset)
		 (let ((runkey     (car runkeydatset))
		       (runkeydats (cadr runkeydatset)))
		   (for-each
		    (lambda (runkeydat)
		      (for-each
		       (lambda (area)
			 (if (area-allowed? area areas runkey contour mode-patt) ;; is this area to be handled (from areas=a,b,c OR using areafn=abcfn and *area-checks* ...)
                             (let* ((aval       (or (configf:lookup mtconf "areas" area) ""))
                                    (aval-alist (common:val->alist aval))
                                    (aval-alist (configf:val->alist aval))
                                    (runname    (alist-ref 'runname runkeydat))
                                    (runtrans   (alist-ref 'runtrans runkeydat))
                                    
                                    (reason     (alist-ref 'message runkeydat))
                                    (sched      (alist-ref 'sched   runkeydat))
                                    (action     (alist-ref 'action  runkeydat))
                                    (dbdest     (alist-ref 'dbdest  runkeydat))
︙
1208
1209
1210
1211
1212
1213
1214
1215

1216
1217
1218
1219
1220
1221
1222

1208
1209
1210
1211
1212
1213
1214

1215
1216
1217
1218
1219
1220
1221
1222








-
+








(if *action*
    (case (string->symbol *action*)
      ((run remove rerun rerun-clean rerun-all set-ss archive kill list)
       (let* ((mtconfdat (simple-setup (args:get-arg "-start-dir")))
	      (mtconf    (car mtconfdat))
	      (area      (args:get-arg "-area")) ;; look up the area to dispatch to from [areas] section
	      (areasec   (if area (configf:lookup mtconf "areas" area) #f))
	      (areadat   (if areasec (common:val->alist areasec) #f))
	      (areadat   (if areasec (configf:val->alist areasec) #f))
	      (area-path (if areadat (alist-ref 'path areadat) #f))
	      (pktsdirs  (configf:lookup mtconf "setup" "pktsdirs"))
	      (pktsdir   (if pktsdirs (car (string-split pktsdirs " ")) #f))
	      (adjargs   (hash-table-copy args:arg-hash))
	      (new-ss    (args:get-arg "-new")))
	 ;; check a few things
	 (cond
︙
1347
1348
1349
1350
1351
1352
1353
1354

1355
1356
1357
1358
1359











1360
1361
1362
1363
1364
1365
1366

1347
1348
1349
1350
1351
1352
1353

1354
1355
1356



1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374








-
+


-
-
-
+
+
+
+
+
+
+
+
+
+
+








                         
                         (let loop ((instr (nn-recv rep)))
                           (print "received " instr ", running \"" script " " instr "\"")
                           (system (conc script " '" instr "'"))
                           (nn-send rep "ok")
                           (loop (nn-recv rep))))
		       (print "ERROR: Port " portnum " already in use. Try another port")))))))
      ((gather) ;; gather all area db's into /tmp/$USER_megatest/alldbs
      ((gatherdb) ;; gather all area db's into /tmp/$USER_megatest/alldbs
       (let* ((mtconfdat (simple-setup (args:get-arg "-start-dir")))
              (mtconf    (car mtconfdat))
              (areas     (get-area-names mtconf)))
         (print "areas: " areas)))
      
              ;; (areas     (get-area-names mtconf))
              (areas     (configf:section->val-alist mtconf "areas")))
         (for-each
          (lambda (area)
            (let* ((area-name (car area))
                   (area-info (cdr area))
                   (area-path (alist-ref 'path area-info)))
              (print "Area: " area)
              (print "   path: " area-path)))
          areas)))
            
      (else
       (let ((all-actions (sort (map conc (delete-duplicates (append *other-actions* (map car *action-keys*)))) string<=?)))
	 (print "unrecognised action: \"" *action* "\", try one of; \"" (string-intersperse all-actions "\", \"") "\"")))
      
      )) ;; the end
︙

This page was generated in about 0.138s by Fossil 2.24 [452f8e1ffc] 2024-03-26 15:36:00