Megatest

Check-in [8c1d89ef36]
Login
Overview
Comment:main.db starting again.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | v1.6584-ck5
Files: files | file ages | folders
SHA1: 8c1d89ef36bf7cbdc1046889e1ae362773f7281d
User & Date: matt on 2021-05-18 22:20:04
Other Links: branch diff | manifest | tags
Context
2021-05-20
05:41
wip check-in: db05dadd93 user: matt tags: v1.6584-ck5
2021-05-18
22:20
main.db starting again. check-in: 8c1d89ef36 user: matt tags: v1.6584-ck5
00:01
wip check-in: e3fed709f0 user: matt tags: v1.6584-ck5
Changes

Modified Makefile from [026e510d16] to [e7f5161936].

49
50
51
52
53
54
55
56

57
58
59
60
61
62
63
49
50
51
52
53
54
55

56
57
58
59
60
61
62
63







-
+







# compiled import files
MOIMPFILES = $(MSRCFILES:%.scm=%.import.o)

%.import.o : %.import.scm
	csc $(CSCOPTS) -unit $*.import -c $*.import.scm -o $*.import.o

mofiles/%.o : %.scm
	mkdir -p mofiles
	@mkdir -p mofiles
	csc $(CSCOPTS) -J -c $< -o mofiles/$*.o

# module dependencies
mofiles/apimod.o : mofiles/commonmod.o
mofiles/apimod.o : mofiles/servermod.o
mofiles/apimod.o : mofiles/tasksmod.o
mofiles/archivemod.o : mofiles/launchmod.o

Modified commonmod.scm from [69a8ca9141] to [5348abd36a].

3640
3641
3642
3643
3644
3645
3646
3647

3648
3649
3650
3651
3652
3653
3654

3655
3656
3657
3658
3659
3660
3661
3640
3641
3642
3643
3644
3645
3646

3647
3648
3649
3650
3651
3652
3653

3654
3655
3656
3657
3658
3659
3660
3661







-
+






-
+







    (lambda (p)
      (let loop ((line (read-line p))
                 (result '()))
        (if (eof-object? line)
            (reverse result)
            (loop (read-line p) (cons line result)))))))

;; timeout is hms string: 1h 5m 3s, default is 1 minute
;; timeout is hms string: 1h 5m 3s, default is 10 minutes
;;
(define (server:expiration-timeout)
  (let ((tmo (configf:lookup *configdat* "server" "timeout")))
    (if (and (string? tmo)
	     (common:hms-string->seconds tmo)) ;; BUG: hms-string->seconds is broken, if given "10" returns 0. Also, it doesn't belong in this logic unless the string->number is changed below
        (* 3600 (string->number tmo))
	60)))
	600))) ;; default is ten minutes

(define (runs:get-mt-env-alist run-id runname target testname itempath)
  ;;(bb-check-path msg: "runs:set-megatest-env-vars entry")
  `(("MT_TEST_NAME"     . ,testname)
    
    ("MT_ITEMPATH"      . ,itempath)

Modified fullrununit.sh from [a13af07ac4] to [3ffa0b3716].

1
2
3
4

5
6
1
2
3

4
5
6



-
+


#!/bin/bash

(killall mtest -v;sleep 1;killall mtest -v -9;rm tests/simplerun/logs/*;rm tests/basicserver.log) &
ck5 make install &&
ck5 make -j install &&
wait  &&
ck5 make unit

Modified megatest.scm from [496e14dd89] to [e1591c4c2e].

1139
1140
1141
1142
1143
1144
1145
1146

1147
1148
1149
1150
1151
1152
1153
1139
1140
1141
1142
1143
1144
1145

1146
1147
1148
1149
1150
1151
1152
1153







-
+







     ;; Server? Start up here.
     ;;
     (if (args:get-arg "-server")
	 (if  (not (args:get-arg "-db"))
	      (debug:print 0 *default-log-port* "ERROR: -db required to start server")
	      (let ((tl        (launch:setup))
		    (dbname    (args:get-arg "-db"))) ;; transport-type (string->symbol (or (args:get-arg "-transport") "http"))))
		(rmt:launch dbname)
		(rmt:server-launch dbname)
		(set! *didsomething* #t))))
	 
     ;; The adjutant is a bit different, it does NOT run (launch:setup) as it is not necessarily tied to
     ;; a specific Megatest area. Detail are being hashed out and this may change.
     ;;
     (if (args:get-arg "-adjutant")
         (begin

Modified rmtmod.scm from [5a65f46de3] to [0b46126861].

215
216
217
218
219
220
221
222
223
224



225
226
227
228
229
230
231
232
233
234
235

236
237
238
239
240
241
242
215
216
217
218
219
220
221



222
223
224
225
226
227
228
229
230
231
232
233
234

235
236
237
238
239
240
241
242







-
-
-
+
+
+










-
+







	     (< (current-seconds) (rmt:conn-expires conn)))
	conn
	#f)))

(define (rmt:find-main-server apath dbname)
  (let* ((pktsdir     (get-pkts-dir apath))
	 (all-srvpkts (get-all-server-pkts pktsdir *srvpktspec*))
	 (dbpath      (conc apath "/" dbname))
	 (viable-srvs (get-viable-servers all-srvpkts dbpath)))
    (get-the-server viable-srvs)))
	 ;; (dbpath      (conc apath "/" dbname))
	 (viable-srvs (get-viable-servers all-srvpkts dbname)))
    (get-the-server apath viable-srvs)))

;; looks for a connection to main
;; connections for other servers happens by requesting from main
;;
(define (rmt:open-main-connection remote apath)
  (let* ((dbname         (db:run-id->dbname #f))
	 (the-srv        (rmt:find-main-server apath dbname))
	 (start-main-srv (lambda ()
			   ;; srv not ready, delay a little and try again
			   (api:run-server-process apath dbname)
			   (thread-sleep! 2)
			   (thread-sleep! 4)
			   (rmt:open-main-connection remote apath) ;; TODO: Add limit to number of tries
			   )))
    (if the-srv ;; yes, we have a server, now try connecting to it
	(let* ((srv-addr (server-address the-srv))
	       (ipaddr   (alist-ref 'ipaddr the-srv))
	       (port     (alist-ref 'port   the-srv))
	       (fullpath (db:dbname->path apath dbname))
317
318
319
320
321
322
323
324




325
326
327
328
329
330
331
317
318
319
320
321
322
323

324
325
326
327
328
329
330
331
332
333
334







-
+
+
+
+







	      res))
	;; no conn yet, start it up
	(begin
	  (rmt:general-open-connection remote apath dbname)
	  (rmt:send-receive-real remote apath dbname rid cmd params)))))

;; db is at apath/.db/dbname, rid is an intermediary solution and will be removed
;; sometime in the future
;; sometime in the future.
;;
;; Purpose - call the main.db server and request a server be started
;; for the given area path and dbname
;;
(define (rmt:send-receive-server-start remote apath dbname)
  (let* ((conn (rmt:get-connection remote apath dbname)))
    (assert conn "FATAL: Unable to connect to db "apath"/"dbname)
    (let* (;; (host    (rmt:conn-ipaddr conn))
	   ;; (port    (rmt:conn-port   conn))
	   ;; (payload (sexpr->string params))
2164
2165
2166
2167
2168
2169
2170
2171

2172
2173
2174
2175
2176
2177
2178
2179
2180

2181
2182
2183
2184
2185
2186
2187
2167
2168
2169
2170
2171
2172
2173

2174
2175
2176
2177
2178
2179
2180
2181
2182

2183
2184
2185
2186
2187
2188
2189
2190







-
+








-
+







	  (loop (cdr tail)
		(if (equal? dbpath (alist-ref 'dbpath spkt))
		    (cons spkt res)
		    res))))))

;; from viable servers get one that is alive and ready
;;
(define (get-the-server serv-pkts)
(define (get-the-server apath serv-pkts)
  (let loop ((tail serv-pkts))
    (if (null? tail)
	#f
	(let* ((spkt  (car tail))
	       (host  (alist-ref 'ipaddr spkt))
	       (port  (alist-ref 'port spkt))
	       (dbpth (alist-ref 'dbpath spkt))
	       (addr  (server-address spkt)))
	  (if (server-ready? host port dbpth)
	  (if (server-ready? host port (conc apath"/"dbpth))
	      spkt
	      (loop (cdr tail)))))))

;; am I the "first" in line server? I.e. my D card is smallest
;; use Z card as tie breaker
;;
(define (get-best-candidate serv-pkts dbpath)
2297
2298
2299
2300
2301
2302
2303
2304

2305
2306
2307
2308


2309
2310
2311
2312
2313

2314
2315
2316
2317
2318
2319
2320
2300
2301
2302
2303
2304
2305
2306

2307
2308
2309


2310
2311
2312
2313
2314
2315

2316
2317
2318
2319
2320
2321
2322
2323







-
+


-
-
+
+




-
+







	     (tries 0))
    ;; first we verify port and interface, update *server-info* in need be.
    (cond
     ((> tries num-tries-allowed)
      (debug:print 0 *default-log-port* "http-transport:keep-running, giving up after trying for several minutes.")
      (exit 1))
     ((not *server-info*)
      (thread-sleep! 1.5)
      (thread-sleep! 0.25)
      (loop *server-info* (+ tries 1)))
     ((not sdat)
      (debug:print 0 *default-log-port* "http-transport:keep-running, impossible, should never get here.")
      (thread-sleep! 1.5)
      (debug:print 0 *default-log-port* "http-transport:keep-running, still no interface, tries="tries)
      (thread-sleep! 0.25)
      (loop *server-info* (+ tries 1)))
     ((or (not (equal? (servdat-host sdat)(servdat-host *server-info*)))
	  (not (equal? (servdat-port sdat)(servdat-port *server-info*))))
      (debug:print-info 0 *default-log-port* "WARNING: interface changed, refreshing iface and port info")
      (thread-sleep! 1.5)
      (thread-sleep! 0.25)
      (loop *server-info* (+ tries 1)))
     (else
      (if (not *server-id*)(set! *server-id* (server:mk-signature)))
      (debug:print 0 *default-log-port*
		   "SERVER STARTED: " (servdat-host *server-info*)
		   ":" (servdat-port *server-info*)
		   " AT " (current-seconds) " server-id: " *server-id*)
2439
2440
2441
2442
2443
2444
2445
2446

2447
2448
2449
2450
2451
2452
2453
2442
2443
2444
2445
2446
2447
2448

2449
2450
2451
2452
2453
2454
2455
2456







-
+








;; Call this to start the actual server
;;
;; all routes though here end in exit ...
;;
;; This is the point at which servers are started
;;
(define (rmt:launch dbname)
(define (rmt:server-launch dbname)
  ;;(let* ((tmp-area            (common:get-db-tmp-area))
  ;;       (server-start        (conc tmp-area "/.server-start"))
  ;;       (server-started      (conc tmp-area "/.server-started"))
  ;;       (start-time          (common:lazy-modification-time server-start))
  ;;       (started-time        (common:lazy-modification-time server-started))
  ;;       (server-starting     (< start-time started-time)) ;; if start-time is less than started-time then a server is still starting
  ;;       (start-time-old      (> (- (current-seconds) start-time) 5))

Modified tests/unittests/basicserver.scm from [fb0b1abb4c] to [28c3a719a7].

25
26
27
28
29
30
31








32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58



59
60
61
62
63
64
65







+
+
+
+
+
+
+
+



















-
-
-







(import rmtmod trace http-client apimod dbmod)
(trace-call-sites #t)
(trace
 ;; db:get-dbdat
 ;; rmt:find-main-server
 ;; rmt:send-receive-real
 ;; sexpr->string
;; server-ready?
;; rmt:register-server
;; rmt:open-main-connection
;; rmt:find-main-server
;; get-all-server-pkts
;; get-viable-servers
;; get-best-candidate
;; api:run-server-process
 )

(test #f #t (rmt:remote? (let ((r (make-rmt:remote)))
			   (set! *rmt:remote* r)
			   r)))
(test #f #f (rmt:get-connection *rmt:remote* *toppath* ".db/main.db"))
(test #f #f (rmt:find-main-server *toppath* ".db/main.db"))
(test #f #t (rmt:open-main-connection *rmt:remote* *toppath*))
(pp (hash-table->alist (rmt:remote-conns *rmt:remote*)))
(test #f #t (rmt:conn? (rmt:get-connection *rmt:remote* *toppath* ".db/main.db")))

(define *main*  (rmt:get-connection *rmt:remote* *toppath* ".db/main.db"))

(for-each (lambda (tdat)
	    (test #f tdat (loop-test (rmt:conn-ipaddr *main*)
				     (rmt:conn-port *main*) tdat)))
	  (list 'a
		'(a "b" 123 1.23 )))
(test #f #t (number? (rmt:send-receive 'ping #f 'hello)))
(trace
 rmt:register-server
 )

(define *db* (db:setup #f))
(test #f 'server-started (api:execute-requests *db* 'get-server (list *toppath* ".db/1.db")))
(set! *dbstruct-db* #f)
(test #f 'server-started (rmt:general-open-connection *rmt:remote* *toppath* ".db/1.db"))
(thread-sleep! 2)
(test #f 'server-started (rmt:general-open-connection *rmt:remote* *toppath* ".db/1.db"))