Changes In Branch v2.0001 Through [49f0afc304] Excluding Merge-Ins
This is equivalent to a diff from e80ba42389 to 49f0afc304
2022-01-08
| ||
20:46 | Switch to using threads instead of mailbox for worker calls. Seems to not block as much. check-in: ba5884c651 user: matt tags: v2.0001 | |
2022-01-07
| ||
17:11 | wip. not much improvement... check-in: 49f0afc304 user: matt tags: v2.0001 | |
2022-01-06
| ||
18:18 | Ensure api calls to db do NOT occur on non-server processes. check-in: 7696fcfff8 user: matt tags: v2.0001 | |
2021-12-26
| ||
20:14 | Attempt to switch from nng to ulex check-in: 6c52169286 user: matt tags: v2.0001 | |
2021-12-23
| ||
20:16 | get host name automatically in test Leaf check-in: e80ba42389 user: matt tags: v2.001, Uses-nng-nanomsg | |
19:36 | Fixed typo, added a bit to the test check-in: a0652877c1 user: matt tags: v2.001 | |
Modified Makefile from [89e357068d] to [a7d80e1a40].
︙ | ︙ | |||
25 26 27 28 29 30 31 | # all : $(PREFIX)/bin/.$(ARCHSTR) mtest dboard mtut ndboard all : $(PREFIX)/bin/.$(ARCHSTR) mtest # add dboard mtut and tcmt back later # module source files MSRCFILES = autoload.scm dbi.scm ducttape-lib.scm pkts.scm stml2.scm \ | | | 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 | # all : $(PREFIX)/bin/.$(ARCHSTR) mtest dboard mtut ndboard all : $(PREFIX)/bin/.$(ARCHSTR) mtest # add dboard mtut and tcmt back later # module source files MSRCFILES = autoload.scm dbi.scm ducttape-lib.scm pkts.scm stml2.scm \ cookie.scm mutils.scm mtargs.scm apimod.scm ulex.scm \ configfmod.scm commonmod.scm dbmod.scm rmtmod.scm \ debugprint.scm mtver.scm csv-xml.scm servermod.scm \ hostinfo.scm adjutant.scm processmod.scm testsmod.scm \ itemsmod.scm keysmod.scm mtmod.scm rmtmod.scm \ tasksmod.scm pgdb.scm launchmod.scm runsmod.scm \ portloggermod.scm archivemod.scm ezstepsmod.scm \ subrunmod.scm bigmod.scm testsmod.scm |
︙ | ︙ | |||
97 98 99 100 101 102 103 | mofiles/launchmod.o : mofiles/bigmod.o mofiles/launchmod.o : mofiles/ezstepsmod.o mofiles/launchmod.o : mofiles/rmtmod.o mofiles/servermod.o mofiles/mtmod.o : mofiles/debugprint.o mofiles/portloggermod.o : mofiles/tasksmod.o mofiles/rmtmod.o : mofiles/apimod.o mofiles/rmtmod.o : mofiles/commonmod.o mofiles/portloggermod.o | | > > > > | 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 | mofiles/launchmod.o : mofiles/bigmod.o mofiles/launchmod.o : mofiles/ezstepsmod.o mofiles/launchmod.o : mofiles/rmtmod.o mofiles/servermod.o mofiles/mtmod.o : mofiles/debugprint.o mofiles/portloggermod.o : mofiles/tasksmod.o mofiles/rmtmod.o : mofiles/apimod.o mofiles/rmtmod.o : mofiles/commonmod.o mofiles/portloggermod.o mofiles/rmtmod.o : mofiles/itemsmod.o mofiles/ulex.o mofiles/runsmod.o : mofiles/rmtmod.o mofiles/archivemod.o mofiles/servermod.o : mofiles/commonmod.o mofiles/stml2.o : mofiles/cookie.o mofiles/dbi.o mofiles/tasksmod.o : mofiles/pgdb.o mofiles/dbmod.o mofiles/testsmod.o : mofiles/commonmod.o mofiles/testsmod.o : mofiles/itemsmod.o mofiles/rmtmod.o mofiles/tasksmod.o # split modules. Note: we can switch between ulex and ulex simple. mofiles/ulex.o : ulex/ulex.scm ulex-simple/ulex.scm dashboard.o megatest.o : db_records.scm megatest-fossil-hash.scm ADTLSCR=mt_laststep mt_runstep mt_ezstep HELPERS=$(addprefix $(PREFIX)/bin/,$(ADTLSCR)) DEPLOYHELPERS=$(addprefix deploytarg/,$(ADTLSCR)) MTESTHASH=$(shell fossil info|grep checkout:| awk '{print $$2}') ifeq ($(MTESTHASH),) |
︙ | ︙ |
Modified TODO from [1f8790aebc] to [8c7ba74ce3].
︙ | ︙ | |||
16 17 18 19 20 21 22 23 24 25 26 27 28 29 | # along with Megatest. If not, see <http://www.gnu.org/licenses/>. TODO ==== Loose ends ---------- . -list-servers not correct . move *remotedat* into bigdata . add back server stats on exit (look in rmt:run in rmtmod.scm) WW15 | > > > | 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 | # along with Megatest. If not, see <http://www.gnu.org/licenses/>. TODO ==== Loose ends ---------- 15:09:29 error in calling find-and-mark-incomplete for run-id 5, exn=#<condition: (exn type)> might be related to initial conditions in the db. (no run entry in runs table?). . -list-servers not correct . move *remotedat* into bigdata . add back server stats on exit (look in rmt:run in rmtmod.scm) WW15 |
︙ | ︙ |
Modified apimod.scm from [f47a08f057] to [97d3b608d8].
︙ | ︙ | |||
27 28 29 30 31 32 33 | (module apimod ( api:run-server-process api:start-server api:dispatch-cmd api:execute-requests | | | 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 | (module apimod ( api:run-server-process api:start-server api:dispatch-cmd api:execute-requests ;; api:process-request ) (import scheme chicken.base chicken.process-context.posix chicken.string chicken.time |
︙ | ︙ | |||
372 373 374 375 376 377 378 | ((have-incompletes?) (apply db:have-incompletes? dbstruct params)) ((login) (apply db:login dbstruct params)) ((general-call) (let ((stmtname (car params)) (run-id (cadr params)) (realparams (cddr params))) (db:general-call dbstruct stmtname run-id realparams))) ((sdb-qry) (apply sdb:qry params)) | | | 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 | ((have-incompletes?) (apply db:have-incompletes? dbstruct params)) ((login) (apply db:login dbstruct params)) ((general-call) (let ((stmtname (car params)) (run-id (cadr params)) (realparams (cddr params))) (db:general-call dbstruct stmtname run-id realparams))) ((sdb-qry) (apply sdb:qry params)) ((ping) `(#t ,(current-process-id) ,(cadr params))) ;; (current-process-id)) ((get-changed-record-ids) (apply db:get-changed-record-ids dbstruct params)) ((get-run-record-ids) (apply db:get-run-record-ids dbstruct params)) ;; TESTMETA ((testmeta-get-record) (apply db:testmeta-get-record dbstruct params)) ;; TASKS ((find-task-queue-records) (apply tasks:find-task-queue-records dbstruct params)) |
︙ | ︙ | |||
418 419 420 421 422 423 424 | ;; http-server send-response ;; api:process-request ;; db:* ;; ;; NB// Runs on the server as part of the server loop ;; | | | 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 | ;; http-server send-response ;; api:process-request ;; db:* ;; ;; NB// Runs on the server as part of the server loop ;; #;(define (api:process-request dbstruct indat) ;; the $ is the request vars proc (let* ((cmd-in (alist-ref 'cmd indat)) ;; ($ 'cmd)) (cmd (if (string? cmd-in)(string->symbol cmd-in) cmd-in)) (params (alist-ref 'params indat)) (key (alist-ref 'key indat)) ;; TODO - add this back ;; (doprint (apply common:low-noise-print 10 params)) ) ;; (if doprint (debug:print 0 *default-log-port* "cmd: " cmd " with params: " params ", key: " key)) |
︙ | ︙ |
Modified build-assist/README from [d108fc0f5b] to [ac3067734e].
|
| < | < < < < | < | < | < < | < < | < | < < < | < | 1 2 3 4 5 6 7 8 9 10 11 12 | README for IUP IUP is a portable toolkit for building graphical user interfaces. It offers a configuration API in three basic languages: C, Lua and LED. IUP's purpose is to allow a program to be executed in different systems without any modification, therefore it is highly portable. Its main advantages are: * high performance, due to the fact that it uses native interface elements. * fast learning by the user, due to the simplicity of its API. Build instructions and usage are available in the IUP documentation. For complete information, visit IUP's web site at http://www.tecgraf.puc-rio.br/iup or access its documentation in the HTML folder. (end of README) |
Modified build-assist/ck5-eggs.list from [339d250117] to [da39a3ee5e].
|
| < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < |
Added build-assist/ck5-full-eggs.list version [3bac908752].
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 | csm address-info ansi-escape-sequences apropos base64 breadline crypt csv-abnf directory-utils dot-locking filepath fmt format http-client itemsmod json linenoise mailbox md5 message-digest nanomsg postgresql queues regex regex-case rfc3339 s11n sha1 simple-exceptions slice sparse-vectors spiffy spiffy-directory-listing spiffy-request-vars sql-de-lite sqlite3 sql-null srfi-1 srfi-13 srfi-19 sxml-modifications sxml-serializer sxml-transforms system-information tcp6 test typed-records uri-common z3 |
Modified build-assist/iup-compile.sh from [f2fe37f920] to [8fc93bba73].
︙ | ︙ | |||
13 14 15 16 17 18 19 | echo " rsync -av include/ $PREFIX/include/" echo " or (depending on versions and what you see in the iup tars - they seem to vary" echo " cp *.a *.so $PREFIX/lib" echo " cp include/*.h $PREFIX/include" echo " 4. run the chicken-install like this:" echo "If you use a wrapper (e.g. ck5) to create the chicken environment:" | | | < | 13 14 15 16 17 18 19 20 21 22 23 24 25 | echo " rsync -av include/ $PREFIX/include/" echo " or (depending on versions and what you see in the iup tars - they seem to vary" echo " cp *.a *.so $PREFIX/lib" echo " cp include/*.h $PREFIX/include" echo " 4. run the chicken-install like this:" echo "If you use a wrapper (e.g. ck5) to create the chicken environment:" echo "CSC_OPTIONS=\"-I$PREFIX/include -I$PREFIX/include/im -I$PREFIX/include/cd -I$PREFIX/include/iup -L$PREFIX/lib -C -std=gnu99\" ck5 chicken-install iup canvas-draw -feature disable-iup-matrixex" echo "else:" echo "CSC_OPTIONS=\"-I$PREFIX/include -I$PREFIX/include/im -I$PREFIX/include/cd -I$PREFIX/include/iup -L$PREFIX/lib -C -std=gnu99\" chicken-install iup canvas-draw" # (export PREFIX=/home/matt/data/buildall/ck5.2;CSC_OPTIONS="-I/home/matt/data/buildall/ck5.2/include -I/home/matt/data/buildall/ck5.2/include/im -I/home/matt/data/buildall/ck5.2/include/cd -I/home/matt/data/buildall/ck5.2/include/iup -L/home/matt/data/buildall/ck5.2/lib -C -std=gnu99" ck5 chicken-install iup -feature disable-iup-matrixex) |
Modified commonmod.scm from [787a13f0a3] to [9a2ccbe33e].
︙ | ︙ | |||
384 385 386 387 388 389 390 391 392 393 394 | *task-db* *db-access-allowed* *db-access-mutex* *db-transaction-mutex* *db-cache-path* *db-with-db-mutex* *db-api-call-time* *no-sync-db* *my-signature* *transport-type* *logged-in-clients* | > < | 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 | *task-db* *db-access-allowed* *db-access-mutex* *db-transaction-mutex* *db-cache-path* *db-with-db-mutex* *db-api-call-time* *didsomething* *no-sync-db* *my-signature* *transport-type* *logged-in-clients* *server-run* *run-id* *server-kind-run* *home-host* *heartbeat-mutex* *api-process-request-count* *max-api-process-requests* |
︙ | ︙ | |||
923 924 925 926 927 928 929 930 931 932 933 934 935 936 | ;; (define *watchdog* #f) ;; A hash table that can be accessed by #{scheme ...} calls in ;; config files. Allows communicating between confgs ;; (define *user-hash-data* (make-hash-table)) (define *db-keys* #f) (define *pkts-info* (make-hash-table)) ;; store stuff like the last parent here (define *configinfo* #f) ;; raw results from setup, includes toppath and table from megatest.config (define *runconfigdat* #f) ;; run configs data (define *configdat* #f) ;; megatest.config data ==> moved to configfmod (define *configstatus* #f) ;; status of data; 'fulldata : all processing done, #f : no data yet, 'partialdata : partial read done (define *toppath* #f) | > | 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 | ;; (define *watchdog* #f) ;; A hash table that can be accessed by #{scheme ...} calls in ;; config files. Allows communicating between confgs ;; (define *user-hash-data* (make-hash-table)) (define *db-keys* #f) (define *didsomething* #f) (define *pkts-info* (make-hash-table)) ;; store stuff like the last parent here (define *configinfo* #f) ;; raw results from setup, includes toppath and table from megatest.config (define *runconfigdat* #f) ;; run configs data (define *configdat* #f) ;; megatest.config data ==> moved to configfmod (define *configstatus* #f) ;; status of data; 'fulldata : all processing done, #f : no data yet, 'partialdata : partial read done (define *toppath* #f) |
︙ | ︙ | |||
972 973 974 975 976 977 978 | (define *my-signature* #f) (define *transport-type* 'http) ;; override with [server] transport http|rpc|nmsg ;; replaced by *rmt:remote* ;; (define *runremote* #f) ;; if set up for server communication this will hold <host port> ;; (define *max-cache-size* 0) (define *logged-in-clients* (make-hash-table)) ;; (define *server-id* #f) | < < | 973 974 975 976 977 978 979 980 981 982 983 984 985 986 | (define *my-signature* #f) (define *transport-type* 'http) ;; override with [server] transport http|rpc|nmsg ;; replaced by *rmt:remote* ;; (define *runremote* #f) ;; if set up for server communication this will hold <host port> ;; (define *max-cache-size* 0) (define *logged-in-clients* (make-hash-table)) ;; (define *server-id* #f) (define *server-run* #t) (define *run-id* #f) (define *server-kind-run* (make-hash-table)) (define *home-host* #f) ;; (define *total-non-write-delay* 0) (define *heartbeat-mutex* (make-mutex)) (define *api-process-request-count* 0) |
︙ | ︙ |
Modified dashboard.scm from [feba132a9c] to [cc72246a09].
︙ | ︙ | |||
96 97 98 99 100 101 102 103 104 105 106 107 108 109 | srfi-1 regex regex-case srfi-69 typed-records sparse-vectors format srfi-4 srfi-14 ) ;; (include "common_records.scm") ;; (include "db_records.scm") ;; (include "run_records.scm") ;; (include "task_records.scm") ;; (include "megatest-version.scm") | > | 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 | srfi-1 regex regex-case srfi-69 typed-records sparse-vectors format srfi-4 srfi-14 srfi-18 ) ;; (include "common_records.scm") ;; (include "db_records.scm") ;; (include "run_records.scm") ;; (include "task_records.scm") ;; (include "megatest-version.scm") |
︙ | ︙ | |||
3752 3753 3754 3755 3756 3757 3758 | (import dashboard) ;; ease debugging by loading ~/.dashboardrc (let ((debugcontrolf (get-debugcontrolf))) (if debugcontrolf (load debugcontrolf))) | > | > > > > | 3753 3754 3755 3756 3757 3758 3759 3760 3761 3762 3763 3764 3765 3766 | (import dashboard) ;; ease debugging by loading ~/.dashboardrc (let ((debugcontrolf (get-debugcontrolf))) (if debugcontrolf (load debugcontrolf))) (import srfi-18) (thread-join! (thread-start! (make-thread main "main"))) |
Modified dbmod.scm from [f6dcc2b111] to [4d2069b432].
︙ | ︙ | |||
3443 3444 3445 3446 3447 3448 3449 | ;; "SELECT count(id) FROM tests WHERE state in ('RUNNING','LAUNCHED','REMOTEHOSTSTART') AND run_id NOT IN (SELECT id FROM runs WHERE state='deleted') AND NOT (uname = 'n/a' AND item_path = '');") "SELECT count(id) FROM tests WHERE state in ('RUNNING','REMOTEHOSTSTART','LAUNCHED') AND run_id=?;" run-id)))) ;; NOT IN (SELECT id FROM runs WHERE state='deleted');") ;; NEW BEHAVIOR: Look only at single run with run-id ;; ;; (define (db:get-running-stats dbstruct run-id) | | < < | | 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452 3453 3454 3455 3456 3457 3458 | ;; "SELECT count(id) FROM tests WHERE state in ('RUNNING','LAUNCHED','REMOTEHOSTSTART') AND run_id NOT IN (SELECT id FROM runs WHERE state='deleted') AND NOT (uname = 'n/a' AND item_path = '');") "SELECT count(id) FROM tests WHERE state in ('RUNNING','REMOTEHOSTSTART','LAUNCHED') AND run_id=?;" run-id)))) ;; NOT IN (SELECT id FROM runs WHERE state='deleted');") ;; NEW BEHAVIOR: Look only at single run with run-id ;; ;; (define (db:get-running-stats dbstruct run-id) (define (db:get-count-tests-running-for-run-id dbstruct run-id) (let* ((qry "SELECT count(id) FROM tests WHERE state in ('RUNNING','LAUNCHED','REMOTEHOSTSTART') AND run_id=?;")) ;; ) (db:with-db dbstruct run-id #f (lambda (db) (let* ((stmth (db:get-cache-stmth dbstruct db qry))) (sqlite3:first-result stmth run-id)))))) |
︙ | ︙ |
Modified debugprint.scm from [2f3aa7f0ad] to [e5877ebcfd].
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 | (declare (unit debugprint)) (declare (uses mtargs)) (module debugprint * ;;(import scheme chicken data-structures extras files ports) (import scheme chicken.base chicken.string chicken.port chicken.process-context chicken.process-context.posix (prefix mtargs args:) srfi-1 system-information ) ;;====================================================================== ;; debug stuff ;;====================================================================== (define verbosity (make-parameter '())) (define *default-log-port* (current-error-port)) | > > | | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 | (declare (unit debugprint)) (declare (uses mtargs)) (module debugprint * ;;(import scheme chicken data-structures extras files ports) (import scheme chicken.base chicken.string chicken.time chicken.time.posix chicken.port chicken.process-context chicken.process-context.posix (prefix mtargs args:) srfi-1 system-information ) ;;====================================================================== ;; debug stuff ;;====================================================================== (define verbosity (make-parameter '())) (define *default-log-port* (current-error-port)) (define debug:print-logger (make-parameter #f)) ;; set to a proc to call on every logging print (define (debug:setup) (let ((debugstr (or (args:get-arg "-debug") (args:get-arg "-debug-noprop") (get-environment-variable "MT_DEBUG_MODE")))) (verbosity (debug:calc-verbosity debugstr 'q)) (debug:check-verbosity (verbosity) debugstr) |
︙ | ︙ | |||
107 108 109 110 111 112 113 | (define (debug:handle-remote-logging params) (if (debug:print-logger) ;; NOTE: turn params into string to keep things simple for now ((debug:print-logger)(conc "REMOTE ("(get-host-name)", pid="(current-process-id)") " (string-intersperse (map conc params) " ") "; " (string-intersperse (command-line-arguments) " "))))) | > > > > > > > > | | | | | | | 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 | (define (debug:handle-remote-logging params) (if (debug:print-logger) ;; NOTE: turn params into string to keep things simple for now ((debug:print-logger)(conc "REMOTE ("(get-host-name)", pid="(current-process-id)") " (string-intersperse (map conc params) " ") "; " (string-intersperse (command-line-arguments) " "))))) (define debug:enable-timestamp (make-parameter #t)) (define (debug:timestamp) (if (debug:enable-timestamp) (conc (time->string (seconds->local-time (current-seconds)) "%H:%M:%S") " ") "")) (define (debug:print n e . params) (if (debug:debug-mode n) (with-output-to-port (or e (current-error-port)) (lambda () ;; (if *logging* ;; (db:log-event (apply conc params)) (apply print (debug:timestamp) params) (debug:handle-remote-logging params) ))) #t ;; only here to make remote stuff happy. It'd be nice to fix that ... ) (define (debug:print-error n e . params) ;; normal print (if (debug:debug-mode n) (with-output-to-port (if (port? e) e (current-error-port)) (lambda () (apply print "ERROR: " (debug:timestamp) params) (debug:handle-remote-logging (cons "ERROR: " params)) ))) ;; pass important messages to stderr (if (and (eq? n 0)(not (eq? e (current-error-port)))) (with-output-to-port (current-error-port) (lambda () (apply print "ERROR: " (debug:timestamp) params) )))) (define (debug:print-info n e . params) (if (debug:debug-mode n) (with-output-to-port (if (port? e) e (current-error-port)) (lambda () (apply print "INFO: (" n ") "(debug:timestamp) params) ;; res) (debug:handle-remote-logging (cons "INFO: " params)) )))) (define (debug:print-warn n e . params) (if (debug:debug-mode n) (with-output-to-port (if (port? e) e (current-error-port)) (lambda () (apply print "WARN: (" n ") " (debug:timestamp) params) ;; res) (debug:handle-remote-logging (cons "WARN: " params)) )))) ) |
Modified megatest.scm from [c1c08ee586] to [be96ddd230].
︙ | ︙ | |||
159 160 161 162 163 164 165 | tasksmod testsmod ) ;; fake out readline usage of toplevel-command (define (toplevel-command . a) #f) | | | 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 | tasksmod testsmod ) ;; fake out readline usage of toplevel-command (define (toplevel-command . a) #f) (define *didsomething* #f) (define *db* #f) ;; this is only for the repl, do not use in general!!!! ;; (include "common_records.scm") ;; (include "key_records.scm") ;; (include "db_records.scm") (include "run_records.scm") ;; (include "test_records.scm") |
︙ | ︙ | |||
317 318 319 320 321 322 323 | newlogf) logpath-in))) (if (not (directory-exists? log-dir)) (system (conc "mkdir -p " log-dir))) (open-output-file logpath)) (exn () (debug:print-error 0 *default-log-port* "Could not open log file for write: "logpath-in) | | | 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 | newlogf) logpath-in))) (if (not (directory-exists? log-dir)) (system (conc "mkdir -p " log-dir))) (open-output-file logpath)) (exn () (debug:print-error 0 *default-log-port* "Could not open log file for write: "logpath-in) (set! *didsomething* #t) (exit 1)))) ;; Disabled help items ;; -rollup : (currently disabled) fill run (set by :runname) with latest test(s) ;; from prior runs with same keys ;; -daemonize : fork into background and disconnect from stdin/out |
︙ | ︙ | |||
501 502 503 504 505 506 507 | Called as " (string-intersperse (argv) " ") " Version " megatest-version ", built from " megatest-fossil-hash )) (define (main) (make-and-init-bigdata) | < < < < < < < < | 501 502 503 504 505 506 507 508 509 510 511 512 513 514 | Called as " (string-intersperse (argv) " ") " Version " megatest-version ", built from " megatest-fossil-hash )) (define (main) (make-and-init-bigdata) ;; load the ~/.megatestrc file, put (use trace)(trace-call-sites #t)(trace function-you-want-to-trace) in this file ;; (let ((debugcontrolf (conc (get-environment-variable "HOME") "/.megatestrc"))) (if (common:file-exists? debugcontrolf) (load debugcontrolf))) ;; usage logging, careful with this, it is not designed to deal with all real world challenges! |
︙ | ︙ | |||
826 827 828 829 830 831 832 | (exit))) (if (args:get-arg "-version") (begin (print (common:version-signature)) ;; (print megatest-version) (exit))) | < < | 818 819 820 821 822 823 824 825 826 827 828 829 830 831 | (exit))) (if (args:get-arg "-version") (begin (print (common:version-signature)) ;; (print megatest-version) (exit))) ;; Overall exit handling setup immediately ;; (if (or (args:get-arg "-process-reap")) ;; (args:get-arg "-runtests") ;; (args:get-arg "-execute") ;; (args:get-arg "-remove-runs") ;; (args:get-arg "-runstep")) |
︙ | ︙ | |||
1065 1066 1067 1068 1069 1070 1071 | sheetname sectionname varname val))) (sqlite3:finalize! db))) (else (pp data)))))) (if out-file (close-output-port out-port)) (exit) ;; yes, bending the rules here - need to exit since this is a utility )) | | > > | | | 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 | sheetname sectionname varname val))) (sqlite3:finalize! db))) (else (pp data)))))) (if out-file (close-output-port out-port)) (exit) ;; yes, bending the rules here - need to exit since this is a utility )) ;; disabled for now #;(if (args:get-arg "-ping") (let* ((server-id (string->number (args:get-arg "-ping"))) ;; extract run-id (i.e. no ":" (host:port (args:get-arg "-ping"))) (server-ready? (or server-id host:port) #f do-exit: #t))) ;;====================================================================== ;; Capture, save and manipulate environments ;;====================================================================== ;; NOTE: Keep these above the section where the server or client code is setup |
︙ | ︙ | |||
2636 2637 2638 2639 2640 2641 2642 | (bdat-time-to-exit-set! *bdat* #t) ) ;;(debug:print-info 13 *default-log-port* "thread-join! watchdog") ;; join the watchdog thread if it has been thread-start!ed (it may not have been started in the case of a server that never enters running state) ;; (symbols returned by thread-state: created ready running blocked suspended sleeping terminated dead) ;; TODO: for multiple areas, we will have multiple watchdogs; and multiple threads to manage | | | 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 | (bdat-time-to-exit-set! *bdat* #t) ) ;;(debug:print-info 13 *default-log-port* "thread-join! watchdog") ;; join the watchdog thread if it has been thread-start!ed (it may not have been started in the case of a server that never enters running state) ;; (symbols returned by thread-state: created ready running blocked suspended sleeping terminated dead) ;; TODO: for multiple areas, we will have multiple watchdogs; and multiple threads to manage #;(let* ((watchdog (bdat-watchdog *bdat*))) (if (thread? watchdog) (case (thread-state watchdog) ((ready running blocked sleeping terminated dead) (thread-join! watchdog))))) (bdat-time-to-exit-set! *bdat* #t) |
︙ | ︙ | |||
2658 2659 2660 2661 2662 2663 2664 | ((1)(exit 1)) ((2)(exit 2)) (else (exit 3))))) ) ) | | | | > > > > | 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 | ((1)(exit 1)) ((2)(exit 2)) (else (exit 3))))) ) ) (import megatest-main commonmod) (import srfi-18) (thread-join! (thread-start! (make-thread main))) |
Modified pgdb.scm from [33d4965c91] to [d568c66421].
︙ | ︙ | |||
10 11 12 13 14 15 16 | (import scheme chicken.base chicken.condition chicken.string chicken.sort list-utils | | | 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 | (import scheme chicken.base chicken.condition chicken.string chicken.sort list-utils ;; postgresql srfi-1 srfi-69 typed-records (prefix dbi dbi:) configfmod (prefix mtargs args:) |
︙ | ︙ |
Modified rmtmod.scm from [e5bd6bd6c5] to [941de4f2ec].
︙ | ︙ | |||
14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 | ;; GNU General Public License for more details. ;; ;; You should have received a copy of the GNU General Public License ;; along with Megatest. If not, see <http://www.gnu.org/licenses/>. ;;====================================================================== (declare (unit rmtmod)) (declare (uses apimod)) (declare (uses commonmod)) (declare (uses configfmod)) (declare (uses dbmod)) (declare (uses debugprint)) (declare (uses itemsmod)) (declare (uses mtargs)) (declare (uses mtver)) (declare (uses pgdb)) (declare (uses portloggermod)) (declare (uses servermod)) (declare (uses tasksmod)) (module rmtmod * (import scheme chicken.base | > > > > > | 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 | ;; GNU General Public License for more details. ;; ;; You should have received a copy of the GNU General Public License ;; along with Megatest. If not, see <http://www.gnu.org/licenses/>. ;;====================================================================== ;; generate entries for ~/.megatestrc with the following ;; ;; grep define ../rmt.scm | grep rmt: |perl -pi -e 's/\(define\s+\((\S+)\W.*$/\1/'|sort -u (declare (unit rmtmod)) (declare (uses apimod)) (declare (uses commonmod)) (declare (uses configfmod)) (declare (uses dbmod)) (declare (uses debugprint)) (declare (uses itemsmod)) (declare (uses mtargs)) (declare (uses mtver)) (declare (uses pgdb)) (declare (uses portloggermod)) (declare (uses servermod)) (declare (uses tasksmod)) (declare (uses ulex)) (module rmtmod * (import scheme chicken.base |
︙ | ︙ | |||
61 62 63 64 65 66 67 | directory-utils format ;; http-client ;; intarweb matchable md5 message-digest | | | 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 | directory-utils format ;; http-client ;; intarweb matchable md5 message-digest ;; nng ;; nanomsg (prefix base64 base64:) (prefix sqlite3 sqlite3:) regex s11n ;; spiffy ;; spiffy-directory-listing ;; spiffy-request-vars |
︙ | ︙ | |||
94 95 96 97 98 99 100 | pgdb pkts portloggermod (prefix mtargs args:) servermod stml2 tasksmod | | < < < | | < < < < < < < < | > < | | < > > > | < < < < < < < | < < < < | < < < | < > | | | | | < | | > | | < | | | | > > > > > > > > > > > > > > > > > > | < < > | | > | | > > | | | | | > | > > > > | < < < < < < | | | | | | | > > < < | < < < | < < | | < | > | < < < < < | | > | | > | | | < < < | < | > > > > | < < > | < < < < < < < < | 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 | pgdb pkts portloggermod (prefix mtargs args:) servermod stml2 tasksmod ulex ) ;; Configurations for server ;; (tcp-buffer-size 2048) ;; (max-connections 2048) ;; info about me as a listener and my connections to db servers ;; stored (for now) in *db-serv-info* ;; (defstruct servdat (host #f) (port #f) (uuid #f) (dbfile #f) (uconn #f) ;; this is the listener *FOR THIS PROCESS* (mode #f) (status 'starting) (trynum 0) ;; count the number of ports we've tried (conns (make-hash-table)) ;; apath/dbname => conndat ) (define *db-serv-info* (make-servdat)) (define (servdat->url sdat) (conc (servdat-host sdat)":"(servdat-port sdat))) ;; db servers contact info ;; (defstruct conndat (apath #f) (dbname #f) (fullname #f) (hostport #f) (ipaddr #f) (port #f) (srvpkt #f) (srvkey #f) (lastmsg 0) (expires 0)) (define *srvpktspec* `((server (host . h) (port . p) (servkey . k) (pid . i) (ipaddr . a) (dbpath . d)))) ;;====================================================================== ;; S U P P O R T F U N C T I O N S ;;====================================================================== ;; set up the api proc, seems like there should be a better place for this? ;; ;; IS THIS NEEDED ANYMORE? TODO - REMOVE IF POSSIBLE ;; ;; (define api-proc (make-parameter conc)) ;; (api-proc api:execute-requests) ;; do we have a connection to apath dbname and ;; is it not expired? then return it ;; ;; else setup a connection ;; ;; if that fails, return '(#f "some reason") ;; NB// convert to raising an exception ;; (define (rmt:get-conn remdat apath dbname) (let* ((fullname (db:dbname->path apath dbname))) (hash-table-ref/default (servdat-conns remdat) fullname #f))) (define (rmt:drop-conn remdat apath dbname) (let* ((fullname (db:dbname->path apath dbname))) (hash-table-delete! (servdat-conns remdat) fullname))) (define (rmt:find-main-server uconn apath dbname) (let* ((pktsdir (get-pkts-dir apath)) (all-srvpkts (get-all-server-pkts pktsdir *srvpktspec*)) (viable-srvs (get-viable-servers all-srvpkts dbname))) (get-the-server uconn apath viable-srvs))) (define *connstart-mutex* (make-mutex)) (define *last-main-start* 0) ;; looks for a connection to main, returns if have and not exired ;; creates new otherwise ;; ;; connections for other servers happens by requesting from main ;; ;; TODO: This is unnecessarily re-creating the record in the hash table ;; (define (rmt:open-main-connection remdat apath) (let* ((fullpath (db:dbname->path apath ".db/main.db")) (conns (servdat-conns remdat)) (conn (rmt:get-conn remdat apath ".db/main.db")) ;; (hash-table-ref/default conns fullpath #f)) ;; TODO - create call for this (start-rmt:run (lambda () (let* ((th1 (make-thread (lambda ()(rmt:run (get-host-name))) "non-db mode server"))) (thread-start! th1) (thread-sleep! 1) (let loop ((count 0)) (assert (< count 30) "FATAL: responder failed to initialize in rmt:open-main-connection") (if (or (not *db-serv-info*) (not (servdat-uconn *db-serv-info*))) (begin (thread-sleep! 1) (loop (+ count 1))) (begin (servdat-mode-set! *db-serv-info* 'non-db) (servdat-uconn *db-serv-info*))))))) (myconn (servdat-uconn *db-serv-info*))) (cond ((not myconn) (start-rmt:run) (rmt:open-main-connection remdat apath)) ((and conn ;; conn is NOT a socket, just saying ... (< (current-seconds) (conndat-expires conn))) #t) ;; we are current and good to go - we'll deal elsewhere with a server that was killed or died ((and conn (>= (current-seconds)(conndat-expires conn))) (debug:print-info 0 *default-log-port* "connection to "fullpath" server expired. Reconnecting.") (rmt:drop-conn remdat apath ".db/main.db") ;; (rmt:open-main-connection remdat apath)) (else ;; Below we will find or create and connect to main (debug:print-info 0 *default-log-port* "rmt:open-main-connection - starting from scratch") (let* ((dbname (db:run-id->dbname #f)) (the-srv (rmt:find-main-server myconn apath dbname)) (start-main-srv (lambda () ;; call IF there is no the-srv found (mutex-lock! *connstart-mutex*) (if (> (- (current-seconds) *last-main-start*) 5) ;; at least four seconds since last attempt to start main server (begin (api:run-server-process apath dbname) (set! *last-main-start* (current-seconds)) (thread-sleep! 1)) (thread-sleep! 0.25)) (mutex-unlock! *connstart-mutex*) (rmt:open-main-connection remdat apath) ;; TODO: Add limit to number of tries ))) (if (not the-srv) ;; have server, try connecting to it (start-main-srv) (let* ((srv-addr (server-address the-srv)) ;; need serv (ipaddr (alist-ref 'ipaddr the-srv)) (port (alist-ref 'port the-srv)) (srvkey (alist-ref 'servkey the-srv)) (fullpath (db:dbname->path apath dbname)) (new-the-srv (make-conndat apath: apath dbname: dbname fullname: fullpath hostport: srv-addr ;; socket: (open-nn-connection srv-addr) - TODO - open ulex connection? ipaddr: ipaddr port: port srvpkt: the-srv srvkey: srvkey ;; generated by rmt:get-signature on the server side lastmsg: (current-seconds) expires: (+ (current-seconds) (server:expiration-timeout) -2) ;; this needs to be gathered during the ping ))) (hash-table-set! conns fullpath new-the-srv))) #t))))) ;; NB// sinfo is a servdat struct ;; (define (rmt:general-open-connection sinfo apath dbname #!key (num-tries 5)) (assert (not (equal? dbname ".db/main.db")) "ERROR: general-open-connection should never be called with main as the db") (let* ((mdbname ".db/main.db") ;; (db:run-id->dbname #f)) TODO: put this back to the lookup when stable (fullname (db:dbname->path apath dbname)) (conns (servdat-conns sinfo)) (mconn (rmt:get-conn sinfo apath ".db/main.db")) (dconn (rmt:get-conn sinfo apath dbname))) #;(if (and mconn (not (debug:print-logger))) (begin (debug:print-info 0 *default-log-port* "Turning on logging to main, look in logs dir for main log.") (debug:print-logger rmt:log-to-main))) (cond ((and mconn dconn (< (current-seconds)(conndat-expires dconn))) #t) ;; good to go ((not mconn) ;; no channel open to main? open it... (rmt:open-main-connection sinfo apath) (rmt:general-open-connection sinfo apath dbname num-tries: (- num-tries 1))) ((not dconn) ;; no channel open to dbname? (let* ((res (rmt:send-receive-real sinfo apath mdbname 'get-server `(,apath ,dbname)))) (case res ((server-started) (if (> num-tries 0) (begin (thread-sleep! 2) (rmt:general-open-connection sinfo apath dbname num-tries: (- num-tries 1))) (begin (debug:print-error 0 *default-log-port* "Failed to start servers needed or open channel to "apath", "dbname) (exit 1)))) (else (if (list? res) ;; server has been registered and the info was returned. pass it on. (begin ;; ("192.168.0.9" 53817 ;; "5e34239f48e8973b3813221e54701a01" "24310" ;; "192.168.0.9" ;; "/home/matt/data/megatest/tests/simplerun" ;; ".db/1.db") (match res ((host port servkey pid ipaddr apath dbname) (debug:print-info 0 *default-log-port* "got "res) (hash-table-set! conns fullname (make-conndat apath: apath dbname: dbname hostport: (conc host":"port) ;; socket: (open-nn-connection (conc host":"port)) ;; TODO - open ulex connection? ipaddr: ipaddr port: port srvkey: servkey lastmsg: (current-seconds) expires: (+ (current-seconds) (server:expiration-timeout) -2)))) (else (debug:print-info 0 *default-log-port* "return data from starting server did not match host port servkey pid ipaddr apath dbname " res))) res) (begin (debug:print-info 0 *default-log-port* "Unexpected result: " res) res))))))) #t)) ;;====================================================================== ;; FOR DEBUGGING SET TO #t ;; (define *localmode* #t) (define *localmode* #f) (define *dbstruct* (make-dbr:dbstruct)) ;; Defaults to current area ;; (define (rmt:send-receive cmd rid params #!key (attemptnum 1)(area-dat #f)) (let* ((apath *toppath*) (sinfo *db-serv-info*) (dbname (db:run-id->dbname rid))) (if *localmode* (api:execute-requests *dbstruct* cmd params) (begin (rmt:open-main-connection sinfo apath) (if rid (rmt:general-open-connection sinfo apath dbname)) #;(if (not (member cmd '(log-to-main))) (debug:print-info 0 *default-log-port* "rmt:send-receive "cmd" params="params)) (rmt:send-receive-real sinfo apath dbname cmd params))))) ;; db is at apath/.db/dbname, rid is an intermediary solution and will be removed ;; sometime in the future ;; (define (rmt:send-receive-real sinfo apath dbname cmd params) (assert (not (eq? 'primordial (thread-name (current-thread)))) "FATAL: Do not call rmt:send-receive-real in the primodial thread.") (let* ((cdat (rmt:get-conn sinfo apath dbname))) (assert cdat "FATAL: rmt:send-receive-real called without the needed channels opened") (let* ((uconn (servdat-uconn sinfo)) ;; get the interface to ulex ;; then send-receive using the ulex layer to host-port stored in cdat (res (send-receive uconn (conndat-hostport cdat) cmd params)) #;(th1 (make-thread (lambda () (set! res (send-receive uconn (conndat-hostport cdat) cmd params))) "send-receive thread"))) ;; (thread-start! th1) ;; (thread-join! th1) ;; gratuitious thread stuff is so that mailbox is not used in primordial thead ;; since we accessed the server we can bump the expires time up (conndat-expires-set! cdat (+ (current-seconds) (server:expiration-timeout) -2)) ;; two second margin for network time misalignments etc. res))) ;; db is at apath/.db/dbname, rid is an intermediary solution and will be removed ;; sometime in the future. ;; ;; Purpose - call the main.db server and request a server be started ;; for the given area path and dbname ;; (define (rmt:print-db-stats) (let ((fmtstr "~40a~7-d~9-d~20,2-f")) ;; "~20,2-f" (debug:print 18 *default-log-port* "DB Stats, "(seconds->year-week/day-time (current-seconds))"\n=====================") (debug:print 18 *default-log-port* (format #f "~40a~8a~10a~10a" "Cmd" "Count" "TotTime" "Avg")) (for-each (lambda (cmd) (let ((cmd-dat (hash-table-ref *db-stats* cmd))) |
︙ | ︙ | |||
452 453 454 455 456 457 458 | (define (rmt:kill-server run-id) (rmt:send-receive 'kill-server #f (list run-id))) (define (rmt:start-server run-id) (rmt:send-receive 'start-server #f (list run-id))) | | | 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 | (define (rmt:kill-server run-id) (rmt:send-receive 'kill-server #f (list run-id))) (define (rmt:start-server run-id) (rmt:send-receive 'start-server #f (list run-id))) (define (rmt:server-info apath dbname) (rmt:send-receive 'get-server-info #f (list apath dbname))) ;;====================================================================== ;; M I S C ;;====================================================================== (define (rmt:login run-id) |
︙ | ︙ | |||
1520 1521 1522 1523 1524 1525 1526 | (define (common:api-changed?) (not (equal? (substring (->string megatest-version) 0 4) (substring (conc (common:get-last-run-version)) 0 4)))) ;; host and port are used to ensure we are remove proper records (define (rmt:server-shutdown host port) | | | | | | | | | | | | > > > | > | | 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 | (define (common:api-changed?) (not (equal? (substring (->string megatest-version) 0 4) (substring (conc (common:get-last-run-version)) 0 4)))) ;; host and port are used to ensure we are remove proper records (define (rmt:server-shutdown host port) (let ((dbfile (servdat-dbfile *db-serv-info*))) (debug:print-info 0 *default-log-port* "dbfile is "dbfile) (if dbfile (let* ((am-server (args:get-arg "-server")) (dbfile (args:get-arg "-db")) (apath *toppath*) #;(sinfo *remotedat*)) ;; foundation for future fix (if *dbstruct-db* (let* ((dbdat (db:get-dbdat *dbstruct-db* apath dbfile)) (db (dbr:dbdat-db dbdat)) (inmem (dbr:dbdat-db dbdat)) ;; WRONG ) ;; do a final sync here (debug:print-info 0 *default-log-port* "Doing final sync for "apath" "dbfile" at "(current-seconds)) (db:sync-inmem->disk *dbstruct-db* apath dbfile force-sync: #t) ;; let's finalize here (debug:print-info 0 *default-log-port* "Finalizing db and inmem") (if (sqlite3:database? db) (sqlite3:finalize! db) (debug:print-info 0 *default-log-port* "in rmt:server-shutdown, db is not a database, not finalizing...")) (if (sqlite3:database? inmem) (sqlite3:finalize! inmem) (debug:print-info 0 *default-log-port* "in rmt:server-shutdown, inmem is not a database, not finalizing...")) (debug:print-info 0 *default-log-port* "Finalizing db and inmem complete")) (debug:print-info 0 *default-log-port* "Db was never opened, no cleanup to do.")) (if (not am-server) (debug:print-info 0 *default-log-port* "I am not a server, should NOT get here!") (if (string-match ".*/main.db$" dbfile) (let ((pkt-file (conc (get-pkts-dir *toppath*) "/" (servdat-uuid *db-serv-info*) ".pkt"))) (debug:print-info 0 *default-log-port* "removing pkt "pkt-file) (delete-file* pkt-file) (debug:print-info 0 *default-log-port* "Releasing lock (if any) for "dbfile ", host "host", port "port) (db:with-lock-db (servdat-dbfile *db-serv-info*) (lambda (dbh dbfile) (db:release-lock dbh dbfile host port)))) ;; I'm not the server - should not have a lock to remove (let* ((sdat *db-serv-info*) ;; we have a run-id server (host (servdat-host sdat)) (port (servdat-port sdat)) (uuid (servdat-uuid sdat)) (res (rmt:deregister-server *db-serv-info* *toppath* host port uuid dbfile))) (debug:print-info 0 *default-log-port* "deregistered-server, res="res) (debug:print-info 0 *default-log-port* "deregistering server "host":"port" with uuid "uuid) ))))))) (define (std-exit-procedure) ;;(common:telemetry-log-close) (on-exit (lambda () 0)) ;;(debug:print-info 13 *default-log-port* "std-exit-procedure called; *time-to-exit*="*time-to-exit*) (let ((no-hurry (if (bdat-time-to-exit *bdat*) ;; hurry up #f (begin (bdat-time-to-exit-set! *bdat* #t) #t)))) (debug:print-info 4 *default-log-port* "starting exit process, finalizing databases.") (if (and no-hurry (debug:debug-mode 18)) (rmt:print-db-stats)) (let ((th1 (make-thread (lambda () ;; thread for cleaning up, give it five seconds (let* ((start-time (current-seconds))) (if *db-serv-info* (let* ((host (servdat-host *db-serv-info*)) (port (servdat-port *db-serv-info*))) (debug:print-info 0 *default-log-port* "Shutting down server/responder.") ;; ;; TODO - add flushing/waiting on the work queue ;; (rmt:server-shutdown host port) (portlogger:open-run-close portlogger:set-port port "released"))) (debug:print-info 0 *default-log-port* "Shutdown activities completed in "(- (current-seconds) start-time)" seconds")) ;; (if *dbstruct-db* (db:close-all *dbstruct-db*)) ;; one second allocated #;(if (bdat-task-db *bdat*) ;; TODO: Check that this is correct for task db (let ((db (cdr (bdat-task-db *bdat*)))) (if (sqlite3:database? db) (begin (debug:print-info 0 *default-log-port* "Closing down task db "db) |
︙ | ︙ | |||
1633 1634 1635 1636 1637 1638 1639 | ;; called in megatest.scm, host-port is string hostname:port ;; ;; NOTE: This is NOT called directly from clients as not all transports support a client running ;; in the same process as the server. ;; ;; conn is a conndat record ;; | | < | | > > > > > > > > > > > | | | | < < | < | > | < | < < < < < < < < < < < < < < | < < < < < | < < < < < < < < < < < < < < > > | < < < < > | < < | > > | | | | | | < | | < < < < | < < | < < < < < < < < < | | < | < < < < < < < < < < | | < < < < < < | | 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 | ;; called in megatest.scm, host-port is string hostname:port ;; ;; NOTE: This is NOT called directly from clients as not all transports support a client running ;; in the same process as the server. ;; ;; conn is a conndat record ;; #;(define (server:ping uconn #!key (do-exit #f)) (let* ((srvkey (conndat-srvkey uconn)) (msg (sexpr->string '(ping ,srvkey)))) (send-receive uconn 'ping msg))) ;; (server-ready? host port server-id)) ;;====================================================================== ;; http-transportmod.scm contents moved here ;;====================================================================== (define (http-transport:make-server-url hostport) (if (not hostport) #f (conc "http://" (car hostport) ":" (cadr hostport)))) ;;====================================================================== ;; S E R V E R ;; ====================================================================== (define (http-get-function fnkey) (hash-table-ref/default *http-functions* fnkey (lambda () "nothing here yet"))) (define *rmt:run-mutex* (make-mutex)) (define *rmt:run-flag* #f) ;; Main entry point to start a server. was start-server (define (rmt:run hostn) (mutex-lock! *rmt:run-mutex*) (if *rmt:run-flag* (begin (debug:print-warn 0 *default-log-port* "rmt:run already running.") (mutex-unlock! *rmt:run-mutex*)) (begin (set! *rmt:run-flag* #t) (mutex-unlock! *rmt:run-mutex*) ;; ;; Configurations for server ;; (tcp-buffer-size 2048) ;; (max-connections 2048) (debug:print 2 *default-log-port* "PID: "(current-process-id)". Attempting to start the server ...") (if (and *db-serv-info* (servdat-uconn *db-serv-info*)) (let* ((uconn (servdat-uconn *db-serv-info*))) (wait-and-close uconn)) (let* ((port (portlogger:open-run-close portlogger:find-port)) (handler-proc (lambda (rem-host-port qrykey cmd params) ;; (set! *db-last-access* (current-seconds)) (assert (list? params) "FATAL: handler called with non-list params") (assert (args:get-arg "-server") "FATAL: handler called on non-server side. cmd="cmd", params="params) (debug:print 0 *default-log-port* "handler call: "cmd", params="params) (api:execute-requests *dbstruct-db* cmd params)))) ;; (api:process-request *dbstuct-db* (if (not *db-serv-info*) (set! *db-serv-info* (make-servdat host: hostn port: port))) (let* ((uconn (run-listener handler-proc port)) (rport (udat-port uconn))) ;; the real port (servdat-host-set! *db-serv-info* hostn) (servdat-port-set! *db-serv-info* rport) (servdat-uconn-set! *db-serv-info* uconn) (wait-and-close uconn) (db:print-current-query-stats) ))) (let* ((host (servdat-host *db-serv-info*)) (port (servdat-port *db-serv-info*)) (mode (or (servdat-mode *db-serv-info*) "non-db"))) ;; server exit stuff here ;; (rmt:server-shutdown host port) - always do in on-exit ;; (portlogger:open-run-close portlogger:set-port port "released") ;; moved to on-exit (debug:print-info 0 *default-log-port* "Server "host":"port" mode "mode"shutdown complete. Exiting") )))) ;;====================================================================== ;; S E R V E R U T I L I T I E S ;;====================================================================== ;;====================================================================== ;; C L I E N T S |
︙ | ︙ | |||
1822 1823 1824 1825 1826 1827 1828 | ;; only use for main.db - need to re-write some of this :( ;; (define (get-lock-db sdat dbfile host port) (assert host "FATAL: get-lock-db called with host not set.") (assert port "FATAL: get-lock-db called with port not set.") (let* ((dbh (db:open-run-db dbfile db:initialize-db)) ;; open-run-db creates a standard db with schema used by all situations | | > | | 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 | ;; only use for main.db - need to re-write some of this :( ;; (define (get-lock-db sdat dbfile host port) (assert host "FATAL: get-lock-db called with host not set.") (assert port "FATAL: get-lock-db called with port not set.") (let* ((dbh (db:open-run-db dbfile db:initialize-db)) ;; open-run-db creates a standard db with schema used by all situations (res (db:get-iam-server-lock dbh dbfile host port)) (uconn (servdat-uconn sdat))) ;; res => list then already locked, check server is responsive ;; => #t then sucessfully got the lock ;; => #f reserved for future use as to indicate something went wrong (match res ((owner_pid owner_host owner_port event_time) (if (server-ready? uconn (conc owner_host":"owner_port) "abc") #f ;; locked by someone else (begin ;; locked by someone dead and gone (debug:print 0 *default-log-port* "WARNING: stale lock - have to steal it. This may fail.") (db:steal-lock-db dbh dbfile port)))) (#t #t) ;; placeholder so that we don't touch res if it is #t (else (set! res #f))) (sqlite3:finalize! dbh) |
︙ | ︙ | |||
1882 1883 1884 1885 1886 1887 1888 | (read-pkt->alist pkt-file pktspec: pktspec)) all-pkt-files))) (define (server-address srv-pkt) (conc (alist-ref 'host srv-pkt) ":" (alist-ref 'port srv-pkt))) | | > | | | | > | < | > | > | | > | | 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 | (read-pkt->alist pkt-file pktspec: pktspec)) all-pkt-files))) (define (server-address srv-pkt) (conc (alist-ref 'host srv-pkt) ":" (alist-ref 'port srv-pkt))) (define (server-ready? uconn host-port key) ;; server-address is host:port (let* ((params `((cmd . ping)(key . ,key))) (data `((cmd . ping) (key . ,key) (params . ,params))) ;; I don't get it. (res (send-receive uconn host-port 'ping data))) (if (eq? res 'ack) ;; yep, likely it is who we want on the other end res #f))) ;; (begin (debug:print-info 0 *default-log-port* "server-ready? => "res) #f)))) ; from the pkts return servers associated with dbpath ;; NOTE: Only one can be alive - have to check on each ;; in the list of pkts returned ;; (define (get-viable-servers serv-pkts dbpath) (let loop ((tail serv-pkts) (res '())) (if (null? tail) res ;; NOTE: sort by age so oldest is considered first (let* ((spkt (car tail))) (loop (cdr tail) (if (equal? dbpath (alist-ref 'dbpath spkt)) (cons spkt res) res)))))) (define (remove-pkts-if-not-alive uconn serv-pkts) (filter (lambda (pkt) (let* ((host (alist-ref 'host pkt)) (port (alist-ref 'port pkt)) (host-port (conc host":"port)) (key (alist-ref 'servkey pkt)) (pktz (alist-ref 'Z pkt)) (res (server-ready? uconn host-port key))) (if res res (let* ((pktsdir (get-pkts-dir *toppath*)) (pktpath (conc pktsdir"/"pktz".pkt"))) (debug:print 0 *default-log-port* "WARNING: pkt with no server "pktpath) (delete-file* pktpath) #f)))) serv-pkts)) ;; from viable servers get one that is alive and ready ;; (define (get-the-server uconn apath serv-pkts) (let loop ((tail serv-pkts)) (if (null? tail) #f (let* ((spkt (car tail)) (host (alist-ref 'ipaddr spkt)) (port (alist-ref 'port spkt)) (host-port (conc host":"port)) (dbpth (alist-ref 'dbpath spkt)) (srvkey (alist-ref 'Z spkt)) ;; (alist-ref 'srvkey spkt)) (addr (server-address spkt))) (if (server-ready? uconn host-port srvkey) spkt (loop (cdr tail))))))) ;; am I the "first" in line server? I.e. my D card is smallest ;; use Z card as tie breaker ;; (define (get-best-candidate serv-pkts dbpath) |
︙ | ︙ | |||
1975 1976 1977 1978 1979 1980 1981 | ;;====================================================================== ;; END NEW SERVER METHOD ;;====================================================================== ;; if .db/main.db check the pkts ;; (define (rmt:wait-for-server pkts-dir db-file server-key) | | | | | | | 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 | ;;====================================================================== ;; END NEW SERVER METHOD ;;====================================================================== ;; if .db/main.db check the pkts ;; (define (rmt:wait-for-server pkts-dir db-file server-key) (let* ((sdat *db-serv-info*)) (let loop ((start-time (current-seconds)) (changed #t) (last-sdat "not this")) (begin ;; let ((sdat #f)) (thread-sleep! 0.01) (debug:print-info 0 *default-log-port* "Waiting for server alive signature") (mutex-lock! *heartbeat-mutex*) (set! sdat *db-serv-info*) (mutex-unlock! *heartbeat-mutex*) (if (and sdat (not changed) (> (- (current-seconds) start-time) 2)) (let* ((uconn (servdat-uconn sdat))) (servdat-status-set! sdat 'iface-stable) (debug:print-info 0 *default-log-port* "Received server alive signature, now attempting to lock in server") ;; create a server pkt in *toppath*/.meta/srvpkts ;; TODO: ;; 1. change sdat to stuct ;; 2. add uuid to struct ;; 3. update uuid in sdat here ;; (servdat-uuid-set! sdat (register-server pkts-dir *srvpktspec* (get-host-name) (servdat-port sdat) server-key (servdat-host sdat) db-file)) ;; (set! *my-signature* (servdat-uuid sdat)) ;; replace with Z, no, stick with proper key ;; now read pkts and see if we are a contender (let* ((all-pkts (get-all-server-pkts pkts-dir *srvpktspec*)) (viables (get-viable-servers all-pkts db-file)) (alive (remove-pkts-if-not-alive uconn viables)) (best-srv (get-best-candidate alive db-file)) (best-srv-key (if best-srv (alist-ref 'servkey best-srv) #f)) (i-am-srv (equal? best-srv-key server-key)) (delete-pkt (lambda () (let* ((pktfile (conc (get-pkts-dir *toppath*) "/" (servdat-uuid *db-serv-info*) ".pkt"))) (debug:print-info 0 *default-log-port* "Attempting to remove bogus pkt file "pktfile) (delete-file* pktfile))))) ;; remove immediately instead of waiting for on-exit (debug:print 0 *default-log-port* "best-srv-key: "best-srv-key", server-key: "server-key", i-am-srv: "i-am-srv) ;; am I the best-srv, compare server-keys to know (if i-am-srv (if (get-lock-db sdat db-file (servdat-host sdat)(servdat-port sdat)) ;; (db:get-iam-server-lock *dbstruct-db* *toppath* run-id) |
︙ | ︙ | |||
2051 2052 2053 2054 2055 2056 2057 | (begin (debug:print-error 0 *default-log-port* "transport appears to have died, exiting server") (exit)) (loop start-time (equal? sdat last-sdat) sdat)))))))) | | | | | | | | | | < | | | | | | | | | | | | | | | | > | < > | | | | | | | | | | | | | | | | | | | < | | | | | | | > > > < < < | | > > > > > | | < < | > | | | < < | < | 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 | (begin (debug:print-error 0 *default-log-port* "transport appears to have died, exiting server") (exit)) (loop start-time (equal? sdat last-sdat) sdat)))))))) (define (rmt:register-server sinfo apath iface port server-key dbname) (servdat-conns sinfo) ;; just checking types (rmt:open-main-connection sinfo apath) ;; we need a channel to main.db (rmt:send-receive-real sinfo apath ;; params: host port servkey pid ipaddr dbpath (db:run-id->dbname #f) 'register-server `(,iface ,port ,server-key ,(current-process-id) ,iface ,apath ,dbname))) (define (rmt:get-count-servers sinfo apath) (servdat-conns sinfo) ;; just checking types (rmt:open-main-connection sinfo apath) ;; we need a channel to main.db (rmt:send-receive-real sinfo apath ;; params: host port servkey pid ipaddr dbpath (db:run-id->dbname #f) 'get-count-servers `(,apath))) (define (rmt:get-servers-info apath) (rmt:send-receive 'get-servers-info #f `(,apath))) (define (rmt:deregister-server db-serv-info apath iface port server-key dbname) (rmt:open-main-connection db-serv-info apath) ;; we need a channel to main.db (rmt:send-receive-real db-serv-info apath ;; params: host port servkey pid ipaddr dbpath (db:run-id->dbname #f) 'deregister-server `(,iface ,port ,server-key ,(current-process-id) ,iface ,apath ,dbname))) (define (rmt:wait-for-stable-interface #!optional (num-tries-allowed 100)) ;; wait until *db-serv-info* stops changing (let* ((stime (current-seconds))) (let loop ((last-host #f) (last-port #f) (tries 0)) (let* ((curr-host (and *db-serv-info* (servdat-host *db-serv-info*))) (curr-port (and *db-serv-info* (servdat-port *db-serv-info*)))) ;; first we verify port and interface, update *db-serv-info* in need be. (cond ((> tries num-tries-allowed) (debug:print 0 *default-log-port* "rmt:keep-running, giving up after trying for several minutes.") (exit 1)) ((not *db-serv-info*) (thread-sleep! 0.25) (loop curr-host curr-port (+ tries 1))) ((or (not last-host)(not last-port)) (debug:print 0 *default-log-port* "rmt:keep-running, still no interface, tries="tries) (thread-sleep! 0.25) (loop curr-host curr-port (+ tries 1))) ((or (not (equal? last-host curr-host)) (not (equal? last-port curr-port))) (debug:print-info 0 *default-log-port* "WARNING: interface changed, refreshing iface and port info") (thread-sleep! 0.25) (loop curr-host curr-port (+ tries 1))) ((< (- (current-seconds) stime) 1) ;; keep up the looping until at least 3 seconds have passed (thread-sleep! 0.5) (loop curr-host curr-port (+ tries 1))) (else (rmt:get-signature) ;; sets *my-signature* as side effect (servdat-status-set! *db-serv-info* 'interface-stable) (debug:print 0 *default-log-port* "SERVER STARTED: " curr-host ":" curr-port " AT " (current-seconds) " server signature: " *my-signature* " with "(servdat-trynum *db-serv-info*)" port changes") (flush-output *default-log-port*) #t)))))) ;; run rmt:keep-running in a parallel thread to monitor that the db is being ;; used and to shutdown after sometime if it is not. ;; (define (rmt:keep-running dbname) ;; if none running or if > 20 seconds since ;; server last used then start shutdown ;; This thread waits for the server to come alive (debug:print-info 0 *default-log-port* "Starting the sync-back, keep alive thread in server") (let* ((sinfo *db-serv-info*) (server-start-time (current-seconds)) (pkts-dir (get-pkts-dir)) (server-key (rmt:get-signature)) ;; This servers key (is-main (equal? (args:get-arg "-db") ".db/main.db")) (last-access 0) (server-timeout (server:expiration-timeout)) (shutdown-server-sequence (lambda (host port) (set! *unclean-shutdown* #f) ;; Should not be needed anymore (debug:print-info 0 *default-log-port* "Starting to shutdown the server. pid="(current-process-id)) ;; (rmt:server-shutdown host port) -- called in on-exit ;; (portlogger:open-run-close portlogger:set-port port "released") called in on-exit (exit))) (timed-out? (lambda () (<= (+ last-access server-timeout) (current-seconds))))) (servdat-dbfile-set! *db-serv-info* (args:get-arg "-db")) ;; main and run db servers have both got wait logic (could/should merge it) (if is-main (rmt:wait-for-server pkts-dir dbname server-key) (rmt:wait-for-stable-interface)) ;; this is our forever loop (let* ((iface (servdat-host *db-serv-info*)) (port (servdat-port *db-serv-info*)) (uconn (servdat-uconn *db-serv-info*))) (let loop ((count 0) (bad-sync-count 0) (start-time (current-milliseconds))) (if (and (not is-main) (common:low-noise-print 60 "servdat-status")) (debug:print-info 0 *default-log-port* "servdat-status is " (servdat-status *db-serv-info*))) (mutex-lock! *heartbeat-mutex*) ;; set up the database handle (if (not *dbstruct-db*) ;; no db opened yet, open the db and register with main if appropriate (let ((watchdog (bdat-watchdog *bdat*))) (debug:print 0 *default-log-port* "SERVER: dbprep") (db:setup dbname) ;; sets *dbstruct-db* as side effect (servdat-status-set! *db-serv-info* 'db-opened) ;; IFF I'm not main, call into main and register self (if (not is-main) (let ((res (rmt:register-server sinfo *toppath* iface port server-key dbname))) (if res ;; we are the server (servdat-status-set! *db-serv-info* 'have-interface-and-db) ;; now check that the db locker is alive, clear it out if not (let* ((serv-info (rmt:server-info *toppath* dbname))) (match serv-info ((host port servkey pid ipaddr apath dbpath) (if (not (server-ready? uconn (conc host":"port) servkey)) (begin (debug:print-info 0 *default-log-port* "Server registered but not alive. Removing and trying again.") (rmt:deregister-server sinfo apath host port servkey dbpath) ;; servkey pid ipaddr apath dbpath) (loop (+ count 1) bad-sync-count start-time)))) (else (debug:print 0 *default-log-port* "We are not the server for "dbname", exiting. Server info is: "serv-info) (exit))))))) (debug:print 0 *default-log-port* "SERVER: running, db "dbname" opened, megatest version: " (common:get-full-version)) ;; start the watchdog ;; is this really needed? #;(if watchdog (if (not (member (thread-state watchdog) '(ready running blocked sleeping dead))) (begin (debug:print-info 0 *default-log-port* "Starting watchdog thread (in state "(thread-state watchdog)")") (thread-start! watchdog)) (debug:print-info 0 *default-log-port* "Not starting watchdog thread (in state "(thread-state watchdog)")")) (debug:print 0 *default-log-port* "ERROR: *watchdog* not setup, cannot start it.")) #;(loop (+ count 1) bad-sync-count start-time) )) (debug:print-info 0 *default-log-port* "syncing "*toppath*" "dbname" at "(current-seconds)) (db:sync-inmem->disk *dbstruct-db* *toppath* dbname force-sync: #t) (mutex-unlock! *heartbeat-mutex*) ;; when things go wrong we don't want to be doing the various ;; queries too often so we strive to run this stuff only every ;; four seconds or so. (let* ((sync-time (- (current-milliseconds) start-time)) (rem-time (quotient (- 4000 sync-time) 1000))) (if (and (<= rem-time 4) (> rem-time 0)) (thread-sleep! rem-time))) ;; Transfer *db-last-access* to last-access to use in checking that we are still alive (set! last-access *db-last-access*) (if (< count 1) ;; 3x3 = 9 secs aprox (loop (+ count 1) bad-sync-count (current-milliseconds))) (if (common:low-noise-print 60 "dbstats") (begin (debug:print 0 *default-log-port* "Server stats:") (db:print-current-query-stats))) (let* ((hrs-since-start (/ (- (current-seconds) server-start-time) 3600))) (cond ((not *server-run*) (debug:print-info 0 *default-log-port* "*server-run* set to #f. Shutting down.") (shutdown-server-sequence (get-host-name) port)) ((timed-out?) (debug:print-info 0 *default-log-port* "Server timed out. seconds since last db access: " (- (current-seconds) last-access)) (shutdown-server-sequence (get-host-name) port)) ((and *server-run* (or (not (timed-out?)) (if is-main ;; do not exit if there are other servers (keep main open until all others gone) (> (rmt:get-count-servers sinfo *toppath*) 1) #f))) (if (common:low-noise-print 120 "server continuing") (debug:print-info 0 *default-log-port* "Server continuing, seconds since last db access: " (- (current-seconds) last-access))) (loop 0 bad-sync-count (current-milliseconds))) (else (set! *unclean-shutdown* #f) (debug:print-info 0 *default-log-port* "Server timed out. seconds since last db access: " (- (current-seconds) last-access)) (shutdown-server-sequence (get-host-name) port) #;(debug:print-info 0 *default-log-port* "Sending 'quit to server, received: " (open-send-receive-nn (conc iface":"port) ;; do this here and not in server-shutdown (sexpr->string 'quit)))))))))) (define (rmt:get-reasonable-hostname) (let* ((inhost (or (args:get-arg "-server") "-"))) (if (equal? inhost "-") (get-host-name) inhost))) ;; Call this to start the actual server ;; ;; all routes though here end in exit ... ;; ;; This is the point at which servers are started ;; (define (rmt:server-launch dbname) (debug:print-info 0 *default-log-port* "Entered rmt:server-launch") (let* ((th2 (make-thread (lambda () (debug:print-info 0 *default-log-port* "Server run thread started") (rmt:run (rmt:get-reasonable-hostname))) "Server run")) (th3 (make-thread (lambda () (debug:print-info 0 *default-log-port* "Server monitor thread started") (if (args:get-arg "-server") (rmt:keep-running dbname))) "Keep running"))) (thread-start! th2) (thread-sleep! 0.252) ;; give the server time to settle before starting the keep-running monitor. (thread-start! th3) (set! *didsomething* #t) (thread-join! th2) (thread-join! th3)) #f) ;; Generate a unique signature for this process, used at both client and ;; server side (define (rmt:mk-signature) (message-digest-string (md5-primitive) (with-output-to-string (lambda () |
︙ | ︙ | |||
2324 2325 2326 2327 2328 2329 2330 | (if (string-search (regexp (conc ":" port-num)) inl) (begin ;(print "Output: " inl) (set! ret #t)) (loop (read-line inp))))))) ret)) | < < < < < < < < < < < < < < | | | | 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 | (if (string-search (regexp (conc ":" port-num)) inl) (begin ;(print "Output: " inl) (set! ret #t)) (loop (read-line inp))))))) ret)) #;(define (open-nn-connection host-port) (let ((req (make-req-socket)) (uri (conc "tcp://" host-port))) (nng-dial req uri) (socket-set! req 'nng/recvtimeo 2000) req)) #;(define (send-receive-nn req msg) (nng-send req msg) (nng-recv req)) #;(define (close-nn-connection req) (nng-close! req)) ;; ;; open connection to server, send message, close connection ;; ;; ;; (define (open-send-close-nn host-port msg #!key (timeout 3) ) ;; default timeout is 3 seconds ;; (let ((req (make-req-socket 'req)) ;; (uri (conc "tcp://" host-port)) |
︙ | ︙ | |||
2388 2389 2390 2391 2392 2393 2394 | ;; (thread-terminate! th1)) ;; "timer thread"))) ;; (thread-start! th1) ;; (thread-start! th2) ;; (thread-join! th1) ;; res)))) ;; | | | 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 | ;; (thread-terminate! th1)) ;; "timer thread"))) ;; (thread-start! th1) ;; (thread-start! th2) ;; (thread-join! th1) ;; res)))) ;; #;(define (open-send-receive-nn host-port msg #!key (timeout 3) ) ;; default timeout is 3 seconds (let ((req (make-req-socket)) (uri (conc "tcp://" host-port)) (res #f)) (handle-exceptions exn (let ((emsg ((condition-property-accessor 'exn 'message) exn))) ;; Send notification |
︙ | ︙ |
Modified runsmod.scm from [e0bbf25b11] to [7c62c7e318].
︙ | ︙ | |||
2554 2555 2556 2557 2558 2559 2560 | (debug:print-error 0 *default-log-port* "Attempted to " action-desc " but run area config file not found") (exit 1)) ;; Extract out stuff needed in most or many calls ;; here then call proc (let* ((keyvals (keys:target->keyval keys target))) (proc target runname keys keyvals))) ;; (if db (sqlite3:finalize! db)) | | > | 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 | (debug:print-error 0 *default-log-port* "Attempted to " action-desc " but run area config file not found") (exit 1)) ;; Extract out stuff needed in most or many calls ;; here then call proc (let* ((keyvals (keys:target->keyval keys target))) (proc target runname keys keyvals))) ;; (if db (sqlite3:finalize! db)) (set! *didsomething* #t) ))))) ;;====================================================================== ;; Lock/unlock runs ;;====================================================================== (define (runs:handle-locking target keys runname lock unlock user) (let* ((db #f) |
︙ | ︙ |
Added rununit.sh version [0dba7a0924].
> > > > > | 1 2 3 4 5 | #!/bin/bash rm tests/*log tests/simplerun/logs/* script -c 'ck5 make unit' |
Modified tests/simplerun/Makefile from [38acf6b450] to [18ac57f19f].
1 2 | cleanup : | | > > | 1 2 3 4 5 | cleanup : killall mtest dboard -v -9 || true rm -rf *.log *.bak NB* logs/* .meta .db |
Added tests/simplerun/debug.scm version [f6de86f926].
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 | (import big-chicken trace rmtmod apimod dbmod ulex srfi-18) (trace-call-sites #t) (trace ;; db:get-tests-for-run ;; rmt:general-open-connection ;; rmt:open-main-connection ;; rmt:drop-conn ;; rmt:send-receive ;; rmt:log-to-main ) (module junk * (import big-chicken rmtmod apimod dbmod srfi-18) (define (make-run-id) #;(let* ((s (conc (current-process-id))) (l (string-length s))) (string->number (conc (string-ref s (- l 1)))) ) 1) (define (run) (let* ((th1 (make-thread (lambda () (let loop ((r 1) ;; (* 20 (make-run-id))) (i 1)) (print "register-test "r" test"i) (rmt:register-test r "test1" (conc "item_" i)) (if (< i 100000) (loop r (+ i 1)) (if (< r 100) (begin (print "get-tests-for-run "r) (rmt:get-tests-for-run r "%" '() '() 0 #f #f #f #f #f 0 #f) (loop (+ r 1) 0))))) )))) (thread-start! th1) (thread-join! th1))) ) (import junk) (run) |
Modified tests/simplerun/megatest.config from [3e9fa2e5ac] to [de09ea7f96].
︙ | ︙ | |||
19 20 21 22 23 24 25 26 27 28 29 30 31 32 | SYSTEM TEXT RELEASE TEXT [setup] # Adjust max_concurrent_jobs to limit how much you load your machines max_concurrent_jobs 50 # Uncomment this to make the in-mem db into a disk based db (slower but good for debug) # be aware that some unit tests will fail with this due to persistent data # # tmpdb /tmp # This is your link path, you can move it but it is generally better to keep it stable linktree #{getenv MT_RUN_AREA_HOME}/../simplelinks | > > > | 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 | SYSTEM TEXT RELEASE TEXT [setup] # Adjust max_concurrent_jobs to limit how much you load your machines max_concurrent_jobs 50 [server] timeout 3600 # Uncomment this to make the in-mem db into a disk based db (slower but good for debug) # be aware that some unit tests will fail with this due to persistent data # # tmpdb /tmp # This is your link path, you can move it but it is generally better to keep it stable linktree #{getenv MT_RUN_AREA_HOME}/../simplelinks |
︙ | ︙ |
Modified tests/tests.scm from [7467445dd0] to [3fa28c6a70].
︙ | ︙ | |||
47 48 49 50 51 52 53 54 55 56 57 58 59 60 | ;; read in all the _record files ;; (let ((files (glob "*_records.scm"))) ;; (for-each ;; (lambda (file) ;; (print "Loading " file) ;; (load file)) ;; files)) (let* ((unit-test-name (list-ref (argv) 4)) (fname (conc "../unittests/" unit-test-name ".scm"))) (if (file-exists? fname) (load fname) (print "ERROR: Unit test " unit-test-name " not found in unittests directory"))) | > > > > > > > > > > | 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 | ;; read in all the _record files ;; (let ((files (glob "*_records.scm"))) ;; (for-each ;; (lambda (file) ;; (print "Loading " file) ;; (load file)) ;; files)) (define-syntax run-in-thread (syntax-rules () ((_ body ...) (let ((th1 (make-thread (lambda () body ...) "the thread"))) (thread-start! th1) (thread-join! th1))))) (let* ((unit-test-name (list-ref (argv) 4)) (fname (conc "../unittests/" unit-test-name ".scm"))) (if (file-exists? fname) (load fname) (print "ERROR: Unit test " unit-test-name " not found in unittests directory"))) |
Modified tests/unittests/all-rmt.scm from [3a8e222d0c] to [170e763023].
︙ | ︙ | |||
89 90 91 92 93 94 95 | ;; (test-batch rmt:get-latest-host-load ;; "rmt:get-latest-host-load" ;; (list (list "localhost" #t (get-host-name)) ;; (list "not-a-host" #t "not-a-host" )) ;; post-proc: pair?) ;; ;; (test #f #t (list? (rmt:get-changed-record-ids 0))) | | | > | > | 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 | ;; (test-batch rmt:get-latest-host-load ;; "rmt:get-latest-host-load" ;; (list (list "localhost" #t (get-host-name)) ;; (list "not-a-host" #t "not-a-host" )) ;; post-proc: pair?) ;; ;; (test #f #t (list? (rmt:get-changed-record-ids 0))) ;; (run-in-thread ;; (test #f #f (begin (runs:update-all-test_meta #f) #f)) (test #f '("test1" "test2")(sort (alist-ref "tagtwo" (rmt:get-tests-tags) equal?) string<=?)) (test #f '() (rmt:get-key-val-pairs 0)) (test #f '("SYSTEM" "RELEASE") (rmt:get-keys)) (test #f '("SYSTEM" "RELEASE") (rmt:get-keys-write)) ;; dummy query to force server start (test #f '() (rmt:get-key-vals 1)) (test #f (vector '("SYSTEM" "RELEASE") '()) (rmt:get-targets)) (test #f "" (rmt:get-target 1)) |
︙ | ︙ | |||
144 145 146 147 148 149 150 | (test #f #t (vector? (rmt:get-runs "%" 10 0 keypatts))) (test #f '(1)(rmt:get-all-run-ids)) (test #f '()(rmt:get-prev-run-ids 1)) (test #f #t (begin (rmt:lock/unlock-run 1 #t #f "mikey") #t)) (test #f "JUSTFINE" (rmt:get-run-status 1)) (test #f #t (begin (rmt:set-run-status 1 "NOTFINE" msg: "A message") #t)) (test #f #t (begin (rmt:update-run-event_time 1) #t)) | | | 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 | (test #f #t (vector? (rmt:get-runs "%" 10 0 keypatts))) (test #f '(1)(rmt:get-all-run-ids)) (test #f '()(rmt:get-prev-run-ids 1)) (test #f #t (begin (rmt:lock/unlock-run 1 #t #f "mikey") #t)) (test #f "JUSTFINE" (rmt:get-run-status 1)) (test #f #t (begin (rmt:set-run-status 1 "NOTFINE" msg: "A message") #t)) (test #f #t (begin (rmt:update-run-event_time 1) #t)) ) ;; (rmt:get-runs-by-patt keys runnamepatt targpatt offset limit fields last-runs-update) ;; fields of #f uses default ;; (let ((keys (rmt:get-keys)) (rnp "%") ;; run name patt (tpt "%/%")) ;; target patt (test-batch rmt:get-runs-by-patt "rmt:get-runs-by-patt" |
︙ | ︙ |
Modified tests/unittests/basicserver.scm from [0bda564ab1] to [7886f1227c].
︙ | ︙ | |||
19 20 21 22 23 24 25 | ;; along with Megatest. If not, see <http://www.gnu.org/licenses/>. ;; Run like this: ;; ;; ./rununittest.sh server 1;(cd simplerun;megatest -stop-server 0) (import rmtmod trace http-client apimod dbmod | | > > > > > > > | > | | | | > | > > > > > | > | < > > | < < < < < | | | < < < | > | < < | 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 | ;; along with Megatest. If not, see <http://www.gnu.org/licenses/>. ;; Run like this: ;; ;; ./rununittest.sh server 1;(cd simplerun;megatest -stop-server 0) (import rmtmod trace http-client apimod dbmod launchmod srfi-69 ulex system-information) (trace-call-sites #t) (trace ;; get-the-server ;; db:get-dbdat ;; rmt:find-main-server ;; rmt:send-receive-real ;; rmt:send-receive ;; sexpr->string ;; server-ready? ;; rmt:register-server ;; api:run-server-process ;; rmt:open-main-connection ;; rmt:general-open-connection ;; rmt:get-conny ;; common:watchdog ;; rmt:find-main-server ;; get-all-server-pkts ;; get-viable-servers ;; get-best-candidate ;; api:run-server-process ;; rmt:run ;; rmt:try-start-server ;; ;; ulex ;; ;; wait-and-close ;; run-listener ) (test #f #t (servdat? (let ((s (make-servdat))) (set! *servdat* s) s))) (test #f #f (rmt:get-conn *servdat* *toppath* ".db/main.db")) (test #f #f (rmt:find-main-server *servdat* *toppath* ".db/main.db")) (define th1 (make-thread (lambda () (rmt:run (get-host-name))) "rmt:run thread")) (thread-start! th1) (thread-sleep! 0.5) ;; give things some time to get going ;; switch to *db-serv-info* instead of *servdat* (define *uconn* (servdat-uconn *db-serv-info*)) (print "*uconn*: " *uconn*) (test #f #t (ulex-listener? (servdat-uconn *db-serv-info*))) (test #f #t (string? (udat-host-port *uconn*))) (run-in-thread (test #f 'ack (server-ready? *uconn* (udat-host-port *uconn*) (servdat-uuid *db-serv-info*)))) (test #f #t (rmt:open-main-connection *db-serv-info* *toppath*)) ;; (pp (hash-table->alist (remotedat-conns *db-serv-info*))) (test #f #t (conndat? (rmt:get-conn *db-serv-info* *toppath* ".db/main.db"))) (define remote *db-serv-info*) (define keyvals '(("SYSTEM" "a")("RELEASE" "b"))) (run-in-thread (test #f (map car keyvals) (rmt:get-keys))) (exit) |
Modified tests/unittests/server.scm from [70755134bd] to [49807e069f].
︙ | ︙ | |||
51 52 53 54 55 56 57 | ) (define *db* (db:setup ".db/main.db")) ;; these let me cut and paste from source easily (define apath *toppath*) (define dbname ".db/2.db") | | | | > | < | | | | 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 | ) (define *db* (db:setup ".db/main.db")) ;; these let me cut and paste from source easily (define apath *toppath*) (define dbname ".db/2.db") (define remote *db-serv-info*) (define keyvals '(("SYSTEM" "a")("RELEASE" "b"))) (test #f #t (rmt:open-main-connection remote apath)) (test #f #t (conndat? (rmt:get-conn *db-serv-info* *toppath* ".db/main.db"))) (test #f #t (conndat? (rmt:get-conn *db-serv-info* *toppath* ".db/main.db"))) (test #f 'server-started (rmt:send-receive-real *db-serv-info* *toppath* ".db/main.db" 'start-server `(,apath ,dbname))) (thread-sleep! 2) (test #f #t (rmt:general-open-connection *db-serv-info* *toppath* ".db/2.db")) ;; (let loop ((end-time (+ (current-seconds) 61))) (test #f #t (list? (rmt:get-servers-info *toppath*))) (test #f '("SYSTEM" "RELEASE") (rmt:get-keys)) (test #f 1 (rmt:send-receive 'register-run #f (list keyvals "run1" "new" "n/a" "justme" #f))) ;; (print "Got here.") (test #f 1 (rmt:send-receive 'register-run 1 (list keyvals "run1" "new" "n/a" "justme" #f))) (test #f 2 (rmt:register-run keyvals "run2" "new" "n/a" "justme" #f)) ;; (test #f 2 (rmt:deregister-server *db-serv-info* *toppath* iface port server-key dbname (test #f 2 (rmt:get-count-servers *db-serv-info* *toppath*)) (test #f "run2" (rmt:get-run-name-from-id 2)) (test #f #f (rmt:send-receive 'get-test-info-by-id 2 '(2 1))) (test #f #t (rmt:general-call 'update-cpuload-diskfree 2 1.5 1e6 1)) ;; (if (< (current-seconds) end-time)(loop end-time))) |
︙ | ︙ |
Added ulex-simple/ulex.scm version [496e2d5b2f].
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 | ;; ulex: Distributed sqlite3 db ;;; ;; Copyright (C) 2018-2021 Matt Welland ;; Redistribution and use in source and binary forms, with or without ;; modification, is permitted. ;; ;; THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS ;; OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED ;; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ;; ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE ;; LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR ;; CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT ;; OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR ;; BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF ;; LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ;; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE ;; USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH ;; DAMAGE. ;;====================================================================== ;; ABOUT: ;; See README in the distribution at https://www.kiatoa.com/fossils/ulex ;; NOTES: ;; Why sql-de-lite and not say, dbi? - performance mostly, then simplicity. ;; ;;====================================================================== (module ulex ( ;; NOTE: looking for the handler proc - find the run-listener :) run-listener ;; (run-listener handler-proc [port]) => uconn ;; NOTE: handler-proc params; ;; (handler-proc rem-host-port qrykey cmd params) send-receive ;; (send-receive uconn host-port cmd data) ;; NOTE: cmd can be any plain text symbol except for these; ;; 'ping 'ack 'goodbye 'response set-work-handler ;; (set-work-handler proc) wait-and-close ;; (wait-and-close uconn) ulex-listener? ;; needed to get the interface:port that was automatically found udat-port udat-host-port ;; for testing only ;; pp-uconn ) (import scheme chicken.base chicken.file chicken.time chicken.condition chicken.string chicken.sort chicken.pretty-print address-info mailbox matchable ;; queues regex regex-case s11n srfi-1 srfi-18 srfi-4 srfi-69 system-information tcp6 typed-records ) ;; udat struct, used by both caller and callee ;; instantiated as uconn by convention ;; (defstruct udat ;; the listener side (port #f) (host-port #f) (socket #f) ;; the peers (peers (make-hash-table)) ;; host:port->peer ;; work handling (work-queue (make-mailbox)) (work-proc #f) ;; set by user (cnum 0) ;; cookie number (mboxes (make-hash-table)) ;; for the replies (avail-cmboxes '()) ;; list of (<cookie> . <mbox>) for re-use ;; threads (numthreads 50) (cmd-thread #f) (work-queue-thread #f) ) ;; ;; struct for keeping track of others we are talking to ;; ;; ;; (defstruct pdat ;; (host-port #f) ;; (conns '()) ;; list of pcon structs, pop one off when calling the peer ;; ) ;; ;; ;; struct for peer connections, keep track of expiration etc. ;; ;; ;; (defstruct pcon ;; (inp #f) ;; (oup #f) ;; (exp (+ (current-seconds) 59)) ;; expires at this time, set to (+ (current-seconds) 59) ;; (lifetime (+ (current-seconds) 600)) ;; throw away and create new after five minutes ;; ) ;;====================================================================== ;; listener ;;====================================================================== ;; is uconn a ulex connector (listener) ;; (define (ulex-listener? uconn) (udat? uconn)) ;; create a tcp listener and return a populated udat struct with ;; my port, address, hostname, pid etc. ;; return #f if fail to find a port to allocate. ;; ;; if udata-in is #f create the record ;; if there is already a serv-listener return the udata ;; (define (setup-listener uconn #!optional (port 4242)) (handle-exceptions exn (if (< port 65535) (setup-listener uconn (+ port 1)) #f) (connect-listener uconn port))) (define (connect-listener uconn port) ;; (tcp-listener-socket LISTENER)(socket-name so) ;; sockaddr-address, sockaddr-port, sockaddr->string (let* ((tlsn (tcp-listen port 1000 #f)) ;; (tcp-listen TCPPORT [BACKLOG [HOST]]) (addr (get-my-best-address))) ;; (hostinfo-addresses (host-information (current-hostname))) (udat-port-set! uconn port) (udat-host-port-set! uconn (conc addr":"port)) (udat-socket-set! uconn tlsn) uconn)) ;; run-listener does all the work of starting a listener in a thread ;; it then returns control ;; (define (run-listener handler-proc #!optional (port-suggestion 4242)) (let* ((uconn (make-udat))) (udat-work-proc-set! uconn handler-proc) (if (setup-listener uconn port-suggestion) (let* ((th1 (make-thread (lambda ()(ulex-cmd-loop uconn)) "Ulex command loop")) #;(th2 (make-thread (lambda ()(process-work-queue uconn)) "Ulex work queue processor"))) (tcp-buffer-size 2048) ;; (max-connections 2048) (thread-start! th1) #;(thread-start! th2) (udat-cmd-thread-set! uconn th1) #;(udat-work-queue-thread-set! uconn th2) (print "cmd loop and process workers started") uconn) (assert #f "ERROR: run-listener called without proper setup.")))) (define (wait-and-close uconn) (thread-join! (udat-cmd-thread uconn)) (tcp-close (udat-socket uconn))) ;;====================================================================== ;; peers and connections ;;====================================================================== (define *send-mutex* (make-mutex)) ;; send structured data to recipient ;; ;; NOTE: qrykey is what was called the "cookie" previously ;; ;; retval tells send to expect and wait for return data (one line) and return it or time out ;; this is for ping where we don't want to necessarily have set up our own server yet. ;; ;; NOTE: see below for beginnings of code to allow re-use of tcp connections ;; - I believe (without substantial evidence) that re-using connections will ;; be beneficial ... ;; (define (send udata host-port qrykey cmd params) (mutex-lock! *send-mutex*) (let* ((my-host-port (udat-host-port udata)) ;; remote will return to this (isme #f #;(equal? host-port my-host-port)) ;; calling myself? ;; dat is a self-contained work block that can be sent or handled locally (dat (list my-host-port qrykey cmd params))) (if isme (ulex-handler udata dat) ;; no transmission needed (handle-exceptions ;; TODO - MAKE THIS EXCEPTION CMD SPECIFIC? exn #f (let-values (((inp oup)(tcp-connect host-port))) (let ((res (if (and inp oup) (begin (serialize dat oup) (deserialize inp)) ;; yes, we always want an ack (begin (print "ERROR: send called but no receiver has been setup. Please call setup first!") #f)))) (close-input-port inp) (close-output-port oup) (mutex-unlock! *send-mutex*) res)))))) ;; res will always be 'ack ;; send a request to the given host-port and register a mailbox in udata ;; wait for the mailbox data and return it ;; (define (send-receive uconn host-port cmd data) (cond ((member cmd '(ping goodbye)) ;; these are immediate (send uconn host-port 'ping cmd data)) (else (let* ((cmbox (get-cmbox uconn)) ;; would it be better to keep a stack of mboxes to reuse? (qrykey (car cmbox)) (mbox (cdr cmbox)) (mbox-time (current-milliseconds)) (sres (send uconn host-port qrykey cmd data))) ;; short res sres)))) ;;====================================================================== ;; responder side ;;====================================================================== ;; take a request, rdat, and if not immediate put it in the work queue ;; ;; Reserved cmds; ack ping goodbye response ;; (define (ulex-handler uconn rdat) (assert (list? rdat) "FATAL: ulex-handler give rdat as not list") (match rdat ;; (string-split controldat) ((rem-host-port qrykey cmd params) ;; (print "ulex-handler got: "rem-host-port" qrykey: "qrykey" cmd: "cmd" params: "params) (let ((mbox (hash-table-ref/default (udat-mboxes uconn) qrykey #f))) (case cmd ;; ((ack )(print "Got ack! But why? Should NOT get here.") 'ack) ((ping) ;; (print "Got Ping!") ;; (add-to-work-queue uconn rdat) 'ack) (else (do-work uconn rdat))))) (else (print "BAD DATA? controldat=" rdat) 'ack) ;; send ack anyway? )) ;; given an already set up uconn start the cmd-loop ;; (define (ulex-cmd-loop uconn) (let* ((serv-listener (udat-socket uconn))) (let loop ((state 'start)) (let-values (((inp oup)(tcp-accept serv-listener))) (let* ((rdat (deserialize inp)) ;; '(my-host-port qrykey cmd params) (resp (ulex-handler uconn rdat))) (if resp (serialize resp oup)) (close-input-port inp) (close-output-port oup)) (loop state))))) ;; add a proc to the cmd list, these are done symetrically (i.e. in all instances) ;; so that the proc can be dereferenced remotely ;; (define (set-work-handler uconn proc) (udat-work-proc-set! uconn proc)) ;;====================================================================== ;; work queues - this is all happening on the listener side ;;====================================================================== ;; rdat is (rem-host-port qrykey cmd params) (define (add-to-work-queue uconn rdat) #;(queue-add! (udat-work-queue uconn) rdat) (mailbox-send! (udat-work-queue uconn) rdat)) (define (do-work uconn rdat) (let* ((proc (udat-work-proc uconn))) ;; get it each time - conceivebly it could change ;; put this following into a do-work procedure (match rdat ((rem-host-port qrykey cmd params) (let* ((start-time (current-milliseconds)) (result (proc rem-host-port qrykey cmd params)) (end-time (current-milliseconds)) (run-time (- end-time start-time))) result)) (else (print "ERROR: rdat "rdat", did not match rem-host-port qrykey cmd params") #f)))) (define (process-work-queue uconn) (let ((wqueue (udat-work-queue uconn)) (proc (udat-work-proc uconn)) (numthr (udat-numthreads uconn))) (let loop ((thnum 1) (threads '())) (let ((thlst (cons (make-thread (lambda () (let work-loop () (let ((rdat (mailbox-receive! wqueue 24000 'MBOX_TIMEOUT))) (do-work uconn rdat)) (work-loop))) (conc "work thread " thnum)) threads))) (if (< thnum numthr) (loop (+ thnum 1) thlst) (begin (print "ULEX: Starting "(length thlst)" worker threads.") (map thread-start! thlst) (print "ULEX: Threads started. Joining all.") (map thread-join! thlst))))))) ;; below was to enable re-use of connections. This seems non-trivial so for ;; now lets open on each call ;; ;; ;; given host-port get or create peer struct ;; ;; ;; (define (udat-get-peer uconn host-port) ;; (or (hash-table-ref/default (udat-peers uconn) host-port #f) ;; ;; no peer, so create pdat and init it ;; ;; ;; NEED stack of connections, pop and use; inp, oup, ;; ;; creation_time (remove and create new if over 24hrs old ;; ;; ;; (let ((pdat (make-pdat host-port: host-port))) ;; (hash-table-set! (udat-peers uconn) host-port pdat) ;; pdat))) ;; ;; ;; is pcon alive ;; ;; ;; given host-port and pdat get a pcon ;; ;; ;; (define (pdat-get-pcon pdat host-port) ;; (let loop ((conns (pdat-conns pdat))) ;; (if (null? conns) ;; none? make and return - do NOT add - it will be pushed back on list later ;; (init-pcon (make-pcon)) ;; (let* ((conn (pop conns))) ;; ;; ;; given host-port get a pcon struct ;; ;; ;; (define (udat-get-pcon ;;====================================================================== ;; misc utils ;;====================================================================== (define (make-cookie uconn) (let ((newcnum (+ (udat-cnum uconn) 1))) (udat-cnum-set! uconn newcnum) (conc (udat-host-port uconn) ":" newcnum))) ;; cookie/mboxes ;; we store each mbox with a cookie (<cookie> . <mbox>) ;; (define (get-cmbox uconn) (if (null? (udat-avail-cmboxes uconn)) (let ((cookie (make-cookie uconn)) (mbox (make-mailbox))) (hash-table-set! (udat-mboxes uconn) cookie mbox) `(,cookie . ,mbox)) (let ((cmbox (car (udat-avail-cmboxes uconn)))) (udat-avail-cmboxes-set! uconn (cdr (udat-avail-cmboxes uconn))) cmbox))) (define (put-cmbox uconn cmbox) (udat-avail-cmboxes-set! uconn (cons cmbox (udat-avail-cmboxes uconn)))) (define (pp-uconn uconn) (pp (udat->alist uconn))) ;;====================================================================== ;; network utilities ;;====================================================================== ;; NOTE: Look at address-info egg as alternative to some of this (define (rate-ip ipaddr) (regex-case ipaddr ( "^127\\..*" _ 0 ) ( "^(10\\.0|192\\.168)\\..*" _ 1 ) ( else 2 ) )) ;; Change this to bias for addresses with a reasonable broadcast value? ;; (define (ip-pref-less? a b) (> (rate-ip a) (rate-ip b))) (define (get-my-best-address) (let ((all-my-addresses (get-all-ips))) (cond ((null? all-my-addresses) (get-host-name)) ;; no interfaces? ((eq? (length all-my-addresses) 1) (car all-my-addresses)) ;; only one to choose from, just go with it (else (car (sort all-my-addresses ip-pref-less?)))))) (define (get-all-ips-sorted) (sort (get-all-ips) ip-pref-less?)) (define (get-all-ips) (map address-info-host (filter (lambda (x) (equal? (address-info-type x) "tcp")) (address-infos (get-host-name))))) ) |
Added ulex-trials/Makefile version [e108f86d9a].
> > > > > > > > | 1 2 3 4 5 6 7 8 | ulex-test : ulex-test.scm csc ulex-test.scm test : ulex-test ./ulex-test do-test clean : rm -f .runners/* NBFAKE* |
Added ulex-trials/ulex-test.scm version [563b467581].
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 | (include "../ulex/ulex.scm") (module ulex-test * (import scheme (chicken io) (chicken base) (chicken time) (chicken file) (chicken file posix) (chicken string) (chicken process-context) (chicken process-context posix) miscmacros ;; nng srfi-18 srfi-69 test matchable typed-records system-information directory-utils ulex ) (define help "Usage: ulex-test COMMAND where COMMAND is one of: do-test : run the basic req/rep test run tcp://host:port : start test server - start several in same dir ") (define address-tcp-1 "tcp://localhost:5555") (define address-tcp-2 "tcp://localhost:6666") (define address-inproc-1 "inproc://local1") (define address-inproc-2 "inproc://local2") ;;; ;;; Req-Rep ;;; (define (make-listening-reply-socket address) (let ((socket (make-rep-socket))) (socket-set! socket 'nng/recvtimeo 2000) (nng-listen socket address) socket)) (define (make-dialed-request-socket address) (let ((socket (make-req-socket))) (socket-set! socket 'nng/recvtimeo 2000) (nng-dial socket address) socket)) (define (req-rep-test address) (let ((rep (make-listening-reply-socket address)) (req (make-dialed-request-socket address))) (nng-send req "message 1") (nng-recv rep) (nng-send rep "message") (begin0 (nng-recv req) (nng-close! rep)))) (define (do-test) (test-group "nng" (test "tcp req-rep" "message" (req-rep-test address-tcp-1)) (test "inproc req-rep" "message" (req-rep-test address-inproc-1))) (test-exit)) ;; this should be run in a thread (define (run-listener-responder socket myaddr) (let loop ((status 'running)) (let* ((msg (nng-recv socket)) (response (process-message msg))) (if (not (eq? response 'done)) (begin (nng-send socket response) (loop status)))))) (define *channels* (make-hash-table)) (define (call channels msg addr) (let* ((csocket (hash-table-ref/default channels addr #f)) (socket (or csocket (make-dialed-request-socket addr)))) (nng-send socket msg) (print "Sent: "msg", received: "(nng-recv socket)) (if (not (hash-table-exists? channels addr)) (hash-table-set! channels addr socket)))) ;; start => hello 0 ;; hello 0 => hello 1 ;; hello 1 => hello 2 ;; ... ;; hello 11 => 'done ;; (define (process-message mesg) (let ((parts (string-split mesg))) (match parts ((msg c) (let ((count (string->number c))) (if (> count 10) 'done (conc msg " " (if count count 0))))) ((msg) (conc msg " 0")) (else "hello 0")))) (define (main) (match (command-line-arguments) (("do-test")(do-test)) ((run myaddr) ;; start listener ;; put myaddr into file by host-pid in .runners ;; for 1 minute ;; get all in .runners ;; call each with a message ;; (let* ((endtimes (+ (current-seconds) 20)) ;; run for 20 seconds (socket (make-listening-reply-socket myaddr)) (rfile (conc ".runners/"(get-host-name)"-"(current-process-id))) (th1 (make-thread (lambda () (run-listener-responder socket myaddr) ) "responder"))) (if (not (and (file-exists? ".runners") (directory? ".runners"))) (create-directory ".runners" #t)) (with-output-to-file rfile (lambda () (print myaddr))) (thread-start! th1) (let loop ((entries '())) (if (> (current-seconds) endtimes) (begin (delete-file* rfile) (sleep 1) (exit)) (if (null? entries) (loop (glob ".runners/*")) (let* ((entry (car entries)) (destaddr (with-input-from-file entry read-line))) (call *channels* (conc "hello-from-"destaddr) destaddr) ;; (thread-sleep! 0.025) (loop (cdr entries)))))))) ((cmd)(print "ERROR: command "cmd", not recognised.\n\n"help)) (else (print help)))) ) ;; end module (import nng-test) (main) |
Modified ulex.scm from [39353b5283] to [f004a2cedd].
︙ | ︙ | |||
15 16 17 18 19 20 21 | ;; ;; You should have received a copy of the GNU General Public License ;; along with Megatest. If not, see <http://www.gnu.org/licenses/>. ;;====================================================================== (declare (unit ulex)) | < > | 15 16 17 18 19 20 21 22 23 24 | ;; ;; You should have received a copy of the GNU General Public License ;; along with Megatest. If not, see <http://www.gnu.org/licenses/>. ;;====================================================================== (declare (unit ulex)) (include "ulex/ulex.scm") ;; (include "ulex-simple/ulex.scm") |
Added ulex/Makefile version [1659f80ce4].
> > > > > > > > > | 1 2 3 4 5 6 7 8 9 | all : ulex.pdf ulex.png ulex.pdf : ulex.dot dot -Tpdf ulex.dot -o ulex.pdf ulex.png : ulex.dot dot -Tpng ulex.dot -o ulex.png |
Deleted ulex/ulex-prev-orig.scm version [42b648b50c].
|
| < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < |
Added ulex/ulex.dot version [08f8e06c6d].
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 | // Copyright 2006-2017, Matthew Welland. // // This file is part of Megatest. // // Megatest is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // Megatest is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU General Public License // along with Megatest. If not, see <http://www.gnu.org/licenses/>. digraph G { // graph[center=true, margin=0.2, nodesep=0.1, ranksep=0.3] layout=neato; // layout=fdp; // overlap=scalexy; //false, compress, ... overlap=scalexy; // sep="+1"; // 0.1, +1 sep="-0.1"; user_program [label="user program"]; subgraph cluster_1 { node [style=filled,shape=oval]; label = "caller"; color=brown; send_receive [label="(send-receive uconn\n host-port qrykey cmd data)"]; send [label="(send uconn host-port\n qrykey cmd data)"]; ulex_cmd_loopcaller [label="(ulex-cmd-loop uconn)"]; ulex_handlercaller [label="(ulex-handler uconn rdat)"]; mailbox [label="mailbox\n\nrdat\n...",shape=box]; send_receive -> send; ulex_cmd_loopcaller -> ulex_handlercaller; ulex_handlercaller -> mailbox; mailbox -> send_receive; } subgraph cluster_2 { node [shape=oval]; label = "listener"; color=green; ulex_cmd_loop [label="(ulex-cmd-loop uconn)"]; ulex_handler [label="(ulex-handler \nuconn rdat)"]; add_to_work_queue [label="(add-to-work-queue\n uconn rdat)"]; queue [label="queue\n\nrdat\n...",shape=box]; process_work_queue [label="(process-work-queue uconn)"]; do_work [label="(do-work uconn rdat)\nrdat: '(rem-host-port qrykey cmd params)"]; user_proc [label="(proc rem-host-port\n qrykey cmd params)\n;; proc supplied by user"]; sendlis [label="(send uconn host-port\n qrykey 'response result)"]; ulex_cmd_loop -> ulex_handler [label="rdat"]; ulex_handler -> add_to_work_queue [label="rdat"]; add_to_work_queue -> queue [label="rdat"]; subgraph cluster_3 { label = "remote work"; color=blue; queue -> process_work_queue [label="rdat"]; process_work_queue -> do_work [label="rdat"]; do_work -> user_proc; // [label="rdat: '(rem-host-port\n qrykey cmd params)"]; } } user_proc -> sendlis; user_program -> send_receive; send_receive -> user_program; send -> ulex_cmd_loop [label="rdat: '(host-port\n qrykey cmd data)"]; sendlis -> ulex_cmd_loopcaller [label="rdat: '(host-port qrykey\n 'response result)"]; ulex_handler -> send [label="'ack"]; ulex_handlercaller -> sendlis [label="'ack"]; } // check_available_queue -> remove_entries_over_10s_old; // remove_entries_over_10s_old -> set_available [label="num_avail < 3"]; // remove_entries_over_10s_old -> exit [label="num_avail > 2"]; // // set_available -> delay_2s; // delay_2s -> check_place_in_queue; // // check_place_in_queue -> "http:transport-launch" [label="at head"]; // check_place_in_queue -> exit [label="not at head"]; // // "client:login" -> "server:shutdown" [label="login failed"]; // "server:shutdown" -> exit; // // subgraph cluster_2 { // "http:transport-launch" -> "http:transport-run"; // "http:transport-launch" -> "http:transport-keep-running"; // // "http:transport-keep-running" -> "tests running?"; // "tests running?" -> "client:login" [label=yes]; // "tests running?" -> "server:shutdown" [label=no]; // "client:login" -> delay_5s [label="login ok"]; // delay_5s -> "http:transport-keep-running"; // } // // start_server -> "server_running?"; // "server_running?" -> set_available [label="no"]; // "server_running?" -> delay_2s [label="yes"]; // delay_2s -> "still_running?"; // "still_running?" -> ping_server [label=yes]; // "still_running?" -> set_available [label=no]; // ping_server -> exit [label=alive]; // ping_server -> remove_server_record [label=dead]; // remove_server_record -> set_available; // set_available -> avail_delay [label="delay 3s"]; // avail_delay -> "first_in_queue?"; // // "first_in_queue?" -> set_running [label=yes]; // set_running -> get_next_port -> handle_requests; // "first_in_queue?" -> "dead_entry_in_queue?" [label=no]; // "dead_entry_in_queue?" -> "server_running?" [label=no]; // "dead_entry_in_queue?" -> "remove_dead_entries" [label=yes]; // remove_dead_entries -> "server_running?"; // // handle_requests -> start_shutdown [label="no traffic\nno running tests"]; // handle_requests -> shutdown_request; // start_shutdown -> shutdown_delay; // shutdown_request -> shutdown_delay; // shutdown_delay -> exit; |
Added ulex/ulex.pdf version [a11f1937d3].
cannot compute difference between binary files
Added ulex/ulex.png version [1ca21b08ef].
cannot compute difference between binary files
Modified ulex/ulex.scm from [88c5ce4192] to [3ca1071e09].
1 2 | ;; ulex: Distributed sqlite3 db ;;; | | | 1 2 3 4 5 6 7 8 9 10 | ;; ulex: Distributed sqlite3 db ;;; ;; Copyright (C) 2018-2021 Matt Welland ;; Redistribution and use in source and binary forms, with or without ;; modification, is permitted. ;; ;; THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS ;; OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED ;; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ;; ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE |
︙ | ︙ | |||
22 23 24 25 26 27 28 | ;; See README in the distribution at https://www.kiatoa.com/fossils/ulex ;; NOTES: ;; Why sql-de-lite and not say, dbi? - performance mostly, then simplicity. ;; ;;====================================================================== (module ulex | > | > > > > > > > > > > > > > > > > > > > > > > > > > | > | | | | | > > > > | | | | | < | > | | | | | | | > | < > > > > > > | 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 | ;; See README in the distribution at https://www.kiatoa.com/fossils/ulex ;; NOTES: ;; Why sql-de-lite and not say, dbi? - performance mostly, then simplicity. ;; ;;====================================================================== (module ulex ( ;; NOTE: looking for the handler proc - find the run-listener :) run-listener ;; (run-listener handler-proc [port]) => uconn ;; NOTE: handler-proc params; ;; (handler-proc rem-host-port qrykey cmd params) send-receive ;; (send-receive uconn host-port cmd data) ;; NOTE: cmd can be any plain text symbol except for these; ;; 'ping 'ack 'goodbye 'response set-work-handler ;; (set-work-handler proc) wait-and-close ;; (wait-and-close uconn) ulex-listener? ;; needed to get the interface:port that was automatically found udat-port udat-host-port ;; for testing only ;; pp-uconn ) (import scheme chicken.base chicken.file chicken.time chicken.condition chicken.string chicken.sort chicken.pretty-print address-info mailbox matchable ;; queues regex regex-case s11n srfi-1 srfi-18 srfi-4 srfi-69 system-information tcp6 typed-records ) ;; udat struct, used by both caller and callee ;; instantiated as uconn by convention ;; (defstruct udat ;; the listener side (port #f) (host-port #f) (socket #f) ;; the peers (peers (make-hash-table)) ;; host:port->peer ;; work handling (work-queue (make-mailbox)) (work-proc #f) ;; set by user (cnum 0) ;; cookie number (mboxes (make-hash-table)) ;; for the replies (avail-cmboxes '()) ;; list of (<cookie> . <mbox>) for re-use ;; threads (numthreads 10) (cmd-thread #f) (work-queue-thread #f) ) ;; ;; struct for keeping track of others we are talking to ;; ;; ;; (defstruct pdat ;; (host-port #f) ;; (conns '()) ;; list of pcon structs, pop one off when calling the peer ;; ) ;; ;; ;; struct for peer connections, keep track of expiration etc. ;; ;; ;; (defstruct pcon ;; (inp #f) ;; (oup #f) ;; (exp (+ (current-seconds) 59)) ;; expires at this time, set to (+ (current-seconds) 59) ;; (lifetime (+ (current-seconds) 600)) ;; throw away and create new after five minutes ;; ) ;;====================================================================== ;; listener ;;====================================================================== ;; is uconn a ulex connector (listener) ;; (define (ulex-listener? uconn) (udat? uconn)) ;; create a tcp listener and return a populated udat struct with ;; my port, address, hostname, pid etc. ;; return #f if fail to find a port to allocate. ;; ;; if udata-in is #f create the record ;; if there is already a serv-listener return the udata ;; |
︙ | ︙ | |||
109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 | ;; sockaddr-address, sockaddr-port, sockaddr->string (let* ((tlsn (tcp-listen port 1000 #f)) ;; (tcp-listen TCPPORT [BACKLOG [HOST]]) (addr (get-my-best-address))) ;; (hostinfo-addresses (host-information (current-hostname))) (udat-port-set! uconn port) (udat-host-port-set! uconn (conc addr":"port)) (udat-socket-set! uconn tlsn) uconn)) ;;====================================================================== ;; peers and connections ;;====================================================================== ;; send structured data to recipient ;; ;; NOTE: qrykey is what was called the "cookie" previously ;; ;; retval tells send to expect and wait for return data (one line) and return it or time out ;; this is for ping where we don't want to necessarily have set up our own server yet. | > > > > > > > > > > > > > > > > > > > > > > > > > | 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 | ;; sockaddr-address, sockaddr-port, sockaddr->string (let* ((tlsn (tcp-listen port 1000 #f)) ;; (tcp-listen TCPPORT [BACKLOG [HOST]]) (addr (get-my-best-address))) ;; (hostinfo-addresses (host-information (current-hostname))) (udat-port-set! uconn port) (udat-host-port-set! uconn (conc addr":"port)) (udat-socket-set! uconn tlsn) uconn)) ;; run-listener does all the work of starting a listener in a thread ;; it then returns control ;; (define (run-listener handler-proc #!optional (port-suggestion 4242)) (let* ((uconn (make-udat))) (udat-work-proc-set! uconn handler-proc) (if (setup-listener uconn port-suggestion) (let* ((th1 (make-thread (lambda ()(ulex-cmd-loop uconn)) "Ulex command loop")) (th2 (make-thread (lambda ()(process-work-queue uconn)) "Ulex work queue processor"))) (tcp-buffer-size 2048) ;; (max-connections 2048) (thread-start! th1) (thread-start! th2) (udat-cmd-thread-set! uconn th1) (udat-work-queue-thread-set! uconn th2) (print "cmd loop and process workers started") uconn) (assert #f "ERROR: run-listener called without proper setup.")))) (define (wait-and-close uconn) (thread-join! (udat-cmd-thread uconn)) (tcp-close (udat-socket uconn))) ;;====================================================================== ;; peers and connections ;;====================================================================== (define *send-mutex* (make-mutex)) ;; send structured data to recipient ;; ;; NOTE: qrykey is what was called the "cookie" previously ;; ;; retval tells send to expect and wait for return data (one line) and return it or time out ;; this is for ping where we don't want to necessarily have set up our own server yet. |
︙ | ︙ | |||
136 137 138 139 140 141 142 143 144 145 | (dat (list my-host-port qrykey cmd params)) ) (if isme (ulex-handler udata dat) ;; no transmission needed (handle-exceptions ;; TODO - MAKE THIS EXCEPTION CMD SPECIFIC? exn #f (let-values (((inp oup)(tcp-connect host-port))) (let ((res (if (and inp oup) (begin | > > | | > | > > > > | | | | | > | > > | | | > > | > > | | | | | | | > | | | | | | | < < < < < < < < < < < < < < < < < | | | > | | > | > > > | > > | < | > | < | > > | | | > > > > > > > > > > | 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 | (dat (list my-host-port qrykey cmd params)) ) (if isme (ulex-handler udata dat) ;; no transmission needed (handle-exceptions ;; TODO - MAKE THIS EXCEPTION CMD SPECIFIC? exn #f (begin ; (mutex-lock! *send-mutex*) (let-values (((inp oup)(tcp-connect host-port))) (let ((res (if (and inp oup) (begin (serialize dat oup) (deserialize inp)) ;; yes, we always want an ack (begin (print "ERROR: send called but no receiver has been setup. Please call setup first!") #f)))) (close-input-port inp) (close-output-port oup) ; (mutex-unlock! *send-mutex*) res))))))) ;; res will always be 'ack ;; send a request to the given host-port and register a mailbox in udata ;; wait for the mailbox data and return it ;; (define (send-receive uconn host-port cmd data) (cond ((member cmd '(ping goodbye)) ;; these are immediate (send uconn host-port 'ping cmd data)) (else (let* ((cmbox (get-cmbox uconn)) ;; would it be better to keep a stack of mboxes to reuse? (qrykey (car cmbox)) (mbox (cdr cmbox)) (mbox-time (current-milliseconds)) (sres (send uconn host-port qrykey cmd data))) ;; short res (if (eq? sres 'ack) (let* ((mbox-timeout-secs 120 #;(if (eq? 'primordial (thread-name (current-thread))) #f 120)) ;; timeout) (mbox-timeout-result 'MBOX_TIMEOUT) (res (mailbox-receive! mbox mbox-timeout-secs mbox-timeout-result)) (mbox-receive-time (current-milliseconds))) ;; (put-cmbox uconn cmbox) ;; reuse mbox and cookie. is it worth it? (hash-table-delete! (udat-mboxes uconn) qrykey) (if (eq? res 'MBOX_TIMEOUT) (begin (print "WARNING: mbox timed out for query "cmd", with data "data) #f) ;; convert to raising exception? res)) (begin ;; (print "ERROR: Communication failed? Got "sres) #f)))))) ;;====================================================================== ;; responder side ;;====================================================================== ;; take a request, rdat, and if not immediate put it in the work queue ;; ;; Reserved cmds; ack ping goodbye response ;; (define (ulex-handler uconn rdat) (assert (list? rdat) "FATAL: ulex-handler give rdat as not list") (match rdat ;; (string-split controldat) ((rem-host-port qrykey cmd params) ;; (print "ulex-handler got: "rem-host-port" qrykey: "qrykey" cmd: "cmd" params: "params) (let ((mbox (hash-table-ref/default (udat-mboxes uconn) qrykey #f))) (case cmd ;; ((ack )(print "Got ack! But why? Should NOT get here.") 'ack) ((ping) ;; (print "Got Ping!") ;; (add-to-work-queue uconn rdat) 'ack) ((goodbye) ;; just clear out references to the caller (add-to-work-queue uconn rdat) 'ack) ((response) ;; this is a result from remote processing, send it as mail ... (if mbox (begin (mailbox-send! mbox params) ;; params here is our result 'ack) (begin (print "ERROR: received result but no associated mbox for cookie "qrykey) #f))) (else ;; (print "Got generic request: "cmd) (add-to-work-queue uconn rdat) 'ack)))) (else (print "BAD DATA? controldat=" rdat) 'ack) ;; send ack anyway? )) ;; given an already set up uconn start the cmd-loop ;; (define (ulex-cmd-loop uconn) (let* ((serv-listener (udat-socket uconn))) (let loop ((state 'start)) (let-values (((inp oup)(tcp-accept serv-listener))) (let* ((rdat (deserialize inp)) ;; '(my-host-port qrykey cmd params) (resp (ulex-handler uconn rdat))) (if resp (serialize resp oup)) (close-input-port inp) (close-output-port oup)) (loop state))))) ;; add a proc to the cmd list, these are done symetrically (i.e. in all instances) ;; so that the proc can be dereferenced remotely ;; (define (set-work-handler uconn proc) (udat-work-proc-set! uconn proc)) ;;====================================================================== ;; work queues - this is all happening on the listener side ;;====================================================================== ;; rdat is (rem-host-port qrykey cmd params) (define (add-to-work-queue uconn rdat) #;(queue-add! (udat-work-queue uconn) rdat) (mailbox-send! (udat-work-queue uconn) rdat)) (define (do-work uconn rdat) (let* ((proc (udat-work-proc uconn))) ;; get it each time - conceivebly it could change ;; put this following into a do-work procedure (match rdat ((rem-host-port qrykey cmd params) (let* ((start-time (current-milliseconds)) (result (proc rem-host-port qrykey cmd params)) (end-time (current-milliseconds)) (run-time (- end-time start-time))) (print "ULEX: work "cmd", "params" done in "run-time" ms") ;; send 'response as cmd and result as params (send uconn rem-host-port qrykey 'response result) ;; could check for ack (print "ULEX: response sent back to "rem-host-port" in "(- (current-milliseconds) end-time)))) (MBOX_TIMEOUT #f) (else (print "ERROR: rdat "rdat", did not match rem-host-port qrykey cmd params"))))) (define (process-work-queue uconn) (let ((wqueue (udat-work-queue uconn)) (proc (udat-work-proc uconn)) (numthr (udat-numthreads uconn))) (let loop ((thnum 1) (threads '())) (let ((thlst (cons (make-thread (lambda () (let work-loop () (let ((rdat (mailbox-receive! wqueue 24000 'MBOX_TIMEOUT))) (do-work uconn rdat)) (work-loop))) (conc "work thread " thnum)) threads))) (if (< thnum numthr) (loop (+ thnum 1) thlst) (begin (print "ULEX: Starting "(length thlst)" worker threads.") (map thread-start! thlst) (print "ULEX: Threads started. Joining all.") (map thread-join! thlst))))))) ;; below was to enable re-use of connections. This seems non-trivial so for ;; now lets open on each call ;; ;; ;; given host-port get or create peer struct ;; ;; ;; (define (udat-get-peer uconn host-port) |
︙ | ︙ | |||
372 373 374 375 376 377 378 | (define (get-all-ips) (map address-info-host (filter (lambda (x) (equal? (address-info-type x) "tcp")) (address-infos (get-host-name))))) | < < < | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | 450 451 452 453 454 455 456 457 | (define (get-all-ips) (map address-info-host (filter (lambda (x) (equal? (address-info-type x) "tcp")) (address-infos (get-host-name))))) ) |