Index: Makefile ================================================================== --- Makefile +++ Makefile @@ -17,11 +17,10 @@ # make install CSCOPTS='-accumulate-profile -profile-name $(PWD)/profile-ww$(shell date +%V.%u)' # rm .o ; make install CSCOPTS='-profile' ; ... ; chicken-profile | less SHELL=/bin/bash PREFIX=$(PWD) -CHICKEN_PREFIX=$(PWD)/new-chicken/ CSCOPTS= INSTALL=install SRCFILES = common.scm items.scm launch.scm ods.scm runconfig.scm \ server.scm configf.scm db.scm keys.scm margs.scm \ megatest-version.scm process.scm runs.scm tasks.scm \ @@ -67,16 +66,19 @@ CSIPATH=$(shell which csi) CKPATH=$(shell dirname $(shell dirname $(CSIPATH))) ARCHSTR=$(shell if [[ -e /usr/bin/sw_vers ]]; then /usr/bin/sw_vers -productVersion; else lsb_release -sr; fi) # ARCHSTR=$(shell bash -c "echo \$$MACHTYPE") +# if have csi on path use that, else use default +CHICKEN_PREFIX=$(or $(CKPATH),$(PREFIX)/$(ARCHSTR)) + PNGFILES = $(shell cd docs/manual;ls *png) -#all : $(PREFIX)/bin/.$(ARCHSTR) mtest dboard mtut ndboard all : $(PREFIX)/bin/.$(ARCHSTR) mtest dboard mtut tcmt -# megatest.o : ducttape-lib.import.o +whatever : + @echo "CHICKEN_PREFIX=$(CHICKEN_PREFIX)" mtest: $(OFILES) readline-fix.scm megatest.o $(MOFILES) $(MOIMPFILES) csc $(CSCOPTS) $(OFILES) $(MOFILES) $(MOIMPFILES) megatest.o -o mtest showmtesthash: @@ -213,49 +215,17 @@ csc $(CSCOPTS) -c common.scm mofiles/commonmod.o configf.o : configf.scm mofiles/commonmod.o csc $(CSCOPTS) -c configf.scm mofiles/commonmod.o -$(PREFIX)/bin/.$(ARCHSTR)/bin/mtest : mtest utils/mk_wrapper +$(PREFIX)/bin/.$(ARCHSTR)/bin/mtest : mtest @echo Installing to PREFIX=$(PREFIX) $(INSTALL) mtest $(PREFIX)/bin/.$(ARCHSTR)/bin/mtest + +$(PREFIX)/bin/megatest : $(PREFIX)/bin/.$(ARCHSTR)/bin/mtest utils/mk_wrapper utils/mk_wrapper $(PREFIX) mtest $(PREFIX)/bin/megatest - utils/mk_wrapper_tool $(PREFIX) chicken $(PREFIX)/bin/chicken - utils/mk_wrapper_tool $(PREFIX) chicken-bind $(PREFIX)/bin/chicken-bind - utils/mk_wrapper_tool $(PREFIX) chicken-bug $(PREFIX)/bin/chicken-bug - utils/mk_wrapper_tool $(PREFIX) chicken-dump $(PREFIX)/bin/chicken-dump - utils/mk_wrapper_tool $(PREFIX) chicken-install $(PREFIX)/bin/chicken-install - utils/mk_wrapper_tool $(PREFIX) chicken-profile $(PREFIX)/bin/chicken-profile - utils/mk_wrapper_tool $(PREFIX) chicken-sqlite3 $(PREFIX)/bin/chicken-sqlite3 - utils/mk_wrapper_tool $(PREFIX) chicken-status $(PREFIX)/bin/chicken-status - utils/mk_wrapper_tool $(PREFIX) chicken-uninstall $(PREFIX)/bin/chicken-uninstall - utils/mk_wrapper_tool $(PREFIX) csc $(PREFIX)/bin/csc - utils/mk_wrapper_tool $(PREFIX) csi $(PREFIX)/bin/csi - utils/mk_wrapper_tool $(PREFIX) feathers $(PREFIX)/bin/feathers - utils/mk_wrapper_tool $(PREFIX) nanocat $(PREFIX)/bin/nanocat - utils/mk_wrapper_tool $(PREFIX) sqlite3 $(PREFIX)/bin/sqlite3 - utils/mk_wrapper_tool $(PREFIX) vacuumdb $(PREFIX)/bin/vacuumdb - utils/mk_wrapper_tool $(PREFIX) logpro $(PREFIX)/bin/logpro - utils/mk_wrapper_tool $(PREFIX) refdb $(PREFIX)/bin/refdb chmod a+x $(PREFIX)/bin/megatest - chmod a+x $(PREFIX)/bin/chicken - chmod a+x $(PREFIX)/bin/chicken-bind - chmod a+x $(PREFIX)/bin/chicken-bug - chmod a+x $(PREFIX)/bin/chicken-dump - chmod a+x $(PREFIX)/bin/chicken-install - chmod a+x $(PREFIX)/bin/chicken-profile - chmod a+x $(PREFIX)/bin/chicken-sqlite3 - chmod a+x $(PREFIX)/bin/chicken-status - chmod a+x $(PREFIX)/bin/chicken-uninstall - chmod a+x $(PREFIX)/bin/csc - chmod a+x $(PREFIX)/bin/csi - chmod a+x $(PREFIX)/bin/feathers - chmod a+x $(PREFIX)/bin/nanocat - chmod a+x $(PREFIX)/bin/sqlite3 - chmod a+x $(PREFIX)/bin/vacuumdb - chmod a+x $(PREFIX)/bin/refdb - chmod a+x $(PREFIX)/bin/logpro $(PREFIX)/bin/.$(ARCHSTR)/bin/ndboard : ndboard $(INSTALL) ndboard $(PREFIX)/bin/.$(ARCHSTR)/bin/ndboard $(PREFIX)/bin/newdashboard : $(PREFIX)/bin/.$(ARCHSTR)/ndboard utils/mk_wrapper @@ -351,14 +321,16 @@ make -C helpers $@ PREFIX=$(PREFIX) INSTALL=$(INSTALL) ARCHSTR=$(ARCHSTR) mtest-reaper: $(PREFIX)/bin/mtest-reaper # install dashboard as dboard so wrapper script can be called dashboard -$(PREFIX)/bin/.$(ARCHSTR)/bin/dboard : dboard $(FILES) utils/mk_wrapper +$(PREFIX)/bin/.$(ARCHSTR)/bin/dboard : dboard $(FILES) + $(INSTALL) dboard $(PREFIX)/bin/.$(ARCHSTR)/bin/dboard + +$(PREFIX)/bin/dashboard : $(PREFIX)/bin/.$(ARCHSTR)/bin/dboard utils/mk_wrapper utils/mk_wrapper $(PREFIX) dboard $(PREFIX)/bin/dashboard chmod a+x $(PREFIX)/bin/dashboard - $(INSTALL) dboard $(PREFIX)/bin/.$(ARCHSTR)/bin/dboard install : $(PREFIX)/bin/.$(ARCHSTR) $(PREFIX)/bin/.$(ARCHSTR)/bin/mtest $(PREFIX)/bin/megatest \ $(PREFIX)/bin/.$(ARCHSTR)/bin/dboard $(PREFIX)/bin/dashboard $(HELPERS) $(PREFIX)/bin/nbfake \ $(PREFIX)/bin/.$(ARCHSTR)/bin/mtexec $(PREFIX)/bin/mtexec $(PREFIX)/bin/serialize-env \ $(PREFIX)/bin/nbfind $(PREFIX)/bin/mtrunner $(PREFIX)/bin/viewscreen $(PREFIX)/bin/mt_xterm \ @@ -561,12 +533,30 @@ $(CHICKEN_PREFIX)/bin/csi : $(CHICKEN_PREFIX)/bin/sqlite3 $(CHICKEN_PREFIX)/lib/libiupweb.so $(CHICKEN_PREFIX)/chicken-4.13.0/LICENSE cd build-$(USER)/chicken-4.13.0;make PLATFORM=linux PREFIX=$(CHICKEN_PREFIX) cd build-$(USER)/chicken-4.13.0;make PLATFORM=linux PREFIX=$(CHICKEN_PREFIX) install -chicken : $(CHICKEN_PREFIX)/bin/csi $(EGGSTARG2) - echo "Fake target to build prefix chicken" +ALL_CKBIN=chicken chicken-bind chicken-bug chicken-dump \ +chicken-install chicken-profile chicken-sqlite3 chicken-status \ +chicken-uninstall csc csi feathers nanocat sqlite3 vacuumdb logpro \ +refdb + +# CHICKEN_BIN_DIR=$(shell dirname $(shell which csi)) +CKBIN_WRAPPERS=$(addprefix $(PREFIX)/bin/,$(ALL_CKBIN)) + + +$(PREFIX)/bin/% : $(CHICKEN_PREFIX)/bin/% $(CHICKEN_PREFIX)/bin/csi $(EGGSTARG2) + utils/mk_wrapper_tool $(PREFIX) $* $(PREFIX)/bin/$* + chmod a+x $(PREFIX)/bin/$* + +$(PREFIX)/bin : + mkdir -p $(PREFIX)/bin + +chicken : $(CKBIN_WRAPPERS) $(PREFIX)/bin + @echo "Fake target to build prefix chicken" + +binwrappers : $(CKBIN_WRAPPERS) postgresql.done : $(CHICKEN_PREFIX)/bin/pg_config CSC_OPTIONS="-I$(CHICKEN_PREFIX)/include -L$(CHICKEN_PREFIX)/lib -L$(CHICKEN_PREFIX)/lib64" $(CHICKEN_PREFIX)/bin/chicken-install postgresql > postgresql.done nanomsg.done : $(CHICKEN_PREFIX)/lib/libnanomsg.so @@ -578,11 +568,11 @@ canvas-draw.done : CSC_OPTIONS="-I$(CHICKEN_PREFIX)/include -L$(CHICKEN_PREFIX)/lib" $(CHICKEN_PREFIX)/bin/chicken-install -D no-library-checks canvas-draw > canvas-draw.done sqlite3.done : CSC_OPTIONS="-I$(CHICKEN_PREFIX)/include -L$(CHICKEN_PREFIX)/lib" $(CHICKEN_PREFIX)/bin/chicken-install sqlite3 > sqlite3.done - + sql-de-lite.done : CSC_OPTIONS="-I$(CHICKEN_PREFIX)/include -L$(CHICKEN_PREFIX)/lib" $(CHICKEN_PREFIX)/bin/chicken-install sql-de-lite > sql-de-lite.done dbi.done : postgresql.done sqlite3.done sql-de-lite.done CSC_OPTIONS="-I$(CHICKEN_PREFIX)/include -L$(CHICKEN_PREFIX)/lib" $(CHICKEN_PREFIX)/bin/chicken-install dbi > dbi.done @@ -590,10 +580,12 @@ %.done : $(CHICKEN_PREFIX)/bin/chicken-install $* > $*.done build-$(USER)/eggs-installed/%.done : $(CHICKEN_PREFIX)/bin/csi $(EGGS) $(CHICKEN_PREFIX)/bin/chicken-install $* > build-$(USER)/eggs-installed/$*.done + + #====================================================================== # Attic #====================================================================== Index: common.scm ================================================================== --- common.scm +++ common.scm @@ -1766,11 +1766,11 @@ ;; get cpu load by reading from /proc/loadavg, return all three values ;; (define (common:get-cpu-load remote-host) (handle-exceptions exn - '(99 99 99) + '(-99 -99 -99) (let* ((actual-hostname (or remote-host (get-host-name) "localhost"))) (or (common:get-cached-info actual-hostname "cpu-load") (let ((result (if remote-host (map (lambda (res) (if (eof-object? res) 9e99 res)) @@ -1777,12 +1777,21 @@ (with-input-from-pipe (conc "ssh " remote-host " cat /proc/loadavg") (lambda ()(list (read)(read)(read))))) (with-input-from-file "/proc/loadavg" (lambda ()(list (read)(read)(read))))))) - (common:write-cached-info actual-hostname "cpu-load" result) - result))))) + (match + result + ((l1 l2 l3) + (if (and (number? l1) + (number? l2) + (number? l3)) + (begin + (common:write-cached-info actual-hostname "cpu-load" result) + result) + '(-1 -1 -1))) ;; -1 is bad result + (else '(-2 -2 -2)))))))) ;; get normalized cpu load by reading from /proc/loadavg and /proc/cpuinfo return all three values and the number of real cpus and the number of threads ;; returns alist '((adj-cpu-load . normalized-proc-load) ... etc. ;; keys: adj-proc-load, adj-core-load, 1m-load, 5m-load, 15m-load ;; @@ -2028,38 +2037,63 @@ (common:wait-for-normalized-load maxload msg remote-host (- rem-tries 1)) #f))))) ;; DO NOT CALL THIS DIRECTLY. It is called from common:wait-for-normalized-load ;; -(define (common:wait-for-cpuload maxload-in numcpus-in waitdelay #!key (count 1000) (msg #f)(remote-host #f)(force-maxload #f)) +(define (common:wait-for-cpuload maxload-in numcpus-in waitdelay #!key (count 1000) (msg #f)(remote-host #f)(force-maxload #f)(num-tries 5)) (let* ((loadavg (common:get-cpu-load remote-host)) (numcpus (if (<= 1 numcpus-in) ;; not possible to have zero. If we get 1, it's possible that we got the previous default, and we should check again (common:get-num-cpus remote-host) numcpus-in)) (maxload (if force-maxload maxload-in (max maxload-in 0.5))) ;; so maxload must be greater than 0.5 for now BUG - FIXME? (first (car loadavg)) (next (cadr loadavg)) - (adjload (* maxload (max 1 numcpus))) ;; possible bug where numcpus (or could be maxload) is zero, crude fallback is to at least use 1 + (adjmaxload (* maxload (max 1 numcpus))) ;; possible bug where + ;; numcpus (or could be + ;; maxload) is zero, + ;; crude fallback is to + ;; at least use 1 (loadjmp (- first next)) - (adjwait (min (+ 300 (random 10)) (abs (* (+ (random 10)(/ (- 1000 count) 10) waitdelay) (- first adjload) )) )));; add some randomness to the time to break any alignment where netbatch dumps many jobs to machines simultaneously - ;; let's let the user know once in a long while that load checking is happening but not constantly report it + ;; add some randomness to the time to break any alignment + ;; where netbatch dumps many jobs to machines simultaneously + (adjwait (min (+ 300 (random 10)) (abs (* (+ (random 10) + (/ (- 1000 count) 10) + waitdelay) + (- first adjmaxload) )) ))) + ;; let's let the user know once in a long while that load checking + ;; is happening but not constantly report it (if (> (random 100) 75) ;; about 25% of the time (debug:print-info 1 *default-log-port* "Checking cpuload on " (or remote-host "localhost") ", maxload: " maxload - ", load: " first ", adjload: " adjload ", loadjmp: " loadjmp)) + ", load: " first ", adjmaxload: " adjmaxload ", loadjmp: " loadjmp)) (cond - ((and (> first adjload) + ((and (< first 0) ;; this indicates the loadavg data is bad - machine may not be reachable + (> num-tries 0)) + (debug:print 0 *default-log-port* "WARNING: received bad data from get-cpu-load " first ", we'll sleep 10s and try " num-tries " more times.") + (thread-sleep! 10) + (common:wait-for-cpuload maxload-in numcpus-in waitdelay + count: count remote-host: remote-host force-maxload: force-maxload num-tries: (- num-tries 1))) + ((and (> first adjmaxload) (> count 0)) - (debug:print-info 0 *default-log-port* "server start delayed " adjwait " seconds due to load " first " exceeding max of " adjload " on server " (or remote-host (get-host-name)) " (normalized load-limit: " maxload ") " (if msg msg "")) + (debug:print-info 0 *default-log-port* + "server start delayed " adjwait + " seconds due to load " first + " exceeding max of " adjmaxload + " on server " (or remote-host (get-host-name)) + " (normalized load-limit: " maxload ") " (if msg msg "")) (thread-sleep! adjwait) (common:wait-for-cpuload maxload numcpus waitdelay count: (- count 1) msg: msg remote-host: remote-host)) ((and (> loadjmp numcpus) (> count 0)) (debug:print-info 0 *default-log-port* "waiting " adjwait " seconds due to load jump " loadjmp " > numcpus " numcpus (if msg msg "")) (thread-sleep! adjwait) - (common:wait-for-cpuload maxload numcpus waitdelay count: (- count 1) msg: msg remote-host: remote-host))))) + (common:wait-for-cpuload maxload numcpus waitdelay count: (- count 1) msg: msg remote-host: remote-host)) + (else + (if (> num-tries 0) + (debug:print 0 *default-log-port* "Load on " (or remote-host "localhost") " is acceptable at " first " continuing.") + (debug:print 0 *default-log-port* "Load on " (or remote-host "localhost") ", "first" could not be retrieved. Giving up and continuing.")))))) (define (get-uname . params) (let* ((uname-res (process:cmd-run->list (conc "uname " (if (null? params) "-a" (car params))))) (uname #f)) (if (null? (car uname-res)) Index: utils/mk_wrapper_tool ================================================================== --- utils/mk_wrapper_tool +++ utils/mk_wrapper_tool @@ -96,6 +96,6 @@ if [ "$LD_LIBRARY_PATH" != "" ];then echo "source $prefix/bin/.\$lsbr/cfg.sh" >> $target fi # echo "if [[ -e \$lsbr ]];then source \$lsbr;fi" >> $target -echo "exec $prefix/.\$lsbr/bin/$cmd \"\$@\"" >> $target +echo "exec $chicken_bin_dir/$cmd \"\$@\"" >> $target