Overview
| Comment: | Merged v1.65-real branch in and fixed misturk |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | v1.6569-multi-db |
| Files: | files | file ages | folders |
| SHA1: |
cca6340787f608ba30aaa07a26c7990a |
| User & Date: | matt on 2021-02-08 23:13:28 |
| Other Links: | branch diff | manifest | tags |
Context
|
2021-02-09
| ||
| 22:41 | Fixed (maybe) initialization of remote struct check-in: 5c4f0f8019 user: matt tags: v1.6569-multi-db | |
|
2021-02-08
| ||
| 23:13 | Merged v1.65-real branch in and fixed misturk check-in: cca6340787 user: matt tags: v1.6569-multi-db | |
| 19:37 | bit closer check-in: 99ba9a5e40 user: matt tags: v1.6569-multi-db | |
|
2021-02-05
| ||
| 15:36 | merged in changes for wildcard target in remove runs, support for dashboard in sles12, and archival of rerun data. check-in: 6ff4310a7a user: mmgraham tags: v1.65-real, v1.6583 | |
Changes
Modified Makefile from [7bc52ac29e] to [91777713e8].
| ︙ | ︙ | |||
322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 |
mtest-reaper: $(PREFIX)/bin/mtest-reaper
# install dashboard as dboard so wrapper script can be called dashboard
$(PREFIX)/bin/.$(ARCHSTR)/dboard : dboard $(FILES) utils/mk_wrapper
utils/mk_wrapper $(PREFIX) dboard $(PREFIX)/bin/dashboard
chmod a+x $(PREFIX)/bin/dashboard
$(INSTALL) dboard $(PREFIX)/bin/.$(ARCHSTR)/dboard
install : $(PREFIX)/bin/.$(ARCHSTR) $(PREFIX)/bin/.$(ARCHSTR)/mtest $(PREFIX)/bin/megatest \
$(PREFIX)/bin/.$(ARCHSTR)/dboard $(PREFIX)/bin/dashboard $(HELPERS) $(PREFIX)/bin/nbfake \
$(PREFIX)/bin/.$(ARCHSTR)/mtexec $(PREFIX)/bin/mtexec $(PREFIX)/bin/serialize-env \
$(PREFIX)/bin/nbfind $(PREFIX)/bin/mtrunner $(PREFIX)/bin/viewscreen $(PREFIX)/bin/mt_xterm \
$(PREFIX)/share/docs/megatest_manual.html $(PREFIX)/bin/remrun \
$(PREFIX)/share/docs/megatest_manual.html $(PREFIX)/bin/remrun $(PREFIX)/bin/mtutil \
$(PREFIX)/bin/tcmt $(PREFIX)/share/db/mt-pg.sql \
| > > > > > > > > > > > > > > > > > > | > > > | 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 |
mtest-reaper: $(PREFIX)/bin/mtest-reaper
# install dashboard as dboard so wrapper script can be called dashboard
$(PREFIX)/bin/.$(ARCHSTR)/dboard : dboard $(FILES) utils/mk_wrapper
utils/mk_wrapper $(PREFIX) dboard $(PREFIX)/bin/dashboard
chmod a+x $(PREFIX)/bin/dashboard
$(INSTALL) dboard $(PREFIX)/bin/.$(ARCHSTR)/dboard
$(PREFIX)/bin/.$(ARCHSTR)/lib/libpangox-1.0.so : lib/libpangox-1.0.so
if [[ $(ARCHSTR) == 12.5 ]]; then \
mkdir -p $(PREFIX)/bin/.$(ARCHSTR)/lib; \
$(INSTALL) lib/libpangox-1.0.so $(PREFIX)/bin/.$(ARCHSTR)/lib/libpangox-1.0.so; \
fi
$(PREFIX)/bin/.$(ARCHSTR)/lib/libpangox-1.0.so.0 : lib/libpangox-1.0.so.0
if [[ $(ARCHSTR) == 12.5 ]]; then \
mkdir -p $(PREFIX)/bin/.$(ARCHSTR)/lib; \
$(INSTALL) lib/libpangox-1.0.so.0 $(PREFIX)/bin/.$(ARCHSTR)/lib/libpangox-1.0.so.0; \
fi
$(PREFIX)/bin/.$(ARCHSTR)/lib/libxcb-xlib.so.0 : lib/libxcb-xlib.so.0
if [[ $(ARCHSTR) == 12.5 ]]; then \
mkdir -p $(PREFIX)/bin/.$(ARCHSTR)/lib; \
$(INSTALL) lib/libxcb-xlib.so.0 $(PREFIX)/bin/.$(ARCHSTR)/lib/libxcb-xlib.so.0; \
fi
install : $(PREFIX)/bin/.$(ARCHSTR) $(PREFIX)/bin/.$(ARCHSTR)/mtest $(PREFIX)/bin/megatest \
$(PREFIX)/bin/.$(ARCHSTR)/dboard $(PREFIX)/bin/dashboard $(HELPERS) $(PREFIX)/bin/nbfake \
$(PREFIX)/bin/.$(ARCHSTR)/mtexec $(PREFIX)/bin/mtexec $(PREFIX)/bin/serialize-env \
$(PREFIX)/bin/nbfind $(PREFIX)/bin/mtrunner $(PREFIX)/bin/viewscreen $(PREFIX)/bin/mt_xterm \
$(PREFIX)/share/docs/megatest_manual.html $(PREFIX)/bin/remrun \
$(PREFIX)/share/docs/megatest_manual.html $(PREFIX)/bin/remrun $(PREFIX)/bin/mtutil \
$(PREFIX)/bin/tcmt $(PREFIX)/share/db/mt-pg.sql \
$(PREFIX)/share/js/jquery-3.1.0.slim.min.js \
$(PREFIX)/bin/.$(ARCHSTR)/lib/libpangox-1.0.so \
$(PREFIX)/bin/.$(ARCHSTR)/lib/libpangox-1.0.so.0 \
$(PREFIX)/bin/.$(ARCHSTR)/lib/libxcb-xlib.so.0
# $(PREFIX)/bin/.$(ARCHSTR)/ndboard
# $(PREFIX)/bin/newdashboard
$(PREFIX)/bin/.$(ARCHSTR) :
mkdir -p $(PREFIX)/bin/.$(ARCHSTR)
mkdir -p $(PREFIX)/bin/.$(ARCHSTR)/lib
|
| ︙ | ︙ |
Modified api.scm from [90c3181454] to [e67163f915].
| ︙ | ︙ | |||
280 281 282 283 284 285 286 |
;;
(define (api:process-request dbstruct $) ;; the $ is the request vars proc
(debug:print 4 *default-log-port* "server-id:" *server-id*)
(let* ((cmd ($ 'cmd))
(paramsj ($ 'params))
(key ($ 'key))
(params (db:string->obj paramsj transport: 'http))) ;; incoming data from the POST (or is it a GET?)
| | | | 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 |
;;
(define (api:process-request dbstruct $) ;; the $ is the request vars proc
(debug:print 4 *default-log-port* "server-id:" *server-id*)
(let* ((cmd ($ 'cmd))
(paramsj ($ 'params))
(key ($ 'key))
(params (db:string->obj paramsj transport: 'http))) ;; incoming data from the POST (or is it a GET?)
(debug:print 4 *default-log-port* "cmd:" cmd " with params " params "key " key)
(if (equal? key *server-id*)
(begin
(set! *api-process-request-count* (+ *api-process-request-count* 1))
(let* ((resdat (api:execute-requests dbstruct (vector cmd params))) ;; process the request, resdat = #( flag result )
(success (vector-ref resdat 0))
(res (vector-ref resdat 1))) ;; (vector flag payload), get the payload, ignore the flag (why?)
(debug:print 4 *default-log-port* "res:" res)
(if (not success)
(debug:print 0 *default-log-port* "ERROR: success flag is #f for " cmd " with params " params))
(if (> *api-process-request-count* *max-api-process-requests*)
(set! *max-api-process-requests* *api-process-request-count*))
(set! *api-process-request-count* (- *api-process-request-count* 1))
;; This can be here but needs controls to ensure it doesn't run more than every 4 seconds
;; (rmt:dat->json-str
|
| ︙ | ︙ |
Modified archive.scm from [0a572a4ddb] to [1fa1cf0256].
| ︙ | ︙ | |||
121 122 123 124 125 126 127 | (let ((res (cons block-id archive-path))) (hash-table-set! blockid-cache key res) res) (begin (debug:print 0 *default-log-port* "WARNING: no disk found for " target ", " run-name ", " test-name ", archive-path=" archive-path) #f))) (begin | | | 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 | (let ((res (cons block-id archive-path))) (hash-table-set! blockid-cache key res) res) (begin (debug:print 0 *default-log-port* "WARNING: no disk found for " target ", " run-name ", " test-name ", archive-path=" archive-path) #f))) (begin (debug:print 0 *default-log-port* "WARNING: no disk found for " target ", " run-name ", " test-name ) #f)))))) ;; no best disk found ;; archive - run bup ;; ;; 1. create the bup dir if not exists ;; 2. start the du of each directory ;; 3. gen index |
| ︙ | ︙ | |||
225 226 227 228 229 230 231 | (toplevel/children (debug:print 0 *default-log-port* "WARNING: cannot archive " test-name " with id " test-id " as it is a toplevel test with children")) ((not (common:file-exists? test-path)) (debug:print 0 *default-log-port* "WARNING: Cannot archive " test-name "/" item-path " as path " test-path " does not exist")) (else | | | 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 | (toplevel/children (debug:print 0 *default-log-port* "WARNING: cannot archive " test-name " with id " test-id " as it is a toplevel test with children")) ((not (common:file-exists? test-path)) (debug:print 0 *default-log-port* "WARNING: Cannot archive " test-name "/" item-path " as path " test-path " does not exist")) (else (debug:print 2 *default-log-port* "From test-dat=" test-dat " derived the following:\n" "test-partial-path = " test-partial-path "\n" "test-path = " test-path "\n" "test-physical-path = " test-physical-path "\n" "partial-path-index = " partial-path-index "\n" "test-base = " test-base) (hash-table-set! disk-groups test-base |
| ︙ | ︙ | |||
269 270 271 272 273 274 275 | (create-directory archive-dir #t)) (case archiver ((bup) ;; Archive using bup (let* ((bup-init-params (list "-d" archive-dir "init")) (bup-index-params (append (list "-d" archive-dir "index") test-paths)) (bup-save-params (append (list "-d" archive-dir "save" ;; (conc "--strip-path=" linktree) (conc "-" compress) ;; or (conc "--compress=" compress) | | | | | > > > > | | > > > > | | > > > > | 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 |
(create-directory archive-dir #t))
(case archiver
((bup) ;; Archive using bup
(let* ((bup-init-params (list "-d" archive-dir "init"))
(bup-index-params (append (list "-d" archive-dir "index") test-paths))
(bup-save-params (append (list "-d" archive-dir "save" ;; (conc "--strip-path=" linktree)
(conc "-" compress) ;; or (conc "--compress=" compress)
"-n" (conc (common:get-testsuite-name) "-"(string-substitute "/" "-" target " "))
(conc "--strip-path=" (conc test-base target "/" )) ;; if we push to the directory do we need this?
)
test-paths)))
(if (not (common:file-exists? (conc archive-dir "/HEAD")))
(begin
;; replace this with jobrunner stuff enventually
(debug:print-info 2 *default-log-port* "Init bup in " archive-dir)
;; (mutex-lock! bup-mutex)
(let-values (((pid-val exit-status exit-code) (run-n-wait bup-exe params: bup-init-params print-cmd: print-prefix)))
(if (not (eq? exit-code 0))
(begin
(debug:print-error 0 *default-log-port* "There was an error initializing bup. Archive failed.")
(exit 1))))
;; (mutex-unlock! bup-mutex)
))
(debug:print-info 2 *default-log-port* "Indexing data to be archived")
;; (mutex-lock! bup-mutex)
(let-values (((pid-val exit-status exit-code) (run-n-wait bup-exe params: bup-index-params print-cmd: print-prefix)))
(if (not (eq? exit-code 0))
(begin
(debug:print-error 0 *default-log-port* "There was an error Indexing bup. Archive failed.")
(exit 1))))
(debug:print-info 2 *default-log-port* "Archiving data with bup")
(let-values (((pid-val exit-status exit-code) (run-n-wait bup-exe params: bup-save-params print-cmd: print-prefix)))
(if (not (eq? exit-code 0))
(begin
(debug:print-error 0 *default-log-port* "There was an archiving data with bup. Archive failed.")
(exit 1))))))
((7z tar)
(for-each
(lambda (test-dat)
(let* ((test-id (db:test-get-id test-dat))
(test-name (db:test-get-testname test-dat))
(item-path (db:test-get-item-path test-dat))
(test-full-name (db:test-make-full-name test-name item-path))
|
| ︙ | ︙ | |||
337 338 339 340 341 342 343 |
(rsync-exe (or (configf:lookup *configdat* "archive" "rsync") "rsync"))
(print-prefix "Running: ")
(archive-info (archive:allocate-new-archive-block blockid-cache *toppath* tsname min-space target-patt run-patt "megatest-db"))
(archive-dir (if archive-info (cdr archive-info) #f))
(archive-id (if archive-info (car archive-info) -1))
(home-host (common:get-homehost))
(archive-time (seconds->std-time-str (current-seconds)))
| | | | > > > > | | > > > > | | > > > > > | | > | < | 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 |
(rsync-exe (or (configf:lookup *configdat* "archive" "rsync") "rsync"))
(print-prefix "Running: ")
(archive-info (archive:allocate-new-archive-block blockid-cache *toppath* tsname min-space target-patt run-patt "megatest-db"))
(archive-dir (if archive-info (cdr archive-info) #f))
(archive-id (if archive-info (car archive-info) -1))
(home-host (common:get-homehost))
(archive-time (seconds->std-time-str (current-seconds)))
(archive-staging-db (conc *toppath* "/.db-snapshot/archive_" archive-time))
(tmp-db-path (conc (common:get-db-tmp-area) "/megatest.db"))
(dbfile (conc archive-staging-db "/megatest.db")))
(create-directory archive-staging-db #t)
(let-values (((pid-val exit-status exit-code) (run-n-wait rsync-exe params: (list "-v" (conc (car home-host) ":"tmp-db-path) archive-staging-db) print-cmd: print-prefix)))
(if (eq? exit-code 0)
(case archiver
((bup) ;; Archive using bup
(let* ((bup-init-params (list "-d" archive-dir "init"))
(bup-index-params (list "-d" archive-dir "index" archive-staging-db))
(bup-save-params (list "-d" archive-dir "save" ;; (conc "--strip-path=" linktree)
(conc "-" compress) ;; or (conc "--compress=" compress)
"-n" (conc tsname "-megatest-db" )
(conc "--strip-path=" archive-staging-db ) ;; if we push to the directory do we need this?
dbfile)))
(if (not (common:file-exists? (conc archive-dir "/HEAD")))
(begin
;; replace this with jobrunner stuff enventually
(debug:print-info 2 *default-log-port* "Init bup in " archive-dir)
(let-values (((pid-val exit-status exit-code)(run-n-wait bup-exe params: bup-init-params print-cmd: print-prefix)))
(if (not (eq? exit-code 0))
(begin
(debug:print-error 0 *default-log-port* "There was an error initializing bup. Archive failed.")
(exit 1))))))
(debug:print-info 2 *default-log-port* "Indexing data to be archived")
(let-values (((pid-val exit-status exit-code) (run-n-wait bup-exe params: bup-index-params print-cmd: print-prefix)))
(if (not (eq? exit-code 0))
(begin
(debug:print-error 0 *default-log-port* "There was an error Indexing bup. Archive failed.")
(exit 1))))
(debug:print-info 2 *default-log-port* "Archiving data with bup")
(let-values (((pid-val exit-status exit-code) (run-n-wait bup-exe params: bup-save-params print-cmd: print-prefix)))
(if (not (eq? exit-code 0))
(begin
(debug:print-error 0 *default-log-port* "There was an error archiving data with bup. Archive failed.")
(exit 1))
(debug:print-info 0 *default-log-port* "To restore megatest.db run megatest -archive replicacte-db -source archive-dir -time-stamp <ts>. Current timestamp: " (seconds->std-time-str (current-seconds)))))))
(else
(debug:print-info 0 *default-log-port* "No support for databse archiving with " archiver)))
(debug:print-error 0 *default-log-port* "There was an error rsyncing tmp database")))))
(define (archive:restore-db archive-path ts)
(let* ((bup-exe (or (configf:lookup *configdat* "archive" "bup") "bup"))
(archive-internal-path (conc (common:get-testsuite-name) "-megatest-db/" ts "/megatest.db" ))
(bup-restore-params (list "-d" archive-path "restore" "-C" *toppath* archive-internal-path)))
(debug:print-info 2 *default-log-port* "Restoring archived data to " *toppath* " from archive in " archive-path " ... " archive-internal-path)
(run-n-wait bup-exe params: bup-restore-params print-cmd: "Running:"))
(sleep 2)
(db:multi-db-sync
(db:setup #f)
'killservers
;'dejunk
;'adj-testids
'old2new
)
(debug:print-info 1 *default-log-port* "dropping triggers to update linktree")
(rmt:drop-all-triggers)
(let* ((linktree (common:get-linktree)) ;; (configf:lookup *configdat* "setup" "linktree")))
(src-archive-linktree (rmt:get-var "src-archive-linktree")))
(if (not (equal? src-archive-linktree linktree))
(rmt:update-tesdata-on-repilcate-db src-archive-linktree linktree))
(debug:print-info 1 *default-log-port* "creating triggers after updating linktree")
(rmt:create-all-triggers)
))
|
| ︙ | ︙ | |||
413 414 415 416 417 418 419 | (define (seconds->std-time-str sec) (time->string (seconds->local-time sec) "%Y-%m-%d-%H%M%S")) | | | | 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 |
(define (seconds->std-time-str sec)
(time->string
(seconds->local-time sec)
"%Y-%m-%d-%H%M%S"))
(define (archive:get-timestamp-dir bup-exe archive-dir testsuite-name target test-partial-path test-last-update)
(debug:print-info 0 *default-log-port* "Test last update time:" (seconds->std-time-str test-last-update))
(let* ((internal-path (conc testsuite-name "-" target))
(archive-update-delay (string->number (or (configf:lookup *configdat* "archive" "test-update-delay") "900" )))
(ts-list (archive:ls->list bup-exe archive-dir internal-path))
(ds-flag (vector-ref (seconds->local-time) 8)))
(let loop ((hed (car ts-list))
(tail (cdr ts-list)))
(if (and (null? tail) (equal? hed "latest"))
#f
|
| ︙ | ︙ | |||
458 459 460 461 462 463 464 | (test-id (db:test-get-id test-dat)) (run-id (db:test-get-run_id test-dat)) (keyvals (rmt:get-key-val-pairs run-id)) (target (string-intersperse (map cadr keyvals) "/")) (toplevel/children (and (db:test-get-is-toplevel test-dat) (> (rmt:test-toplevel-num-items run-id test-name) 0))) | | | | | 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 |
(test-id (db:test-get-id test-dat))
(run-id (db:test-get-run_id test-dat))
(keyvals (rmt:get-key-val-pairs run-id))
(target (string-intersperse (map cadr keyvals) "/"))
(toplevel/children (and (db:test-get-is-toplevel test-dat)
(> (rmt:test-toplevel-num-items run-id test-name) 0)))
(test-partial-path (conc run-name "/" (db:test-make-full-name test-name item-path)))
;; note the trailing slash to get the dir inspite of it being a link
(test-path (conc linktree "/" test-partial-path))
;; if the old path was not deleted then prev-test-physical-path will end up pointing to a real directory
(mutex-lock! rp-mutex)
(prev-test-physical-path (if (common:file-exists? test-path)
;; (read-symbolic-link test-path #t)
(common:real-path test-path)
#f))
(mutex-unlock! rp-mutex)
(new-test-physical-path (conc best-disk "/" test-partial-path))
(archive-block-id (db:test-get-archived test-dat))
(test-last-update (db:test-get-last_update test-dat))
(archive-block-info (rmt:test-get-archive-block-info archive-block-id))
(archive-path (if (vector? archive-block-info)
(vector-ref archive-block-info 2) ;; look in db.scm for test-get-archive-block-info for the vector record info
#f)) ;; no archive found?
(archive-timestamp-dir (if archive-path (archive:get-timestamp-dir bup-exe archive-path (common:get-testsuite-name) (string-substitute "/" "-" target " ") test-partial-path test-last-update) #f))
(archive-internal-path (conc (common:get-testsuite-name) "-" (string-substitute "/" "-" target " ") "/" archive-timestamp-dir "/" test-partial-path))
(include-paths (args:get-arg "-include"))
(exclude-pattern (args:get-arg "-exclude-rx"))
(exclude-file (args:get-arg "-exclude-rx-from")))
(if (not archive-timestamp-dir)
(debug:print-error 0 *default-log-port* "Archive not found for testsuite" (common:get-testsuite-name) " run/test/itempath" test-partial-path)
(begin
;; some sanity checks, move an existing path out of the way - iif it is not a toplevel with children
|
| ︙ | ︙ |
Added bin/.11/lib/libpangox-1.0.so version [d55c756a93].
cannot compute difference between binary files
Added bin/.11/lib/libpangox-1.0.so.0 version [d55c756a93].
cannot compute difference between binary files
Added bin/.11/lib/libxcb-xlib.so.0 version [b7cbe8e250].
cannot compute difference between binary files
Modified db.scm from [784defadd2] to [83a7d618db].
| ︙ | ︙ | |||
415 416 417 418 419 420 421 |
(lambda (option)
(case option
;; kill servers
((killservers)
(for-each
(lambda (server)
| > > > > > | | | 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 |
(lambda (option)
(case option
;; kill servers
((killservers)
(for-each
(lambda (server)
(handle-exceptions
exn
(begin
(debug:print-info 0 *default-log-port* "Unable to get host and/or port from " servr ", exn=" exn)
#f)
(match-let (((mod-time host port start-time server-id pid) server))
(if (and host pid)
(tasks:kill-server host pid)))))
servers)
;; /tmp/bjbarcla/megatest_localdb/fullrun/.nfs.pdx.disks.icf_env_disk001.bjbarcla.gwa.issues.mtdev.matt-bisect.megatest.ext-tests.runs.all.v1.65.1.6524.dev.bb-24-justrollup-f8.rollup.fullrun/megatest.db.lock
(delete-file* (common:get-sync-lock-filepath))
)
;; clear out junk records
|
| ︙ | ︙ |
Modified dbmod.scm from [3cca16f7e6] to [baab89656e].
| ︙ | ︙ | |||
203 204 205 206 207 208 209 |
(exn (io-error) (debug:print 0 *default-log-port* "ERROR: i/o error with " fname ". Check permissions, disk space etc. and try again."))
(exn (corrupt) (debug:print 0 *default-log-port* "ERROR: database " fname " is corrupt. Repair it to proceed."))
(exn (busy) (debug:print 0 *default-log-port* "ERROR: database " fname " is locked. Try copying to another location, remove original and copy back."))
(exn (permission)(debug:print 0 *default-log-port* "ERROR: database " fname " has some permissions problem."))
(exn () (debug:print 0 *default-log-port* "ERROR: Unknown error with database " fname " message: " ((condition-property-accessor 'exn 'message) exn))))
)))
| | | | | 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 |
(exn (io-error) (debug:print 0 *default-log-port* "ERROR: i/o error with " fname ". Check permissions, disk space etc. and try again."))
(exn (corrupt) (debug:print 0 *default-log-port* "ERROR: database " fname " is corrupt. Repair it to proceed."))
(exn (busy) (debug:print 0 *default-log-port* "ERROR: database " fname " is locked. Try copying to another location, remove original and copy back."))
(exn (permission)(debug:print 0 *default-log-port* "ERROR: database " fname " has some permissions problem."))
(exn () (debug:print 0 *default-log-port* "ERROR: Unknown error with database " fname " message: " ((condition-property-accessor 'exn 'message) exn))))
)))
(define (db:initialize-main-db dbdat)
(assert *db-keys* "Attempt to initialize main db before configs are properly processed")
;; (if (not *configinfo*)(launch:setup)) ;; added because Elena was getting stack dump because *configinfo* below was #f.
(let* ((keys *db-keys*)
(havekeys (> (length keys) 0))
(keystr (keys->keystr keys))
(fieldstr (keys:make-key/field-string keys)) ;;; configdat))
(db (db:dbdat-get-db dbdat)))
(for-each (lambda (key)
(let ((keyn key))
(if (member (string-downcase keyn)
|
| ︙ | ︙ | |||
592 593 594 595 596 597 598 599 600 601 602 603 604 605 |
;(mtdbmodtime (if mtdbexists (common:lazy-sqlite-db-modification-time mtdbpath) #f)) ; moving this before db:open-megatest-db is called. if wal mode is on -WAL and -shm file get created with causing the tmpdbmodtime timestamp always greater than mtdbmodtime
;(tmpdbmodtime (if dbfexists (common:lazy-sqlite-db-modification-time tmpdbfname) #f))
;if wal mode is on -WAL and -shm file get created when db:open-megatest-db is called. modtimedelta will always be < 10 so db in tmp not get synced
;(tmpdbmodtime (if dbfexists (db:get-last-update-time (car tmpdb)) #f))
;(fmt (file-modification-time tmpdbfname))
(modtimedelta (and mtdbmodtime tmpdbmodtime (- mtdbmodtime tmpdbmodtime))))
(when write-access
(sqlite3:execute (car mtdb) "drop trigger if exists update_tests_trigger")
(sqlite3:execute (car mtdb) "drop trigger if exists update_runs_trigger"))
;(print "mtdbmodtime " mtdbmodtime " tmpdbmodtime " tmpdbmodtime " mtdbpath " mtdbpath " " (conc *toppath* "/megatest.db"))
;;(debug:print-info 13 *default-log-port* "db:open-db>> mtdbpath="mtdbpath" mtdbexists="mtdbexists" and write-access="write-access)
(if (and dbexists (not write-access))
| > | 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 |
;(mtdbmodtime (if mtdbexists (common:lazy-sqlite-db-modification-time mtdbpath) #f)) ; moving this before db:open-megatest-db is called. if wal mode is on -WAL and -shm file get created with causing the tmpdbmodtime timestamp always greater than mtdbmodtime
;(tmpdbmodtime (if dbfexists (common:lazy-sqlite-db-modification-time tmpdbfname) #f))
;if wal mode is on -WAL and -shm file get created when db:open-megatest-db is called. modtimedelta will always be < 10 so db in tmp not get synced
;(tmpdbmodtime (if dbfexists (db:get-last-update-time (car tmpdb)) #f))
;(fmt (file-modification-time tmpdbfname))
(modtimedelta (and mtdbmodtime tmpdbmodtime (- mtdbmodtime tmpdbmodtime))))
(assert (sqlite3:database? (car mtdb)) "ERROR: in db:open-db and mtdb does not contain database!")
(when write-access
(sqlite3:execute (car mtdb) "drop trigger if exists update_tests_trigger")
(sqlite3:execute (car mtdb) "drop trigger if exists update_runs_trigger"))
;(print "mtdbmodtime " mtdbmodtime " tmpdbmodtime " tmpdbmodtime " mtdbpath " mtdbpath " " (conc *toppath* "/megatest.db"))
;;(debug:print-info 13 *default-log-port* "db:open-db>> mtdbpath="mtdbpath" mtdbexists="mtdbexists" and write-access="write-access)
(if (and dbexists (not write-access))
|
| ︙ | ︙ |
Added lib/libpangox-1.0.so version [d55c756a93].
cannot compute difference between binary files
Added lib/libpangox-1.0.so.0 version [d55c756a93].
cannot compute difference between binary files
Added lib/libxcb-xlib.so.0 version [b7cbe8e250].
cannot compute difference between binary files
Modified megatest-version.scm from [af5ac77c8c] to [ffc179ede7].
| ︙ | ︙ | |||
16 17 18 19 20 21 22 | ;; along with Megatest. If not, see <http://www.gnu.org/licenses/>. ;; Always use two or four digit decimal ;; 1.01, 1.02...1.10,1.11,1.1101 ... 1.99,2.00.. ;; (declare (unit megatest-version)) | | | 16 17 18 19 20 21 22 23 | ;; along with Megatest. If not, see <http://www.gnu.org/licenses/>. ;; Always use two or four digit decimal ;; 1.01, 1.02...1.10,1.11,1.1101 ... 1.99,2.00.. ;; (declare (unit megatest-version)) (define megatest-version 1.6583) |
Modified megatest.scm from [267351f3b9] to [b9882ee58b].
| ︙ | ︙ | |||
534 535 536 537 538 539 540 |
"-list-disks"
"-list-targets"
"-show-runconfig"
;;"-list-db-targets"
"-show-runconfig"
"-show-config"
"-show-cmdinfo"
| | > > > > > > > > > > | | | 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 |
"-list-disks"
"-list-targets"
"-show-runconfig"
;;"-list-db-targets"
"-show-runconfig"
"-show-config"
"-show-cmdinfo"
"-cleanup-db"
))
(no-watchdog-argvals (list '("-archive" . "replicate-db")))
(start-watchdog-specail-arg-val (let loop ((hed (car no-watchdog-argvals))
(tail (cdr no-watchdog-argvals)))
;; (print "hed" hed " arg " (args:get-arg (car hed)) " val:" (cdr hed) " eql" (equal? (args:get-arg (car hed)) (cdr hed)))
(if (equal? (args:get-arg (car hed)) (cdr hed))
#f
(if (null? tail)
#t
(loop (car tail) (cdr tail))))))
(no-watchdog-args-vals (filter (lambda (x) x)
(map args:get-arg no-watchdog-args)))
(start-watchdog (and (null? no-watchdog-args-vals) start-watchdog-specail-arg-val)))
;(print "no-watchdog-args="no-watchdog-args "no-watchdog-args-vals="no-watchdog-args-vals " start-watchdog-specail-arg-val:" start-watchdog-specail-arg-val " start-watchdog:" start-watchdog)
(if start-watchdog
(thread-start! *watchdog*)))
;; bracket open-output-file with code to make leading directory if it does not exist and handle exceptions
(define (open-logfile logpath-in)
(condition-case
|
| ︙ | ︙ | |||
1984 1985 1986 1987 1988 1989 1990 | (print path)) paths)))))) ;;====================================================================== ;; Archive tests ;;====================================================================== ;; Archive tests matching target, runname, and testpatt | | | | | 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 |
(print path))
paths))))))
;;======================================================================
;; Archive tests
;;======================================================================
;; Archive tests matching target, runname, and testpatt
(if (equal? (args:get-arg "-archive") "replicate-db")
(begin
;; check if source
;; check if megatest.db exist
(launch:setup)
(if (not (args:get-arg "-source"))
(begin
(debug:print-info 1 *default-log-port* "Missing required argument -source <archive path>")
(exit 1)))
(if (common:file-exists? (conc *toppath* "/megatest.db"))
(begin
(debug:print-info 1 *default-log-port* "File " (conc *toppath* "/megatest.db") " already exists. Please remove it before trying to replicate db")
|
| ︙ | ︙ | |||
2015 2016 2017 2018 2019 2020 2021 |
(begin
(archive:restore-db src ts)
(set! *didsomething* #t))
(begin
(debug:print-error 1 *default-log-port* "Path " source " not found")
(exit 1))))))
;; else do a general-run-call
| | | 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 |
(begin
(archive:restore-db src ts)
(set! *didsomething* #t))
(begin
(debug:print-error 1 *default-log-port* "Path " source " not found")
(exit 1))))))
;; else do a general-run-call
(if (and (args:get-arg "-archive") (not (equal? (args:get-arg "-archive") "replicate-db")))
(begin
;; for the archive get we need to preserve the starting dir as part of the target path
(if (and (args:get-arg "-dest")
(not (equal? (substring (args:get-arg "-dest") 0 1) "/")))
(let ((newpath (conc (current-directory) "/" (args:get-arg "-dest"))))
(debug:print-info 1 *default-log-port* "Preserving original path to destination, was " (args:get-arg "-dest") ", now " newpath)
(hash-table-set! args:arg-hash "-dest" newpath)))
|
| ︙ | ︙ |
Modified mtut.scm from [3ab8528d8b] to [5d611c3d90].
| ︙ | ︙ | |||
1324 1325 1326 1327 1328 1329 1330 |
(if (alist-ref key *additional-cards*) ;; these cards do not translate to parameters or switches
res
(begin
(print "ERROR: Unknown key in packet \"" key "\" with value \"" val "\"")
res)))))
(conc "megatest " (if (not (member action '("sync")))
(conc action " " action-param)
| > > | | 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 |
(if (alist-ref key *additional-cards*) ;; these cards do not translate to parameters or switches
res
(begin
(print "ERROR: Unknown key in packet \"" key "\" with value \"" val "\"")
res)))))
(conc "megatest " (if (not (member action '("sync")))
(conc action " " action-param)
"") (if (member action '("-run" "-rerun-clean" "-rerun-all" "-kill-rerun"))
"-rerun DEAD,ABORT,KILLED"
""))
pkta)))
;; (use trace)(trace pkt->cmdline)
(define (write-pkt pktsdir uuid pkt)
(if pktsdir
(with-output-to-file
|
| ︙ | ︙ |
Modified runs.scm from [359b1f9864] to [e3175d31bc].
| ︙ | ︙ | |||
393 394 395 396 397 398 399 |
(print-call-chain *default-log-port*)
(debug:print 0 *default-log-port* "Message: " ((condition-property-accessor 'exn 'message) exn) ", exn=" exn)
(debug:print 0 *default-log-port* "ERROR: failed to run pre-hook " run-pre-hook ", check the log " log-file))
(debug:print-info 0 *default-log-port* "running run-pre-hook: \"" run-pre-hook "\", log is " actual-logf)
(system (conc run-pre-hook " >> " actual-logf " 2>&1"))
(debug:print-info 0 *default-log-port* "pre-hook \"" run-pre-hook "\" took " (- (current-seconds) start-time) " seconds to run.")))
(debug:print 0 *default-log-port* "Skipping pre-hook call \"" run-pre-hook "\" as there are existing tests for this run.")))))
| | > > | 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 |
(print-call-chain *default-log-port*)
(debug:print 0 *default-log-port* "Message: " ((condition-property-accessor 'exn 'message) exn) ", exn=" exn)
(debug:print 0 *default-log-port* "ERROR: failed to run pre-hook " run-pre-hook ", check the log " log-file))
(debug:print-info 0 *default-log-port* "running run-pre-hook: \"" run-pre-hook "\", log is " actual-logf)
(system (conc run-pre-hook " >> " actual-logf " 2>&1"))
(debug:print-info 0 *default-log-port* "pre-hook \"" run-pre-hook "\" took " (- (current-seconds) start-time) " seconds to run.")))
(debug:print 0 *default-log-port* "Skipping pre-hook call \"" run-pre-hook "\" as there are existing tests for this run.")))))
(define (runs:run-post-hook run-id)
(let* ((run-post-hook (configf:lookup *configdat* "runs" "post-hook"))
(existing-tests (if run-post-hook
(rmt:get-tests-for-run run-id "%" '() '() ;; run-id testpatt states statuses
#f #f ;; offset limit
#f ;; not-in
#f ;; sort-by
|
| ︙ | ︙ | |||
432 433 434 435 436 437 438 439 440 441 442 443 444 445 | (begin (print-call-chain *default-log-port*) (debug:print 0 *default-log-port* "Message: " ((condition-property-accessor 'exn 'message) exn) ", exn=" exn) (debug:print 0 *default-log-port* "ERROR: failed to run post-hook " run-post-hook ", check the log " log-file)) (debug:print-info 0 *default-log-port* "running run-post-hook: \"" run-post-hook "\", log is " actual-logf) (system (conc run-post-hook " >> " actual-logf " 2>&1")) (debug:print-info 0 *default-log-port* "post-hook \"" run-post-hook "\" took " (- (current-seconds) start-time) " seconds to run.")))))) ;; return #t when all items in waitors-upon list are represented in test-patt, #f otherwise. (define (runs:testpatts-mention-waitors-upon? test-patt waitors-upon) (null? (tests:filter-test-names-not-matched waitors-upon test-patt))) ;;====================================================================== ;; runs:run-tests is called from megatest.scm and itself | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 |
(begin
(print-call-chain *default-log-port*)
(debug:print 0 *default-log-port* "Message: " ((condition-property-accessor 'exn 'message) exn) ", exn=" exn)
(debug:print 0 *default-log-port* "ERROR: failed to run post-hook " run-post-hook ", check the log " log-file))
(debug:print-info 0 *default-log-port* "running run-post-hook: \"" run-post-hook "\", log is " actual-logf)
(system (conc run-post-hook " >> " actual-logf " 2>&1"))
(debug:print-info 0 *default-log-port* "post-hook \"" run-post-hook "\" took " (- (current-seconds) start-time) " seconds to run."))))))
(define (runs:rerun-hook test-id new-test-path testdat rerunlst)
(let* ((rerun-hook (configf:lookup *configdat* "runs" "rerun-hook"))
(log-dir (conc *toppath* "/reruns/logs"))
(target (getenv "MT_TARGET"))
(runname (common:args-get-runname))
(rundir (db:test-get-rundir testdat))
(tarfiledir (conc *toppath* "/reruns"))
(status (db:test-get-status testdat))
(comment (conc "\"" (db:test-get-comment testdat) "\"" ))
(testname (db:test-get-testname testdat))
(itempath (db:test-get-item-path testdat))
(file-body (conc status "-" (string-translate target "/" "-") "-" runname "." testname (if (not (string=? itempath "")) (conc "." (string-translate itempath "/" "-")) "")))
(log-file (conc file-body ".log"))
;; (log-file (conc status "-" (string-translate target "/" "-") "-" runname "." testname (if (not (string=? itempath "")) (conc "." (string-translate itempath "/" "-")) "") ".log"))
(full-log-fname (conc log-dir "/" log-file))
(tarfilename (conc file-body ".tar"))
;; (tarfilename (conc status "." (string-translate target "/" "-") "." runname "." testname (if (not (string=? itempath "")) (conc "." (string-translate itempath "/" "-")) "") ".tar"))
)
(if rerun-hook
(let* ((use-log-dir (if (not (directory-exists? log-dir))
(handle-exceptions
exn
(begin
(debug:print 0 *default-log-port* "WARNING: Failed to create " log-dir ", exn=" exn)
#f)
(create-directory log-dir #t)
#t)
#t))
(start-time (current-seconds))
(actual-logf (if use-log-dir full-log-fname log-file))
(sys-call-text (conc rerun-hook " " tarfilename " " rundir " " actual-logf " " runname " " tarfiledir " " status " " target " " comment " " testname " " itempath " >> " actual-logf " 2>&1"))
)
(debug:print 2 *default-log-port* "Found rerun-hook in config:" rerun-hook)
(handle-exceptions
exn
(begin
(print-call-chain *default-log-port*)
(debug:print 0 *default-log-port* "Message: " ((condition-property-accessor 'exn 'message) exn) ", exn=" exn)
(debug:print 0 *default-log-port* "ERROR: failed to run rerun-hook " rerun-hook ", check the log " log-file))
(debug:print-info 0 *default-log-port* "running rerun-hook: \"" rerun-hook "\", log is " actual-logf)
;; call the hook
(debug:print-info 0 *default-log-port* "Calling rerun-hook for " test-id new-test-path testdat rerunlst)
(debug:print-info 0 *default-log-port* "rerun hook: " rerun-hook)
(debug:print-info 0 *default-log-port* "tarfilename: " tarfilename)
(debug:print-info 0 *default-log-port* "rundir: " rundir)
(debug:print-info 0 *default-log-port* "actual-logf: " actual-logf)
(debug:print-info 0 *default-log-port* "runname: " runname)
(debug:print-info 0 *default-log-port* "sys-call-text: " sys-call-text)
(system sys-call-text)
(debug:print-info 0 *default-log-port* "rerun-hook \"" rerun-hook "\" took " (- (current-seconds) start-time) " seconds to run."))))))
;; return #t when all items in waitors-upon list are represented in test-patt, #f otherwise.
(define (runs:testpatts-mention-waitors-upon? test-patt waitors-upon)
(null? (tests:filter-test-names-not-matched waitors-upon test-patt)))
;;======================================================================
;; runs:run-tests is called from megatest.scm and itself
|
| ︙ | ︙ | |||
603 604 605 606 607 608 609 |
;; Ensure all tests are registered in the test_meta table
(runs:update-all-test_meta #f)
;; run the run prehook if there are no tests yet run for this run:
;;
(runs:run-pre-hook run-id)
| | | 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 |
;; Ensure all tests are registered in the test_meta table
(runs:update-all-test_meta #f)
;; run the run prehook if there are no tests yet run for this run:
;;
(runs:run-pre-hook run-id)
;; mark all test launched flag as false in the meta table
(rmt:set-var (conc "lunch-complete-" run-id) "no")
(debug:print-info 1 *default-log-port* "Setting end-of-run to no")
(let* ((config-reruns (let ((x (configf:lookup *configdat* "setup" "reruns")))
(if x (string->number x) #f)))
(config-rerun-cnt (if config-reruns
config-reruns
1)))
|
| ︙ | ︙ | |||
765 766 767 768 769 770 771 | (thread-join! th2) ;; if run-count > 0 call, set -preclean and -rerun STUCK/DEAD (if (> run-count 0) ;; handle reruns (begin (if (not (hash-table-ref/default flags "-preclean" #f)) (hash-table-set! flags "-preclean" #t)) (if (not (hash-table-ref/default flags "-rerun" #f)) | | | 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 |
(thread-join! th2)
;; if run-count > 0 call, set -preclean and -rerun STUCK/DEAD
(if (> run-count 0) ;; handle reruns
(begin
(if (not (hash-table-ref/default flags "-preclean" #f))
(hash-table-set! flags "-preclean" #t))
(if (not (hash-table-ref/default flags "-rerun" #f))
(hash-table-set! flags "-rerun" "ABORT,STUCK/DEAD,n/a,ZERO_ITEMS"))
;; recursive call to self
(runs:run-tests target runname test-patts user flags run-count: (- run-count 1)))
(launch:end-of-run-check run-id)))
(debug:print-info 0 *default-log-port* "No tests to run")))
(debug:print-info 4 *default-log-port* "All done by here")
;; TODO: try putting post hook call here
|
| ︙ | ︙ | |||
2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 |
keepgoing)
;; Require to force re-run for COMPLETED or *anything* + PASS,WARN or CHECK
(or (member (test:get-status testdat) '("PASS" "WARN" "CHECK" "SKIP" "WAIVED"))
(member (test:get-state testdat) '("COMPLETED"))))
(debug:print-info 2 *default-log-port* "running test " test-name "/" item-path " suppressed as it is " (test:get-state testdat) " and " (test:get-status testdat))
(hash-table-set! test-registry full-test-name 'DONOTRUN) ;; COMPLETED)
(set! runflag #f))
;; -rerun and status is one of the specifed, run it
((and rerun
(let* ((rerunlst (string-split rerun ","))
(must-rerun (member (test:get-status testdat) rerunlst)))
(debug:print-info 3 *default-log-port* "-rerun list: " rerun ", test-status: " (test:get-status testdat)", must-rerun: " must-rerun)
must-rerun))
(debug:print-info 2 *default-log-port* "Rerun forced for test " test-name "/" item-path)
| > | > > > > > > | > | > | 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 |
keepgoing)
;; Require to force re-run for COMPLETED or *anything* + PASS,WARN or CHECK
(or (member (test:get-status testdat) '("PASS" "WARN" "CHECK" "SKIP" "WAIVED"))
(member (test:get-state testdat) '("COMPLETED"))))
(debug:print-info 2 *default-log-port* "running test " test-name "/" item-path " suppressed as it is " (test:get-state testdat) " and " (test:get-status testdat))
(hash-table-set! test-registry full-test-name 'DONOTRUN) ;; COMPLETED)
(set! runflag #f))
;; -rerun and status is one of the specifed, run it
((and rerun
(let* ((rerunlst (string-split rerun ","))
(must-rerun (member (test:get-status testdat) rerunlst)))
(debug:print-info 3 *default-log-port* "-rerun list: " rerun ", test-status: " (test:get-status testdat)", must-rerun: " must-rerun)
must-rerun))
(debug:print-info 2 *default-log-port* "Rerun forced for test " test-name "/" item-path)
(set! runflag #t)
(debug:print-info 2 *default-log-port* "Calling rerun hook")
(runs:rerun-hook test-id new-test-path testdat rerun)
)
;; -keepgoing, do not rerun FAIL
((and keepgoing
(member (test:get-status testdat) '("FAIL")))
(set! runflag #f))
((and (not rerun)
(member (test:get-status testdat) '("FAIL" "n/a")))
(set! runflag #t))
(else (set! runflag #f)))
(debug:print 4 *default-log-port* "RUNNING => runflag: " runflag " STATE: " (test:get-state testdat) " STATUS: " (test:get-status testdat))
(if (not runflag)
(if (not parent-test)
(if (runs:lownoise (conc "not starting test" full-test-name) 60)
(debug:print 1 *default-log-port* "NOTE: Not starting test " full-test-name " as it is state \"" (test:get-state testdat)
"\" and status \"" (test:get-status testdat) "\", use -rerun \"" (test:get-status testdat)
|
| ︙ | ︙ | |||
2203 2204 2205 2206 2207 2208 2209 | ;; delete redundant runs within a target - N is the input ;; delete redundant runs within a target IFF older than given date/time AND keep at least N ;; (define (runs:remove-all-but-last-n-runs-per-target target-patts runpatt num-to-keep #!key (actions '(print))) (let* ((runs-ht (runs:get-hash-by-target target-patts runpatt)) (age (if (args:get-arg "-age")(common:hms-string->seconds (args:get-arg "-age")) #f)) (age-mark (if age (- (current-seconds) age) (+ (current-seconds) 86400))) | | > > > > > > | > > | | | | | | | | | | | | | | | | | < < | < < | < < | > < < < < < < < < < < < < < < < < < < < | 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 |
;; delete redundant runs within a target - N is the input
;; delete redundant runs within a target IFF older than given date/time AND keep at least N
;;
(define (runs:remove-all-but-last-n-runs-per-target target-patts runpatt num-to-keep #!key (actions '(print)))
(let* ((runs-ht (runs:get-hash-by-target target-patts runpatt))
(age (if (args:get-arg "-age")(common:hms-string->seconds (args:get-arg "-age")) #f))
(age-mark (if age (- (current-seconds) age) (+ (current-seconds) 86400)))
(precmd (or (args:get-arg "-precmd") ""))
(action-chk (member (string->symbol "remove-runs") actions)))
;; check the sequence of actions archive must comme before remove-runs
(if (and action-chk (member (string->symbol "archive") action-chk))
(begin
(debug:print-error 0 *default-log-port* "action remove-runs must come after archive")
(exit 1)))
(print "Actions: " actions " age: " age)
(for-each
(lambda (action)
(for-each
(lambda (target)
(let* ((runs (hash-table-ref runs-ht target))
(sorted (sort runs (lambda (a b)(< (simple-run-event_time a)(simple-run-event_time b)))))
(to-remove (let* ((len (length sorted))
(trim-amt (- len num-to-keep)))
(if (> trim-amt 0)
(take sorted trim-amt)
'()))))
(hash-table-set! runs-ht target to-remove)
(print target ":")
(for-each
(lambda (run)
(let ((remove (member run to-remove (lambda (a b)
(eq? (simple-run-id a)
(simple-run-id b))))))
(if (and age (> (simple-run-event_time run) age-mark))
(print "Skipping handling of " target "/" (simple-run-runname run) " as it is younger than " (args:get-arg "-age"))
(case action
((print)
(print " " (simple-run-runname run)
" " (time->string (seconds->local-time (simple-run-event_time run)) "WW%V.%u %H:%M:%S")
" " (if remove "REMOVE" "")))
((remove-runs)
(if remove (system (conc precmd " megatest -remove-runs -target " target " -runname " (simple-run-runname run) " -testpatt %"
(if (member 'kill-runs actions) ;; if kill-runs is specified then set -kill-wait to 0
" -kill-wait 0"
"")))))
((archive)
(if remove (system (conc precmd " megatest -archive save-remove -target " target " -runname " (simple-run-runname run) " -testpatt %"))))
((kill-runs)
(if remove (system (conc precmd " megatest -kill-runs -target " target " -runname " (simple-run-runname run) " -testpatt %"))))))))
sorted)))
(hash-table-keys runs-ht)))
actions)
runs-ht))
(define (remove-last-path-directory path-in)
(let* ((dparts (string-split path-in "/"))
(path-out (conc "/" (string-intersperse (take dparts (- (length dparts) 1)) "/")))
)
path-out
)
)
(define (runs:operate-on action target runnamepatt testpatt #!key (state #f)(status #f)(new-state-status #f)(mode #f)(options '()))
(common:clear-caches) ;; clear all caches
(let* ((db #f)
;; (tdbdat (tasks:open-db))
(keys (rmt:get-keys))
(rundat (mt:get-runs-by-patt keys runnamepatt target))
(header (vector-ref rundat 0))
|
| ︙ | ︙ | |||
2512 2513 2514 2515 2516 2517 2518 |
(loop new-test-dat tal)
(loop (car tal)(append tal (list new-test-dat)))))
(begin
(let ((rundir (db:test-get-rundir new-test-dat)))
(if (and (not (string= rundir "/tmp/badname"))
(file-exists? rundir)
(substring-index run-name rundir)
| | | > > | 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 |
(loop new-test-dat tal)
(loop (car tal)(append tal (list new-test-dat)))))
(begin
(let ((rundir (db:test-get-rundir new-test-dat)))
(if (and (not (string= rundir "/tmp/badname"))
(file-exists? rundir)
(substring-index run-name rundir)
(tests:glob-like-match (conc "%/" target "/%") rundir)
)
(begin
(set! lasttpath (db:test-get-rundir new-test-dat)) ;; remember this path for run removal
(set! lastrealpath (remove-last-path-directory (resolve-pathname lasttpath)))
(hash-table-set! run-paths-hash lastrealpath 1)
(runs:remove-test-directory new-test-dat mode) ;; 'remove-all)
)
(begin
(debug:print 2 *default-log-port* "Not removing directory " rundir " because either it doesn't exist or has a bad name")
(debug:print 2 *default-log-port* "Is /tmp/badname: " (string= rundir "/tmp/badname"))
(debug:print 2 *default-log-port* "Exists: " (file-exists? rundir))
(debug:print 2 *default-log-port* "Has run-name: " (substring-index run-name rundir))
(debug:print 2 *default-log-port* "Has target: " (tests:glob-like-match (conc "%/" target "/%") rundir))
(debug:print 2 *default-log-port* "Target: " target)
;;PJH remove record from db no need to cleanup directory
(case mode
((remove-data-only)(mt:test-set-state-status-by-id (db:test-get-run_id test)(db:test-get-id test) (db:test-get-state test)(db:test-get-status test) #f))
((archive-remove) (mt:test-set-state-status-by-id (db:test-get-run_id test)(db:test-get-id test) "ARCHIVED" #f #f))
(else (rmt:delete-test-records (db:test-get-run_id test) (db:test-get-id test))))
)
|
| ︙ | ︙ |
Modified sauth-common.scm from [28ffd8e69e] to [5771575e2e].
| ︙ | ︙ | |||
238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 |
(sauthorize:db-do (lambda (db)
(let* ((data-row (query fetch (sql db (conc "SELECT code, exe_name, id, basepath FROM areas where areas.code = '" code "'")))))
(set! obj data-row))))
;(print obj)
obj))
;; function to validate the users input for target path and resolve the path
;; TODO: Check for restriction in subpath
(define (sauth-common:resolve-path new current allowed-sheets)
(let* ((target-path (append current (string-split new "/")))
(target-path-string (string-join target-path "/"))
(normal-path (normalize-pathname target-path-string))
| > > > > > > > > > | 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 |
(sauthorize:db-do (lambda (db)
(let* ((data-row (query fetch (sql db (conc "SELECT code, exe_name, id, basepath FROM areas where areas.code = '" code "'")))))
(set! obj data-row))))
;(print obj)
obj))
(define (sauth-common:src-size path)
(let ((output (with-input-from-pipe (conc "/usr/bin/du -s " path " | awk '{print $1}'")
(lambda()
(read-line)))))
(string->number output)))
(define (sauth-common:space-left-at-dest path)
(let* ((output (run/string (pipe (df ,path ) (tail -1))))
(size (caddr (cdr (string-split output " ")))))
(string->number size)))
;; function to validate the users input for target path and resolve the path
;; TODO: Check for restriction in subpath
(define (sauth-common:resolve-path new current allowed-sheets)
(let* ((target-path (append current (string-split new "/")))
(target-path-string (string-join target-path "/"))
(normal-path (normalize-pathname target-path-string))
|
| ︙ | ︙ | |||
277 278 279 280 281 282 283 |
base-path
(conc base-path "/" (string-join (cdr resolved-path) "/")))))
(if (and (not (equal? restricted-areas "" ))
(string-match (regexp restrictions) target-path))
(begin
| | | 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 |
base-path
(conc base-path "/" (string-join (cdr resolved-path) "/")))))
(if (and (not (equal? restricted-areas "" ))
(string-match (regexp restrictions) target-path))
(begin
(sauth:print-error (conc "Access denied to " (string-join resolved-path "/")))
;(exit 1)
#f)
target-path)
))
#f)))
|
| ︙ | ︙ |
Modified server.scm from [d838ea12e7] to [a020b6a629].
| ︙ | ︙ | |||
468 469 470 471 472 473 474 |
(define (server:kill servr)
(handle-exceptions
exn
(begin
(debug:print-info 0 *default-log-port* "Unable to get host and/or port from " servr ", exn=" exn)
#f)
| | | 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 |
(define (server:kill servr)
(handle-exceptions
exn
(begin
(debug:print-info 0 *default-log-port* "Unable to get host and/or port from " servr ", exn=" exn)
#f)
(match-let (((mod-time hostname port start-time server-id pid)
servr))
(tasks:kill-server hostname pid))))
;; called in megatest.scm, host-port is string hostname:port
;;
;; NOTE: This is NOT called directly from clients as not all transports support a client running
;; in the same process as the server.
|
| ︙ | ︙ |
Modified spublish.scm from [0af43ce4a9] to [d0bcfc709c].
| ︙ | ︙ | |||
389 390 391 392 393 394 395 396 397 398 399 400 401 402 |
(define (spublish:shell-cp src-path target-path)
(cond
((not (file-exists? target-path))
(sauth:print-error (conc " target Directory " target-path " does not exist!!")))
((not (file-exists? src-path))
(sauth:print-error (conc "Source path " src-path " does not exist!!" )))
(else
(if (is_directory src-path)
(begin
(let* ((parent-dir src-path)
(start-dir target-path))
(run (pipe
(begin (system (conc "cd " parent-dir " ;tar chf - ." )))
(begin (change-directory start-dir)
| > > > > | 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 |
(define (spublish:shell-cp src-path target-path)
(cond
((not (file-exists? target-path))
(sauth:print-error (conc " target Directory " target-path " does not exist!!")))
((not (file-exists? src-path))
(sauth:print-error (conc "Source path " src-path " does not exist!!" )))
(else
(if (< (sauth-common:space-left-at-dest target-path) (sauth-common:src-size src-path))
(begin
(sauth:print-error "Destination does not have enough disk space.")
(exit 1)))
(if (is_directory src-path)
(begin
(let* ((parent-dir src-path)
(start-dir target-path))
(run (pipe
(begin (system (conc "cd " parent-dir " ;tar chf - ." )))
(begin (change-directory start-dir)
|
| ︙ | ︙ |
Modified sretrieve.scm from [c73e7e987b] to [bc076b5abf].
| ︙ | ︙ | |||
695 696 697 698 699 700 701 |
(print (string-substitute (conc base_path "/") "" p "-"))))
((directory? p)
;;do nothing for dirs)
)
(else
(if (not (string-match (regexp exclude) p ))
| | > | 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 |
(print (string-substitute (conc base_path "/") "" p "-"))))
((directory? p)
;;do nothing for dirs)
)
(else
(if (not (string-match (regexp exclude) p ))
(print (string-substitute (conc base_path "/") "" p "-"))))))
dotfiles: #t))
(define (sretrieve:shell-help)
(conc "Usage: " *exe-name* " [action [params ...]]
ls [target path] : list contents of target area. The output of the cmd can be piped into other system cmd. eg ls <path> | grep txt
cd <target directory> : To change the current directory within the sretrive shell.
pwd : Prints the full pathname of the current directory within the sretrive shell.
|
| ︙ | ︙ | |||
1079 1080 1081 1082 1083 1084 1085 |
(else (print 0 "Unrecognised command " action))))
(define (main)
(let* ((args (argv))
(prog (car args))
(rema (cdr args))
(exe-name (pathname-file (car (argv))))
| | | | 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 |
(else (print 0 "Unrecognised command " action))))
(define (main)
(let* ((args (argv))
(prog (car args))
(rema (cdr args))
(exe-name (pathname-file (car (argv))))
;(exe-dir (or (pathname-directory prog)
; (sretrieve:find exe-name (string-split (get-environment-variable "PATH") ":"))))
;(configdat (sretrieve:load-config exe-dir exe-name))
)
;; preserve the exe data in the config file
;(hash-table-set! configdat "exe-info" (list (list "exe-name" exe-name)
; (list "exe-dir" exe-dir)))
(cond
;; one-word commands
|
| ︙ | ︙ |
Modified utils/mk_wrapper from [e11fc37257] to [713ec8f660].
| ︙ | ︙ | |||
17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 |
# You should have received a copy of the GNU General Public License
# along with Megatest. If not, see <http://www.gnu.org/licenses/>.
prefix=$1
cmd=$2
target=$3
cfgfile="$prefix/bin/.$(lsb_release -sr)/cfg.sh"
# we wish to create a var in cfg.sh for finding sqlite3 executable
chicken_bin_dir=$(dirname $(which csi))
if [[ -e $chicken_bin_dir/sqlite3 ]];then
sqlite3_exe=$chicken_bin_dir/sqlite3
else
sqlite3_exe=$(which sqlite3)
fi
if [ "$LD_LIBRARY_PATH" != "" ];then
echo "INFO: Using LD_LIBRARY_PATH=$LD_LIBRARY_PATH" >&2
( cat << __EOF
if [ -z \$MT_ORIG_ENV ]; then
export MT_ORIG_ENV=\$( $prefix/bin/serialize-env )
fi
if [ "\$LD_LIBRARY_PATH" != "" ];then
| > > | | | 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 |
# You should have received a copy of the GNU General Public License
# along with Megatest. If not, see <http://www.gnu.org/licenses/>.
prefix=$1
cmd=$2
target=$3
cfgfile="$prefix/bin/.$(lsb_release -sr)/cfg.sh"
libdir="$prefix/bin/.$(lsb_release -sr)/lib"
# we wish to create a var in cfg.sh for finding sqlite3 executable
chicken_bin_dir=$(dirname $(which csi))
if [[ -e $chicken_bin_dir/sqlite3 ]];then
sqlite3_exe=$chicken_bin_dir/sqlite3
else
sqlite3_exe=$(which sqlite3)
fi
if [ "$LD_LIBRARY_PATH" != "" ];then
echo "INFO: Using LD_LIBRARY_PATH=$LD_LIBRARY_PATH" >&2
echo "INFO: Writing $cfgfile" >&2
( cat << __EOF
if [ -z \$MT_ORIG_ENV ]; then
export MT_ORIG_ENV=\$( $prefix/bin/serialize-env )
fi
if [ "\$LD_LIBRARY_PATH" != "" ];then
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:\$LD_LIBRARY_PATH:$libdir
else
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$libdir
fi
export MT_SQLITE3_EXE=$sqlite3_exe
__EOF
) > $cfgfile
echo
else
|
| ︙ | ︙ |