Index: Makefile ================================================================== --- Makefile +++ Makefile @@ -339,10 +339,13 @@ $(PREFIX)/bin/mt-new-to-old.sh : utils/mt-new-to-old.sh $(INSTALL) $< $@ chmod a+x $@ +$(PREFIX)/bin/convert-db.sh : utils/convert-db.sh + $(INSTALL) $< $@ + chmod a+x $@ deploytarg/nbfake : utils/nbfake $(INSTALL) $< $@ chmod a+x $@ @@ -385,10 +388,11 @@ install : $(PREFIX)/bin/.$(ARCHSTR) $(PREFIX)/bin/.$(ARCHSTR)/mtest $(PREFIX)/bin/megatest \ $(PREFIX)/bin/.$(ARCHSTR)/dboard $(PREFIX)/bin/dashboard $(HELPERS) $(PREFIX)/bin/nbfake \ $(PREFIX)/bin/nbfind $(PREFIX)/bin/mtrunner $(PREFIX)/bin/viewscreen $(PREFIX)/bin/mt_xterm \ $(PREFIX)/bin/mt-old-to-new.sh $(PREFIX)/bin/mt-new-to-old.sh \ + $(PREFIX)/bin/convert-db.sh $(PREFIX)/bin/convert-db.sh \ $(PREFIX)/share/docs/megatest_manual.html $(PREFIX)/bin/remrun \ $(PREFIX)/share/docs/megatest_manual.html $(PREFIX)/bin/remrun $(PREFIX)/bin/mtutil \ $(PREFIX)/share/db/mt-pg.sql \ $(PREFIX)/share/js/jquery-3.1.0.slim.min.js \ $(PREFIX)/bin/.$(ARCHSTR)/lib/libpangox-1.0.so \ Index: common.scm ================================================================== --- common.scm +++ common.scm @@ -38,13 +38,15 @@ (import commonmod debugprint rmtmod (prefix mtargs args:)) - + +(define (remove-server-files directory-path) + (let ((files (glob (string-append directory-path "/server*")))) + (for-each delete-file* files))) (include "common_records.scm") - (define (remove-files filespec) (let ((files (glob filespec))) (for-each delete-file* files))) (define (stop-the-train) @@ -53,10 +55,11 @@ (if (and *toppath* (file-exists? (conc *toppath*"/stop-the-train"))) (let* ((msg (conc "ERROR: found file "*toppath*"/stop-the-train, exiting immediately"))) ;; yes, print to current-output-port AND *default-log-port*, annoying but necessary I think (print msg) + (remove-server-files (conc *toppath* "/logs")) (debug:print 0 *default-log-port* msg) (remove-files (conc *toppath* "/logs/server*")) (remove-files (conc *toppath* "/.servinfo/*")) (remove-files (conc *toppath* "/.mtdb/*lock")) (exit 1))) Index: docs/Makefile ================================================================== --- docs/Makefile +++ docs/Makefile @@ -33,5 +33,8 @@ megatest.pdf : megatest.lyx lyx -e pdf2 megatest.lyx pkts.pdf : pkts.dot dot -Tpdf pkts.dot -o pkts.pdf + +stepwise.pdf : stepwise-rpc-via-direct-and-tcp-or-http.dot + dot stepwise-rpc-via-direct-and-tcp-or-http.dot -Tpdf -o stepwise.pdf ADDED docs/stepwise-rpc-via-direct-and-tcp-or-http.dot Index: docs/stepwise-rpc-via-direct-and-tcp-or-http.dot ================================================================== --- /dev/null +++ docs/stepwise-rpc-via-direct-and-tcp-or-http.dot @@ -0,0 +1,98 @@ +// Copyright 2006-2017, Matthew Welland. +// +// This file is part of Megatest. +// +// Megatest is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Megatest is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with Megatest. If not, see . +// + +digraph megatest_state_status { + ranksep=0.05; + // rankdir=LR + + node [shape=box,style=filled]; + + "START" -> "have_server"; + "DONE" [label="ALL DONE"]; + "send_request" -> "receive_response"; + "receive_response" -> "DONE"; + "have_server" -> "start_http_server" [label="no"]; + + subgraph cluster_start_server { + label="Start Server"; + "start_http_server" -> "create_servinfo_file"; + "start_http_server" -> "enable_direct_mode"; + "create_servinfo_file" -> "delay_60_sec"; + "delay_60_sec" -> "set_up_tmp_cache"; + "set_up_tmp_cache" -> "switch_mode_to_tmp"; + } + + subgraph cluster_direct_access { + label="Direct Access"; + + "direct_access" -> "touch_access_file"; + "touch_access_file" -> "touch_host_pid_file"; + "touch_host_pid_file" -> "count_host_pid_files"; + "count_host_pid_files" -> "call_query_proc" [label="count < 5"]; + "wait_for_low_count_host_pid_files" [label="sleep 1"]; + "count_host_pid_files" -> "wait_for_low_count_host_pid_files"[label="count >= 5"]; + "wait_for_low_count_host_pid_files" -> "count_host_pid_files"; + "call_query_proc" -> "DONE"; + } + + "have_server" -> "ping_server" [label="yes"]; + "have_server" -> "direct_access" [label="no" ]; + "ping_server" -> "send_request" [label="server alive"]; + "ping_server" -> "remove_server_file" [label="server not alive"]; + "remove_server_file" -> "START"; + + +} +// subgraph cluster_notstarted { +// label="Not started"; + +// "NOT_STARTED FAILS" [ +// label = "{ NOT_STARTED/FAILS |{ NO_ITEMS | FAIL_PREREQ | FAIL_TIMEOUT }}"; +// shape= "record"; +// ] +// +// "NOT_STARTED n/a" -> "LAUNCHED n/a" [label=" launch"]; +// "NOT_STARTED WAIT" -> "LAUNCHED n/a" +// +// "NOT_STARTED n/a"; +// "NOT_STARTED WAIT" [ +// label = "{NOT_STARTED WAIT|{ NO_SLOTS | WAIT_PREREQ}}"; +// shape = "record"; +// ] +// +// // struct3 [shape=record,label="hello\nworld |{ b |{c| d|e}| f}| g | h"]; +// +// "NOT_STARTED n/a" -> "NOT_STARTED FAILS"; +// "NOT_STARTED n/a" -> "NOT_STARTED WAIT"; +// +// "RUNNING" [ +// shape="record"; +// label="{RUNNING|{n/a| PASS | FAIL}}"; +// ] +// +// "COMPLETED" [ +// shape="record"; +// label = "{COMPLETED|{PASS | SKIP | WAIVED | FAIL | CHECK| ABORT}}"; +// ] +// +// +// "RUNNING" -> "COMPLETED"; +// "RUNNING" -> "INCOMPLETE" [label="test dead for > 24hrs"]; +// +// +// "LAUNCHED n/a" -> "REMOTEHOSTSTART n/a" -> "RUNNING"; Index: launch.scm ================================================================== --- launch.scm +++ launch.scm @@ -380,10 +380,55 @@ ;; (change-file-mode name (bitwise-ior perm/irwxg perm/irwxu)) ))) (else (debug:print-info 0 "Invalid logpro definiton found in [logpro] section of testconfig. \"" logprodat "\"")))) logpros))) + +;; read testconfig and create .logpro and script files +;; - use #f for tconfigreg to re-read the testconfigs from disk +;; +(define (launch:extract-scripts-logpro test-dir test-name item-path tconfigreg-in) + (let* ((tconfigreg (or tconfigreg-in + (tests:get-all))) + (tconfig-fname (conc test-dir "/.testconfig")) + (tconfig-tmpfile (conc tconfig-fname ".tmp")) + (tconfig (tests:get-testconfig test-name item-path tconfigreg #t force-create: #t)) ;; 'return-procs))) + (scripts (configf:get-section tconfig "scripts")) + (logpros (configf:get-section tconfig "logpro"))) + ;; create .testconfig file + (configf:write-alist tconfig tconfig-tmpfile) + (file-move tconfig-tmpfile tconfig-fname #t) + (delete-file* ".final-status") + + ;; extract scripts from testconfig and write them to files in test run dir + (for-each + (lambda (scriptdat) + (match scriptdat + ((name content) + (debug:print-info 2 *default-log-port* "Creating script "(current-directory)"/"name) + (with-output-to-file name + (lambda () + (print content))) + (change-file-mode name (bitwise-ior perm/irwxg perm/irwxu))) + (else + (debug:print-info 0 "Invalid script definiton found in [scripts] section of testconfig. \"" scriptdat "\"")))) + scripts) + + ;; extract logpro from testconfig and write them to files in test run dir + (for-each + (lambda (logprodat) + (match logprodat + ((name content) + (debug:print-info 2 *default-log-port* "Creating logpro file "(current-directory)"/"name) + (with-output-to-file name + (lambda () + (print content) + ;; (change-file-mode name (bitwise-ior perm/irwxg perm/irwxu)) + ))) + (else + (debug:print-info 0 "Invalid logpro definiton found in [logpro] section of testconfig. \"" logprodat "\"")))) + logpros))) (define (launch:execute encoded-cmd) (let* ((cmdinfo (common:read-encoded-string encoded-cmd)) (tconfigreg #f)) (setenv "MT_CMDINFO" encoded-cmd) @@ -733,16 +778,19 @@ (th1 (make-thread monitorjob "monitor job")) (th2 (make-thread runit "run job")) (tconfig (tests:get-testconfig test-name item-path tconfigreg #t)) (propagate-exit-code (configf:lookup *configdat* "setup" "propagate-exit-code")) (propagate-status-list '("FAIL" "KILLED" "ABORT" "DEAD" "CHECK" "SKIP" "WAIVED")) - (test-status "not set") + (test-status "not set") + (test-state "not set") (precmd (configf:lookup tconfig "setup" "precmd")) (postcmd (configf:lookup tconfig "setup" "postcmd"))) ;; first, if set, run the precmd (if precmd ;; (file-exists? precmd)(file-execute-access? precmd)) - (system precmd)) ;; up to test author to put nbfake if desired. + (begin + ;; (save-environment-as-files "precmd-envt") + (system precmd))) ;; up to test author to put nbfake if desired. (set! job-thread th2) (thread-start! th1) (thread-start! th2) (thread-join! th2) (debug:print-info 0 *default-log-port* "Megatest execute of test " test-name ", item path " item-path " complete. Notifying the db ...") @@ -801,25 +849,30 @@ (mutex-unlock! m) (launch:end-of-run-check run-id ) (debug:print 2 *default-log-port* "Output from running " fullrunscript ", pid " (launch:einf-pid exit-info) " in work area " work-area ":\n====\n exit code " (launch:einf-exit-code exit-info) "\n" "====\n") - - (set! test-status (db:test-get-status (rmt:get-testinfo-state-status run-id test-id))) + (let* ((testrec (rmt:get-testinfo-state-status run-id test-id))) + (set! test-status (db:test-get-status testrec)) + (set! test-state (db:test-get-state testrec))) ;; If the propagate-exit-code option has been set in the megatest config, and the test status matches the list, set the exit code to 1. - (if postcmd - (system postcmd)) (if (and propagate-exit-code (string=? propagate-exit-code "yes") (member test-status propagate-status-list)) (begin (debug:print 1 *default-log-port* "Setting exit status to 1 because of test status of " test-status) (set! *globalexitstatus* 1) ) ) + (if postcmd + (begin + (setenv "MT_TEST_STATE" test-state) + (setenv "MT_TEST_STATUS" test-status) + ;; (save-environment-as-files "postcmd-envt") + (system postcmd))) (if (not (launch:einf-exit-status exit-info)) (exit 4)))) ))) ;; Spec for End of test Index: megatest.scm ================================================================== --- megatest.scm +++ megatest.scm @@ -113,11 +113,13 @@ ;; (let ((debugcontrolf (conc (get-environment-variable "HOME") "/.megatestrc"))) (if (common:file-exists? debugcontrolf) (begin ;; for some reason, debug:print does not work here. Had to use print. - (print (conc "WARNING: loading " debugcontrolf)) + (with-output-to-port (current-error-port) + (lambda () + (print (conc "WARNING: loading " debugcontrolf)))) (load debugcontrolf) ) ) ) @@ -263,11 +265,10 @@ -debug N|N,M,O... : enable debug 0-N or N and M and O ... -debug-noprop N|M,M,O...: enable debug but do not propagate to subprocesses via MT_DEBUG -config fname : override the megatest.config file with fname -append-config fname : append fname to the megatest.config file -import-sexpr fname : import a sexpr file (use -list-runs % -dumpmode sexpr to create) - -remove-dbs all : remove Megatest DBs before importing sexpr. (Use only with -import-sexpr) -regen-testfiles : regenerate scripts and logpro files from testconfig, run in test context Utilities -env2file fname : write the environment to fname.csh and fname.sh -envcap a : save current variables labeled as context 'a' in file envdat.db @@ -2130,10 +2131,25 @@ (paths (tests:test-get-paths-matching keys target (args:get-arg "-test-files")))) (for-each (lambda (path) (print path)) paths)))))) +;;====================================================================== +;; Utils for test areas +;;====================================================================== + +(if (args:get-arg "-regen-testfiles") + (if (getenv "MT_TEST_RUN_DIR") + (begin + (launch:setup) + (change-directory (getenv "MT_TEST_RUN_DIR")) + (let* ((testname (getenv "MT_TEST_NAME")) + (itempath (getenv "MT_ITEMPATH"))) + (launch:extract-scripts-logpro (getenv "MT_TEST_RUN_DIR") testname itempath #f)) + (set! *didsomething* #t)) + (debug:print 0 *default-log-port* "ERROR: Must run -regen-testfiles in a test environment (i.e. test xterm from dashboard)"))) + ;;====================================================================== ;; Utils for test areas ;;====================================================================== (if (args:get-arg "-regen-testfiles") Index: portlogger.scm ================================================================== --- portlogger.scm +++ portlogger.scm @@ -15,10 +15,11 @@ ;; ;; You should have received a copy of the GNU General Public License ;; along with Megatest. If not, see . ;; +(require-extension (srfi 18) extras tcp s11n) (declare (unit portlogger)) (declare (uses debugprint)) (declare (uses dbmod)) Index: rmt.scm ================================================================== --- rmt.scm +++ rmt.scm @@ -790,5 +790,8 @@ (let* ((cfg-deadtime (configf:lookup-number *configdat* "setup" "deadtime")) (test-stats-update-period (configf:lookup-number *configdat* "setup" "test-stats-update-period"))) (rmt:find-and-mark-incomplete-engine run-id ovr-deadtime cfg-deadtime test-stats-update-period) ;;call end of eud of run detection for posthook (launch:end-of-run-check run-id))) + +;; orphaned from cherrypick merge +;; (debug:print 0 *default-log-port* "Inserting " (length tests-data) " tests in run " runname) Index: runs.scm ================================================================== --- runs.scm +++ runs.scm @@ -811,31 +811,21 @@ (debug:print-info 1 *default-log-port* "Adding \"" (string-intersperse required-tests " ") "\" to the run queue")) ;; NOTE: these are all parent tests, items are not expanded yet. (debug:print-info 4 *default-log-port* "test-records=" (hash-table->alist test-records)) (let ((reglen (configf:lookup *configdat* "setup" "runqueue"))) (if (> (length (hash-table-keys test-records)) 0) - (let* ((keep-going #t) - (run-queue-retries 5) - (run-ids (rmt:get-all-run-ids))) - #;(for-each (lambda (run-id) - (if keep-going - (handle-exceptions - exn - (debug:print 0 *default-log-port* "error in calling find-and-mark-incomplete for run-id " run-id ", exn=" exn) - (rmt:find-and-mark-incomplete run-id #f)))) ;; ovr-deadtime))) ;; could be root of https://hsdes.intel.com/appstore/article/#/220546828/main -- Title: Megatest jobs show DEAD even though they are still running (1.64/27) - run-ids) + (let* () (runs:run-tests-queue run-id runname test-records keyvals flags test-patts required-tests (any->number reglen) all-tests-registry) - (set! keep-going #f) (if (> run-count 0) ;; handle reruns (begin (if (not (hash-table-ref/default flags "-preclean" #f)) (hash-table-set! flags "-preclean" #t)) (if (not (hash-table-ref/default flags "-rerun" #f)) (hash-table-set! flags "-rerun" "ABORT,STUCK/DEAD,n/a,ZERO_ITEMS")) - (runs:run-tests target runname test-patts user flags run-count: (- run-count 1))) - (launch:end-of-run-check run-id))) + (runs:run-tests target runname test-patts user flags run-count: (- run-count 1))) + (launch:end-of-run-check run-id))) (debug:print-info 0 *default-log-port* "No tests to run"))) (debug:print-info 4 *default-log-port* "All done by here") ;; TODO: try putting post hook call here ; (debug:print-info 2 *default-log-port* " run-count " run-count) @@ -1381,12 +1371,16 @@ #f ;; yes, really (list (car tal)(cdr tal) reg reruns)) (begin (if (runs:lownoise (conc "FAILED prerequisites or other issue" hed) 60) (debug:print 0 *default-log-port* "WARNING: test " hed " has FAILED prerequisites or other issue. Internal state >" nth-try "< will be overridden and we'll retry.")) - ;; was: (mt:test-set-state-status-by-testname run-id test-name item-path "NOT_STARTED" "KEEP_TRYING" #f) - (mt:test-set-state-status-by-testname-unless-completed run-id test-name item-path "COMPLETED" "PREQ_FAIL" #f) + (let* ((test-id (rmt:get-test-id run-id testname item-path)) + (test-info (rmt:get-testinfo-state-status run-id test-id)) ;; we need *current* info + (status (db:test-status test-info))) + (if (equal? status "KEEP_TRYING") + (mt:test-set-state-status-by-testname-unless-completed run-id test-name item-path "COMPLETED" "PREQ_FAIL" #f) + (mt:test-set-state-status-by-testname run-id test-name item-path "NOT_STARTED" "KEEP_TRYING" #f))) (hash-table-set! test-registry hed 'removed) ;; was 0 (if (not (and (null? reg) (null? tal))) (runs:loop-values tal reg reglen regfull reruns) #f)))) (else Index: server.scm ================================================================== --- server.scm +++ server.scm @@ -393,11 +393,11 @@ (set! new-server-key (with-input-from-file start-flag (lambda () (read-line)))) (equal? server-key new-server-key))) #t ;; If either of the above conditions is not true, print a "Gating server start" message, wait + 1, then call this function recursively. (begin - (debug:print-info 0 *default-log-port* "Gating server start, last start: " + (debug:print-info 2 *default-log-port* "Gating server start, last start: " (seconds->time-string fmodtime) ", time since last start: " delta ", required idletime: " idletime ", gating reason:" (if old-enough "another job started a server" "too soon to start another server")) (thread-sleep! ( + 1 idletime)) (server:wait-for-server-start-last-flag areapath))))))) Index: tcp-transportmod.scm ================================================================== --- tcp-transportmod.scm +++ tcp-transportmod.scm @@ -159,14 +159,40 @@ (define (tt:get-conn ttdat dbfname) (hash-table-ref/default (tt-conns ttdat) dbfname #f)) ;; do all the busy work of finding and setting up conn for ;; connecting to a server +;; This function, `tt:client-connect-to-server`, is designed to manage connections between a client and a server within a testing framework. +;; The function takes four arguments: +;; 1. `ttdat`: a data structure that holds information about the testing environment or connections. +;; 2. `dbfname`: The name of the database file that the client wants to connect to. +;; 3. `run-id`: An identifier for the current run of the test suite. +;; 4. `testsuite`: The test suite that is being run. +;; +;; Here's a step-by-step explanation of what the function does: +;; +;; 1. It first asserts that the `run-id` is valid for the given `dbfname` using the `tt:valid-run-id` function. If the `run-id` is not valid, it raises a fatal error. +;; 2. It prints debug information indicating that the function `tt:client-connect-to-server` has been called with the given `dbfname`. +;; 3. It attempts to retrieve an existing connection to the server from a hash table (`tt-conns`) using the `dbfname` as the key. If a connection already exists, it prints debug information and returns the existing connection. +;; 4. If no existing connection is found, it retrieves the current server information from the servinfo file, using the `tt:get-current-server-info` function. +;; 5. It uses pattern matching to destructure the server information into variables (`host`, `port`, `start-time`, `server-id`, `pid`, `dbfname2`, `servinffile`). It then asserts that the `dbfname` from the server info matches the one provided to the function. +;; 6. It constructs a connection object (`conn`) with the server information. +;; 7. It attempts to ping the server using `tt:timed-ping` to verify that the server is running and can be communicated with. +;; 8. Depending on the result of the ping: +;; - If the server is running (`running`), it prints debug information, saves the connection in the hash table, and returns the connection. +;; - If the server is starting (`starting`), it sleeps for 2 seconds and then recursively calls itself to retry the connection. +;; - If the server is neither running nor starting, it checks if it's been more than 10 seconds since the last server start attempt. If so, it attempts to start the server using `server-start-proc` and then sleeps for 1 second before retrying the connection. +;; 9. If no server information is found (`else` case), it checks if it's been more than 3 seconds since the last server start attempt. If so, it starts a new server using `server-start-proc`, updates the last server start time, and sleeps for 4 seconds. +;; 10. It then sleeps for 1 second and prints debug information before recursively calling itself to retry the connection. +;; +;; The function uses recursion to keep trying to connect to the server, with various sleep intervals to prevent overwhelming the system with connection attempts or server starts. +;; It also uses a hash table to cache connections and avoid reconnecting to a server if a connection already exists. +;; The function is designed to handle different server states and ensure that a server is running and available before returning a valid connection to the caller. ;; (define (tt:client-connect-to-server ttdat dbfname run-id testsuite server-start-proc) (assert (tt:valid-run-id run-id dbfname) "FATAL: invalid run-id "run-id) - (debug:print-info 2 *default-log-port* "tt:client-connect-to-server " dbfname " " run-id) + (debug:print-info 2 *default-log-port* "tt:client-connect-to-server " dbfname) (let* ((conn (tt:get-conn ttdat dbfname)) (server-start-proc (or server-start-proc (lambda () (assert (equal? dbfname "main.db") ;; only main.db is started here "FATAL: called server-start-proc for db other than main.db") @@ -175,11 +201,11 @@ testsuite ;; (dbfile:testsuite-name) (common:find-local-megatest) run-id))))) (if conn (begin - (debug:print-info 2 *default-log-port* "already connected to a server") + (debug:print-info 2 *default-log-port* "already connected to a server for " dbfname) conn) ;; we are already connected to the server ;; no conn (let* ((sdats (tt:get-server-info-sorted ttdat dbfname)) (sdat (if (null? sdats) ADDED utils/convert-db.sh Index: utils/convert-db.sh ================================================================== --- /dev/null +++ utils/convert-db.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +if [ -z "megatest.config" ]; then + echo "The file 'megatest.config' does not exist. This must be run in a megatest area." + exit 1 +fi +if [ -d ".mtdb" ]; then + echo "The .mtdb directory already exists. Will not do the conversion" + exit 1 +fi +if [ -d ".megatest" ]; then + echo "Found a .megatest directory. Will convert from megatest 1.70 to 1.71/1.80 format" + /p/foundry/env/pkgs/megatest/1.70/16/bin/megatest -list-runs % -dumpmode sexpr > data.sexpr +else + if [ -f "megatest.db" ]; then + echo "Found megatest.db. Will convert from megatest 1.65 to 1.71/1.80 format" + /p/foundry/env/pkgs/megatest/1.65/92/bin/megatest -list-runs % -dumpmode sexpr > data.sexpr + else + echo "Did not find .megatest or megatest.db. Cannot do the conversion" + exit 1 + fi +fi +which megatest +megatest -import-sexpr data.sexpr +