Overview
Comment: | Unravelled a few ravels. |
---|---|
Downloads: | Tarball | ZIP archive | SQL archive |
Timelines: | family | ancestors | descendants | both | v1.70-defunct-try |
Files: | files | file ages | folders |
SHA1: |
9990ebb12c9167a81e5470c0e62ebe00 |
User & Date: | matt on 2019-12-08 19:46:08 |
Other Links: | branch diff | manifest | tags |
Context
2019-12-08
| ||
20:31 | Moved few items around; api:read/write query lists, megatest-fossil-hash check-in: 34541b6e01 user: matt tags: v1.70-defunct-try | |
19:46 | Unravelled a few ravels. check-in: 9990ebb12c user: matt tags: v1.70-defunct-try | |
17:10 | bump version to v1.7001. check-in: 856bbfa186 user: matt tags: v1.70-defunct-try | |
Changes
Modified common-inc.scm from [64b34ec4d3] to [fa5d5b4ff2].
︙ | ︙ | |||
109 110 111 112 113 114 115 | (define *test-meta-updated* (make-hash-table)) (define *globalexitstatus* 0) ;; attempt to work around possible thread issues (define *passnum* 0) ;; when running track calls to run-tests or similar ;; (define *alt-log-file* #f) ;; used by -log (define *common:denoise* (make-hash-table)) ;; for low noise printing (define *default-log-port* (current-error-port)) | < | 109 110 111 112 113 114 115 116 117 118 119 120 121 122 | (define *test-meta-updated* (make-hash-table)) (define *globalexitstatus* 0) ;; attempt to work around possible thread issues (define *passnum* 0) ;; when running track calls to run-tests or similar ;; (define *alt-log-file* #f) ;; used by -log (define *common:denoise* (make-hash-table)) ;; for low noise printing (define *default-log-port* (current-error-port)) (define *default-area-tag* "local") ;; DATABASE (define *dbstruct-db* #f) ;; used to cache the dbstruct in db:setup. Goal is to remove this. ;; db stats (define *db-stats* (make-hash-table)) ;; hash of vectors < count duration-total > (define *db-stats-mutex* (make-mutex)) |
︙ | ︙ | |||
908 909 910 911 912 913 914 915 916 917 918 919 920 921 | (define *wdnum* 0) (define *wdnum*mutex (make-mutex)) (define (common:human-time) (time->string (seconds->local-time (current-seconds)) "%Y-%m-%d %H:%M:%S")) ;; currently the primary job of the watchdog is to run the sync back to megatest.db from the db in /tmp ;; if we are on the homehost and we are a server (by definition we are on the homehost if we are a server) ;; (define (common:readonly-watchdog dbstruct) (thread-sleep! 0.05) ;; delay for startup (debug:print-info 13 *default-log-port* "common:readonly-watchdog entered.") | > > > > > > > > > > > | 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 | (define *wdnum* 0) (define *wdnum*mutex (make-mutex)) (define (common:human-time) (time->string (seconds->local-time (current-seconds)) "%Y-%m-%d %H:%M:%S")) (define *time-zero* (current-seconds)) ;; for the watchdog (define *watchdog* (make-thread (lambda () (handle-exceptions exn (begin (print-call-chain) (print " message: " ((condition-property-accessor 'exn 'message) exn))) (common:watchdog))) "Watchdog thread")) ;; currently the primary job of the watchdog is to run the sync back to megatest.db from the db in /tmp ;; if we are on the homehost and we are a server (by definition we are on the homehost if we are a server) ;; (define (common:readonly-watchdog dbstruct) (thread-sleep! 0.05) ;; delay for startup (debug:print-info 13 *default-log-port* "common:readonly-watchdog entered.") |
︙ | ︙ |
Modified common_records.scm from [349188aa4b] to [5084b8d608].
︙ | ︙ | |||
17 18 19 20 21 22 23 | ;; along with Megatest. If not, see <http://www.gnu.org/licenses/>. ;; ;;====================================================================== ;; (use trace) (use typed-records) | < < < < < | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | 17 18 19 20 21 22 23 24 | ;; along with Megatest. If not, see <http://www.gnu.org/licenses/>. ;; ;;====================================================================== ;; (use trace) (use typed-records) ;; moved to commonmod |
Modified commonmod.scm from [1896faeca2] to [e028470746].
︙ | ︙ | |||
569 570 571 572 573 574 575 | ;; ;; returns list of fd count, socket count ;; (define (get-file-descriptor-count #!key (pid (current-process-id ))) ;; (list ;; (length (glob (conc "/proc/" pid "/fd/*"))) ;; (length (filter identity (map socket? (glob (conc "/proc/" pid "/fd/*"))))) ;; ) ;; ) | | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 | ;; ;; returns list of fd count, socket count ;; (define (get-file-descriptor-count #!key (pid (current-process-id ))) ;; (list ;; (length (glob (conc "/proc/" pid "/fd/*"))) ;; (length (filter identity (map socket? (glob (conc "/proc/" pid "/fd/*"))))) ;; ) ;; ) ;; ;; pulled from common_records.scm ;; globals - modules that include this need these here (define *logging* #f) (define *functions* (make-hash-table)) ;; symbol => fn ### TEMPORARY!!! ;; (define *toppath* #f) (define *transport-type* 'http) #;(define (exec-fn fn . params) (if (hash-table-exists? *functions* fn) (apply (hash-table-ref *functions* fn) params) (begin (debug:print-error 0 "exec-fn " fn " not found") #f))) #;(define (set-fn fn-name fn) (hash-table-set! *functions* fn-name fn)) (include "altdb.scm") ;; Pulled from http-transport.scm (define (make-http-transport:server-dat)(make-vector 6)) (define (http-transport:server-dat-get-iface vec) (vector-ref vec 0)) (define (http-transport:server-dat-get-port vec) (vector-ref vec 1)) (define (http-transport:server-dat-get-api-uri vec) (vector-ref vec 2)) (define (http-transport:server-dat-get-api-url vec) (vector-ref vec 3)) (define (http-transport:server-dat-get-api-req vec) (vector-ref vec 4)) (define (http-transport:server-dat-get-last-access vec) (vector-ref vec 5)) (define (http-transport:server-dat-get-socket vec) (vector-ref vec 6)) (define (http-transport:server-dat-make-url vec) (if (and (http-transport:server-dat-get-iface vec) (http-transport:server-dat-get-port vec)) (conc "http://" (http-transport:server-dat-get-iface vec) ":" (http-transport:server-dat-get-port vec)) #f)) (define (http-transport:server-dat-update-last-access vec) (if (vector? vec) (vector-set! vec 5 (current-seconds)) (begin (print-call-chain (current-error-port)) (debug:print-error 0 (current-error-port) "call to http-transport:server-dat-update-last-access with non-vector!!")))) ;;====================================================================== ;; ;;====================================================================== ;; allow these queries through without starting a server ;; (define api:read-only-queries '(get-key-val-pairs get-var get-keys get-key-vals test-toplevel-num-items get-test-info-by-id get-steps-info-by-id get-data-info-by-id test-get-rundir-from-test-id get-count-tests-running-for-testname get-count-tests-running get-count-tests-running-in-jobgroup get-previous-test-run-record get-matching-previous-test-run-records test-get-logfile-info test-get-records-for-index-file get-testinfo-state-status test-get-top-process-pid test-get-paths-matching-keynames-target-new get-prereqs-not-met get-count-tests-running-for-run-id get-run-info get-run-status get-run-state get-run-stats get-run-times get-targets get-target ;; register-run get-tests-tags get-test-times get-tests-for-run get-test-id get-tests-for-runs-mindata get-tests-for-run-mindata get-run-name-from-id get-runs simple-get-runs get-num-runs get-runs-cnt-by-patt get-all-run-ids get-prev-run-ids get-run-ids-matching-target get-runs-by-patt get-steps-data get-steps-for-test read-test-data read-test-data* login tasks-get-last testmeta-get-record have-incompletes? synchash-get get-changed-record-ids get-run-record-ids get-not-completed-cnt)) (define api:write-queries '( get-keys-write ;; dummy "write" query to force server start ;; SERVERS start-server kill-server ;; TESTS test-set-state-status-by-id delete-test-records delete-old-deleted-test-records test-set-state-status test-set-top-process-pid set-state-status-and-roll-up-items update-pass-fail-counts top-test-set-per-pf-counts ;; (db:top-test-set-per-pf-counts (db:get-db *db* 5) 5 "runfirst") ;; RUNS register-run set-tests-state-status delete-run lock/unlock-run update-run-event_time mark-incomplete set-state-status-and-roll-up-run ;; STEPS teststep-set-status! delete-steps-for-test ;; TEST DATA test-data-rollup csv->test-data ;; MISC sync-inmem->db ;; TESTMETA testmeta-add-record testmeta-update-field ;; TASKS tasks-add tasks-set-state-given-param-key )) ;;====================================================================== ;; ALLDATA ;;====================================================================== ;; ;; attempt to consolidate a bunch of global information into one struct to toss around (defstruct alldat ;; misc (denoise (make-hash-table)) (areapath #f) ;; i.e. toppath (mtconfig #f) (log-port #f) (areadat #f) ;; i.e. runremote (rmt-mutex (make-mutex)) (db-sync-mutex (make-mutex)) (db-with-db-mutex (make-mutex)) (read-only-queries api:read-only-queries) (write-queries api:write-queries) (max-api-process-requests 0) (api-process-request-count 0) (db-keys #f) (megatest-version "1.6536") (megatest-fossil-hash #f) ;; database related (tmppath #f) ;; tmp path for dbs ;; runremote fields (hh-dat #f) ;; (exec-fn 'common:get-homehost)) ;; homehost record ( addr . hhflag ) (server-url #f) ;; (if *toppath* (exec-fn 'server:check-if-running *toppath*))) ;; (server:check-if-running *toppath*) #f)) (last-server-check 0) ;; last time we checked to see if the server was alive (conndat #f) (transport *transport-type*) (server-timeout #f) ;; (exec-fn 'server:expiration-timeout)) (force-server #f) (ro-mode #f) (ro-mode-checked #f) ;; flag that indicates we have checked for ro-mode (ulex:conn #f) ;; ulex db conn is not exactly a db connector, more like a network connector ;; dbstruct (tmpdb #f) (dbstack #f) ;; stack for tmp db handles, do not initialize with a stack (mtdb #f) (refndb #f) (homehost #f) ;; not used yet (on-homehost #f) ;; not used yet (read-only #f) ) (define *alldat* (make-alldat)) ;; Some of these routines use: ;; ;; http://www.cs.toronto.edu/~gfb/scheme/simple-macros.html ;; ;; Syntax for defining macros in a simple style similar to function definiton, ;; when there is a single pattern for the argument list and there are no keywords. ;; ;; (define-simple-syntax (name arg ...) body ...) ;; (define-syntax define-simple-syntax (syntax-rules () ((_ (name arg ...) body ...) (define-syntax name (syntax-rules () ((name arg ...) (begin body ...))))))) ;; (define-syntax common:handle-exceptions ;; (syntax-rules () ;; ((_ exn-in errstmt ...)(handle-exceptions exn-in errstmt ...)))) (define-syntax common:debug-handle-exceptions (syntax-rules () ((_ debug exn errstmt body ...) (if debug (begin body ...) (handle-exceptions exn errstmt body ...))))) (define-syntax common:handle-exceptions (syntax-rules () ((_ exn errstmt body ...) (begin body ...)))) ;; (define handle-exceptions common:handle-exceptions) ;; iup callbacks are not dumping the stack, this is a work-around ;; (define-simple-syntax (debug:catch-and-dump proc procname) (handle-exceptions exn (begin (print-call-chain (current-error-port)) (with-output-to-port (current-error-port) (lambda () (print ((condition-property-accessor 'exn 'message) exn)) (print "Callback error in " procname) (print "Full condition info:\n" (condition->list exn))))) (proc))) ;; Need a mutex protected way to get and set values ;; or use (define-simple-syntax ?? ;; (define-inline (with-mutex mtx accessor record . val) (mutex-lock! mtx) (let ((res (apply accessor record val))) (mutex-unlock! mtx) res)) ;; Brandon's debug printer shortcut (indulge me :) ;; (define *BB-process-starttime* (current-milliseconds)) #;(define (BB> . in-args) (let* ((stack (get-call-chain)) (location "??")) (for-each (lambda (frame) (let* ((this-loc (vector-ref frame 0)) (temp (string-split (->string this-loc) " ")) (this-func (if (and (list? temp) (> (length temp) 1)) (cadr temp) "???"))) (if (equal? this-func "BB>") (set! location this-loc)))) stack) (let* ((color-on "\x1b[1m") (color-off "\x1b[0m") (dp-args (append (list 0 *default-log-port* (conc color-on location "@"(/ (- (current-milliseconds) *BB-process-starttime*) 1000) color-off " ") ) in-args))) (apply debug:print dp-args)))) ;; (define *BBpp_custom_expanders_list* (make-hash-table)) ;; register hash tables with BBpp. #;(hash-table-set! *BBpp_custom_expanders_list* HASH_TABLE: (cons hash-table? hash-table->alist)) ;; test name converter #;(define (BBpp_custom_converter arg) (let ((res #f)) (for-each (lambda (custom-type-name) (let* ((custom-type-info (hash-table-ref *BBpp_custom_expanders_list* custom-type-name)) (custom-type-test (car custom-type-info)) (custom-type-converter (cdr custom-type-info))) (when (and (not res) (custom-type-test arg)) (set! res (custom-type-converter arg))))) (hash-table-keys *BBpp_custom_expanders_list*)) (if res (BBpp_ res) arg))) #;(define (BBpp_ arg) (cond ;;((SOMESTRUCT? arg) (cons SOMESTRUCT: (SOMESTRUCT->alist arg))) ;;((dboard:tabdat? arg) (cons dboard:tabdat: (dboard:tabdat->alist arg))) ((hash-table? arg) (let ((al (hash-table->alist arg))) (BBpp_ (cons HASH_TABLE: al)))) ((null? arg) '()) ;;((list? arg) (cons (BBpp_ (car arg)) (BBpp_ (cdr arg)))) ((pair? arg) (cons (BBpp_ (car arg)) (BBpp_ (cdr arg)))) (else (BBpp_custom_converter arg)))) ;; Brandon's pretty printer. It expands hashes and custom types in addition to regular pp #;(define (BBpp arg) (pp (BBpp_ arg))) ;(use define-macro) #;(define-syntax inspect (syntax-rules () [(_ x) ;; (with-output-to-port (current-error-port) (printf "~a is: ~a\n" 'x (with-output-to-string (lambda () (BBpp x)))) ;; ) ] [(_ x y ...) (begin (inspect x) (inspect y ...))])) ;; if a value is printable (i.e. string or number) return the value ;; else return an empty string (define-inline (printable val) (if (or (number? val)(string? val)) val "")) ) |
Modified http-transport-inc.scm from [8e338cca1d] to [42769b04e9].
︙ | ︙ | |||
309 310 311 312 313 314 315 | (print-call-chain *default-log-port*) (debug:print-error 0 *default-log-port* " closing connection failed with error: " ((condition-property-accessor 'exn 'message) exn))) (close-connection! api-dat) ;;(close-idle-connections!) #t)) #f))) | | > | | | | | | | | | | | | | | | | | | | | | | | | | 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 | (print-call-chain *default-log-port*) (debug:print-error 0 *default-log-port* " closing connection failed with error: " ((condition-property-accessor 'exn 'message) exn))) (close-connection! api-dat) ;;(close-idle-connections!) #t)) #f))) ;; Moved to commonmod ;; ;; (define (make-http-transport:server-dat)(make-vector 6)) ;; (define (http-transport:server-dat-get-iface vec) (vector-ref vec 0)) ;; (define (http-transport:server-dat-get-port vec) (vector-ref vec 1)) ;; (define (http-transport:server-dat-get-api-uri vec) (vector-ref vec 2)) ;; (define (http-transport:server-dat-get-api-url vec) (vector-ref vec 3)) ;; (define (http-transport:server-dat-get-api-req vec) (vector-ref vec 4)) ;; (define (http-transport:server-dat-get-last-access vec) (vector-ref vec 5)) ;; (define (http-transport:server-dat-get-socket vec) (vector-ref vec 6)) ;; ;; (define (http-transport:server-dat-make-url vec) ;; (if (and (http-transport:server-dat-get-iface vec) ;; (http-transport:server-dat-get-port vec)) ;; (conc "http://" ;; (http-transport:server-dat-get-iface vec) ;; ":" ;; (http-transport:server-dat-get-port vec)) ;; #f)) ;; ;; (define (http-transport:server-dat-update-last-access vec) ;; (if (vector? vec) ;; (vector-set! vec 5 (current-seconds)) ;; (begin ;; (print-call-chain (current-error-port)) ;; (debug:print-error 0 *default-log-port* "call to http-transport:server-dat-update-last-access with non-vector!!")))) ;; ;; connect ;; (define (http-transport:client-connect iface port) (let* ((api-url (conc "http://" iface ":" port "/api")) (api-uri (uri-reference (conc "http://" iface ":" port "/api"))) |
︙ | ︙ |
Modified megatest.scm from [39c16682aa] to [8a122fe918].
︙ | ︙ | |||
489 490 491 492 493 494 495 | (let ((targ (or (args:get-arg "-reqtarg")(args:get-arg "-target")))) (if targ (setenv "MT_TARGET" targ))) ;; The watchdog is to keep an eye on things like db sync etc. ;; ;; TODO: for multiple areas, we will have multiple watchdogs; and multiple threads to manage | | | 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 | (let ((targ (or (args:get-arg "-reqtarg")(args:get-arg "-target")))) (if targ (setenv "MT_TARGET" targ))) ;; The watchdog is to keep an eye on things like db sync etc. ;; ;; TODO: for multiple areas, we will have multiple watchdogs; and multiple threads to manage #;(define *watchdog* (make-thread (lambda () (handle-exceptions exn (begin (print-call-chain) (print " message: " ((condition-property-accessor 'exn 'message) exn))) (common:watchdog))) |
︙ | ︙ |