Overview
Comment: | enabled taging on areas with pgdb sync |
---|---|
Downloads: | Tarball | ZIP archive | SQL archive |
Timelines: | family | ancestors | descendants | both | v1.65 |
Files: | files | file ages | folders |
SHA1: |
a9bdb539385321ec79cace958a816d39 |
User & Date: | pjhatwal on 2017-09-14 16:40:38 |
Other Links: | branch diff | manifest | tags |
Context
2017-09-15
| ||
11:01 | Merged html-gen branch check-in: 67151b5753 user: mrwellan tags: v1.65 | |
2017-09-14
| ||
16:40 | enabled taging on areas with pgdb sync check-in: a9bdb53938 user: pjhatwal tags: v1.65 | |
2017-09-12
| ||
12:21 | added web tables to mt-pg.sql check-in: 5c94f3c919 user: pjhatwal tags: v1.65 | |
Changes
Modified cgisetup/models/pgdb.scm from [875743c283] to [297bdb043a].
︙ | ︙ | |||
88 89 90 91 92 93 94 95 96 97 98 99 100 101 | exn (begin (print-call-chain) (debug:print 0 *default-log-port* "ERROR: cannot create ttype entry, " ((condition-property-accessor 'exn 'message) exn)) #f) (dbi:exec dbh "INSERT INTO ttype (target_spec) VALUES (?);" target-spec)) (pgdb:get-ttype dbh target-spec))))) ;;====================================================================== ;; R U N S ;;====================================================================== ;; given a target spec id, target and run-name return the run-id ;; if no run found return #f | > > > > > > > > > > > > > > > > > > > > > > > > > > > | 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 | exn (begin (print-call-chain) (debug:print 0 *default-log-port* "ERROR: cannot create ttype entry, " ((condition-property-accessor 'exn 'message) exn)) #f) (dbi:exec dbh "INSERT INTO ttype (target_spec) VALUES (?);" target-spec)) (pgdb:get-ttype dbh target-spec))))) ;;====================================================================== ;; T A G S ;;====================================================================== (define (pgdb:get-tag-info-by-name dbh tag) (dbi:get-one-row dbh "SELECT id,tag_name FROM tags where tag_name=?;" tag)) (define (pgdb:insert-tag dbh name ) (dbi:exec dbh "INSERT INTO tags (tag_name) VALUES (?)" name )) (define (pgdb:insert-area-tag dbh tag-id area-id ) (dbi:exec dbh "INSERT INTO area_tags (tag_id, area_id) VALUES (?,?)" tag-id area-id )) (define (pgdb:is-area-taged dbh area-id) (let ((area-tag-id (dbi:get-one dbh "SELECT id FROM area_tags WHERE area_id=?;" area-id))) (if area-tag-id #t #f))) (define (pgdb:is-area-taged-with-a-tag dbh tag-id area-id) (let ((area-tag-id (dbi:get-one dbh "SELECT id FROM area_tags WHERE area_id=? and tag_id=?;" area-id tag-id))) (if area-tag-id #t #f))) ;;====================================================================== ;; R U N S ;;====================================================================== ;; given a target spec id, target and run-name return the run-id ;; if no run found return #f |
︙ | ︙ |
Modified common.scm from [1cf92d093e] to [2d6fcc61d7].
︙ | ︙ | |||
112 113 114 115 116 117 118 119 120 121 122 123 124 125 | (define *test-meta-updated* (make-hash-table)) (define *globalexitstatus* 0) ;; attempt to work around possible thread issues (define *passnum* 0) ;; when running track calls to run-tests or similar ;; (define *alt-log-file* #f) ;; used by -log (define *common:denoise* (make-hash-table)) ;; for low noise printing (define *default-log-port* (current-error-port)) (define *time-zero* (current-seconds)) ;; for the watchdog ;; DATABASE (define *dbstruct-db* #f) ;; used to cache the dbstruct in db:setup. Goal is to remove this. ;; db stats (define *db-stats* (make-hash-table)) ;; hash of vectors < count duration-total > (define *db-stats-mutex* (make-mutex)) ;; db access | > | 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 | (define *test-meta-updated* (make-hash-table)) (define *globalexitstatus* 0) ;; attempt to work around possible thread issues (define *passnum* 0) ;; when running track calls to run-tests or similar ;; (define *alt-log-file* #f) ;; used by -log (define *common:denoise* (make-hash-table)) ;; for low noise printing (define *default-log-port* (current-error-port)) (define *time-zero* (current-seconds)) ;; for the watchdog (define *default-area-tag* "local") ;; DATABASE (define *dbstruct-db* #f) ;; used to cache the dbstruct in db:setup. Goal is to remove this. ;; db stats (define *db-stats* (make-hash-table)) ;; hash of vectors < count duration-total > (define *db-stats-mutex* (make-mutex)) ;; db access |
︙ | ︙ |
Modified megatest.scm from [a6fd98e3db] to [172e06516c].
︙ | ︙ | |||
168 169 170 171 172 173 174 175 176 177 178 179 180 181 | -fields fieldspec : fields to include in json dump; runs:id,runame+tests:testname+steps -sort fieldname : in -list-runs sort tests by this field -testdata-csv [categorypatt/]varpatt : dump testdata for given category Misc -start-dir path : switch to this directory before running megatest -contour cname : add a level of hierarcy to the linktree and run paths -rebuild-db : bring the database schema up to date -cleanup-db : remove any orphan records, vacuum the db -import-megatest.db : push data from megatest.db to cache db files in /tmp/$USER -sync-to-megatest.db : pull data from cache files in /tmp/$USER to megatest.db -sync-to dest : sync to new postgresql central style database -update-meta : update the tests metadata for all tests -setvars VAR1=val1,VAR2=val2 : Add environment variables to a run NB// these are | > | 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 | -fields fieldspec : fields to include in json dump; runs:id,runame+tests:testname+steps -sort fieldname : in -list-runs sort tests by this field -testdata-csv [categorypatt/]varpatt : dump testdata for given category Misc -start-dir path : switch to this directory before running megatest -contour cname : add a level of hierarcy to the linktree and run paths -area-tag tagname : add a tag to an area while syncking to pgdb -rebuild-db : bring the database schema up to date -cleanup-db : remove any orphan records, vacuum the db -import-megatest.db : push data from megatest.db to cache db files in /tmp/$USER -sync-to-megatest.db : pull data from cache files in /tmp/$USER to megatest.db -sync-to dest : sync to new postgresql central style database -update-meta : update the tests metadata for all tests -setvars VAR1=val1,VAR2=val2 : Add environment variables to a run NB// these are |
︙ | ︙ | |||
276 277 278 279 280 281 282 283 284 285 286 287 288 289 | ":value" ":expected" ":tol" ":units" ;; misc "-start-dir" "-contour" "-server" "-transport" "-port" "-extract-ods" "-pathmod" "-env2file" "-envcap" | > | 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 | ":value" ":expected" ":tol" ":units" ;; misc "-start-dir" "-contour" "-area-tag" "-server" "-transport" "-port" "-extract-ods" "-pathmod" "-env2file" "-envcap" |
︙ | ︙ |
Modified mt-pg.sql from [c919648701] to [59ce0b208a].
︙ | ︙ | |||
39 40 41 42 43 44 45 46 47 48 49 50 51 52 | CREATE TABLE IF NOT EXISTS areas ( id SERIAL PRIMARY KEY, area_name TEXT NOT NULL, area_path TEXT NOT NULL, last_sync INTEGER DEFAULT 0, CONSTRAINT areaconstraint UNIQUE (area_name)); INSERT INTO areas (id,area_name,area_path) VALUES (0,'local','.'); CREATE TABLE IF NOT EXISTS ttype ( id SERIAL PRIMARY KEY, target_spec TEXT DEFAULT ''); CREATE TABLE IF NOT EXISTS runs ( | > > > > > > > > > > > | 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 | CREATE TABLE IF NOT EXISTS areas ( id SERIAL PRIMARY KEY, area_name TEXT NOT NULL, area_path TEXT NOT NULL, last_sync INTEGER DEFAULT 0, CONSTRAINT areaconstraint UNIQUE (area_name)); CREATE TABLE IF NOT EXISTS tags ( id SERIAL PRIMARY KEY, tag_name TEXT NOT NULL, CONSTRAINT tagconstraint UNIQUE (tag_name)); CREATE TABLE IF NOT EXISTS area_tags ( id SERIAL PRIMARY KEY, tag_id INTEGER DEFAULT 0, area_id INTEGER DEFAULT 0, CONSTRAINT areatagconstraint UNIQUE (tag_id, area_id)); INSERT INTO areas (id,area_name,area_path) VALUES (0,'local','.'); CREATE TABLE IF NOT EXISTS ttype ( id SERIAL PRIMARY KEY, target_spec TEXT DEFAULT ''); CREATE TABLE IF NOT EXISTS runs ( |
︙ | ︙ |
Modified tasks.scm from [4926b0b4a0] to [59fe011a46].
︙ | ︙ | |||
792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 | (print "Inserting test with run-id: " run-id " and test-id: " test-id) (pgdb:insert-test dbh pgdb-run-id test-name item-path state status host cpuload diskfree uname run-dir log-file run-duration comment event-time archived) (set! pgdb-test-id (pgdb:get-test-id dbh pgdb-run-id test-name item-path)))) (hash-table-set! test-ht test-id pgdb-test-id)) (print "WARNING: Skipping run with run-id:" run-id ". This run was created after privious sync and removed before this sync.")))) test-ids))) ;; get runs changed since last sync ;; (define (tasks:sync-test-data dbh cached-info area-info) ;; (let* (( (define (tasks:sync-to-postgres configdat dest) (let* ((dbh (pgdb:open configdat dbname: dest)) (area-info (pgdb:get-area-by-path dbh *toppath*)) (cached-info (make-hash-table)) (start (current-seconds))) (for-each (lambda (dtype) (hash-table-set! cached-info dtype (make-hash-table))) '(runs targets tests steps data)) (hash-table-set! cached-info 'start start) ;; when done we'll set sync times to this (if area-info (let* ((last-sync-time (vector-ref area-info 3)) (changed (rmt:get-changed-record-ids last-sync-time)) (run-ids (alist-ref 'runs changed)) (test-ids (alist-ref 'tests changed)) (test-step-ids (alist-ref 'test_steps changed)) (test-data-ids (alist-ref 'test_data changed)) | > > > > > > > > > > > > > > > > > > > > > | > > > | | > > > | 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 | (print "Inserting test with run-id: " run-id " and test-id: " test-id) (pgdb:insert-test dbh pgdb-run-id test-name item-path state status host cpuload diskfree uname run-dir log-file run-duration comment event-time archived) (set! pgdb-test-id (pgdb:get-test-id dbh pgdb-run-id test-name item-path)))) (hash-table-set! test-ht test-id pgdb-test-id)) (print "WARNING: Skipping run with run-id:" run-id ". This run was created after privious sync and removed before this sync.")))) test-ids))) (define (task:add-area-tag dbh area-info tag) (let* ((tag-info (pgdb:get-tag-info-by-name dbh tag))) (if (not tag-info) (begin (if (handle-exceptions exn (begin (print ((condition-property-accessor 'exn 'message) exn)) #f) (pgdb:insert-tag dbh tag)) (set! tag-info (pgdb:get-tag-info-by-name dbh tag)) #f))) ;;add to area_tags (handle-exceptions exn (begin (print ((condition-property-accessor 'exn 'message) exn)) #f) (if (not (pgdb:is-area-taged-with-a-tag dbh (vector-ref tag-info 0) (vector-ref area-info 0))) (pgdb:insert-area-tag dbh (vector-ref tag-info 0) (vector-ref area-info 0)))))) ;; get runs changed since last sync ;; (define (tasks:sync-test-data dbh cached-info area-info) ;; (let* (( (define (tasks:sync-to-postgres configdat dest) (let* ((dbh (pgdb:open configdat dbname: dest)) (area-info (pgdb:get-area-by-path dbh *toppath*)) (cached-info (make-hash-table)) (start (current-seconds))) (for-each (lambda (dtype) (hash-table-set! cached-info dtype (make-hash-table))) '(runs targets tests steps data)) (hash-table-set! cached-info 'start start) ;; when done we'll set sync times to this (if area-info (let* ((last-sync-time (vector-ref area-info 3)) (changed (rmt:get-changed-record-ids last-sync-time)) (run-ids (alist-ref 'runs changed)) (test-ids (alist-ref 'tests changed)) (test-step-ids (alist-ref 'test_steps changed)) (test-data-ids (alist-ref 'test_data changed)) (run-stat-ids (alist-ref 'run_stats changed)) (area-tag (if (args:get-arg "-area-tag") (args:get-arg "-area-tag") ""))) ; (print "area-info: " area-info) (if (and (equal? area-tag "") (not (pgdb:is-area-taged dbh (vector-ref area-info 0)))) (set! area-tag *default-area-tag*)) (if (not (equal? area-tag "")) (task:add-area-tag dbh area-info area-tag)) (if (not (null? test-ids)) (begin (print "Syncing " (length test-step-ids) " changed tests") ;;Assumption here is that if test-step or test data is changed then the test last update time is changed ;; not syncing run stats at this time as they can be derived from tests table. (tasks:sync-tests-data dbh cached-info test-ids area-info) ;(exit) |
︙ | ︙ |