Overview
Comment: | Misc. work on route to megatest.db -> mt pg db sync |
---|---|
Downloads: | Tarball | ZIP archive | SQL archive |
Timelines: | family | ancestors | descendants | both | v1.64 |
Files: | files | file ages | folders |
SHA1: |
6a8906cbd3c37bcaa09d929673462ca3 |
User & Date: | matt on 2017-02-26 10:54:42 |
Other Links: | branch diff | manifest | tags |
Context
2017-02-26
| ||
11:27 | Fixed schema issue. transfering/updating individual run entries done. check-in: 266b1a7dc4 user: matt tags: v1.64 | |
10:54 | Misc. work on route to megatest.db -> mt pg db sync check-in: 6a8906cbd3 user: matt tags: v1.64 | |
2017-02-25
| ||
23:18 | Added placeholder for pgdb sync check-in: 096366f99c user: matt tags: v1.64 | |
Changes
Modified common.scm from [316e289358] to [ad8f90ccc3].
︙ | ︙ | |||
547 548 549 550 551 552 553 | (if val val default))) (define (assoc/default key lst . default) (let ((res (assoc key lst))) (if res (cadr res)(if (null? default) #f (car default))))) (define (common:get-testsuite-name) | > | > > | 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 | (if val val default))) (define (assoc/default key lst . default) (let ((res (assoc key lst))) (if res (cadr res)(if (null? default) #f (car default))))) (define (common:get-testsuite-name) (or (configf:lookup *configdat* "setup" "area-name") ;; megatest is a flexible tool, testsuite is too limiting a description. (configf:lookup *configdat* "setup" "testsuite" ) (if *toppath* (pathname-file *toppath*) (pathname-file (current-directory))))) (define common:get-area-name common:get-testsuite-name) (define (common:get-db-tmp-area) (if *db-cache-path* *db-cache-path* (let ((dbpath (create-directory (conc "/tmp/" (current-user-name) "/megatest_localdb/" (common:get-testsuite-name) "/" |
︙ | ︙ |
Modified db.scm from [af28187c70] to [82052000b6].
︙ | ︙ | |||
2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 | (cons (list->vector r) res)) '() db qry-str runnamepatt))))))) ;; use (get-value-by-header (db:get-header runinfo)(db:get-rows runinfo)) (define (db:get-run-info dbstruct run-id) ;;(if (hash-table-ref/default *run-info-cache* run-id #f) ;; (hash-table-ref *run-info-cache* run-id) (let* ((res (vector #f #f #f #f)) (keys (db:get-keys dbstruct)) (remfields (list "id" "runname" "state" "status" "owner" "event_time")) (header (append keys remfields)) | > > > | 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 | (cons (list->vector r) res)) '() db qry-str runnamepatt))))))) ;; use (get-value-by-header (db:get-header runinfo)(db:get-rows runinfo)) ;; NOTE: Does NOT return a list of rows (or one row) for the first slot of the vector ;; this is inconsistent with get-runs but it makes some sense. ;; (define (db:get-run-info dbstruct run-id) ;;(if (hash-table-ref/default *run-info-cache* run-id #f) ;; (hash-table-ref *run-info-cache* run-id) (let* ((res (vector #f #f #f #f)) (keys (db:get-keys dbstruct)) (remfields (list "id" "runname" "state" "status" "owner" "event_time")) (header (append keys remfields)) |
︙ | ︙ |
Modified mt-pg.sql from [97be168469] to [e9ac1a40ea].
1 | -- CREATE TABLE IF NOT EXISTS keys ( | | | 1 2 3 4 5 6 7 8 9 | -- CREATE TABLE IF NOT EXISTS keys ( -- id SERIAL PRIMARY KEY, -- fieldname TEXT, -- fieldtype TEXT, -- CONSTRAINT keyconstraint UNIQUE (fieldname)); DROP TABLE IF EXISTS areas; DROP TABLE IF EXISTS ttype; DROP TABLE IF EXISTS runs; |
︙ | ︙ | |||
19 20 21 22 23 24 25 | DROP TABLE IF EXISTS tests; DROP TABLE IF EXISTS test_steps; DROP TABLE IF EXISTS test_data; DROP TABLE IF EXISTS test_rundat; DROP TABLE IF EXISTS archives; CREATE TABLE IF NOT EXISTS areas ( | | | | | | | | | | | | | | | | | 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 | DROP TABLE IF EXISTS tests; DROP TABLE IF EXISTS test_steps; DROP TABLE IF EXISTS test_data; DROP TABLE IF EXISTS test_rundat; DROP TABLE IF EXISTS archives; CREATE TABLE IF NOT EXISTS areas ( id SERIAL PRIMARY KEY, area_name TEXT NOT NULL, area_path TEXT NOT NULL, last_sync INTEGER DEFAULT 0, CONSTRAINT areaconstraint UNIQUE (area_name)); INSERT INTO areas (id,area_name,area_path) VALUES (0,'local','.'); CREATE TABLE IF NOT EXISTS ttype ( id SERIAL PRIMARY KEY, target_spec TEXT DEFAULT ''); CREATE TABLE IF NOT EXISTS runs ( id SERIAL PRIMARY KEY, target TEXT DEFAULT '', ttype_id INTEGER DEFAULT 0, runname TEXT DEFAULT 'norun', state TEXT DEFAULT '', status TEXT DEFAULT '', owner TEXT DEFAULT '', event_time INTEGER DEFAULT extract(epoch from now()), comment TEXT DEFAULT '', fail_count INTEGER DEFAULT 0, pass_count INTEGER DEFAULT 0, last_update INTEGER DEFAULT extract(epoch from now()), area_id INTEGER DEFAULT 0, CONSTRAINT runsconstraint UNIQUE (runname)); CREATE TABLE IF NOT EXISTS run_stats ( id SERIAL PRIMARY KEY, run_id INTEGER, state TEXT, status TEXT, count INTEGER, last_update INTEGER DEFAULT extract(epoch from now())); CREATE TABLE IF NOT EXISTS test_meta ( id SERIAL PRIMARY KEY, testname TEXT DEFAULT '', author TEXT DEFAULT '', owner TEXT DEFAULT '', description TEXT DEFAULT '', reviewed TEXT, iterated TEXT DEFAULT '', avg_runtime REAL, avg_disk REAL, tags TEXT DEFAULT '', jobgroup TEXT DEFAULT 'default', CONSTRAINT test_meta_constraint UNIQUE (testname)); CREATE TABLE IF NOT EXISTS tasks_queue ( id SERIAL PRIMARY KEY, action TEXT DEFAULT '', owner TEXT, state TEXT DEFAULT 'new', target TEXT DEFAULT '', name TEXT DEFAULT '', testpatt TEXT DEFAULT '', keylock TEXT, params TEXT, creation_time INTEGER DEFAULT extract(epoch from now()), execution_time INTEGER); CREATE TABLE IF NOT EXISTS archive_disks ( id SERIAL PRIMARY KEY, archive_area_name TEXT, disk_path TEXT, last_df INTEGER DEFAULT -1, last_df_time INTEGER DEFAULT extract(epoch from now()), creation_time INTEGER DEFAULT extract(epoch from now())); CREATE TABLE IF NOT EXISTS archive_blocks ( id SERIAL PRIMARY KEY, archive_disk_id INTEGER, disk_path TEXT, last_du INTEGER DEFAULT -1, last_du_time INTEGER DEFAULT extract(epoch from now()), creation_time INTEGER DEFAULT extract(epoch from now())); CREATE TABLE IF NOT EXISTS archive_allocations ( id SERIAL PRIMARY KEY, archive_block_id INTEGER, testname TEXT, item_path TEXT, creation_time INTEGER DEFAULT extract(epoch from now())); CREATE TABLE IF NOT EXISTS extradat ( id SERIAL PRIMARY KEY, run_id INTEGER, key TEXT, val TEXT); CREATE TABLE IF NOT EXISTS metadat ( id SERIAL PRIMARY KEY, var TEXT, val TEXT); CREATE TABLE IF NOT EXISTS access_log ( id SERIAL PRIMARY KEY, "user" TEXT, accessed TIMESTAMP, args TEXT); CREATE TABLE IF NOT EXISTS tests ( id SERIAL PRIMARY KEY, run_id INTEGER DEFAULT -1, testname TEXT DEFAULT 'noname', host TEXT DEFAULT 'n/a', cpuload REAL DEFAULT -1, diskfree INTEGER DEFAULT -1, uname TEXT DEFAULT 'n/a', rundir TEXT DEFAULT '/tmp/badname', |
︙ | ︙ | |||
148 149 150 151 152 153 154 | fail_count INTEGER DEFAULT 0, pass_count INTEGER DEFAULT 0, archived INTEGER DEFAULT 0, -- 0=no, > 1=archive block id where test data can be found last_update INTEGER DEFAULT extract(epoch from now()), CONSTRAINT testsconstraint UNIQUE (run_id, testname, item_path)); CREATE TABLE IF NOT EXISTS test_steps ( | | | | | | | | | 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 | fail_count INTEGER DEFAULT 0, pass_count INTEGER DEFAULT 0, archived INTEGER DEFAULT 0, -- 0=no, > 1=archive block id where test data can be found last_update INTEGER DEFAULT extract(epoch from now()), CONSTRAINT testsconstraint UNIQUE (run_id, testname, item_path)); CREATE TABLE IF NOT EXISTS test_steps ( id SERIAL PRIMARY KEY, test_id INTEGER, stepname TEXT, state TEXT DEFAULT 'NOT_STARTED', status TEXT DEFAULT 'n/a', event_time INTEGER DEFAULT extract(epoch from now()), comment TEXT DEFAULT '', logfile TEXT DEFAULT '', last_update INTEGER DEFAULT extract(epoch from now()), CONSTRAINT test_steps_constraint UNIQUE (test_id,stepname,state)); CREATE TABLE IF NOT EXISTS test_data ( id SERIAL PRIMARY KEY, test_id INTEGER, category TEXT DEFAULT '', variable TEXT, value REAL, expected REAL, tol REAL, units TEXT, comment TEXT DEFAULT '', status TEXT DEFAULT 'n/a', type TEXT DEFAULT '', last_update INTEGER DEFAULT extract(epoch from now()), CONSTRAINT test_data_constraint UNIQUE (test_id,category,variable)); CREATE TABLE IF NOT EXISTS test_rundat ( id SERIAL PRIMARY KEY, test_id INTEGER, update_time INTEGER, cpuload INTEGER DEFAULT -1, diskfree INTEGER DEFAULT -1, diskusage INTEGER DEFAULT -1, run_duration INTEGER DEFAULT 0); CREATE TABLE IF NOT EXISTS archives ( id SERIAL PRIMARY KEY, test_id INTEGER, state TEXT DEFAULT 'new', status TEXT DEFAULT 'n/a', archive_type TEXT DEFAULT 'bup', du INTEGER, archive_path TEXT); -- TRUNCATE archive_blocks, archive_allocations, extradat, metadat, -- access_log, tests, test_steps, test_data, test_rundat, archives, runs, -- run_stats, test_meta, tasks_queue, archive_disks; |
Modified tasks.scm from [b57542e266] to [de638fd380].
︙ | ︙ | |||
583 584 585 586 587 588 589 | ;;====================================================================== ;; S Y N C T O P O S T G R E S Q L ;;====================================================================== ;; In the spirit of "dump your junk in the tasks module" I'll put the ;; sync to postgres here for now. | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | | > > > > > > | > | > > > > > > > > | 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 | ;;====================================================================== ;; S Y N C T O P O S T G R E S Q L ;;====================================================================== ;; In the spirit of "dump your junk in the tasks module" I'll put the ;; sync to postgres here for now. ;; attempt to automatically set up an area. call only if get area by path ;; returns naught of interest ;; (define (tasks:set-area dbh configdat #!key (toppath #f)) ;; could I safely put *toppath* in for the default for toppath? when would it be evaluated? (let loop ((area-name (or (configf:lookup configdat "setup" "area-name") (common:get-area-name))) (modifier 'none)) (let ((success (handle-exceptions exn (begin (debug:print 0 *default-log-port* "ERROR: cannot create area entry, " ((condition-property-accessor 'exn 'message) exn)) #f) ;; FIXME: I don't care for now but I should look at *why* there was an exception (pgdb:add-area dbh area-name (or toppath *toppath*))))) (or success (case modifier ((none)(loop (conc (current-user-name) "_" area-name) 'user)) ((user)(loop (conc (substring (common:get-area-path-signature) 0 4) area-name) 'areasig)) (else #f)))))) ;; give up ;; gets mtpg-run-id and syncs the record if different ;; (define (tasks:run-id->mtpg-run-id dbh cached-info run-id) (let* ((runs-ht (hash-table-ref cached-info 'runs)) (runinf (hash-table-ref runs-ht run-id))) (if runinf runinf ;; already cached (let* ((keytarg (string-intersperse (rmt:get-keys) "/")) ;; e.g. version/iteration/platform (spec-id (pgdb:get-ttype dbh keytarg)) (target (rmt:get-target run-id)) ;; e.g. v1.63/a3e1/ubuntu (run-dat (rmt:get-run-info run-id)) ;; NOTE: get-run-info returns a vector < row header > (run-name (rmt:get-run-name-from-id run-id)) (new-run-id (pgdb:get-run-id dbh spec-id target run-name)) (row (db:get-rows run-dat)) ;; yes, this returns a single row (header (db:get-header run-dat)) (state (db:get-value-by-header rows header "state ")) (status (db:get-value-by-header row header "status")) (owner (db:get-value-by-header row header "owner")) (event-time (db:get-value-by-header row header "event_time")) (comment (db:get-value-by-header row header "comment")) (fail-count (db:get-value-by-header row header "fail_count")) (pass-count (db:get-value-by-header row header "pass_count")) (area-id (db:get-value-by-header row header "area_id)"))) (if new-run-id (begin ;; let ((run-record (pgdb:get-run-info dbh new-run-id)) (hash-table-set! runs-ht run-id new-run-id) ;; ensure key fields are up to date (pgdb:refresh-run-info dbh new-run-id state status owner event-time comment fail-count pass-count area-id)) (if (handle-exceptions exn (begin (print-call-chain) #f) (pgdb:insert-run dbh spec-id target state status owner event-time comment fail-count pass-count area-id)) (tasks:run-id->mtpg-run-id dbh cached-info run-id) #f)))))) ;;(define (tasks:sync-test-data dbh cached-info area-info) ;; (let* (( (define (tasks:sync-to-postgres configdat) (let* ((dbh (pgdb:open configdat)) (area-info (pgdb:get-area-by-path dbh *toppath*)) (cached-info (make-hash-table))) (for-each (lambda (dtype) (hash-table-set! cached-info dtype (make-hash-table))) '(runs targets tests)) (hash-table-set! cached-info 'start (current-seconds)) (if area-info (begin (print "area-info: " area-info) (tasks:sync-test-data dbh cached-info area-info) ) (if (tasks:set-area dbh configdat) (tasks:sync-to-postgres configdat) (begin (debug:print 0 *default-log-port* "ERROR: unable to create an area record") #f))))) |