Overview
| Comment: | Got pgdb sync working by querying for test related data in the correct dbs, passing the test ids, test_step ids and test_data ids as pairs with the run-id, and adding run-id to a few test data query functions. |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | v1.70 |
| Files: | files | file ages | folders |
| SHA1: |
6fb02466de173ac66531ec71657640cc |
| User & Date: | mmgraham on 2022-06-13 17:44:53 |
| Other Links: | branch diff | manifest | tags |
Context
|
2022-06-13
| ||
| 19:25 | Changed megatest version to v1.7004 check-in: ba39e7edd8 user: mmgraham tags: v1.70, v1.7004 | |
| 17:44 | Got pgdb sync working by querying for test related data in the correct dbs, passing the test ids, test_step ids and test_data ids as pairs with the run-id, and adding run-id to a few test data query functions. check-in: 6fb02466de user: mmgraham tags: v1.70 | |
|
2022-06-10
| ||
| 23:36 | Changed .db directories to .megatest check-in: be8e6a1b38 user: mmgraham tags: v1.70, v1.7003 | |
Changes
Modified db.scm from [624e4507df] to [8c84d35a2c].
| ︙ | ︙ | |||
3120 3121 3122 3123 3124 3125 3126 |
(lambda (id test-id stepname state status event-time logfile comment)
(set! res (cons (vector id test-id stepname state status event-time (if (string? logfile) logfile "") comment) res)))
db
"SELECT id,test_id,stepname,state,status,event_time,logfile,comment FROM test_steps WHERE status != 'DELETED' AND test_id=? ORDER BY id ASC;" ;; event_time DESC,id ASC;
test-id)
(reverse res)))))
| | < > | 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 |
(lambda (id test-id stepname state status event-time logfile comment)
(set! res (cons (vector id test-id stepname state status event-time (if (string? logfile) logfile "") comment) res)))
db
"SELECT id,test_id,stepname,state,status,event_time,logfile,comment FROM test_steps WHERE status != 'DELETED' AND test_id=? ORDER BY id ASC;" ;; event_time DESC,id ASC;
test-id)
(reverse res)))))
(define (db:get-steps-info-by-id dbstruct run-id test-step-id)
(db:with-db
dbstruct
run-id
#f
(lambda (dbdat db)
(let* ((res (vector #f #f #f #f #f #f #f #f #f)))
(sqlite3:for-each-row
(lambda (id test-id stepname state status event-time logfile comment last-update)
(set! res (vector id test-id stepname state status event-time (if (string? logfile) logfile "") comment last-update)))
db
|
| ︙ | ︙ | |||
3154 3155 3156 3157 3158 3159 3160 |
test-id)
(reverse res)))))
;;======================================================================
;; T E S T D A T A
;;======================================================================
| | < > | 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 |
test-id)
(reverse res)))))
;;======================================================================
;; T E S T D A T A
;;======================================================================
(define (db:get-data-info-by-id dbstruct run-id test-data-id)
(let* ((stmt "SELECT id,test_id, category, variable, value, expected, tol, units, comment, status, type, last_update FROM test_data WHERE id=? ORDER BY id ASC;")) ;; event_time DESC,id ASC;
(db:with-db
dbstruct
run-id
#f
(lambda (dbdat db)
(let* ((stmth (db:get-cache-stmth dbdat #f db stmt))
(res (sqlite3:fold-row
(lambda (res id test-id category variable value expected tol units comment status type last-update)
(vector id test-id category variable value expected tol units comment status type last-update))
(vector #f #f #f #f #f #f #f #f #f #f #f #f)
|
| ︙ | ︙ | |||
4372 4373 4374 4375 4376 4377 4378 |
(test_data . ,(sqlite3:fold-row backcons '() db (conc "SELECT id FROM test_data WHERE test_id in (" test-qry ")" )))
))))))
;;======================================================================
;; Just for sync, procedures to make sync easy
;;======================================================================
| | | | > > > > > | > > > > > | | | > > > > > > > > > > | > > > > > > > > > > | > > > > > > > > > > | > > > > > > > > > > | > > > > | < > > > | 4372 4373 4374 4375 4376 4377 4378 4379 4380 4381 4382 4383 4384 4385 4386 4387 4388 4389 4390 4391 4392 4393 4394 4395 4396 4397 4398 4399 4400 4401 4402 4403 4404 4405 4406 4407 4408 4409 4410 4411 4412 4413 4414 4415 4416 4417 4418 4419 4420 4421 4422 4423 4424 4425 4426 4427 4428 4429 4430 4431 4432 4433 4434 4435 4436 4437 4438 4439 4440 4441 4442 4443 4444 4445 4446 4447 4448 4449 4450 4451 4452 4453 4454 4455 4456 4457 |
(test_data . ,(sqlite3:fold-row backcons '() db (conc "SELECT id FROM test_data WHERE test_id in (" test-qry ")" )))
))))))
;;======================================================================
;; Just for sync, procedures to make sync easy
;;======================================================================
;; get an alist of run ids and test/run, test_step/run pairs changed since time since-time
;; '((runs . (1 2 3 ...))(tests . ((5 . 1) (6 . 3) (6 . 2) (7 . 1) ...
;;
(define (db:get-changed-record-ids dbstruct since-time)
;; no transaction, allow the db to be accessed between the big queries
(let* ((backcons (lambda (lst item)(cons item lst)))
(all_tests '())
(all_test_steps '())
(all_test_data '())
(run_ids
(db:with-db dbstruct #f #f
(lambda (dbdat db)
(sqlite3:fold-row backcons '() db "SELECT id FROM runs WHERE last_update>=?" since-time))
)
)
(run_stat_ids
(db:with-db dbstruct #f #f
(lambda (dbdat db)
(sqlite3:fold-row backcons '() db "SELECT id FROM run_stats WHERE last_update>=?" since-time))
)
)
)
(for-each
(lambda (run_id)
(set! all_tests
(append
(map (lambda (x) (cons x run_id))
(db:with-db dbstruct run_id #f
(lambda (dbdat db)
(sqlite3:fold-row backcons '() db "SELECT id FROM tests WHERE last_update>=?" since-time)
)
)
) all_tests
)
)
(set! all_test_steps
(append
(map (lambda (x) (cons x run_id))
(db:with-db dbstruct run_id #f
(lambda (dbdat db)
(sqlite3:fold-row backcons '() db "SELECT id FROM test_steps WHERE last_update>=?" since-time)
)
)
) all_test_steps
)
)
(set! all_test_data
(append
(map (lambda (x) (cons x run_id))
(db:with-db dbstruct run_id #f
(lambda (dbdat db)
(sqlite3:fold-row backcons '() db "SELECT id FROM test_data WHERE last_update>=?" since-time)
)
)
) all_test_data
)
)
)
run_ids
)
(debug:print 2 *default-log-port* "run_ids = " run_ids)
(debug:print 2 *default-log-port* "all_tests = " all_tests)
`((runs . ,run_ids)
(tests . ,all_tests)
(test_steps . ,all_test_steps)
(test_data . ,all_test_data)
(run_stats . ,run_stat_ids)
)
)
)
;;======================================================================
;; Extract ods file from the db
;;======================================================================
;; NOT REWRITTEN YET!!!!!
|
| ︙ | ︙ |
Modified rmt.scm from [fa25f8e8fb] to [3a21be5e2c].
| ︙ | ︙ | |||
889 890 891 892 893 894 895 | (define (rmt:delete-steps-for-test! run-id test-id) (rmt:send-receive 'delete-steps-for-test! run-id (list run-id test-id))) (define (rmt:get-steps-for-test run-id test-id) (rmt:send-receive 'get-steps-for-test run-id (list run-id test-id))) | | | | | | 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 | (define (rmt:delete-steps-for-test! run-id test-id) (rmt:send-receive 'delete-steps-for-test! run-id (list run-id test-id))) (define (rmt:get-steps-for-test run-id test-id) (rmt:send-receive 'get-steps-for-test run-id (list run-id test-id))) (define (rmt:get-steps-info-by-id run-id test-step-id) (rmt:send-receive 'get-steps-info-by-id #f (list run-id test-step-id))) ;;====================================================================== ;; T E S T D A T A ;;====================================================================== (define (rmt:read-test-data run-id test-id categorypatt #!key (work-area #f)) (rmt:send-receive 'read-test-data run-id (list run-id test-id categorypatt))) (define (rmt:read-test-data-varpatt run-id test-id categorypatt varpatt #!key (work-area #f)) (rmt:send-receive 'read-test-data-varpatt run-id (list run-id test-id categorypatt varpatt))) (define (rmt:get-data-info-by-id run-id test-data-id) (rmt:send-receive 'get-data-info-by-id #f (list run-id test-data-id))) (define (rmt:testmeta-add-record testname) (rmt:send-receive 'testmeta-add-record #f (list testname))) (define (rmt:testmeta-get-record testname) (rmt:send-receive 'testmeta-get-record #f (list testname))) |
| ︙ | ︙ |
Modified tasks.scm from [fd2a2e05e7] to [1f7b346573].
| ︙ | ︙ | |||
836 837 838 839 840 841 842 |
(if (not (pgdb:is-run-taged-with-a-tag dbh (vector-ref tag-info 0) run-id))
(pgdb:insert-run-tag dbh (vector-ref tag-info 0) run-id)))))
(define (tasks:sync-test-steps dbh cached-info test-step-ids smallest-last-update-time)
; (print "Sync Steps " test-step-ids )
(let ((test-ht (hash-table-ref cached-info 'tests))
| | > > > > > > | | 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 |
(if (not (pgdb:is-run-taged-with-a-tag dbh (vector-ref tag-info 0) run-id))
(pgdb:insert-run-tag dbh (vector-ref tag-info 0) run-id)))))
(define (tasks:sync-test-steps dbh cached-info test-step-ids smallest-last-update-time)
; (print "Sync Steps " test-step-ids )
(let ((test-ht (hash-table-ref cached-info 'tests))
(step-ht (hash-table-ref cached-info 'steps))
(run-id-in #f)
)
(for-each
(lambda (test-step-id)
(set! run-id-in (cdr test-step-id))
(set! test-step-id (car test-step-id))
(let* ((test-step-info (rmt:get-steps-info-by-id run-id-in test-step-id))
(step-id (tdb:step-get-id test-step-info))
(test-id (tdb:step-get-test_id test-step-info))
(stepname (tdb:step-get-stepname test-step-info))
(state (tdb:step-get-state test-step-info))
(status (tdb:step-get-status test-step-info))
(event_time (tdb:step-get-event_time test-step-info))
(comment (tdb:step-get-comment test-step-info))
|
| ︙ | ︙ | |||
937 938 939 940 941 942 943 |
(debug:print-info 1 *default-log-port* "Error: Could not get test data info for data id " test-data-id )))) ;; this is a wierd senario need to debug
test-data-ids)))
(define (tasks:sync-tests-data dbh cached-info test-ids area-info smallest-last-update-time)
| | > > | > > | | 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 |
(debug:print-info 1 *default-log-port* "Error: Could not get test data info for data id " test-data-id )))) ;; this is a wierd senario need to debug
test-data-ids)))
(define (tasks:sync-tests-data dbh cached-info test-ids area-info smallest-last-update-time)
(let ((test-ht (hash-table-ref cached-info 'tests))
(run-id-in #f))
(for-each
(lambda (test-id)
(set! run-id-in (cdr test-id))
(set! test-id (car test-id))
(debug:print 0 *default-log-port* "test-id: " test-id " run-id: " run-id-in)
(let* ((test-info (rmt:get-test-info-by-id run-id-in test-id))
(run-id (db:test-get-run_id test-info)) ;; look these up in db_records.scm
(test-id (db:test-get-id test-info))
(test-name (db:test-get-testname test-info))
(item-path (db:test-get-item-path test-info))
(state (db:test-get-state test-info))
(status (db:test-get-status test-info))
(host (db:test-get-host test-info))
|
| ︙ | ︙ | |||
971 972 973 974 975 976 977 |
;(print pgdb-run-id)
(pgdb:get-test-id dbh pgdb-run-id test-name item-path))
#f)))
;; "id" "run_id" "testname" "state" "status" "event_time"
;; "host" "cpuload" "diskfree" "uname" "rundir" "item_path"
;; "run_duration" "final_logf" "comment" "shortdir" "attemptnum" "archived"
(if (or (not item-path) (string-null? item-path))
| | | 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 |
;(print pgdb-run-id)
(pgdb:get-test-id dbh pgdb-run-id test-name item-path))
#f)))
;; "id" "run_id" "testname" "state" "status" "event_time"
;; "host" "cpuload" "diskfree" "uname" "rundir" "item_path"
;; "run_duration" "final_logf" "comment" "shortdir" "attemptnum" "archived"
(if (or (not item-path) (string-null? item-path))
(debug:print-info 0 *default-log-port* "Working on Run id : " run-id " and test name : " test-name))
(if pgdb-run-id
(begin
(if pgdb-test-id ;; have a record
(begin ;; let ((key-name (conc run-id "/" test-name "/" item-path)))
(debug:print-info 4 *default-log-port* "Updating existing test with run-id: " run-id " and test-id: " test-id " pgdb run id: " pgdb-run-id " pgdb-test-id " pgdb-test-id)
(let* ((pgdb-last-update (pgdb:get-test-last-update dbh pgdb-test-id)))
(if (and (> last-update pgdb-last-update) (or (not smallest-time) (< last-update smallest-time))) ;;if last-update is same as pgdb-last-update then it is safe to assume the records are identical and we can use a larger last update time.
|
| ︙ | ︙ | |||
1073 1074 1075 1076 1077 1078 1079 |
(if (args:get-arg "-area")
(args:get-arg "-area")
""))))
(if (and (equal? area-tag "") (not (pgdb:is-area-taged dbh (vector-ref area-info 0))))
(set! area-tag *default-area-tag*))
(if (not (equal? area-tag ""))
(task:add-area-tag dbh area-info area-tag))
| | | | | > > > > | | | | | | < > > | 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 |
(if (args:get-arg "-area")
(args:get-arg "-area")
""))))
(if (and (equal? area-tag "") (not (pgdb:is-area-taged dbh (vector-ref area-info 0))))
(set! area-tag *default-area-tag*))
(if (not (equal? area-tag ""))
(task:add-area-tag dbh area-info area-tag))
(if (not (null? run-ids))
(begin
(debug:print-info 0 *default-log-port* "syncing runs: " run-ids)
(tasks:sync-run-data dbh cached-info run-ids area-info smallest-last-update-time)
)
)
(if (not (null? test-ids))
(begin
(debug:print-info 0 *default-log-port* "syncing tests: " test-ids)
(tasks:sync-tests-data dbh cached-info test-ids area-info smallest-last-update-time)
(debug:print-info 0 *default-log-port* "syncing test steps")
(tasks:sync-test-steps dbh cached-info test-step-ids smallest-last-update-time)
(debug:print-info 0 *default-log-port* "syncing test data")
(tasks:sync-test-gen-data dbh cached-info test-data-ids smallest-last-update-time)
)
)
(let* ((smallest-time (hash-table-ref/default smallest-last-update-time "smallest-time" (current-seconds))))
(debug:print-info 0 "smallest-time :" smallest-time " last-sync-time " last-sync-time)
(if (not (and target run-name))
(if (or (and smallest-time (> smallest-time last-sync-time)) (and smallest-time (eq? last-sync-time 0)))
(pgdb:write-sync-time dbh area-info smallest-time))))) ;;this needs to be changed
(if (tasks:set-area dbh configdat)
(tasks:sync-to-postgres configdat dest)
(begin
(debug:print 0 *default-log-port* "ERROR: unable to create an area record")
#f)))))
|