Megatest

Check-in [6a8906cbd3]
Login
☰Home Timeline Files Branches Tags Tickets Wiki Login
Overview
Comment:Misc. work on route to megatest.db -> mt pg db sync
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | v1.64
Files: files | file ages | folders
SHA1: 6a8906cbd3c37bcaa09d929673462ca38feea198
User & Date: matt on 2017-02-26 10:54:42
Other Links: branch diff | manifest | tags
Context
2017-02-26
11:27
Fixed schema issue. transfering/updating individual run entries done. check-in: 266b1a7dc4 user: matt tags: v1.64
10:54
Misc. work on route to megatest.db -> mt pg db sync check-in: 6a8906cbd3 user: matt tags: v1.64
2017-02-25
23:18
Added placeholder for pgdb sync check-in: 096366f99c user: matt tags: v1.64
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Modified common.scm from [316e289358] to [ad8f90ccc3].

︙
547
548
549
550
551
552
553

554

555
556
557


558
559
560
561
562
563
564

547
548
549
550
551
552
553
554

555
556
557
558
559
560
561
562
563
564
565
566
567








+
-
+



+
+








    (if val val default)))

(define (assoc/default key lst . default)
  (let ((res (assoc key lst)))
    (if res (cadr res)(if (null? default) #f (car default)))))

(define (common:get-testsuite-name)
  (or (configf:lookup *configdat* "setup" "area-name") ;; megatest is a flexible tool, testsuite is too limiting a description.
  (or (configf:lookup *configdat* "setup" "testsuite" )
      (configf:lookup *configdat* "setup" "testsuite" )
      (if *toppath* 
          (pathname-file *toppath*)
          (pathname-file (current-directory)))))

(define common:get-area-name common:get-testsuite-name)

(define (common:get-db-tmp-area)
  (if *db-cache-path*
      *db-cache-path*
      (let ((dbpath (create-directory (conc "/tmp/" (current-user-name)
					    "/megatest_localdb/"
					    (common:get-testsuite-name) "/"
︙

Modified db.scm from [af28187c70] to [82052000b6].

︙
2193
2194
2195
2196
2197
2198
2199



2200
2201
2202
2203
2204
2205
2206

2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209








+
+
+








                              (cons (list->vector r) res))
                            '()
                            db
                            qry-str
                            runnamepatt)))))))

;; use (get-value-by-header (db:get-header runinfo)(db:get-rows runinfo))
;; NOTE: Does NOT return a list of rows (or one row) for the first slot of the vector
;;       this is inconsistent with get-runs but it makes some sense.
;;
(define (db:get-run-info dbstruct run-id)
  ;;(if (hash-table-ref/default *run-info-cache* run-id #f)
  ;;    (hash-table-ref *run-info-cache* run-id)
  (let* ((res       (vector #f #f #f #f))
	 (keys      (db:get-keys dbstruct))
	 (remfields (list "id" "runname" "state" "status" "owner" "event_time"))
	 (header    (append keys remfields))
︙

Modified mt-pg.sql from [97be168469] to [e9ac1a40ea]. 

1
2

3
4
5
6
7
8
9

1

2
3
4
5
6
7
8
9


-
+








-- CREATE TABLE IF NOT EXISTS keys (
--        id INTEGER PRIMARY KEY,
--        id SERIAL PRIMARY KEY,
--        fieldname TEXT,
--        fieldtype TEXT,
--        CONSTRAINT keyconstraint UNIQUE (fieldname));

DROP TABLE IF EXISTS areas;
DROP TABLE IF EXISTS ttype;
DROP TABLE IF EXISTS runs;
︙
19
20
21
22
23
24
25
26
27
28



29
30
31
32
33
34
35

36
37
38
39

40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55

56
57
58
59
60
61
62
63

64
65
66
67
68
69
70
71
72
73
74
75
76
77

78
79
80
81
82
83
84
85
86
87
88
89
90

91
92
93
94
95
96
97
98

99
100
101
102
103
104
105
106

107
108
109
110
111
112
113

114
115
116
117
118
119

120
121
122
123
124

125
126
127
128
129
130

131
132
133
134
135
136
137

19
20
21
22
23
24
25



26
27
28
29
30
31
32
33
34

35
36
37
38

39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54

55
56
57
58
59
60
61
62

63
64
65
66
67
68
69
70
71
72
73
74
75
76

77
78
79
80
81
82
83
84
85
86
87
88
89

90
91
92
93
94
95
96
97

98
99
100
101
102
103
104
105

106
107
108
109
110
111
112

113
114
115
116
117
118

119
120
121
122
123

124
125
126
127
128
129

130
131
132
133
134
135
136
137








-
-
-
+
+
+






-
+



-
+















-
+







-
+













-
+












-
+







-
+







-
+






-
+





-
+




-
+





-
+








DROP TABLE IF EXISTS tests;
DROP TABLE IF EXISTS test_steps;
DROP TABLE IF EXISTS test_data;
DROP TABLE IF EXISTS test_rundat;
DROP TABLE IF EXISTS archives;

CREATE TABLE IF NOT EXISTS areas (
       id INTEGER PRIMARY KEY,
       area_name TEXT DEFAULT 'local',
       area_path TEXT DEFAULT '.',
       id SERIAL PRIMARY KEY,
       area_name TEXT NOT NULL,
       area_path TEXT NOT NULL,
       last_sync INTEGER DEFAULT 0,
       CONSTRAINT areaconstraint UNIQUE (area_name));

INSERT INTO areas (id,area_name,area_path) VALUES (0,'local','.');

CREATE TABLE IF NOT EXISTS ttype (
       id INTEGER PRIMARY KEY,
       id SERIAL PRIMARY KEY,
       target_spec TEXT DEFAULT '');
       
CREATE TABLE IF NOT EXISTS runs (
       id INTEGER PRIMARY KEY,
       id SERIAL PRIMARY KEY,
       target     TEXT DEFAULT '',
       ttype_id   INTEGER DEFAULT 0,
       runname    TEXT DEFAULT 'norun',
       state      TEXT DEFAULT '',
       status     TEXT DEFAULT '',
       owner      TEXT DEFAULT '',
       event_time INTEGER DEFAULT extract(epoch from now()),
       comment    TEXT DEFAULT '',
       fail_count INTEGER DEFAULT 0,
       pass_count INTEGER DEFAULT 0,
       last_update INTEGER DEFAULT extract(epoch from now()),
       area_id     INTEGER DEFAULT 0,
       CONSTRAINT runsconstraint UNIQUE (runname));

CREATE TABLE IF NOT EXISTS run_stats (
       id     INTEGER PRIMARY KEY,
       id     SERIAL PRIMARY KEY,
       run_id INTEGER,
       state  TEXT,
       status TEXT,
       count  INTEGER,
       last_update INTEGER DEFAULT extract(epoch from now()));

CREATE TABLE IF NOT EXISTS test_meta (
       id          INTEGER PRIMARY KEY,
       id          SERIAL PRIMARY KEY,
       testname    TEXT DEFAULT '',
       author      TEXT DEFAULT '',
       owner       TEXT DEFAULT '',
       description TEXT DEFAULT '',
       reviewed    TEXT,
       iterated    TEXT DEFAULT '',
       avg_runtime REAL,
       avg_disk    REAL,
       tags        TEXT DEFAULT '',
       jobgroup    TEXT DEFAULT 'default',
       CONSTRAINT test_meta_constraint UNIQUE (testname));

CREATE TABLE IF NOT EXISTS tasks_queue (
       id INTEGER PRIMARY KEY,
       id SERIAL PRIMARY KEY,
       action TEXT DEFAULT '',
       owner TEXT,
       state TEXT DEFAULT 'new',
       target TEXT DEFAULT '',
       name TEXT DEFAULT '',
       testpatt TEXT DEFAULT '',
       keylock TEXT,
       params TEXT,
       creation_time INTEGER DEFAULT extract(epoch from now()),
       execution_time INTEGER);

CREATE TABLE IF NOT EXISTS archive_disks (
       id INTEGER PRIMARY KEY,
       id SERIAL PRIMARY KEY,
       archive_area_name TEXT,
       disk_path TEXT,
       last_df INTEGER DEFAULT -1,
       last_df_time INTEGER DEFAULT extract(epoch from now()),
       creation_time INTEGER DEFAULT extract(epoch from now()));

CREATE TABLE IF NOT EXISTS archive_blocks (
       id INTEGER PRIMARY KEY,
       id SERIAL PRIMARY KEY,
       archive_disk_id INTEGER,
       disk_path TEXT,
       last_du INTEGER DEFAULT -1,
       last_du_time INTEGER DEFAULT extract(epoch from now()),
       creation_time INTEGER DEFAULT extract(epoch from now()));

CREATE TABLE IF NOT EXISTS archive_allocations (
       id INTEGER PRIMARY KEY,
       id SERIAL PRIMARY KEY,
       archive_block_id INTEGER,
       testname TEXT,
       item_path TEXT,
       creation_time INTEGER DEFAULT extract(epoch from now()));

CREATE TABLE IF NOT EXISTS extradat (
       id INTEGER PRIMARY KEY,
       id SERIAL PRIMARY KEY,
       run_id INTEGER,
       key TEXT,
       val TEXT);

CREATE TABLE IF NOT EXISTS metadat (
       id INTEGER PRIMARY KEY,
       id SERIAL PRIMARY KEY,
       var TEXT,
       val TEXT);

CREATE TABLE IF NOT EXISTS access_log (
       id INTEGER PRIMARY KEY,
       id SERIAL PRIMARY KEY,
       "user" TEXT,
       accessed TIMESTAMP,
       args TEXT);

CREATE TABLE IF NOT EXISTS tests  (
       id INTEGER PRIMARY KEY,                                                                            
       id SERIAL PRIMARY KEY,                                                                            
       run_id       INTEGER   DEFAULT -1,                                                                 
       testname     TEXT      DEFAULT 'noname',                                                           
       host         TEXT      DEFAULT 'n/a',                                                              
       cpuload      REAL      DEFAULT -1,                                                                 
       diskfree     INTEGER   DEFAULT -1,                                                                 
       uname        TEXT      DEFAULT 'n/a',                                                              
       rundir       TEXT      DEFAULT '/tmp/badname',
︙
148
149
150
151
152
153
154
155

156
157
158
159
160
161
162
163
164
165
166
167

168
169
170
171
172
173
174
175
176
177
178
179
180
181
182

183
184
185
186
187
188
189
190
191

192
193
194
195
196
197
198
199
200
201
202




148
149
150
151
152
153
154

155
156
157
158
159
160
161
162
163
164
165
166

167
168
169
170
171
172
173
174
175
176
177
178
179
180
181

182
183
184
185
186
187
188
189
190

191
192
193
194
195
196
197
198
199



200
201
202








-
+











-
+














-
+








-
+








-
-
-
+
+
+

       fail_count   INTEGER   DEFAULT 0,                                                                  
       pass_count   INTEGER   DEFAULT 0,                                                                  
       archived     INTEGER   DEFAULT 0, -- 0=no, > 1=archive block id where test data can be found       
       last_update  INTEGER DEFAULT extract(epoch from now()),                                               
       CONSTRAINT testsconstraint UNIQUE (run_id, testname, item_path));  

CREATE TABLE IF NOT EXISTS test_steps (
       id INTEGER PRIMARY KEY,
       id SERIAL PRIMARY KEY,
       test_id INTEGER, 
       stepname TEXT, 
       state TEXT DEFAULT 'NOT_STARTED', 
       status TEXT DEFAULT 'n/a',
       event_time INTEGER DEFAULT extract(epoch from now()),
       comment TEXT DEFAULT '',
       logfile TEXT DEFAULT '',
       last_update  INTEGER DEFAULT extract(epoch from now()),
       CONSTRAINT test_steps_constraint UNIQUE (test_id,stepname,state));

CREATE TABLE IF NOT EXISTS test_data (
       id INTEGER PRIMARY KEY,
       id SERIAL PRIMARY KEY,
       test_id INTEGER,
       category TEXT DEFAULT '',
       variable TEXT,
       value REAL,
       expected REAL,
       tol REAL,
       units TEXT,
       comment TEXT DEFAULT '',
       status TEXT DEFAULT 'n/a',
       type TEXT DEFAULT '',
       last_update  INTEGER DEFAULT extract(epoch from now()),
       CONSTRAINT test_data_constraint UNIQUE (test_id,category,variable));

CREATE TABLE IF NOT EXISTS test_rundat (
       id           INTEGER PRIMARY KEY,
       id           SERIAL PRIMARY KEY,
       test_id      INTEGER,
       update_time  INTEGER,
       cpuload      INTEGER DEFAULT -1,
       diskfree     INTEGER DEFAULT -1,
       diskusage    INTEGER DEFAULT -1,
       run_duration INTEGER DEFAULT 0);

CREATE TABLE IF NOT EXISTS archives (
       id           INTEGER PRIMARY KEY,
       id           SERIAL PRIMARY KEY,
       test_id      INTEGER,
       state        TEXT DEFAULT 'new',
       status       TEXT DEFAULT 'n/a',
       archive_type TEXT DEFAULT 'bup',
       du           INTEGER,
       archive_path TEXT);
 

TRUNCATE archive_blocks, archive_allocations, extradat, metadat,
access_log, tests, test_steps, test_data, test_rundat, archives, runs,
run_stats, test_meta, tasks_queue, archive_disks;
-- TRUNCATE archive_blocks, archive_allocations, extradat, metadat,
-- access_log, tests, test_steps, test_data, test_rundat, archives, runs,
-- run_stats, test_meta, tasks_queue, archive_disks;

Modified tasks.scm from [b57542e266] to [de638fd380].

︙
583
584
585
586
587
588
589

































































590
591
592
593




















583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654




655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673








+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

;;======================================================================
;;  S Y N C   T O   P O S T G R E S Q L
;;======================================================================

;; In the spirit of "dump your junk in the tasks module" I'll put the
;; sync to postgres here for now.

;; attempt to automatically set up an area. call only if get area by path
;; returns naught of interest
;;
(define (tasks:set-area dbh configdat #!key (toppath #f)) ;; could I safely put *toppath* in for the default for toppath? when would it be evaluated?
  (let loop ((area-name (or (configf:lookup configdat "setup" "area-name")
			    (common:get-area-name)))
	     (modifier  'none))
    (let ((success (handle-exceptions
		       exn
		       (begin
			 (debug:print 0 *default-log-port* "ERROR: cannot create area entry, " ((condition-property-accessor 'exn 'message) exn))
			 #f) ;; FIXME: I don't care for now but I should look at *why* there was an exception
		     (pgdb:add-area dbh area-name (or toppath *toppath*)))))
      (or success
	  (case modifier
	    ((none)(loop (conc (current-user-name) "_" area-name) 'user))
	    ((user)(loop (conc (substring (common:get-area-path-signature) 0 4)
			       area-name) 'areasig))
	    (else #f)))))) ;; give up

;; gets mtpg-run-id and syncs the record if different
;;
(define (tasks:run-id->mtpg-run-id dbh cached-info run-id)
  (let* ((runs-ht (hash-table-ref cached-info 'runs))
	 (runinf  (hash-table-ref runs-ht run-id)))
    (if runinf
	runinf ;; already cached
	(let* ((keytarg    (string-intersperse (rmt:get-keys) "/")) ;; e.g. version/iteration/platform
	       (spec-id    (pgdb:get-ttype dbh keytarg))
	       (target     (rmt:get-target run-id))                 ;; e.g. v1.63/a3e1/ubuntu
	       (run-dat    (rmt:get-run-info run-id))               ;; NOTE: get-run-info returns a vector < row header >
	       (run-name   (rmt:get-run-name-from-id run-id))
	       (new-run-id (pgdb:get-run-id dbh spec-id target run-name))
	       (row        (db:get-rows run-dat))                   ;; yes, this returns a single row
	       (header     (db:get-header run-dat))
	       (state      (db:get-value-by-header rows header "state "))
	       (status     (db:get-value-by-header row header "status"))
	       (owner      (db:get-value-by-header row header "owner"))
	       (event-time (db:get-value-by-header row header "event_time"))
	       (comment    (db:get-value-by-header row header "comment"))
	       (fail-count (db:get-value-by-header row header "fail_count"))
	       (pass-count (db:get-value-by-header row header "pass_count"))
	       (area-id    (db:get-value-by-header row header "area_id)")))
	  (if new-run-id
	      (begin ;; let ((run-record (pgdb:get-run-info dbh new-run-id))
		(hash-table-set! runs-ht run-id new-run-id)
		;; ensure key fields are up to date
		(pgdb:refresh-run-info
		 dbh
		 new-run-id
		 state status owner event-time comment fail-count pass-count area-id))
	      (if (handle-exceptions
		      exn
		      (begin (print-call-chain) #f)
		    (pgdb:insert-run
		     dbh
		     spec-id target state status owner event-time comment fail-count pass-count area-id))
		  (tasks:run-id->mtpg-run-id dbh cached-info run-id)
		  #f))))))
		
		 
	       
  ;;(define (tasks:sync-test-data dbh cached-info area-info)
  ;; (let* ((

(define (tasks:sync-to-postgres)
  (let* ((dbh (pgdb:open *configdat*))
	 (area-info (pgdb:area-path->area-info dbh *toppath*)))
    (print "area-info: " area-info)))
(define (tasks:sync-to-postgres configdat)
  (let* ((dbh         (pgdb:open configdat))
	 (area-info   (pgdb:get-area-by-path dbh *toppath*))
	 (cached-info (make-hash-table)))
    (for-each (lambda (dtype)
		(hash-table-set! cached-info dtype (make-hash-table)))
	      '(runs targets tests))
    (hash-table-set! cached-info 'start (current-seconds))
    (if area-info
	(begin
	  (print "area-info: " area-info)
	  (tasks:sync-test-data dbh cached-info area-info)
	  )
	(if (tasks:set-area dbh configdat)
	    (tasks:sync-to-postgres configdat)
	    (begin
	      (debug:print 0 *default-log-port* "ERROR: unable to create an area record")
	      #f)))))

This page was generated in about 0.13s by Fossil 2.24 [452f8e1ffc] 2024-03-26 15:36:00