Megatest

Check-in [5de2fd17b5]
Login
☰Home Timeline Files Branches Tags Tickets Wiki Login
Overview
Comment:Get data from archive partially working
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | v1.65-broken
Files: files | file ages | folders
SHA1: 5de2fd17b5348d5283f874e7ec9bf2c53cad12a0
User & Date: mrwellan on 2020-04-21 04:13:36
Other Links: branch diff | manifest | tags
Context
2020-04-27
17:46
Archive get now works well check-in: 0645166f9c user: mrwellan tags: v1.65-broken
2020-04-21
04:13
Get data from archive partially working check-in: 5de2fd17b5 user: mrwellan tags: v1.65-broken
2020-04-20
23:38
Fixed typo check-in: 2a858c5054 user: mrwellan tags: v1.65-broken
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Modified archive.scm from [618f9a591e] to [bb0fd0a689].

︙
384
385
386
387
388
389
390
391















































384
385
386
387
388
389
390

391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436








-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

		 (debug:print-info 0 *default-log-port* "Restoring archived data to " new-test-physical-path " from archive in " archive-path " ... " archive-internal-path)
		 ;; (mutex-lock! bup-mutex)
		 (run-n-wait bup-exe params: bup-restore-params print-cmd: #f)
		 ;; (mutex-unlock! bup-mutex)
		 (mt:test-set-state-status-by-id run-id test-id "COMPLETED" #f #f)))
	     (debug:print-error 0 *default-log-port* "No archive path in the record for run-id=" run-id " test-id=" test-id))))
     (filter vector? tests))))
	 

;; from an archive get a specific path - works ONLY with bup for now
;;
(define (archive:bup-get-data archive-command run-id run-name tests rp-mutex bup-mutex)
  (if (null? tests)
      (debug:print-info 0 *default-log-port* "get-data called with no matching tests to operate on.")
      (let* ((bup-exe      (or (configf:lookup *configdat* "archive" "bup") "bup"))
	     (linktree     (common:get-linktree)) ;; (configf:lookup *configdat* "setup" "linktree")))
	     (test-dat     (car tests))
	     (destpath     (args:get-arg "-dest")))

	;; When restoring test-dat will initially contain an old and invalid path to the test
	(let* ((item-path         (db:test-get-item-path test-dat))
	       (test-name         (db:test-get-testname  test-dat))
	       (test-id           (db:test-get-id        test-dat))
	       (run-id            (db:test-get-run_id    test-dat))
	       (keyvals           (rmt:get-key-val-pairs run-id))
	       (target            (string-intersperse (map cadr keyvals) "/"))
	       
	       (toplevel/children (and (db:test-get-is-toplevel test-dat)
				       (> (rmt:test-toplevel-num-items run-id test-name) 0)))
	       (test-partial-path (conc target "/" run-name "/" (db:test-make-full-name test-name item-path)))
	       ;; note the trailing slash to get the dir inspite of it being a link
	       (test-path         (conc linktree "/" test-partial-path))
	       (mutex-lock! rp-mutex)
	       (mutex-unlock! rp-mutex)
	       (archive-block-id        (db:test-get-archived test-dat))
	       (archive-block-info      (rmt:test-get-archive-block-info archive-block-id))
	       (archive-path            (if (vector? archive-block-info)
					    (vector-ref archive-block-info 2) ;; look in db.scm for test-get-archive-block-info for the vector record info
					    #f)) ;; no archive found?
	       (archive-internal-path   (conc (common:get-testsuite-name) "-" run-id "/latest/" test-partial-path)))
	  
	  (if (and archive-path ;; no point in proceeding if there is no actual archive
		   (not toplevel/children))
	      (begin
		;; bup -d /tmp/matt/adisk1/2015_q1/fullrun_e1a40/ restore -C /tmp/seeme fullrun-30/latest/ubuntu/nfs/none/w02.1.20.54_b/
		
		;; DO BUP RESTORE
		(let* (;; new-test-path won't work - must use best-disk instead? Nope, new-test-path but tack on /..
		       (bup-restore-params  (list "-d" archive-path "restore" "-C" (or destpath "data") " " archive-internal-path)))
		  (debug:print-info 0 *default-log-port* "Restoring archived data to " (or destpath "data") " from archive in " archive-path " ... " archive-internal-path)
		  ;; (mutex-lock! bup-mutex)
		  (run-n-wait bup-exe params: bup-restore-params print-cmd: #f)))
	      (debug:print-error 0 *default-log-port* "No archive path in the record for run-id=" run-id " test-id=" test-id))))))

Modified megatest.scm from [23508e9e3f] to [cd4b9c8593].

︙
238
239
240
241
242
243
244
245

246
247
248
249
250
251
252

238
239
240
241
242
243
244

245
246
247
248
249
250
251
252








-
+








  -refdb2dat refdb        : convert refdb to sexp or to format specified by s-dumpmode
                            formats: perl, ruby, sqlite3, csv (for csv the -o param
                            will substitute %s for the sheet name in generating 
                            multiple sheets)
  -o                      : output file for refdb2dat (defaults to stdout)
  -archive cmd            : archive runs specified by selectors to one of disks specified
                            in the [archive-disks] section.
                            cmd: keep-html, restore, save, save-remove
                            cmd: keep-html, restore, save, save-remove, get (use -dest to set destination)
  -generate-html          : create a simple html dashboard for browsing your runs
  -generate-html-structure  : create a top level html veiw to list targets/runs and a Run view within each run directory.  
  -list-run-time          : list time requered to complete runs. It supports following switches
                            -run-patt <patt> -target-patt <patt> -dumpmode <csv,json,plain-text>
  -list-test-time	  : list time requered to complete each test in a run. It following following arguments
                            -runname <patt> -target <patt> -dumpmode <csv,json,plain-text>
  -syscheck               : do some very basic checks; write access and space in tmp, home, runs, links and
︙
310
311
312
313
314
315
316

317
318
319
320
321
322
323

310
311
312
313
314
315
316
317
318
319
320
321
322
323
324








+








			"-logpro"
			"-m"
			"-rerun"

			"-days"
			"-rename-run"
			"-to"
			"-dest"
			;; values and messages
			":category"
			":variable"
			":value"
			":expected"
			":tol"
			":units"
︙

Modified runs.scm from [38b916e9a8] to [f48ffb3bcd].

︙
2167
2168
2169
2170
2171
2172
2173
2174


2175
2176

2177
2178



2179
2180
2181


2182
2183
2184
2185
2186
2187
2188

2167
2168
2169
2170
2171
2172
2173

2174
2175
2176

2177
2178

2179
2180
2181
2182


2183
2184
2185
2186
2187
2188
2189
2190
2191








-
+
+

-
+

-
+
+
+

-
-
+
+








		   ((run-wait)
		    (debug:print 1 *default-log-port* "Waiting for run " runkey ", run=" runnamepatt " to complete"))
		   ((archive)
		    (debug:print 1 *default-log-port* "Archiving/restoring (" (args:get-arg "-archive") ") data for run: " runkey " " (db:get-value-by-header run header "runname"))
		    (set! worker-thread
			  (make-thread
			   (lambda ()
			     (case (string->symbol (args:get-arg "-archive"))
			     (let ((op (string->symbol (args:get-arg "-archive"))))
			       (case op
			       ((save save-remove keep-html)
				(archive:run-bup (args:get-arg "-archive") run-id run-name tests rp-mutex bup-mutex))
				(archive:run-bup op run-id run-name tests rp-mutex bup-mutex))
			       ((restore)
				(archive:bup-restore (args:get-arg "-archive") run-id run-name tests rp-mutex bup-mutex))
				(archive:bup-restore op run-id run-name tests rp-mutex bup-mutex))
			       ((get)
				(archive:bup-get-data op run-id run-name tests rp-mutex bup-mutex))
			       (else 
				(debug:print-error 0 *default-log-port* "unrecognised sub command to -archive. Run \"megatest\" to see help")
				(exit))))
				(debug:print-error 0 *default-log-port* "unrecognised sub command " op " for -archive. Run \"megatest\" to see help")
				(exit)))))
			   "archive-bup-thread"))
		    (thread-start! worker-thread))
		   (else
		    (debug:print-info 0 *default-log-port* "action not recognised " action)))
		 
		 ;; actions that operate on one test at a time can be handled below
		 ;;
︙

This page was generated in about 0.122s by Fossil 2.24 [452f8e1ffc] 2024-03-26 15:36:00