Megatest

Check-in [2a182fc8db]
Login
Overview
Comment:db2db sync almost working
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | v1.80-servload
Files: files | file ages | folders
SHA1: 2a182fc8db012dc0ed63ea288c333f029651da8b
User & Date: matt on 2023-04-30 20:59:41
Other Links: branch diff | manifest | tags
Context
2023-05-01
09:00
db2db works (at least with attach mode syncing) check-in: 9361d9fe7b user: matt tags: v1.80-servload
2023-04-30
20:59
db2db sync almost working check-in: 2a182fc8db user: matt tags: v1.80-servload
10:57
wip check-in: f22ffbf700 user: matt tags: v1.80-servload
Changes

Modified dbmod.scm from [89117c0f03] to [6ce3b69143].

27
28
29
30
31
32
33

34
35
36
37
38
39
40
(module dbmod
	*
	
(import scheme
	chicken
	data-structures
	extras


	(prefix sqlite3 sqlite3:)
	posix
	typed-records
	srfi-1
	srfi-18
	srfi-69







>







27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
(module dbmod
	*
	
(import scheme
	chicken
	data-structures
	extras
	files

	(prefix sqlite3 sqlite3:)
	posix
	typed-records
	srfi-1
	srfi-18
	srfi-69
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
;;        (dbmod:sync-tables tables last-update inmem db)
;;        (dbmod:sync-tables tables last-update db inmem))))

;; direction: 'fromdest 'todest
;;
(define (dbmod:sync-gasket tables last-update inmem dbh dbfname direction)
  (assert (sqlite3:database? inmem) "FATAL: sync-gasket: inmem is not a db")
  (assert (sqlite3:database? inmem) "FATAL: sync-gasket: dbh is not a db")
  (case (dbfile:sync-method)
    ((none) #f)
    ((attach)
     (dbmod:attach-sync tables inmem dbfname direction))
    ((newsync)
     (dbmod:new-sync tables inmem dbh dbfname direction))
    (else







|







241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
;;        (dbmod:sync-tables tables last-update inmem db)
;;        (dbmod:sync-tables tables last-update db inmem))))

;; direction: 'fromdest 'todest
;;
(define (dbmod:sync-gasket tables last-update inmem dbh dbfname direction)
  (assert (sqlite3:database? inmem) "FATAL: sync-gasket: inmem is not a db")
  (assert (sqlite3:database? dbh) "FATAL: sync-gasket: dbh is not a db")
  (case (dbfile:sync-method)
    ((none) #f)
    ((attach)
     (dbmod:attach-sync tables inmem dbfname direction))
    ((newsync)
     (dbmod:new-sync tables inmem dbh dbfname direction))
    (else
787
788
789
790
791
792
793
794









795
796
797
798
799
800
801
802
			(or newstate  currstate "NOT_STARTED")
			(or newstatus currstate "UNKNOWN")
			run-id testname)))))
;;======================================================================
;; db to db sync
;;======================================================================

(define (dbmod:db-to-db-sync src-db dest-db last-update)









  (let ((sdb #f) ;;
	(ddb #f))
    (dbmod:sync-gasket tables last-update inmem db
		       dbfullname syncdir)
    ))


)







|
>
>
>
>
>
>
>
>
>
|
|
|
<
<

|
<
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807


808
809

			(or newstate  currstate "NOT_STARTED")
			(or newstatus currstate "UNKNOWN")
			run-id testname)))))
;;======================================================================
;; db to db sync
;;======================================================================

(define (dbmod:db-to-db-sync src-db dest-db last-update init-proc keys)
  (if (and (file-exists? src-db)
	   (file-read-access? src-db))
      (let* ((d-wr   (or (and (file-exists? dest-db)
			      (file-write-access? dest-db)) ;; exists and writable
			 (let* ((dirname (pathname-directory dest-db)))
			   (and dirname
				(file-exists? dirname)
				(file-write-access? dirname)))))
	     (tables (db:sync-all-tables-list keys))
	     (sdb    (dbmod:safely-open-db src-db init-proc #t))
	     (ddb    (dbmod:safely-open-db dest-db init-proc d-wr)))
	(dbmod:sync-gasket tables last-update sdb ddb dest-db 'todest))))



)

Modified megatest.scm from [adec828972] to [a35d824cd5].

282
283
284
285
286
287
288

289
290
291
292
293
294
295
  -list-run-time          : list time requered to complete runs. It supports following switches
                            -run-patt <patt> -target-patt <patt> -dumpmode <csv,json,plain-text>
  -list-test-time	  : list time requered to complete each test in a run. It following following arguments
                            -runname <patt> -target <patt> -dumpmode <csv,json,plain-text>
  -syscheck               : do some very basic checks; write access and space in tmp, home, runs, links and 
                            is $DISPLAY valid 
  -list-waivers           : dump waivers for specified target, runname, testpatt to stdout


Diff report
  -diff-rep               : generate diff report (must include -src-target, -src-runname, -target, -runname
                                                  and either -diff-email or -diff-html)
  -src-target <target>
  -src-runname <target>
  -diff-email <emails>    : comma separated list of email addresses to send diff report







>







282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
  -list-run-time          : list time requered to complete runs. It supports following switches
                            -run-patt <patt> -target-patt <patt> -dumpmode <csv,json,plain-text>
  -list-test-time	  : list time requered to complete each test in a run. It following following arguments
                            -runname <patt> -target <patt> -dumpmode <csv,json,plain-text>
  -syscheck               : do some very basic checks; write access and space in tmp, home, runs, links and 
                            is $DISPLAY valid 
  -list-waivers           : dump waivers for specified target, runname, testpatt to stdout
  -db2db                  : sync db to db, use -from and -to to specify the databases

Diff report
  -diff-rep               : generate diff report (must include -src-target, -src-runname, -target, -runname
                                                  and either -diff-email or -diff-html)
  -src-target <target>
  -src-runname <target>
  -diff-email <emails>    : comma separated list of email addresses to send diff report
344
345
346
347
348
349
350

351
352
353
354
355
356
357
			"-runstep"
			"-logpro"
			"-m"
			"-rerun"

			"-days"
			"-rename-run"

			"-to"
			"-dest"
                        "-source" 
                        "-time-stamp" 
			;; values and messages
			":category"
			":variable"







>







345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
			"-runstep"
			"-logpro"
			"-m"
			"-rerun"

			"-days"
			"-rename-run"
			"-from"
			"-to"
			"-dest"
                        "-source" 
                        "-time-stamp" 
			;; values and messages
			":category"
			":variable"
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
			"-clean-cache"
			"-no-cache"
			"-cache-db"
			"-cp-eventtime-to-publishtime"
                        "-use-db-cache"
                        "-prepend-contour"


			;; misc
			"-repl"
			"-lock"
			"-unlock"
			"-list-servers"
			"-kill-servers"
                        "-run-wait"      ;; wait on a run to complete (i.e. no RUNNING)







<







450
451
452
453
454
455
456

457
458
459
460
461
462
463
			"-clean-cache"
			"-no-cache"
			"-cache-db"
			"-cp-eventtime-to-publishtime"
                        "-use-db-cache"
                        "-prepend-contour"


			;; misc
			"-repl"
			"-lock"
			"-unlock"
			"-list-servers"
			"-kill-servers"
                        "-run-wait"      ;; wait on a run to complete (i.e. no RUNNING)
493
494
495
496
497
498
499

500
501
502
503
504
505
506
			"-create-megatest-area"
			"-mark-incompletes"

			"-convert-to-norm"
			"-convert-to-old"
			"-import-megatest.db"
			"-sync-to-megatest.db"

                        "-sync-brute-force"
			"-logging"
			"-v" ;; verbose 2, more than normal (normal is 1)
			"-q" ;; quiet 0, errors/warnings only

                        "-diff-rep"








>







494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
			"-create-megatest-area"
			"-mark-incompletes"

			"-convert-to-norm"
			"-convert-to-old"
			"-import-megatest.db"
			"-sync-to-megatest.db"
			"-db2db"
                        "-sync-brute-force"
			"-logging"
			"-v" ;; verbose 2, more than normal (normal is 1)
			"-q" ;; quiet 0, errors/warnings only

                        "-diff-rep"

2558
2559
2560
2561
2562
2563
2564

























2565
2566
2567
2568
2569
2570
2571
      (set! *didsomething* #t)))

(if (args:get-arg "-sync-to")
    (let ((toppath (launch:setup)))
      (tasks:sync-to-postgres *configdat* (args:get-arg "-sync-to"))
      (set! *didsomething* #t)))


























(if (args:get-arg "-list-test-time")
     (let* ((toppath (launch:setup))) 
     (task:get-test-times)  
     (set! *didsomething* #t)))

(if (args:get-arg "-list-run-time")
     (let* ((toppath (launch:setup))) 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
      (set! *didsomething* #t)))

(if (args:get-arg "-sync-to")
    (let ((toppath (launch:setup)))
      (tasks:sync-to-postgres *configdat* (args:get-arg "-sync-to"))
      (set! *didsomething* #t)))

;; use with -from and -to
;;
(if (args:get-arg "-db2db")
    (let* ((duh      (launch:setup))
	   (src-db   (args:get-arg "-from"))
	   (dest-db  (args:get-arg "-to"))
	   (lockfile (conc dest-db".lock"))
	   ;; (locked   (common:simple-file-lock lockfile))
	   (keys     (db:get-keys #f))
	   (res      ;; (if locked
			 (dbmod:db-to-db-sync src-db dest-db 0 (dbfile:db-init-proc) keys)
			 ;;	 #f)))
			 ))
      
      (if (and src-db dest-db)
	  (begin
	    (debug:print-info 0 *default-log-port* "Attempting to sync data from "src-db" to "dest-db"...")
	    (if res
		(begin
		  (common:simple-file-release-lock lockfile)
		  (debug:print 0 *default-log-port* "Synced " res " records from "src-db" to "dest-db))
		(debug:print 0 *default-log-port* "Skipping sync, there is a sync in progress."))
	    (set! *didsomething* #t))
	  (debug:print 0 *default-log-port* "Usage for -db2db; -to and -from must be specified"))))

(if (args:get-arg "-list-test-time")
     (let* ((toppath (launch:setup))) 
     (task:get-test-times)  
     (set! *didsomething* #t)))

(if (args:get-arg "-list-run-time")
     (let* ((toppath (launch:setup)))