412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
|
;; just tests, test_steps and test_data tables
(define db:sync-tests-only
(list
;; (list "strs"
;; '("id" #f)
;; '("str" #f))
(list "tests"
'("id" #f)
'("run_id" #f)
'("testname" #f)
'("host" #f)
'("cpuload" #f)
'("diskfree" #f)
'("uname" #f)
'("rundir" #f)
'("shortdir" #f)
'("item_path" #f)
'("state" #f)
'("status" #f)
'("attemptnum" #f)
'("final_logf" #f)
'("logdat" #f)
'("run_duration" #f)
'("comment" #f)
'("event_time" #f)
'("fail_count" #f)
'("pass_count" #f)
'("archived" #f))
'("id" #f)
'("test_id" #f)
'("stepname" #f)
'("state" #f)
'("status" #f)
'("event_time" #f)
'("comment" #f)
'("logfile" #f))
'("id" #f)
'("test_id" #f)
'("category" #f)
'("variable" #f)
'("value" #f)
'("expected" #f)
'("tol" #f)
'("units" #f)
'("comment" #f)
'("status" #f)
'("type" #f))))
'("id" #f)
'("fieldname" #f)
'("fieldtype" #f))
(list "metadat" '("var" #f) '("val" #f))
(append (list "runs"
'("id" #f))
(map (lambda (k)(list k #f))
(append keys
(list "runname" "state" "status" "owner" "event_time" "comment" "fail_count" "pass_count"))))
'("id" #f)
'("testname" #f)
'("owner" #f)
'("description" #f)
'("reviewed" #f)
'("iterated" #f)
'("avg_runtime" #f)
'("avg_disk" #f)
'("tags" #f)
'("jobgroup" #f)))))
(define (db:sync-all-tables-list dbstruct)
(append (db:sync-main-list dbstruct)
db:sync-tests-only))
;; use bunch of Unix commands to try to break the lock and recreate the db
;;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
>
>
>
>
>
>
|
>
>
>
|
|
|
|
|
|
|
|
|
|
|
|
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
|
;; just tests, test_steps and test_data tables
(define db:sync-tests-only
(list
;; (list "strs"
;; '("id" #f)
;; '("str" #f))
(list "tests"
'("id" "INTEGER")
'("run_id" "INTEGER")
'("testname" "TEXT")
'("host" "TEXT")
'("cpuload" "REAL")
'("diskfree" "INTEGER")
'("uname" "TEXT")
'("rundir" "TEXT")
'("shortdir" "TEXT")
'("item_path" "TEXT")
'("state" "TEXT")
'("status" "TEXT")
'("attemptnum" "INTEGER")
'("final_logf" "TEXT")
'("logdat" "TEXT")
'("run_duration" "INTEGER")
'("comment" "TEXT")
'("event_time" "INTEGER")
'("fail_count" "INTEGER")
'("pass_count" "INTEGER")
'("archived" "INTEGER"))
(list "test_steps"
'("id" "INTEGER")
'("test_id" "INTEGER")
'("stepname" "TEXT")
'("state" "TEXT")
'("status" "TEXT")
'("event_time" "INTEGER")
'("comment" "TEXT")
'("logfile" "TEXT"))
(list "test_data"
'("id" "INTEGER")
'("test_id" "INTEGER")
'("category" "TEXT")
'("variable" "TEXT")
'("value" "REAL")
'("expected" "REAL")
'("tol" "REAL")
'("units" "TEXT")
'("comment" "TEXT")
'("status" "TEXT")
'("type" "TEXT"))))
;; needs db to get keys, this is for syncing all tables
;;
(define (db:sync-main-list dbstruct)
(let ((keys (db:get-keys dbstruct)))
(list
(list "keys"
'("id" "INTEGER")
'("fieldname" "TEXT")
'("fieldtype" "TEXT"))
(list "metadat" '("var" "TEXT") '("val" "TEXT"))
(list "runs"
'("id" "INTEGER")
'("release" "TEXT")
'("iteration" "TEXT")
'("testsuite_mode" "TEXT")
'("runname" "TEXT")
'("state" "TEXT")
'("status" "TEXT")
'("owner" "TEXT")
'("event_time" "INTEGER")
'("comment" "TEXT")
'("fail_count" "INTEGER")
'("pass_count" "INTEGER"))
(list "test_meta"
'("id" "INTEGER")
'("testname" "TEXT")
'("owner" "TEXT")
'("description" "TEXT")
'("reviewed" "INTEGER")
'("iterated" "TEXT")
'("avg_runtime" "REAL")
'("avg_disk" "REAL")
'("tags" "TEXT")
'("jobgroup" "TEXT")))))
(define (db:sync-all-tables-list dbstruct)
(append (db:sync-main-list dbstruct)
db:sync-tests-only))
;; use bunch of Unix commands to try to break the lock and recreate the db
;;
|
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
|
;; first pass implementation, just insert all changed rows
(for-each
(lambda (targdb)
(set! targdb (dbi:convert (db:dbdat-get-db targdb)))
(if (eqv? (dbi:db-dbtype targdb) 'pg)
(let* ((prep ""))
(for-each
(lambda (row)
(set! tabletypes (cons (cons (string->symbol (vector-ref row 1)) (vector-ref row 2)) tabletypes)))
(dbi:pull-metadata (db:dbdat-get-db fromdb) tablename))
(set! prep (string-intersperse (map (lambda (x) (alist-ref (string->symbol (car x)) tabletypes)) fields) ","))
(set! prep (conc "PREPARE fullins (" prep ") AS INSERT INTO " tablename " ( " (string-intersperse (map car fields) ",") " ) VALUES ( "))
(set! prep (conc prep "$" (+ i 1) " );"))))
(set! full-ins prep)))
(let* ((db (dbi:convert (db:dbdat-get-db targdb)))
(stmth (dbi:prepare db full-ins)))
;; (db:delay-if-busy targdb) ;; NO WAITING
(for-each
(lambda (fromdat-lst)
(dbi:with-transaction
|
<
<
<
<
|
|
|
|
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
|
;; first pass implementation, just insert all changed rows
(for-each
(lambda (targdb)
(set! targdb (dbi:convert (db:dbdat-get-db targdb)))
(if (eqv? (dbi:db-dbtype targdb) 'pg)
(let* ((prep ""))
(set! prep (string-intersperse (map cadr fields) ","))
(set! prep (conc "PREPARE fullins (" prep ") AS INSERT INTO " tablename " ( " (string-intersperse (map car fields) ",") " ) VALUES ( ")) ;;maybe add lookup in the future depending on where types are needed
(let loop ((i 1))
(set! prep (conc prep "$" i ","))
(if (< i (- num-fields 1))
(loop (+ i 1))
(set! prep (conc prep "$" (+ i 1) " ) ON CONFLICT DO NOTHING;"))))
(set! full-ins prep)))
(let* ((db (dbi:convert (db:dbdat-get-db targdb)))
(stmth (dbi:prepare db full-ins)))
;; (db:delay-if-busy targdb) ;; NO WAITING
(for-each
(lambda (fromdat-lst)
(dbi:with-transaction
|