Megatest

Check-in [07ba2ed8da]
Login
Overview
Comment:Converted couple more files to ck5
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | v1.80-mbi
Files: files | file ages | folders
SHA1: 07ba2ed8daac814782ef6cab946d3b1b31c71e09
User & Date: matt on 2023-04-01 18:08:04
Other Links: branch diff | manifest | tags
Context
2023-04-01
21:20
Another file (mostly) ported to chicken 5 check-in: 3e5fcece74 user: matt tags: v1.80-mbi
18:08
Converted couple more files to ck5 check-in: 07ba2ed8da user: matt tags: v1.80-mbi
2023-03-31
21:32
More cleanup check-in: 57d442213a user: matt tags: v1.80-mbi
Changes

Modified dbfile.scm from [0fed86f471] to [6b81b8fa20].

22
23
24
25
26
27
28
29
30
31





32
33
34
35
36
37

38
39
40
41
42
43
44
45
46
47
48
























49










50
51
52
53
54
55
56
22
23
24
25
26
27
28



29
30
31
32
33
34
35

36


37
38




39
40




41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82







-
-
-
+
+
+
+
+


-

-
-
+

-
-
-
-


-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+
+
+
+
+
+
+
+
+
+








(declare (unit dbfile))
(declare (uses debugprint))
(declare (uses commonmod))

(module dbfile
	*
	
  (import scheme
	  chicken

(import scheme)
(cond-expand
 (chicken-4
  (import chicken
	  data-structures
	  extras
	  matchable
  
	  (prefix sqlite3 sqlite3:)
	  posix typed-records
	  posix

	  srfi-18
	  srfi-1
	  srfi-69
	  stack
	  files
	  ports
	  
	  commonmod
	  debugprint
	  )
	  )
  (define current-process-milliseconds current-milliseconds)
  )
  (chicken-5
   (import chicken.base
	   chicken.condition
	   chicken.file
	   chicken.file.posix
	   chicken.format
	   chicken.io
	   chicken.pathname
	   chicken.port
	   chicken.process
	   chicken.process-context.posix
	   chicken.sort
	   chicken.string
	   chicken.time
	   chicken.time.posix

	   system-information
	   )
   (define file-move move-file)
   (define file-write-access? file-writable?)
   ))

  (import (prefix sqlite3 sqlite3:))
  (import typed-records)
  (import srfi-18)
  (import srfi-1)
  (import srfi-69)
  (import stack)
  (import commonmod)
  (import debugprint)
  (import matchable)
  
;; parameters
;;
(define dbfile:testsuite-name (make-parameter #f))

(define keep-age-param        (make-parameter 10))      ;; qif file age, if over move to attic
(define num-run-dbs           (make-parameter 10))      ;; number of db's in .mtdb
(define dbfile:sync-method    (make-parameter 'attach)) ;; 'attach or 'original
840
841
842
843
844
845
846
847

848
849
850
851
852
853
854
866
867
868
869
870
871
872

873
874
875
876
877
878
879
880







-
+







                   readonly-slave-dbs))) -6)
    (else
     ;; (dbfile:print-err "db:sync-tables: args are good")

     (let ((stmts       (make-hash-table)) ;; table-field => stmt
	   (all-stmts   '())              ;; ( ( stmt1 value1 ) ( stml2 value2 ))
	   (numrecs     (make-hash-table))
	   (start-time  (current-milliseconds))
	   (start-time  (current-process-milliseconds))
	   (tot-count   0))
       (for-each ;; table
	(lambda (tabledat)
	  (let* ((tablename        (car tabledat))
		 (fields           (cdr tabledat))
		 (has-last-update  (member "last_update" fields))
		 (use-last-update  (cond
984
985
986
987
988
989
990
991

992
993
994
995
996
997
998
1010
1011
1012
1013
1014
1015
1016

1017
1018
1019
1020
1021
1022
1023
1024







-
+







                 (if (member "last_update" field-names)
                    (db:create-trigger db tablename))))
	     (append (list todb) slave-dbs)
           )
          )
        )
	tbls)
       (let* ((runtime      (- (current-milliseconds) start-time))
       (let* ((runtime      (- (current-process-milliseconds) start-time))
	      (should-print (or ;; (debug:debug-mode 12)
			     (common:low-noise-print 120 "db sync")
			     (> runtime 500)))) ;; low and high sync times treated as separate.
	 (for-each 
	  (lambda (dat)
	    (let ((tblname (car dat))
		  (count   (cdr dat)))

Modified dbmod.scm from [b286c5dacd] to [13ce171945].

23
24
25
26
27
28
29
30
31
32
33
34
35




















36
37
38
39
40




41
42
43
44



45
46
47
48
49
50
51
52
23
24
25
26
27
28
29






30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49





50
51
52
53




54
55
56

57
58
59
60
61
62
63







-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-
-
-
-
+
+
+
+
-
-
-
-
+
+
+
-







(declare (uses dbfile))
(declare (uses commonmod))
(declare (uses debugprint))

(module dbmod
	*
	
(import scheme
	chicken
	data-structures
	extras

	(prefix sqlite3 sqlite3:)
(import scheme)
(cond-expand
 (chicken-4
  (import chicken
	  data-structures
	  extras
	  posix
	  )
  (define current-process-milliseconds current-milliseconds)
  )
 (chicken-5
  (import chicken.base
	  chicken.file
	  chicken.sort
	  chicken.string
	  chicken.time
	  
	  )))

(import (prefix sqlite3 sqlite3:))
	posix
	typed-records
	srfi-1
	srfi-18
	srfi-69
(import typed-records)
(import srfi-1)
(import srfi-18)
(import srfi-69)

	commonmod
	dbfile
	debugprint
(import commonmod)
(import dbfile)
(import debugprint)
	)

;; NOTE: This returns only the name "1.db", "main.db", not the path
;;
(define (dbmod:run-id->dbfname run-id)
  (conc (dbfile:run-id->dbnum run-id)".db"))

(define (dbmod:get-dbdir dbstruct)
234
235
236
237
238
239
240
241

242
243
244
245
246
247
248
245
246
247
248
249
250
251

252
253
254
255
256
257
258
259







-
+







;;
;; Use (db:sync-all-tables-list keys) to get the tbls input
;;
(define (dbmod:sync-tables tbls last-update fromdb todb)
  (let ((stmts       (make-hash-table)) ;; table-field => stmt
	(all-stmts   '())              ;; ( ( stmt1 value1 ) ( stml2 value2 ))
	(numrecs     (make-hash-table))
	(start-time  (current-milliseconds))
	(start-time  (current-process-milliseconds))
	(tot-count   0))
    (for-each ;; table
     (lambda (tabledat)
       (let* ((tablename        (car tabledat))
	      (fields           (cdr tabledat))
	      (has-last-update  (member "last_update" fields))
	      (use-last-update  (dbmod:calc-use-last-update has-last-update fields last-update))
349
350
351
352
353
354
355
356

357
358
359
360
361
362
363
360
361
362
363
364
365
366

367
368
369
370
371
372
373
374







-
+







	    fromdats)
	   
	   (sqlite3:finalize! stmth)
           (if (member "last_update" field-names)
               (db:create-trigger db tablename)))
	 ))
     tbls)
    (let* ((runtime      (- (current-milliseconds) start-time))
    (let* ((runtime      (- (current-process-milliseconds) start-time))
	   (should-print (or ;; (debug:debug-mode 12)
			  (common:low-noise-print 120 "db sync")
			  (> runtime 500)))) ;; low and high sync times treated as separate.
      (for-each 
       (lambda (dat)
	 (let ((tblname (car dat))
	       (count   (cdr dat)))
415
416
417
418
419
420
421
422

423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440

441
442
443
444
445
446
447
426
427
428
429
430
431
432

433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450

451
452
453
454
455
456
457
458







-
+

















-
+







			    ","))
		  (stmt1 (conc "INSERT OR IGNORE INTO "todb table
			       " SELECT * FROM "fromdb table";"))
		  (stmt8 (conc "UPDATE "todb table" SET ("no-id-fields-str") = (SELECT "no-id-fields-str" FROM "fromdb table" WHERE "todb table".id="fromdb table".id"
			       (if (member "last_update" fields)
				   (conc " AND "fromdb table".last_update > "todb table".last_update);")
				   ");")))
		  (start-ms (current-milliseconds)))
		  (start-ms (current-process-milliseconds)))
	     ;; (debug:print 0 *default-log-port* "stmt8="stmt8)
	     ;; (if (sqlite3:auto-committing? dbh)
	     ;; (begin
	     (mutex-lock! *db-transaction-mutex*)
	     (sqlite3:with-transaction
	      dbh
	      (lambda ()
		(debug:print-info 0 *default-log-port* "Sync from "fromdb table" to "todb table" using "stmt1)
		(sqlite3:execute dbh stmt1)    ;; get all new rows
		
		#;(if (member "last_update" fields)
		(sqlite3:execute dbh stmt8))    ;; get all updated rows
		;; (sqlite3:execute dbh stmt5)
		;; (sqlite3:execute dbh stmt4) ;; if it worked this would be better for incremental up
		;; (sqlite3:execute dbh stmt6)
		))
	     (debug:print 0 *default-log-port* "Synced table "table
			  " in "(- (current-milliseconds) start-ms)"ms") ;; )
			  " in "(- (current-process-milliseconds) start-ms)"ms") ;; )
	     (mutex-unlock! *db-transaction-mutex*)))
	     
	 ;; (debug:print 0 *default-log-port* "Skipping sync of table "table" due to transaction in flight."))))
	 table-names)
	(sqlite3:execute dbh "DETACH auxdb;"))))

;; FAILED ATTEMPTS
506
507
508
509
510
511
512
513

514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529

530
531
532
533
534
535
536
537
538
539
540
541
542
517
518
519
520
521
522
523

524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539

540
541
542
543
544
545
546
547
548
549
550
551
552
553







-
+















-
+













		  ;; (stmt8 (conc "UPDATE "todb table" SET ("no-id-fields-str") = (SELECT "no-id-fields-str" FROM "fromdb table " WHERE "todb table".id="fromdb table".id"
		  ;; 	       (if (member "last_update" fields)
		  ;; 		   (conc " AND "fromdb table".last_update > "todb table".last_update);")
		  ;; 		   ");")))
		  (stmt1    (conc "SELECT MAX(last_update) FROM "table";")) ;; use the highest last_update as your time reference
		  (stmt2    (conc "SELECT no-id-fields-str FROM "table" WHERE last_update>?;"))
		  (stmt3    (conc "UPDATE "table" SET ("no-id-fields-str") = ("questionmarks") WHERE id=?;"))
		  (start-ms (current-milliseconds)))
		  (start-ms (current-process-milliseconds)))
	     (debug:print 0 *default-log-port* "stmt3="stmt3)
	     (if (sqlite3:auto-committing? dbh1)
		 (begin
		   (sqlite3:with-transaction
		    dbh1
		    (lambda ()
		      (sqlite3:execute dbh1 stmt1)    ;; get all new rows

		      #;(if (member "last_update" fields)
			  (sqlite3:execute dbh1 stmt8))    ;; get all updated rows
		      ;; (sqlite3:execute dbh stmt5)
		      ;; (sqlite3:execute dbh stmt4) ;; if it worked this would be better for incremental up
		      ;; (sqlite3:execute dbh stmt6)
		      ))
		   (debug:print 0 *default-log-port* "Synced table "table
				" in "(- (current-milliseconds) start-ms)"ms"))
				" in "(- (current-process-milliseconds) start-ms)"ms"))
		 (debug:print 0 *default-log-port* "Skipping sync of table "table" due to transaction in flight."))))
	 table-names)
	(sqlite3:execute dbh1 "DETACH auxdb;"))))




;;======================================================================
;; Moved from dbfile
;;======================================================================


)