Megatest

Check-in [b045e9649e]
Login
☰Home Timeline Files Branches Tags Tickets Wiki Login
Overview
Comment:Cleaned up messages on server startup. Servers started only if write frequency is high.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | v1.60
Files: files | file ages | folders
SHA1: b045e9649ebf1624a0106644e0626f53ad3cf8de
User & Date: mrwellan on 2014-08-26 10:08:08
Other Links: branch diff | manifest | tags
Context
2014-08-26
22:56
Added partially implemented portlogger check-in: ce1f2b5ce1 user: matt tags: v1.60
10:08
Cleaned up messages on server startup. Servers started only if write frequency is high. check-in: b045e9649e user: mrwellan tags: v1.60
00:02
Added message on read-only query bypassing server check-in: aeed6c5c75 user: matt tags: v1.60
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Modified common.scm from [8133b36933] to [ad4c5ec07f].

︙
42
43
44
45
46
47
48

49
50
51
52
53
54
55

42
43
44
45
46
47
48
49
50
51
52
53
54
55
56








+








(define *configdat*  #f)
(define *toppath*    #f)
(define *already-seen-runconfig-info* #f)
(define *waiting-queue*     (make-hash-table))
(define *test-meta-updated* (make-hash-table))
(define *globalexitstatus*  0) ;; attempt to work around possible thread issues
(define *passnum*           0) ;; when running track calls to run-tests or similar
(define *write-frequency*   (make-hash-table)) ;; run-id => (vector (current-seconds) 0))

;; DATABASE
(define *open-dbs* (vector #f (make-hash-table))) ;; megatestdb run-id-dbs

;; SERVER
(define *my-client-signature* #f)
(define *transport-type*    'http)
︙

Modified datashare.scm from [600b946e0a] to [f6eb31ba0d].

︙
253
254
255
256
257
258
259
260

261
262

263
264
265
266
267
268
269

253
254
255
256
257
258
259

260
261

262
263
264
265
266
267
268
269








-
+

-
+








	 (targ-path   (conc disk-path "/" area "/" version "/" iteration))
	 (id          (datastore:get-id db area version iteration))
	 (db          (datashare:open-db configdat)))
    (if (> space-avail 10000) ;; dumb heuristic
	(begin
	  (create-directory targ-path #t)
	  (datastore:set-stored-path db id targ-path)
	  (print "Running command: rsync -av " source-path " " targ-path)
	  (print "Running command: rsync -av " source-path "/ " targ-path "/")
	  (let ((th1 (make-thread (lambda ()
				    (let ((pid (process-run "rsync" (list "-av" source-path targ-path))))
				    (let ((pid (process-run "rsync" (list "-av" (conc source-path "/") (conc targ-path "/")))))
				      (process-wait pid)
				      (datastore:set-copied db id "yes")
				      (sqlite3:finalize! db)))
				   "Data copy")))
	    (thread-start! th1))
	  #t)
	(begin
︙
338
339
340
341
342
343
344
345

346
347
348
349
350
351
352

338
339
340
341
342
343
344

345
346
347
348
349
350
351
352








-
+








		       ;;  					     ;; set positon using coordinates or center, start, top, left, end, bottom, right, parent-center, current
		       ;;  					     ;; #:x 'mouse
		       ;;  					     ;; #:y 'mouse
		       ;;  )					     
		       ))))

(define (datashare:publish-view configdat)
  (pp (hash-table->alist configdat))
  ;; (pp (hash-table->alist configdat))
  (let* ((areas       (configf:get-section configdat "areas"))
	 (label-size  "70x")
	 (areas-sel   (iup:listbox #:expand "HORIZONTAL" #:dropdown "YES"))
	 (version-tb  (iup:textbox #:expand "HORIZONTAL")) ;;  #:size "50x"))
	 (areas-sel   (iup:listbox #:expand "HORIZONTAL" #:dropdown "YES"))
	 (component   (iup:listbox #:expand "HORIZONTAL" #:dropdown "YES" ))
	 (version-val (iup:textbox #:expand "HORIZONTAL" #:size "50x"))
︙
395
396
397
398
399
400
401
402

403
404
405
406
407
408
409

395
396
397
398
399
400
401

402
403
404
405
406
407
408
409








-
+








				  #:action (lambda (obj)
					     (let* ((fd  (iup:file-dialog #:dialogtype "DIR"))
						    (top (iup:show fd #:modal? "YES")))
					       (iup:attribute-set! source-tb "VALUE"
								   (iup:attribute fd "VALUE"))
					       (iup:destroy! fd))))))
    (print "areas")
    (pp areas)
    ;; (pp areas)
    (fold (lambda (areadat num)
	    ;; (print "Adding num=" num ", areadat=" areadat)
	    (iup:attribute-set! areas-sel (conc num) (car areadat))
	    (+ 1 num))
	  1 areas)
    (iup:vbox
     (iup:hbox (iup:label "Area:"        #:size label-size)   areas-sel)
︙
463
464
465
466
467
468
469
470

471
472
473
474
475
476
477

463
464
465
466
467
468
469

470
471
472
473
474
475
476
477








-
+








				      (set! curr-record record)
				      (iup:attribute-set! submitter      "TITLE" (datastore:pkg-get-submitter record))
				      (iup:attribute-set! date-submitted "TITLE" (time->string (seconds->local-time (datastore:pkg-get-datetime record))))
				      (iup:attribute-set! comment        "TITLE" (datastore:pkg-get-comment record))
				      (iup:attribute-set! quality        "TITLE" (datastore:pkg-get-quality record))
				      (iup:attribute-set! copy-link      "TITLE" (datastore:pkg-get-store_type record))
				      ))
				(print  "id=" id " path=" path " record=" record);; (tree:node->path obj id) " run-id: " run-id)
				;; (print  "id=" id " path=" path " record=" record);; (tree:node->path obj id) " run-id: " run-id)
				))))
	   (tb2             (iup:treebox
			    #:value 0
			    #:name "Installed"
			    #:expand "YES"
			    #:addexpanded "NO"
			    #:selection-cb
︙
571
572
573
574
575
576
577















578
579
580
581
582
583

584
585

586
587
588
589
590
591
592





593
594
595
596



597

598
599
600
601
602
603
604
605
606
607
608
609
610

571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597

598


599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618

619
620
621
622
623
624
625
626
627
628
629
630
631
632








+
+
+
+
+
+
+
+
+
+
+
+
+
+
+





-
+
-
-
+







+
+
+
+
+




+
+
+
-
+














	;; (set! (iup:callback tabs tabchange-cb:) (lambda (a b c)(print "SWITCHED TO TAB: " a " " b " " c)))
	(iup:attribute-set! tabs "TABTITLE0" "Publish")
	(iup:attribute-set! tabs "TABTITLE1" "Get")
	(iup:attribute-set! tabs "TABTITLE2" "Manage")
	;; (iup:attribute-set! tabs "BGCOLOR" "190 190 190")
	tabs)))
  (iup:main-loop))

;;======================================================================
;; MISC
;;======================================================================

(define (datastore:find name paths)
  (if (null? paths)
      #f
      (let loop ((hed (car paths))
		 (tal (cdr paths)))
	(if (file-exists? (conc hed "/" name))
	    hed
	    (if (null? tal)
		#f
		(loop (car tal)(cdr tal)))))))

;;======================================================================
;; MAIN
;;======================================================================

(define (datashare:load-config path)
(define (datashare:load-config exe-dir exe-name)
  (let* ((exename (pathname-file (car (argv))))
	 (fname   (conc path "/." exename ".config")))
  (let* ((fname   (conc exe-dir "/." exe-name ".config")))
    (ini:property-separator-patt " *  *")
    (ini:property-separator #\space)
    (if (file-exists? fname)
	;; (ini:read-ini fname)
	(read-config fname #f #t)
	(make-hash-table))))

;; ease debugging by loading ~/.dashboardrc - remove from production!
(let ((debugcontrolf (conc (get-environment-variable "HOME") "/.datasharerc")))
  (if (file-exists? debugcontrolf)
      (load debugcontrolf)))

(define (main)
  (let* ((args      (argv))
	 (prog      (car args))
	 (rema      (cdr args))
	 (exe-name  (pathname-file (car (argv))))
	 (exe-dir   (or (pathname-directory prog)
			(datastore:find exe-name (string-split (get-environment-variable "PATH") ":"))))
	 (configdat (datashare:load-config (pathname-directory prog))))
	 (configdat (datashare:load-config exe-dir exe-name)))
    (cond
     ((eq? (length rema) 1)
      (case (string->symbol (car rema))
	((help -h -help --h --help)
	 (print datashare:help))
	(else
	 (print "ERROR: Unrecognised command. Try \"datashare help\""))))
     ((null? rema)(datashare:gui configdat))
     ((>= (length rema) 2)
      (apply process-action (car rema)(cdr rema)))
     (else (print "ERROR: Unrecognised command. Try \"datashare help\"")))))

(main)

Modified db.scm from [d8f685dae7] to [02c1e91472].

︙
240
241
242
243
244
245
246
247

248
249
250
251
252
253
254
255
256
257
258
259



260

261
262
263
264
265
266
267
268
269
270
271
272
273
274
275

276
277
278
279
280
281
282

240
241
242
243
244
245
246

247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262

263
264
265
266
267
268
269
270
271
272
273
274
275
276
277

278
279
280
281
282
283
284
285








-
+












+
+
+
-
+














-
+








	(rundb  (dbr:dbstruct-get-rundb dbstruct))
	(inmem  (dbr:dbstruct-get-inmem dbstruct))
	(maindb (dbr:dbstruct-get-main  dbstruct))
	(refdb  (dbr:dbstruct-get-refdb dbstruct))
	(olddb  (dbr:dbstruct-get-olddb dbstruct))
	;; (runid  (dbr:dbstruct-get-run-id dbstruct))
	)
    (debug:print-info 0 "Syncing for run-id " run-id)
    (debug:print-info 4 "Syncing for run-id: " run-id)
    (if (eq? run-id 0)
	;; runid equal to 0 is main.db
	(if maindb
	    (if (or (not (number? mtime))
		    (not (number? stime))
		    (> mtime stime)
		    force-sync)
		(let ((num-synced (db:sync-tables (db:sync-main-list maindb) maindb olddb)))
		  (dbr:dbstruct-set-stime! dbstruct (current-milliseconds))
		  num-synced)
		0)
	    (begin
	      ;; this can occur when using local access (i.e. not in a server)
	      ;; need a flag to turn it off.
	      ;;
	      (debug:print 0 "WARNING: call to sync main.db to megatest.db but main not initialized")
	      (debug:print 3 "WARNING: call to sync main.db to megatest.db but main not initialized")
	      0))
	;; any other runid is a run
	(if (or (not (number? mtime))
		(not (number? stime))
		(> mtime stime)
		force-sync)
	    (let ((num-synced (db:sync-tables db:sync-tests-only inmem refdb rundb olddb)))
	      (dbr:dbstruct-set-stime! dbstruct (current-milliseconds))
	      num-synced)
	    0))))

;; close all opened run-id dbs
(define (db:close-all dbstruct)
  ;; finalize main.db
  (db:sync-touched dbstruct force-sync: #t)
  (db:sync-touched dbstruct 0 force-sync: #t)
  (sqlite3:finalize! (db:get-db dbstruct #f))
  (let* ((local (dbr:dbstruct-get-local dbstruct))
	 (rundb (dbr:dbstruct-get-rundb dbstruct)))
    (if local
	(for-each
	 (lambda (db)
	   (if (sqlite3:database? db)
︙
369
370
371
372
373
374
375
376
377


378
379
380
381
382
383
384

372
373
374
375
376
377
378


379
380
381
382
383
384
385
386
387








-
-
+
+








	   '("avg_disk"       #f)
	   '("tags"           #f)
	   '("jobgroup"       #f)))))
    
;; tbls is ( ("tablename" ( "field1" [#f|proc1] ) ( "field2" [#f|proc2] ) .... ) )
(define (db:sync-tables tbls fromdb todb . slave-dbs)
  (cond
   ((not fromdb) (debug:print 0 "ERROR: db:sync-tables called with fromdb missing") -1)
   ((not todb)   (debug:print 0 "ERROR: db:sync-tables called with todb missing") -2)
   ((not fromdb) (debug:print 3 "WARNING: db:sync-tables called with fromdb missing") -1)
   ((not todb)   (debug:print 3 "WARNING: db:sync-tables called with todb missing") -2)
   ((not (sqlite3:database? fromdb))
    (debug:print 0 "ERROR: db:sync-tables called with fromdb not a database " fromdb) -3)
   ((not (sqlite3:database? todb))
    (debug:print 0 "ERROR: db:sync-tables called with todb not a database " todb) -4)
   (else
    (let ((stmts       (make-hash-table)) ;; table-field => stmt
	  (all-stmts   '())              ;; ( ( stmt1 value1 ) ( stml2 value2 ))
︙

Modified rmt.scm from [d60558790e] to [5178e075e2].

︙
30
31
32
33
34
35
36



















37
38
39
40
41
42
43
44
45
46
47
48
49
50
51


52
53
54
55
56
57
58

30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68


69
70
71
72
73
74
75
76
77








+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+













-
-
+
+








;; )


;;======================================================================
;;  S U P P O R T   F U N C T I O N S
;;======================================================================

(define (rmt:write-frequency-over-limit? cmd run-id)
  (or (member cmd api:read-only-queries)
      (let* ((tmprec (hash-table-ref/default *write-frequency* run-id #f))
	     (record (if tmprec tmprec 
			 (let ((v (vector (current-seconds) 0)))
			   (hash-table-set! *write-frequency* run-id v)
			   v)))
	     (count  (+ 1 (vector-ref record 1)))
	     (start  (vector-ref record 0)))
	(vector-set! record 1 count)
	(if (and (> count 1) 
		 (< (/ (- (current-seconds) start)
		       count) ;; seconds per count
		    10))
	    (begin
	      (debug:print-info 1 "db write rate too high, starting a server")
	      #t)
	    #f)))) ;; less than 10 seconds per count - start up a server

;; cmd is a symbol
;; vars is a json string encoding the parameters for the call
;;
(define (rmt:send-receive cmd rid params)
  (let* ((run-id          (if rid rid 0))
	 (connection-info (let ((cinfo (hash-table-ref/default *runremote* run-id #f)))
			    (if cinfo
				cinfo
				;; if read only query and server not already running
				;; bypass starting the server. 
				;;
				;; NB// can cache the answer for server running for 10 seconds ...
				;;
				(if (and (member cmd api:read-only-queries)
					 (not (open-run-close tasks:get-server tasks:open-db run-id)))
				(if (and (not (rmt:write-frequency-over-limit? cmd run-id))
					 (not (open-run-close tasks:server-running-or-starting? tasks:open-db run-id)))
				    #f
				    (let loop ((numtries 100))
				      (let ((res (client:setup run-id)))
					(if res 
					    (hash-table-ref/default *runremote* run-id #f) ;; client:setup filled this in (hopefully)
					    (if (> numtries 0)
						(begin
︙
66
67
68
69
70
71
72
73

74
75
76
77
78
79
80

85
86
87
88
89
90
91

92
93
94
95
96
97
98
99








-
+








	(let ((res             (http-transport:client-api-send-receive run-id connection-info cmd jparams)))
	  (if res
	      (db:string->obj res)
	      (let ((new-connection-info (client:setup run-id)))
		(debug:print 0 "WARNING: Communication failed, trying call to http-transport:client-api-send-receive again.")
		(rmt:send-receive cmd run-id params))))
	(begin
	  (debug:print-info 0 "no server and read-only query, bypassing normal channel")
	  (debug:print-info 4 "no server and read-only query, bypassing normal channel")
	  (rmt:open-qry-close-locally cmd run-id params)))))

(define (rmt:open-qry-close-locally cmd run-id params)
  (let* ((dbdir (conc    (configf:lookup *configdat* "setup" "linktree") "/.db"))
	 (dbstruct-local (make-dbr:dbstruct path:  dbdir
					    local: #t))
	 (db-file-path   (db:dbfile-path 0))
︙

Modified tasks.scm from [003a5b308d] to [5715ae88f8].

︙
263
264
265
266
267
268
269









270
271
272
273
274
275
276

263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285








+
+
+
+
+
+
+
+
+








     mdb
     ;; removed:
     ;; strftime('%s','now')-heartbeat < 10 AND mt_version = ?
     "SELECT id,interface,port,pubport,transport,pid,hostname FROM servers
          WHERE run_id=? AND state='running'
          ORDER BY start_time DESC LIMIT 1;" run-id) ;; (common:version-signature) run-id)
    res))

(define (tasks:server-running-or-starting? mdb run-id)
  (let ((res #f))
    (sqlite3:for-each-row
     (lambda (id)
       (set! res id))
     mdb
     "SELECT id FROM servers WHERE run_id=? AND state in ('running','available');" run-id)
    res))

(define (tasks:get-all-servers mdb)
  (let ((res '()))
    (sqlite3:for-each-row
     (lambda (id pid hostname interface port pubport start-time priority state mt-version last-update transport run-id)
       ;;                       0   1        2         3    4       5          6        7     8          9          10        11     12
       (set! res (cons (vector id pid hostname interface port pubport start-time priority state mt-version last-update transport run-id) res)))
︙

This page was generated in about 0.113s by Fossil 2.24 [452f8e1ffc] 2024-03-26 15:36:00