Megatest

Check-in [b9bcceaeae]
Login
☰Home Timeline Files Branches Tags Tickets Wiki Login
Overview
Comment:Added Roberts nbfake changes, use a dotfile for the db ready file flag
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | v1.64-envdebug | v1.6403
Files: files | file ages | folders
SHA1: b9bcceaeae6a702054f1309437941226d5fcd71e
User & Date: mrwellan on 2017-04-04 12:08:52
Other Links: branch diff | manifest | tags
Context
2017-04-04
14:54
Hacks to make newdashboard start - just a jumpstarter check-in: 805f902745 user: mrwellan tags: v1.64-envdebug
12:08
Added Roberts nbfake changes, use a dotfile for the db ready file flag check-in: b9bcceaeae user: mrwellan tags: v1.64-envdebug, v1.6403
2017-04-03
22:05
Joined transactions for initiating databases. This problably fixes many issues! check-in: b158f5ed64 user: matt tags: v1.64-envdebug
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Modified db.scm from [f1b6299092] to [eb8b9ceb35].

︙︙
208
209
210
211
212
213
214

215
216
217
218
219
220
221
222





223
224
225
226
227
228
229
230
231
232









233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257

;; returns: db existed-prior-to-opening
;; RA => Returns a db handler; sets the lock if opened in writable mode
;;
;;(define *db-open-mutex* (make-mutex))

(define (db:lock-create-open fname initproc)
  (let* ((parent-dir   (or (pathname-directory fname)(current-directory))) ;; no parent? go local

	 (dir-writable (file-write-access? parent-dir))
	 (file-exists  (file-exists? fname))
	 (file-write   (if file-exists
			   (file-write-access? fname)
			   dir-writable )))
    ;;(mutex-lock! *db-open-mutex*) ;; tried this mutex, not clear it helped.
    (if file-write ;; dir-writable
	(condition-case





	    (let ((db      (sqlite3:open-database fname)))
	      (sqlite3:set-busy-handler! db (make-busy-timeout 136000))
	      (sqlite3:execute db "PRAGMA synchronous = 0;")
	      (if (not file-exists)
		  (begin
		    (if (and (configf:lookup *configdat* "setup" "use-wal")
			     (string-match "^/tmp/.*" fname)) ;; this is a file in /tmp
			(sqlite3:execute db "PRAGMA journal_mode=WAL;")
			(print "Creating " fname " in NON-WAL mode."))
		    (initproc db)))









	      db)
	  (exn (io-error)  (debug:print 0 *default-log-port* "ERROR: i/o error with " fname ". Check permissions, disk space etc. and try again."))
	  (exn (corrupt)   (debug:print 0 *default-log-port* "ERROR: database " fname " is corrupt. Repair it to proceed."))
	  (exn (busy)      (debug:print 0 *default-log-port* "ERROR: database " fname " is locked. Try copying to another location, remove original and copy back."))
	  (exn (permission)(debug:print 0 *default-log-port* "ERROR: database " fname " has some permissions problem."))
	  (exn () (debug:print 0 *default-log-port* "ERROR: Unknown error with database " fname " message: " ((condition-property-accessor 'exn 'message) exn))))

	(condition-case
	    (begin
	      (debug:print 2 *default-log-port* "WARNING: opening db in non-writable dir " fname)
	      (let ((db (sqlite3:open-database fname)))
		;;(mutex-unlock! *db-open-mutex*)
		db))
	  (exn (io-error)  (debug:print 0 *default-log-port* "ERROR: i/o error with " fname ". Check permissions, disk space etc. and try again."))
	  (exn (corrupt)   (debug:print 0 *default-log-port* "ERROR: database " fname " is corrupt. Repair it to proceed."))
	  (exn (busy)      (debug:print 0 *default-log-port* "ERROR: database " fname " is locked. Try copying to another location, remove original and copy back."))
	  (exn (permission)(debug:print 0 *default-log-port* "ERROR: database " fname " has some permissions problem."))
	  (exn () (debug:print 0 *default-log-port* "ERROR: Unknown error with database " fname " message: " ((condition-property-accessor 'exn 'message) exn))))
	)))














>








>
>
>
>
>
|
|
|
|
|
|
|
|
|
|
>
>
>
>
>
>
>
>
>
|
|
|
|
|
|
|

|
|
|
|
|
|
|
|
|
|








208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272

;; returns: db existed-prior-to-opening
;; RA => Returns a db handler; sets the lock if opened in writable mode
;;
;;(define *db-open-mutex* (make-mutex))

(define (db:lock-create-open fname initproc)
  (let* ((parent-dir   (or (pathname-directory fname)(current-directory))) ;; no parent? go local
         (raw-fname    (pathname-file fname))
	 (dir-writable (file-write-access? parent-dir))
	 (file-exists  (file-exists? fname))
	 (file-write   (if file-exists
			   (file-write-access? fname)
			   dir-writable )))
    ;;(mutex-lock! *db-open-mutex*) ;; tried this mutex, not clear it helped.
    (if file-write ;; dir-writable
	(condition-case
         (let* ((lockfname   (conc fname ".lock"))
                (readyfname  (conc parent-dir "/.ready-" raw-fname))
                (readyexists (file-exists? readyfname)))
           (if (not readyexists)
               (common:simple-file-lock-and-wait lockfname))
           (let ((db      (sqlite3:open-database fname)))
             (sqlite3:set-busy-handler! db (make-busy-timeout 136000))
             (sqlite3:execute db "PRAGMA synchronous = 0;")
             (if (not file-exists)
                 (begin
                   (if (and (configf:lookup *configdat* "setup" "use-wal")
                            (string-match "^/tmp/.*" fname)) ;; this is a file in /tmp
                       (sqlite3:execute db "PRAGMA journal_mode=WAL;")
                       (print "Creating " fname " in NON-WAL mode."))
                   (initproc db)))
             (if (not readyexists)
                 (begin
                   (common:simple-file-release-lock lockfname)
                   (with-output-to-file
                       readyfname
                     (lambda ()
                       (print "Ready at " 
                              (seconds->year-work-week/day-time 
                               (current-seconds)))))))
             db))
         (exn (io-error)  (debug:print 0 *default-log-port* "ERROR: i/o error with " fname ". Check permissions, disk space etc. and try again."))
         (exn (corrupt)   (debug:print 0 *default-log-port* "ERROR: database " fname " is corrupt. Repair it to proceed."))
         (exn (busy)      (debug:print 0 *default-log-port* "ERROR: database " fname " is locked. Try copying to another location, remove original and copy back."))
         (exn (permission)(debug:print 0 *default-log-port* "ERROR: database " fname " has some permissions problem."))
         (exn () (debug:print 0 *default-log-port* "ERROR: Unknown error with database " fname " message: " ((condition-property-accessor 'exn 'message) exn))))
        
	(condition-case
         (begin
           (debug:print 2 *default-log-port* "WARNING: opening db in non-writable dir " fname)
           (let ((db (sqlite3:open-database fname)))
             ;;(mutex-unlock! *db-open-mutex*)
             db))
         (exn (io-error)  (debug:print 0 *default-log-port* "ERROR: i/o error with " fname ". Check permissions, disk space etc. and try again."))
         (exn (corrupt)   (debug:print 0 *default-log-port* "ERROR: database " fname " is corrupt. Repair it to proceed."))
         (exn (busy)      (debug:print 0 *default-log-port* "ERROR: database " fname " is locked. Try copying to another location, remove original and copy back."))
         (exn (permission)(debug:print 0 *default-log-port* "ERROR: database " fname " has some permissions problem."))
         (exn () (debug:print 0 *default-log-port* "ERROR: Unknown error with database " fname " message: " ((condition-property-accessor 'exn 'message) exn))))
	)))
︙︙

Modified megatest-version.scm from [d461ac75ec] to [f28f04440f]. 

1
2
3
4
5
6
7

;; Always use two or four digit decimal
;; 1.01, 1.02...1.10,1.11,1.1101 ... 1.99,2.00..

(declare (unit megatest-version))

(define megatest-version 1.6402)







|


1
2
3
4
5
6
7

;; Always use two or four digit decimal
;; 1.01, 1.02...1.10,1.11,1.1101 ... 1.99,2.00..

(declare (unit megatest-version))

(define megatest-version 1.6403)

Modified utils/nbfake from [df0eb253b8] to [bfff85f08b]. 

1
2
3
4
5
6
7
8


9
10
11
12
13
14
15
16
17
18
19
20
21


22
23
24
25
26
27
28

#!/bin/bash
###############################################################################
#
# nbfake - capture command output in a logfile
#
# nbfake behavior can be changed by setting the following env vars:
#   NBFAKE_HOST       SSH to $NBFAKE_HOST and run command
#   NBFAKE_LOG        Logfile for nbfake output


#
###############################################################################

if [[ -z "$@" ]]; then
  cat <<__EOF

nbfake usage:

nbfake <command to run>

nbfake behavior can be changed by setting the following env vars:
   NBFAKE_HOST       SSH to \$NBFAKE_HOST and run command
   NBFAKE_LOG        Logfile for nbfake output



__EOF
  exit
fi

#==============================================================================
# Setup









>
>













>
>








1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32

#!/bin/bash
###############################################################################
#
# nbfake - capture command output in a logfile
#
# nbfake behavior can be changed by setting the following env vars:
#   NBFAKE_HOST       SSH to $NBFAKE_HOST and run command
#   NBFAKE_LOG        Logfile for nbfake output
#   NB_WASH_GROUPS    comma-separated list of groups to wash into
#   NB_WASH_ENABLED   must be set in order to enable wash groups
#
###############################################################################

if [[ -z "$@" ]]; then
  cat <<__EOF

nbfake usage:

nbfake <command to run>

nbfake behavior can be changed by setting the following env vars:
   NBFAKE_HOST       SSH to \$NBFAKE_HOST and run command
   NBFAKE_LOG        Logfile for nbfake output
   NB_WASH_GROUPS    comma-separated list of groups to wash into
   NB_WASH_ENABLED   must be set in order to enable wash groups

__EOF
  exit
fi

#==============================================================================
# Setup
︙︙
51
52
53
54
55
56
57







58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76

fi

# Set default nbfake log

if [[ -z "$MY_NBFAKE_LOG" ]]; then
  MY_NBFAKE_LOG=NBFAKE-$(date +%GWW%V.%u_%T)
fi








#==============================================================================
# Run and log
#==============================================================================

cat <<__EOF >&2
#======================================================================
# NBFAKE logging command to: $MY_NBFAKE_LOG
#     $*
#======================================================================
__EOF

if [[ -z "$MY_NBFAKE_HOST" ]]; then
  # Run locally
  sh -c "cd $CURRWD;export DISPLAY=$DISPLAY; export PATH=$PATH; nohup $* >> $MY_NBFAKE_LOG 2>&1 &"
else
  # run remotely
  ssh -X -n -f $MY_NBFAKE_HOST "sh -c \"cd $CURRWD;export DISPLAY=$DISPLAY; export PATH=$PATH; nohup $* >> $MY_NBFAKE_LOG 2>&1 &\""
fi








>
>
>
>
>
>
>








|





|


|


55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87

fi

# Set default nbfake log

if [[ -z "$MY_NBFAKE_LOG" ]]; then
  MY_NBFAKE_LOG=NBFAKE-$(date +%GWW%V.%u_%T)
fi

# wash groups handling. Default is no action
WASHCMD=""
if [[ -n ${NB_WASH_ENABLED+1} && -n ${NB_WASH_GROUPS+1} ]]; then
   grouplist=`echo $NB_WASH_GROUPS | tr ',' ' '`
   WASHCMD="wash -q -n $grouplist -X"
fi

#==============================================================================
# Run and log
#==============================================================================

cat <<__EOF >&2
#======================================================================
# NBFAKE logging command to: $MY_NBFAKE_LOG
#     $WASHCMD $*
#======================================================================
__EOF

if [[ -z "$MY_NBFAKE_HOST" ]]; then
  # Run locally
  sh -c "cd $CURRWD;export DISPLAY=$DISPLAY; export PATH=$PATH; nohup $WASHCMD $* >> $MY_NBFAKE_LOG 2>&1 &"
else
  # run remotely
  ssh -X -n -f $MY_NBFAKE_HOST "sh -c \"cd $CURRWD;export DISPLAY=$DISPLAY; export PATH=$PATH; nohup $WASHCMD $* >> $MY_NBFAKE_LOG 2>&1 &\""
fi

This page was generated in about 0.132s by Fossil 2.24 [452f8e1ffc] 2024-03-26 15:36:00