Overview
| Comment: | fixed sretrive get cmd to fetch single file |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | v1.61 |
| Files: | files | file ages | folders |
| SHA1: |
fa7cc0ed23a697ce4659763b83c71fb9 |
| User & Date: | pjhatwal on 2016-04-29 17:58:47 |
| Other Links: | branch diff | manifest | tags |
Context
|
2016-05-01
| ||
| 12:27 | Refactored in prep to fix regression where filters don't work in dashboard check-in: 1ff831a4be user: matt tags: v1.61 | |
|
2016-04-30
| ||
| 23:06 | adding inteldate conversion utility procedures check-in: 1a47cc93d9 user: bb tags: v1.61 | |
|
2016-04-29
| ||
| 17:58 | fixed sretrive get cmd to fetch single file check-in: fa7cc0ed23 user: pjhatwal tags: v1.61 | |
| 11:42 | Merged in Jeff's changes to the chicken/iup install makefile check-in: 9696fb116c user: mrwellan tags: v1.61 | |
Changes
Modified datashare-testing/.sretrieve.config from [f5fc49272d] to [71cb2ce9dc].
1 2 3 4 | [settings] base-dir /tmp/delme_data allowed-users matt allowed-chars [0-9a-zA-Z\-\.]+ | | | 1 2 3 4 5 6 7 8 |
[settings]
base-dir /tmp/delme_data
allowed-users matt
allowed-chars [0-9a-zA-Z\-\.]+
allowed-sub-paths [0-9a-zA-Z\-\.]+
[database]
location #{scheme (create-directory "/tmp/#{getenv USER}" #t)}
|
Modified sretrieve.scm from [76eaef987d] to [1a41688ba0].
| ︙ | ︙ | |||
52 53 54 55 56 57 58 |
(define *sretrieve:current-tab-number* 0)
(define *args-hash* (make-hash-table))
(define sretrieve:help (conc "Usage: " *exe-name* " [action [params ...]]
ls : list contents of target area
get <relversion> : retrieve data for release <version>
-m \"message\" : why retrieved?
| | | 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 |
(define *sretrieve:current-tab-number* 0)
(define *args-hash* (make-hash-table))
(define sretrieve:help (conc "Usage: " *exe-name* " [action [params ...]]
ls : list contents of target area
get <relversion> : retrieve data for release <version>
-m \"message\" : why retrieved?
cp <relative path> : copy file to current directory
log : get listing of recent downloads
Part of the Megatest tool suite.
Learn more at http://www.kiatoa.com/fossils/megatest
Version: " megatest-fossil-hash)) ;; "
|
| ︙ | ︙ | |||
106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 |
;; (call-with-database
;; (lambda (db)
;; (set-busy-handler! db (busy-timeout 10000)) ; 10 second timeout
;; ...))
;; Create the sqlite db
(define (sretrieve:db-do configdat proc)
(let ((path (configf:lookup configdat "database" "location")))
(if (not path)
(begin
(debug:print 0 "[database]\nlocation /some/path\n\n Is missing from the config file!")
(exit 1)))
(if (and path
(directory? path)
(file-read-access? path))
(let* ((dbpath (conc path "/" *exe-name* ".db"))
(writeable (file-write-access? dbpath))
(dbexists (file-exists? dbpath)))
(handle-exceptions
exn
(begin
(debug:print 2 "ERROR: problem accessing db " dbpath
((condition-property-accessor 'exn 'message) exn))
(exit 1))
(call-with-database
dbpath
(lambda (db)
| > > | | | 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 |
;; (call-with-database
;; (lambda (db)
;; (set-busy-handler! db (busy-timeout 10000)) ; 10 second timeout
;; ...))
;; Create the sqlite db
(define (sretrieve:db-do configdat proc)
(let ((path (configf:lookup configdat "database" "location")))
(if (not path)
(begin
(debug:print 0 "[database]\nlocation /some/path\n\n Is missing from the config file!")
(exit 1)))
(if (and path
(directory? path)
(file-read-access? path))
(let* ((dbpath (conc path "/" *exe-name* ".db"))
(writeable (file-write-access? dbpath))
(dbexists (file-exists? dbpath)))
(handle-exceptions
exn
(begin
(debug:print 2 "ERROR: problem accessing db " dbpath
((condition-property-accessor 'exn 'message) exn))
(exit 1))
;;(debug:print 0 "calling proc " proc "db path " dbpath )
(call-with-database
dbpath
(lambda (db)
;;(debug:print 0 "calling proc " proc " on db " db)
(set-busy-handler! db (busy-timeout 10000)) ;; 10 sec timeout
(if (not dbexists)(sretrieve:initialize-db db))
(proc db)))))
(debug:print 0 "ERROR: invalid path for storing database: " path))))
;; copy in directory to dest, validation is done BEFORE calling this
;;
(define (sretrieve:get configdat retriever version comment)
(let* ((base-dir (configf:lookup configdat "settings" "base-dir"))
(datadir (conc base-dir "/" version)))
(if (or (not base-dir)
(not (file-exists? base-dir)))
(begin
|
| ︙ | ︙ | |||
154 155 156 157 158 159 160 |
(sretrieve:db-do
configdat
(lambda (db)
(sretrieve:register-action db "get" retriever datadir comment)))
(sretrieve:do-as-calling-user
(lambda ()
| > > | | | | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 |
(sretrieve:db-do
configdat
(lambda (db)
(sretrieve:register-action db "get" retriever datadir comment)))
(sretrieve:do-as-calling-user
(lambda ()
(if (directory? datadir)
(begin
(change-directory datadir)
(let ((files (filter (lambda (x)
(not (member x '("." ".."))))
(glob "*" ".*"))))
(print "files: " files)
(process-execute "/bin/tar" (append (list "chfv" "-") files))))
(begin
(let* ((parent-dir (pathname-directory datadir) )
(filename (conc(pathname-file datadir) "." (pathname-extension datadir))))
(change-directory parent-dir)
(process-execute "/bin/tar" (list "chfv" "-" filename))
)))
))
))
;; copy in file to dest, validation is done BEFORE calling this
;;
(define (sretrieve:cp configdat retriever file comment)
(let* ((base-dir (configf:lookup configdat "settings" "base-dir"))
(allowed-sub-paths (configf:lookup configdat "settings" "allowed-sub-paths"))
(datadir (conc base-dir "/" file))
(filename (conc(pathname-file datadir) "." (pathname-extension datadir))))
(if (or (not base-dir)
(not (file-exists? base-dir)))
(begin
(debug:print 0 "ERROR: Bad configuration! base-dir " base-dir " not found")
(exit 1)))
(print datadir)
(if (not (file-exists? datadir))
(begin
(debug:print 0 "ERROR: File (" file "), not found at " base-dir "." )
(exit 1)))
(if (directory? datadir)
(begin
(debug:print 0 "ERROR: (" file ") is a dirctory!! cp cmd works only on files ." )
(exit 1)))
(if(not (string-match (regexp allowed-sub-paths) file))
(begin
(debug:print 0 "ERROR: Access denied to file (" file ")!! " )
(exit 1)))
(sretrieve:db-do
configdat
(lambda (db)
(sretrieve:register-action db "cp" retriever datadir comment)))
(sretrieve:do-as-calling-user
;; (debug:print 0 "ph: "(pathname-directory datadir) "!! " )
(change-directory (pathname-directory datadir))
;;(debug:print 0 "ph: /bin/tar" (list "chfv" "-" filename) )
(process-execute "/bin/tar" (list "chfv" "-" filename)))
))
;; ls in file to dest, validation is done BEFORE calling this
;;
(define (sretrieve:ls configdat retriever file comment)
(let* ((base-dir (configf:lookup configdat "settings" "base-dir"))
(allowed-sub-paths (configf:lookup configdat "settings" "allowed-sub-paths"))
(datadir (conc base-dir "/" file))
(filename (conc(pathname-file datadir) "." (pathname-extension datadir))))
(if (or (not base-dir)
(not (file-exists? base-dir)))
(begin
(debug:print 0 "ERROR: Bad configuration! base-dir " base-dir " not found")
(exit 1)))
(print datadir)
(if (not (file-exists? datadir))
(begin
(debug:print 0 "ERROR: File (" file "), not found at " base-dir "." )
(exit 1)))
(if(not (string-match (regexp allowed-sub-paths) file))
(begin
(debug:print 0 "ERROR: Access denied to file (" file ")!! " )
(exit 1)))
(sretrieve:do-as-calling-user
(lambda ()
;;(change-directory datadir)
;; (debug:print 0 "/usr/bin/find" (list datadir "-ls" "|" "grep" "-E" "'"allowed-file-patt"'"))
;; (status (with-input-from-pipe "find " datadir " -ls | grep -E '" allowed-file-patt "'" (lambda () (read-line))))
;; (debug:print 0 status)
(process-execute "/bin/ls" (list "-ls" "-lrt" datadir ))
))))
;;(filter (lambda (x)
;; (not (member x '("." ".."))))
;; (glob "*" ".*"))))))))
(define (sretrieve:validate target-dir targ-mk)
(let* ((normal-path (normalize-pathname targ-mk))
|
| ︙ | ︙ | |||
374 375 376 377 378 379 380 381 382 383 384 385 386 387 |
(make-hash-table))))
(pop-directory)
res)))
(define (sretrieve:process-action configdat action . args)
(let* ((base-dir (configf:lookup configdat "settings" "base-dir"))
(user (current-user-name))
(allowed-users (string-split
(or (configf:lookup configdat "settings" "allowed-users")
"")))
(default-area (configf:lookup configdat "settings" "default-area"))) ;; otherwise known as the package
(if (not base-dir)
(begin
| > | 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 |
(make-hash-table))))
(pop-directory)
res)))
(define (sretrieve:process-action configdat action . args)
(let* ((base-dir (configf:lookup configdat "settings" "base-dir"))
(user (current-user-name))
(allowed-sub-paths (configf:lookup configdat "settings" "allowed-sub-paths"))
(allowed-users (string-split
(or (configf:lookup configdat "settings" "allowed-users")
"")))
(default-area (configf:lookup configdat "settings" "default-area"))) ;; otherwise known as the package
(if (not base-dir)
(begin
|
| ︙ | ︙ | |||
407 408 409 410 411 412 413 414 415 416 417 418 419 420 |
(package-type (or (args:get-arg "-package")
default-area))
(exe-dir (configf:lookup configdat "exe-info" "exe-dir")))
;; (relconfig (sretrieve:load-packages configdat exe-dir package-type)))
(debug:print 0 "retrieving " version " of " package-type " as tar data on stdout")
(sretrieve:get configdat user version msg)))
(else (debug:print 0 "Unrecognised command " action)))))
;; ease debugging by loading ~/.dashboardrc - REMOVE FROM PRODUCTION!
;; (let ((debugcontrolf (conc (get-environment-variable "HOME") "/.sretrieverc")))
;; (if (file-exists? debugcontrolf)
;; (load debugcontrolf)))
| > > > > > > > > > > > > > > > > > > > > > > > | 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 |
(package-type (or (args:get-arg "-package")
default-area))
(exe-dir (configf:lookup configdat "exe-info" "exe-dir")))
;; (relconfig (sretrieve:load-packages configdat exe-dir package-type)))
(debug:print 0 "retrieving " version " of " package-type " as tar data on stdout")
(sretrieve:get configdat user version msg)))
((cp)
(if (< (length args) 1)
(begin
(debug:print 0 "ERROR: Missing arguments; " (string-intersperse args ", "))
(exit 1)))
(let* ((remargs (args:get-args args '("-m" "-i" "-package") '() args:arg-hash 0))
(file (car args))
(msg (or (args:get-arg "-m") "")) )
(debug:print 0 "copinging " file " to current directory " )
(sretrieve:cp configdat user file msg)))
((ls)
(if (< (length args) 1)
(begin
(debug:print 0 "ERROR: Missing arguments; " (string-intersperse args ", "))
(exit 1)))
(let* ((remargs (args:get-args args '("-m" "-i" "-package") '() args:arg-hash 0))
(dir (car args))
(msg (or (args:get-arg "-m") "")) )
(debug:print 0 "Listing files in " )
(sretrieve:ls configdat user dir msg)))
(else (debug:print 0 "Unrecognised command " action)))))
;; ease debugging by loading ~/.dashboardrc - REMOVE FROM PRODUCTION!
;; (let ((debugcontrolf (conc (get-environment-variable "HOME") "/.sretrieverc")))
;; (if (file-exists? debugcontrolf)
;; (load debugcontrolf)))
|
| ︙ | ︙ | |||
440 441 442 443 444 445 446 |
((ls)
(let* ((base-dir (configf:lookup configdat "settings" "base-dir")))
(if base-dir
(begin
(print "Files in " base-dir)
(sretrieve:do-as-calling-user
(lambda ()
| | | 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 |
((ls)
(let* ((base-dir (configf:lookup configdat "settings" "base-dir")))
(if base-dir
(begin
(print "Files in " base-dir)
(sretrieve:do-as-calling-user
(lambda ()
(process-execute "/bin/ls" (list "-lrt" base-dir)))))
(print "ERROR: No base dir specified!"))))
((log)
(sretrieve:db-do configdat (lambda (db)
(print "Logs : ")
(query (for-each-row
(lambda (row)
(apply print (intersperse row " | "))))
|
| ︙ | ︙ |