Megatest

Check-in [706be62179]
Login
Overview
Comment:sretrive added restrictions
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | pjhatwal
Files: files | file ages | folders
SHA1: 706be621797dca6e1d4dc989e7dff75a75972eb0
User & Date: pjhatwal on 2016-07-25 16:34:24
Other Links: branch diff | manifest | tags
Context
2016-08-12
14:39
added sauthorize check-in: 05e4a3078e user: pjhatwal tags: pjhatwal
2016-07-25
16:34
sretrive added restrictions check-in: 706be62179 user: pjhatwal tags: pjhatwal
2016-07-11
12:41
updated sretrive to use shell like setup check-in: f5e2e2d8d1 user: pjhatwal tags: pjhatwal
Changes

Modified Makefile from [1879ee0391] to [b25e0f3a2e].

220
221
222
223
224
225
226
227
228
229
230
231
232
233
234

datashare-testing/sretrieve : sretrieve.scm megatest-version.o margs.o configf.o process.o 
	csc sretrieve.scm megatest-version.o margs.o configf.o process.o -o datashare-testing/sretrieve

sretrieve/sretrieve : datashare-testing/sretrieve
	csc -deploy -deployed sretrieve.scm megatest-version.o margs.o configf.o
	chicken-install -keep-installed $(PROXY) -deploy -prefix sretrieve defstruct srfi-18 format sql-de-lite \
             srfi-1 posix regex regex-case srfi-69

# base64 dot-locking \
#             csv-xml z3

#  "(define (toplevel-command . a) #f)"
# if egrep 'version.*3.0' $(shell dirname $(shell dirname $(shell which csi)))/lib/chicken/7/readline.setup-info;then \








|







220
221
222
223
224
225
226
227
228
229
230
231
232
233
234

datashare-testing/sretrieve : sretrieve.scm megatest-version.o margs.o configf.o process.o 
	csc sretrieve.scm megatest-version.o margs.o configf.o process.o -o datashare-testing/sretrieve

sretrieve/sretrieve : datashare-testing/sretrieve
	csc -deploy -deployed sretrieve.scm megatest-version.o margs.o configf.o
	chicken-install -keep-installed $(PROXY) -deploy -prefix sretrieve defstruct srfi-18 format sql-de-lite \
             srfi-1 posix regex regex-case srfi-69 

# base64 dot-locking \
#             csv-xml z3

#  "(define (toplevel-command . a) #f)"
# if egrep 'version.*3.0' $(shell dirname $(shell dirname $(shell which csi)))/lib/chicken/7/readline.setup-info;then \

Modified datashare-testing/.sretrieve.config from [71cb2ce9dc] to [0866f639ac].

1
2
3
4
5
6
7
8
[settings]
base-dir      /tmp/delme_data
allowed-users matt
allowed-chars [0-9a-zA-Z\-\.]+
allowed-sub-paths [0-9a-zA-Z\-\.]+
[database]
location #{scheme (create-directory "/tmp/#{getenv USER}" #t)}



|





1
2
3
4
5
6
7
8
[settings]
base-dir      /tmp/delme_data
allowed-users matt 
allowed-chars [0-9a-zA-Z\-\.]+
allowed-sub-paths [0-9a-zA-Z\-\.]+
[database]
location #{scheme (create-directory "/tmp/#{getenv USER}" #t)}

Modified sretrieve.scm from [a29b863a93] to [29750f33f2].

1
2
3
4
5
6
7
8
9
10
11

12
13
14
15
16
17
18

;; Copyright 2006-2013, Matthew Welland.
;; 
;;  This program is made available under the GNU GPL version 2.0 or
;;  greater. See the accompanying file COPYING for details.
;; 
;;  This program is distributed WITHOUT ANY WARRANTY; without even the
;;  implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
;;  PURPOSE.

(use defstruct)


;; (use ssax)
;; (use sxml-serializer)
;; (use sxml-modifications)
;; (use regex)
;; (use srfi-69)
;; (use regex-case)











>







1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19

;; Copyright 2006-2013, Matthew Welland.
;; 
;;  This program is made available under the GNU GPL version 2.0 or
;;  greater. See the accompanying file COPYING for details.
;; 
;;  This program is distributed WITHOUT ANY WARRANTY; without even the
;;  implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
;;  PURPOSE.

(use defstruct)
(use scsh-process)

;; (use ssax)
;; (use sxml-serializer)
;; (use sxml-modifications)
;; (use regex)
;; (use srfi-69)
;; (use regex-case)
27
28
29
30
31
32
33


34
35
36
37
38
39
40
(use refdb)
;; (require-library ini-file)
;; (import (prefix ini-file ini:))

(use sql-de-lite srfi-1 posix regex regex-case srfi-69)
;; (import (prefix sqlite3 sqlite3:))
;; 


(declare (uses configf))
;; (declare (uses tree))
(declare (uses margs))
;; (declare (uses dcommon))
;; (declare (uses launch))
;; (declare (uses gutils))
;; (declare (uses db))







>
>







28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
(use refdb)
;; (require-library ini-file)
;; (import (prefix ini-file ini:))

(use sql-de-lite srfi-1 posix regex regex-case srfi-69)
;; (import (prefix sqlite3 sqlite3:))
;; 
(declare (uses common))

(declare (uses configf))
;; (declare (uses tree))
(declare (uses margs))
;; (declare (uses dcommon))
;; (declare (uses launch))
;; (declare (uses gutils))
;; (declare (uses db))
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
          bundle       TEXT NOT NULL,
          release      TEXT NOT NULL,
          status       TEXT NOT NULL,
          event_date   TEXT NOT NULL);"
    )))

(define (sretrieve:register-action db action submitter source-path comment)
  (print "(sretrieve:register-action db " db " action " action " submitter " submitter " source-path " source-path " comment " comment)
  (exec (sql db "INSERT INTO actions (action,retriever,srcpath,comment)
                 VALUES(?,?,?,?)")
	action
	submitter
	source-path
	(or comment "")))








|







97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
          bundle       TEXT NOT NULL,
          release      TEXT NOT NULL,
          status       TEXT NOT NULL,
          event_date   TEXT NOT NULL);"
    )))

(define (sretrieve:register-action db action submitter source-path comment)
 ; (print "(sretrieve:register-action db " db " action " action " submitter " submitter " source-path " source-path " comment " comment)
  (exec (sql db "INSERT INTO actions (action,retriever,srcpath,comment)
                 VALUES(?,?,?,?)")
	action
	submitter
	source-path
	(or comment "")))

316
317
318
319
320
321
322






























323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
;;======================================================================
;; SHELL
;;======================================================================


(define *refdb*     "/p/foundry/env/pkgs/chicken/4.10.0_ext/bin/refdb") 
(define *refdbloc*     "/nfs/site/disks/ch_ciaf_disk023/fdk_gwa_disk003/pjhatwal/fossil/megatest1.60/megatest/datashare-testing/sretrieve_configs") 































;; function to find sheets to which use has access 
(define (sretrieve:has-permission sheet configfile)
  (let* ((users (get-rowcol-names  configfile sheet car))
          (retuser ""))
    (if (member (current-user-name) users)
        #t
        #f))) 
    
 ;; function  to check if user is trying to access a restricted area
 
(define (sretrieve:is-permitted-area dir allowed-list)
    (for-each 
        (lambda (allowed-dir)
          
         (if (equal? dir allowed-dir)
          allowed-dir))
        (cdr allowed-list)))

;; function to validate the users input for target path and resolve the path
;; TODO: Check for restriction in subpath 
(define (sretrieve:resolve-path  new current allowed-sheets)







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>














<







319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369

370
371
372
373
374
375
376
;;======================================================================
;; SHELL
;;======================================================================


(define *refdb*     "/p/foundry/env/pkgs/chicken/4.10.0_ext/bin/refdb") 
(define *refdbloc*     "/nfs/site/disks/ch_ciaf_disk023/fdk_gwa_disk003/pjhatwal/fossil/megatest1.60/megatest/datashare-testing/sretrieve_configs") 

;; Create the sqlite db for shell
(define (sretrieve:shell-db-do path proc) 
    (if (not path)
	(begin
	  (debug:print 0 "[database]\nlocation /some/path\n\n Is missing from the config file!")
	  (exit 1)))
    (if (and path
	     (directory? path)
	     (file-read-access? path))
	(let* ((dbpath    (conc path "/" *exe-name* ".db"))
	       (writeable (file-write-access? dbpath))
	       (dbexists  (file-exists? dbpath)))
	  (handle-exceptions
	   exn
	   (begin
	     (debug:print 2 "ERROR: problem accessing db " dbpath
			  ((condition-property-accessor 'exn 'message) exn))
	     (exit 1))
            ;;(debug:print 0 "calling proc " proc "db path " dbpath )
	   (call-with-database
            dbpath
	    (lambda (db)
	       ;;(debug:print 0 "calling proc " proc " on db " db)
	      (set-busy-handler! db (busy-timeout 10000)) ;; 10 sec timeout
	      (if (not dbexists)(sretrieve:initialize-db db))
	      (proc db)))))
	(debug:print 0 "ERROR: invalid path for storing database: " path)))



;; function to find sheets to which use has access 
(define (sretrieve:has-permission sheet configfile)
  (let* ((users (get-rowcol-names  configfile sheet car))
          (retuser ""))
    (if (member (current-user-name) users)
        #t
        #f))) 
    
 ;; function  to check if user is trying to access a restricted area
 
(define (sretrieve:is-permitted-area dir allowed-list)
    (for-each 
        (lambda (allowed-dir)

         (if (equal? dir allowed-dir)
          allowed-dir))
        (cdr allowed-list)))

;; function to validate the users input for target path and resolve the path
;; TODO: Check for restriction in subpath 
(define (sretrieve:resolve-path  new current allowed-sheets)
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411



412

413




414







415
416
417
418
419
420







421




422
423








424













425
426
















427







428
429
































430
431
432


433
434

435




436
437
438
439
440
441
442
443
444
445
446
447
448

449
450




451
452











453
454
455
456
457
458
459
460
461
462
463

464
465

466
467











468
469



470


471
472
473
474
475
476
477
478

479
480
481

482
483
484
485
486
487
488
489
490
491
492
493


494






495






496
497

498
499
500
501
502

503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531


532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547




548
549
550
551
552
553
554
555
556
557





558






559













560

561
562
563
564
565
566
567
568
569
570
571



572
573
574
575
576
577


578
579
580
581
582
583
584
585
586
587
588

589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
 (if (null? sheets)
      #f
      (let loop ((hed (car sheets))
		 (tal (cdr sheets))
                  (res '()))
        (let* ((user (sretrieve:has-permission hed configfile))
              (access-valid (sretrieve:is-access-valid  hed configfile)))
        ;;(print "access-valid " access-valid)
	(if (and (equal? user #t ) (equal? access-valid #t))
               (begin  
               ;;(print "got perm " (sretrieve:has-permission hed configfile)) 
               (if (null? tal)
		 (cons hed res)
		 (loop (car tal)(cdr tal)(cons hed res))))
                (if (null? tal)
                     res
                   (loop (car tal)(cdr tal) res)))))))

(define (sretrieve:shell-ls-cmd base-path-list ext-path top-areas configfile)
  (if (and (null? base-path-list) (equal? ext-path "") )
      (print (string-intersperse top-areas " "))
  (let* ((resolved-path (sretrieve:resolve-path ext-path base-path-list top-areas )))
           (if (not (equal? resolved-path #f))
           (if (null? resolved-path) 
             (print (string-intersperse top-areas " "))
           (let* (
                    ;(sheet (car resolved-path))
        	   ;(base-path (lookup configfile sheet (current-user-name) "base path")) 
           	   ;(target-path (conc base-path "/" (string-join (cdr resolved-path) "/")))
                    (target-path (sretrieve:get-target-path  base-path-list  ext-path top-areas configfile)))
                (print "Resolved path: " target-path)   



                (sretrieve:do-as-calling-user

                (lambda ()




	            (system (conc "ls " target-path))))))))))








(define (sretrieve:shell-cat-cmd base-path-list ext-path top-areas configfile)
  (let* ((resolved-path (sretrieve:resolve-path ext-path base-path-list top-areas ))
          (data "") )
         (if (not (equal? resolved-path #f))
           (if (null? resolved-path) 







             data




           (let* (
;(sheet (car resolved-path))








 ;       	   (base-path (lookup configfile sheet (current-user-name) "base path"))













                   (target-path (sretrieve:get-target-path  base-path-list  ext-path top-areas configfile)))
           	   
















                







                (sretrieve:do-as-calling-user
                (lambda ()
































                     (if (or (not (file-exists? target-path)) (directory? target-path))
                         (begin
                            (print "Target path does not exist or is a directory!") 


                          data)  
                        (set! data (with-input-from-pipe (conc "cat " target-path) (lambda () (read-all)))))))

                    data))




             data)))

(define (sretrieve:get-target-path base-path-list ext-path top-areas configfile)
  (let* ((resolved-path (sretrieve:resolve-path ext-path base-path-list top-areas ))
          (usr (current-user-name) ) )
         ;;(print resolved-path)  
         (if (not (equal? resolved-path #f))
           (if (null? resolved-path) 
             #f
           (let* ((sheet (car resolved-path))
                   (fname  (conc  configfile "/" sheet ".dat"))
                   (config-data (sretrieve:load-shell-config fname))  
        	   (base-path (configf:lookup config-data "basepath" usr))

           	   (target-path (conc base-path "/" (string-join (cdr resolved-path) "/"))))
                      




                                        target-path))
             #f)))












(define (sretrieve:load-shell-config fname)
          (if (file-exists? fname)
	(read-config fname #f #f)
	))


(define (is_directory target-path) 
  (let* ((retval #f))
  (sretrieve:do-as-calling-user
    	(lambda ()

          (if (directory? target-path)
               (set! retval  #t))))

     retval)) 













(define (sretrieve:get-shell-cmd target-path )



     (use scsh-process)


      (if (is_directory target-path) 
        (begin
           (let* ((parent-dir target-path)
                   (start-dir (current-directory))
                   (files (filter (lambda (x)
			(not (member x '("." ".."))))
		      (glob "*" ".*"))))
                          (change-directory parent-dir)  

              (run (pipe
      		   (tar  "chfv" "-" ".")
          	   (begin(system (conc "cd " start-dir ";tar xf -"))))))) 

        (begin
           (let*((parent-dir (pathname-directory target-path))
                  (start-dir (current-directory))
                (filename (if  (pathname-extension target-path)  
                                      (conc(pathname-file target-path) "." (pathname-extension target-path))
                                      (pathname-file target-path))))
               
               (change-directory parent-dir)  
               (run (pipe
                 (tar "chfv" "-" ,filename)
                 (begin (system (conc "cd " start-dir ";tar xf -")))))
))))









	






(define (toplevel-command . args) #f)
(define (sretrieve:shell)

  (use readline)
  (let* ((path      '())
	 (prompt    "> ")
	 ;(top-areas '("mrwellan" "pjhatwal" "bjbarcla" "ritikaag" "jmoon18"))
         (args      (argv))

	 (prog      (car args)) 
         (exe-name  (pathname-file (car (argv))))
	 (exe-dir   (or (pathname-directory prog)
			(sretrieve:find exe-name (string-split (get-environment-variable "PATH") ":"))))
         (config-file  (conc exe-dir "/sretrieve_configs"))  
         (config-data  (configf:read-refdb config-file))
         (sheets (list-sheets config-file)) 
         (top-areas (sretrieve:get-accessable-projects sheets config-file))
         (old-port (current-input-port)) 
          (close-port     #f)     
	 (iport     (make-readline-port prompt)))
  ;  (install-history-file) ;;  [homedir] [filename] [nlines])
  ;  (with-input-from-port iport
  ;    (lambda ()
	(let loop ((inl (read-line iport)))
	  (if (not (or (or (eof-object? inl)
		       (equal? inl "exit")) (port-closed? iport)))
	      (let* ((parts (string-split inl))
		     (cmd   (if (null? parts) #f (car parts))))
		(if (and (not cmd) (not (port-closed? iport)))
		    (loop (read-line))
		    (case (string->symbol cmd)
		      ((cd)
		       (if (> (length parts) 1) ;; have a parameter
                           (begin
                             (let*((arg (cadr parts))
                                   (resolved-path (sretrieve:resolve-path  arg path top-areas)))
                                  
                                 (if (not (equal? resolved-path #f))    


			         (set! path resolved-path))))  
   			   (set! path '())))
                      ((pwd)
                         (if (null? path)
                           (print "/")  
                           (print "/" (string-join path "/")))) 
		      ((ls)
		       (let* ((thepath (if (> (length parts) 1) ;; have a parameter
					   (cdr parts)
					   `()))
			      (plen    (length thepath)))
			 (cond
			  ((null? thepath)
                           (sretrieve:shell-ls-cmd path "" top-areas config-file))
			  ((< plen 2)
                           (sretrieve:shell-ls-cmd path  (car thepath) top-areas config-file)))))




                       ((cat)
		       (let* ((thepath (if (> (length parts) 1) ;; have a parameter
					   (cdr parts)
					   `()))
			      (plen    (length thepath)))
			 (cond
			  ((null? thepath)
                          (print "Error: Missing argument to cat"))
			  ((< plen 2)
                           (let* ((data (sretrieve:shell-cat-cmd path  (car thepath) top-areas config-file)))





                           (print data)))






			  (else













                            (let* ((data (sretrieve:shell-cat-cmd path  (car thepath) top-areas config-file)))

                             (system (conc "echo '" data "' " (string-join (cdr thepath)))))))))
                      ((get)
                         (let* ((thepath (if (> (length parts) 1) ;; have a parameter
					   (cdr parts)
					   `()))
			      (plen    (length thepath)))
			 (cond
			  ((null? thepath)
                          (print "Error: Missing argument <path> to get"))
			  ((< plen 2)
                           (let* ((target-path (sretrieve:get-target-path path  (car thepath) top-areas config-file)))



                                    
                                  (sretrieve:get-shell-cmd target-path )
                                  ;;(print path)
                           ))
			  (else
                            (print "Error: get cmd takes only one argument ")))))    


		      (else 
		       (print "Got command: " inl)
                       )))
                 (loop (read-line iport))
                )))))
;;))
    

;;======================================================================
;; MAIN
;;======================================================================


(define (sretrieve:load-config exe-dir exe-name)
  (let* ((fname   (conc exe-dir "/." exe-name ".config")))
    ;; (ini:property-separator-patt " *  *")
    ;; (ini:property-separator #\space)
    (if (file-exists? fname)
	;; (ini:read-ini fname)
	(read-config fname #f #t)
	(make-hash-table))))

;; package-type is "megatest", "builds", "kits" etc.
;;

(define (sretrieve:load-packages configdat exe-dir package-type)
  (push-directory exe-dir)







<
|









|






<
<
<
<
|
|
>
>
>
|
>
|
>
>
>
>
|
>
>
>
>
>
>
>

|
|



>
>
>
>
>
>
>
|
>
>
>
>
|
|
>
>
>
>
>
>
>
>
|
>
>
>
>
>
>
>
>
>
>
>
>
>
|
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
|
>
>
>
>
>
>
>
|
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
|
<
|
>
>
|
|
>
|
>
>
>
>
|




<
|






>

|
>
>
>
>
|

>
>
>
>
>
>
>
>
>
>
>











>


>


>
>
>
>
>
>
>
>
>
>
>

|
>
>
>
|
>
>
|


|
<
<
|
|
>
|
|
|
>






<

|
|
|
|
>
>

>
>
>
>
>
>
|
>
>
>
>
>
>


>


|
<
|
>





|


|

















|
|
|
>
>
|












|

|
>
>
>
>









|
>
>
>
>
>
|
>
>
>
>
>
>

>
>
>
>
>
>
>
>
>
>
>
>
>
|
>
|









|
>
>
>

|



|
>
>











>







|







413
414
415
416
417
418
419

420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436




437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559

560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575

576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642


643
644
645
646
647
648
649
650
651
652
653
654
655

656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682

683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
 (if (null? sheets)
      #f
      (let loop ((hed (car sheets))
		 (tal (cdr sheets))
                  (res '()))
        (let* ((user (sretrieve:has-permission hed configfile))
              (access-valid (sretrieve:is-access-valid  hed configfile)))

        (if (and (equal? user #t ) (equal? access-valid #t))
               (begin  
               ;;(print "got perm " (sretrieve:has-permission hed configfile)) 
               (if (null? tal)
		 (cons hed res)
		 (loop (car tal)(cdr tal)(cons hed res))))
                (if (null? tal)
                     res
                   (loop (car tal)(cdr tal) res)))))))

(define (sretrieve:shell-ls-cmd base-path-list ext-path top-areas configfile db-location tail-cmd-list)
  (if (and (null? base-path-list) (equal? ext-path "") )
      (print (string-intersperse top-areas " "))
  (let* ((resolved-path (sretrieve:resolve-path ext-path base-path-list top-areas )))
           (if (not (equal? resolved-path #f))
           (if (null? resolved-path) 
             (print (string-intersperse top-areas " "))




           (let* ((target-path (sretrieve:get-target-path  base-path-list  ext-path top-areas configfile)))
                ;(print "Resolved path: " target-path)
                (if (not (equal? target-path #f))
                (begin 
                 
                (sretrieve:shell-db-do
         	 db-location
     		(lambda (db)
       		(sretrieve:register-action db "ls" (current-user-name) target-path (conc "Executing cmd: ls "  target-path))))
                (cond
		  ((null? tail-cmd-list)
		     (run (pipe
      	      	      (ls "-lrt" ,target-path))))
		  ((not (equal? (car tail-cmd-list) "|"))
                         (print "ls cmd cannot accept "(string-join tail-cmd-list) " as an argument!!"))
                  (else  
                    (run (pipe
      	      	      (ls "-lrt" ,target-path)
                      (begin (system (string-join (cdr tail-cmd-list))))))
      ))))))))))

(define (sretrieve:shell-cat-cmd base-pathlist ext-path top-areas configfile db-location tail-cmd-list)
  (let* ((resolved-path (sretrieve:resolve-path ext-path base-pathlist top-areas ))
          (data "") )
         (if (not (equal? resolved-path #f))
           (if (null? resolved-path) 
             (print "Path could not be resolved!!")
           (let* ((target-path (sretrieve:get-target-path  base-pathlist  ext-path top-areas configfile)))
             (if (not (equal? target-path #f))
               (if (or (not (file-exists? target-path)) (directory? target-path))
               (print "Target path does not exist or is a directory!") 
               (begin 
                 (sretrieve:shell-db-do
         	    db-location
     		    (lambda (db)
       		      (sretrieve:register-action db "cat" (current-user-name) target-path (conc "Executing cmd: cat " target-path))))   
		 (cond
		  ((null? tail-cmd-list)
		     (run (pipe
      	      	      (cat  ,target-path))))
		  ((not (equal? (car tail-cmd-list) "|"))
                         (print "cat cmd cannot accept "(string-join tail-cmd-list) " as an argument!!"))
                  (else  
                    (run (pipe
      	      	      (cat ,target-path)
                      (begin (system (string-join (cdr tail-cmd-list))))))))))
)))
             (print "Path could not be resolved!!"))))

(define (get-options cmd-list split-str)
    (if (null? cmd-list)
              (list '() '())
    (let loop ((hed (car cmd-list))
		 (tal (cdr cmd-list))
                  (res '()))
                   (cond
                   ((equal? hed split-str)
                      (list res tal))
                   ((null? tal)
                      (list (cons hed res) tal)) 
                   (else 
		  	(loop (car tal)(cdr tal)(cons hed res)))))))

  
(define (sretrieve:shell-grep-cmd base-pathlist ext-path top-areas configfile db-location tail-cmd-list)
  (let* ((resolved-path (sretrieve:resolve-path ext-path base-pathlist top-areas ))
          (pattern (car tail-cmd-list))
          (pipe-cmd-list (get-options (cdr tail-cmd-list) "|"))
          (options (string-join (car pipe-cmd-list)))
          (pipe-cmd (cadr  pipe-cmd-list))
          (redirect-split (string-split (string-join  tail-cmd-list) ">"))   )
         (if(and ( > (length redirect-split) 2 ))
            (print "grep  cmd cannot accept > " (string-join redirect-split) " as an argument!!" )  
         (if (not (equal? resolved-path #f))
           (if (null? resolved-path) 
             (print "Path could not be resolved!!")
           (let* ((target-path (sretrieve:get-target-path  base-pathlist  ext-path top-areas configfile))
                  (restrictions (if (equal? target-path #f)
                                                 ""
                                               (sretrieve:shell-lookup base-pathlist ext-path  top-areas configfile)))
                 (rest-str  (string-split (conc " --exclude-dir=" (string-join (string-split restrictions ",") " --exclude-dir=") ))))
             (if (not (file-exists? target-path))
               (print "Target path does not exist!") 
               (begin 
                 (sretrieve:shell-db-do
         	    db-location
     		    (lambda (db)
       		      (sretrieve:register-action db "grep" (current-user-name) target-path (conc "Executing cmd: grep " target-path pattern (string-join tail-cmd-list) )))) 
              ; (sretrieve:do-as-calling-user
    	      ;    (lambda ()
              ;     (if (null? pipe-cmd)
              ;     (process-execute "/usr/bin/grep" (append (list options pattern target-path)  rest-str))
              ;     (process-execute "/usr/bin/grep" (append (append (list options pattern target-path) rest-str) (append (list "|") pipe-cmd))))))
               ;  (print rest-str) 
                 (cond
	 	  ((and (null? pipe-cmd) (string-null? options))
	 	     (run (pipe
      	      	      (grep ,pattern ,target-path ))))
		  ((and (null? pipe-cmd) (not (string-null? options)))
		     (run (pipe
      	      	      (begin (process-execute "/usr/bin/grep" (append (list options pattern target-path) rest-str))))))
                  ((and (not (null? pipe-cmd)) (string-null? options))
                        (run (pipe
      	      	      (grep ,exclude-dir ,pattern ,target-path)
                      (begin (system (string-join pipe-cmd))))))
                  (else   
                      (run (pipe
      	      	      ;(grep  ,options ,exclude-dir ,pattern ,target-path)
                       (begin (process-execute "/usr/bin/grep" (append (list options pattern target-path) rest-str)))

                      (begin (system (string-join  pipe-cmd)))))))
))))
             (print "Path could not be resolved!!")))))


(define (sretrieve:shell-less-cmd base-pathlist ext-path top-areas configfile db-location)
  (let* ((resolved-path (sretrieve:resolve-path ext-path base-pathlist top-areas )))
         (if (not (equal? resolved-path #f))
           (if (null? resolved-path) 
             (print "Path could not be resolved!!")
           (let* ((target-path (sretrieve:get-target-path  base-pathlist  ext-path top-areas configfile)))
               (if (not (equal? target-path #f))
           	 (if (or (not (file-exists? target-path)) (directory? target-path))

                   (print "Target path does not exist or is a directory!") 
                   (begin 
                     (sretrieve:shell-db-do
         	          db-location
     			  (lambda (db)
       				(sretrieve:register-action db "less" (current-user-name) target-path (conc "Executing cmd: less "  target-path))))   
 
                     (setenv "LESSSECURE" "1")
                     (run (pipe
      	      	     (less ,target-path))))))))
             (print "Path could not be resolved!!"))))


(define (sretrieve:get-target-path base-path-list ext-path top-areas configfile)
  (let* ((resolved-path (sretrieve:resolve-path ext-path base-path-list top-areas ))
          (usr (current-user-name) ) )

          (if (not (equal? resolved-path #f))
           (if (null? resolved-path) 
             #f
           (let* ((sheet (car resolved-path))
                   (fname  (conc  configfile "/" sheet ".dat"))
                   (config-data (sretrieve:load-shell-config fname))  
        	   (base-path (configf:lookup config-data "basepath" usr))
                   (restrictions (conc ".*" (string-join (string-split (configf:lookup config-data "restricted areas" usr) ",") ".*|.*") ".*"))
           	   (target-path (conc base-path "/" (string-join (cdr resolved-path) "/"))))
                    
                    (if (string-match (regexp  restrictions) target-path) 
                        (begin
                          (print "Access denied to " (string-join resolved-path "/"))   
                         #f)
                                        target-path)))
             #f)))

(define (sretrieve:shell-lookup base-path-list ext-path top-areas configfile)
  (let* ((resolved-path (sretrieve:resolve-path ext-path base-path-list top-areas ))
          (usr (current-user-name))
          (sheet (car resolved-path))
          (fname  (conc  configfile "/" sheet ".dat"))
          (config-data (sretrieve:load-shell-config fname))  
          (base-path (configf:lookup config-data "basepath" usr))
          (value (configf:lookup config-data "restricted areas" usr)))
                    value))
             

(define (sretrieve:load-shell-config fname)
          (if (file-exists? fname)
	(read-config fname #f #f)
	))


(define (is_directory target-path) 
  (let* ((retval #f))
  (sretrieve:do-as-calling-user
    	(lambda ()
          ;(print (current-effective-user-id) ) 
          (if (directory? target-path)
               (set! retval  #t))))
             ;(print (current-effective-user-id))
     retval)) 

(define (make-exclude-pattern  restriction-list  )
  (if (null? restriction-list)
     ""
   (let loop ((hed (car restriction-list))
		 (tal (cdr restriction-list))
                  (ret-str ""))
                   (cond
                   ((null? tal)
                      (conc ret-str " --exclude='*" hed "*'")) 
                   (else 
		  	(loop (car tal)(cdr tal)(conc ret-str " --exclude='*" hed "*'"))))))    )

(define (sretrieve:get-shell-cmd target-path db-location restrictions)
    (if (not (equal? target-path #f))
    (begin     
    (sretrieve:shell-db-do
    	  db-location
     	  (lambda (db)
       		(sretrieve:register-action db "get" (current-user-name) target-path (conc "Executing cmd: get "  target-path))))
     (if (is_directory target-path) 
        (begin
           (let* ((parent-dir target-path)
                  (start-dir (current-directory))


                  (execlude (make-exclude-pattern (string-split restrictions ","))))
                  (change-directory parent-dir)
                                    
                 (run (pipe
                   (tar "chfv" "-" "." )
                   (begin (system (conc "cd " start-dir ";tar  xf - "   execlude )))))
                 )) 
        (begin
           (let*((parent-dir (pathname-directory target-path))
                  (start-dir (current-directory))
                (filename (if  (pathname-extension target-path)  
                                      (conc(pathname-file target-path) "." (pathname-extension target-path))
                                      (pathname-file target-path))))

               (change-directory parent-dir)  
                 (run (pipe
                   (tar "chfv" "-" ,filename)
                   (begin (system (conc "cd " start-dir ";tar xf -")))))))))))

(define (sretrieve:shell-help)
(conc "Usage: " *exe-name* " [action [params ...]]

  ls    [target path]               	  : list contents of target area. The output of the cmd can be piped into other system cmd. eg ls <path> | grep txt
  cd    <target directory>	     	  : To change the current directory within the sretrive shell. 
  pwd				     	  : Prints the full pathname of the current directory within the sretrive shell.
  get   <file or directory path>     	  : download directory/files into the directory where sretrieve shell cmd was invoked   
  less  <file path>		     	  : Read input file to allows backward movement in the file as well as forward movement 
  cat   <file path>                  	  : show the contents of a file. The output of the cmd can be piped into other system cmd.

  sgrep <search path> <pattern> [options] : Similar to unix grep cmd But with diffrent parameter ordering. The output of the cmd can be piped into other system cmd. 
Part of the Megatest tool suite.
Learn more at http://www.kiatoa.com/fossils/megatest

Version: " megatest-fossil-hash)
)	
(define (toplevel-command . args) #f)
(define (sretrieve:shell)
 ;; (print (current-effective-user-id))
  (use readline)
  (let* ((path      '())
	 (prompt    "sretrieve> ")

	 (args      (argv))
         (usr (current-user-name) )   
	 (prog      (car args)) 
         (exe-name  (pathname-file (car (argv))))
	 (exe-dir   (or (pathname-directory prog)
			(sretrieve:find exe-name (string-split (get-environment-variable "PATH") ":"))))
         (config-file  (conc exe-dir "/sretrieve_configs"))  
         (db-location (conc exe-dir "/db"))
         (sheets (list-sheets config-file)) 
         (top-areas (sretrieve:get-accessable-projects sheets config-file))
       
          (close-port     #f)     
	 (iport     (make-readline-port prompt)))
  ;  (install-history-file) ;;  [homedir] [filename] [nlines])
  ;  (with-input-from-port iport
  ;    (lambda ()
	(let loop ((inl (read-line iport)))
	  (if (not (or (or (eof-object? inl)
		       (equal? inl "exit")) (port-closed? iport)))
	      (let* ((parts (string-split inl))
		     (cmd   (if (null? parts) #f (car parts))))
		(if (and (not cmd) (not (port-closed? iport)))
		    (loop (read-line))
		    (case (string->symbol cmd)
		      ((cd)
		       (if (> (length parts) 1) ;; have a parameter
                           (begin
                             (let*((arg (cadr parts))
                                   (resolved-path (sretrieve:resolve-path  arg path top-areas))
                                   (target-path (sretrieve:get-target-path path  arg top-areas config-file)))
                                 (if (not (equal? target-path #f))
                                 (if (or (equal? resolved-path #f) (not (file-exists? target-path)))    
                                 (print "Invalid argument " arg ".. ")
			         (set! path resolved-path)))))  
   			   (set! path '())))
                      ((pwd)
                         (if (null? path)
                           (print "/")  
                           (print "/" (string-join path "/")))) 
		      ((ls)
		       (let* ((thepath (if (> (length parts) 1) ;; have a parameter
					   (cdr parts)
					   `()))
			      (plen    (length thepath)))
			 (cond
			  ((null? thepath)
                           (sretrieve:shell-ls-cmd path "" top-areas config-file db-location '()))
			  ((< plen 2)
                            (sretrieve:shell-ls-cmd path  (car thepath) top-areas config-file db-location  '()))
                          (else 
                            (if (equal? (car thepath) "|")
                              (sretrieve:shell-ls-cmd path "" top-areas config-file db-location thepath)
                              (sretrieve:shell-ls-cmd path  (car thepath) top-areas config-file db-location (cdr thepath)))))))
                       ((cat)
		       (let* ((thepath (if (> (length parts) 1) ;; have a parameter
					   (cdr parts)
					   `()))
			      (plen    (length thepath)))
			 (cond
			  ((null? thepath)
                          (print "Error: Missing argument to cat"))
			  ((< plen 2)
                            (sretrieve:shell-cat-cmd path  (car thepath) top-areas config-file db-location '()))
			  (else
                             (sretrieve:shell-cat-cmd path  (car thepath) top-areas config-file db-location (cdr thepath))))))
                       ((grep)
		       (let* ((thepath (if (> (length parts) 1) ;; have a parameter
					   (cdr parts)
					   `()))
			      (plen    (length thepath)))
			 (cond
			  ((null? thepath) 
                            (print "Error: Missing arguments to grep!! Useage: grep <search path> <pattern> [options] "))
                          ((< plen  2)
                          (print "Error: Missing arguments to grep!! Useage: grep <search path> <pattern> [options] "))
			  (else
                             (sretrieve:shell-grep-cmd path  (car thepath) top-areas config-file db-location (cdr thepath))))))

                      ((less)
		       (let* ((thepath (if (> (length parts) 1) ;; have a parameter
					   (cdr parts)
					   `()))
			      (plen    (length thepath)))
			 (cond
			  ((null? thepath)
                          (print "Error: Missing argument to cat"))
			  ((< plen 2)
                            (sretrieve:shell-less-cmd path  (car thepath) top-areas config-file db-location))
			  (else
                             ;(sretrieve:shell-cat-cmd path  (car thepath) top-areas config-file))
))))

                      ((get)
                         (let* ((thepath (if (> (length parts) 1) ;; have a parameter
					   (cdr parts)
					   `()))
			      (plen    (length thepath)))
			 (cond
			  ((null? thepath)
                          (print "Error: Missing argument <path> to get"))
			  ((< plen 2)
                           (let* ((target-path (sretrieve:get-target-path path  (car thepath) top-areas config-file))
				 (restrictions (if (equal? target-path #f)
                                                 ""
                                               (sretrieve:shell-lookup path  (car thepath) top-areas config-file))))
                                    
                                  (sretrieve:get-shell-cmd target-path db-location restrictions)
                                  ;;(print path)
                           ))
			  (else
                            (print "Error: get cmd takes only one argument "))))) 
                      ((help)
                          (print (sretrieve:shell-help)))
		      (else 
		       (print "Got command: " inl)
                       )))
                 (loop (read-line iport))
                )))))
;;))
    

;;======================================================================
;; MAIN
;;======================================================================
;;(define *default-log-port* (current-error-port))

(define (sretrieve:load-config exe-dir exe-name)
  (let* ((fname   (conc exe-dir "/." exe-name ".config")))
    ;; (ini:property-separator-patt " *  *")
    ;; (ini:property-separator #\space)
    (if (file-exists? fname)
	;; (ini:read-ini fname)
	(read-config fname #f #f)
	(make-hash-table))))

;; package-type is "megatest", "builds", "kits" etc.
;;

(define (sretrieve:load-packages configdat exe-dir package-type)
  (push-directory exe-dir)
735
736
737
738
739
740
741



     ;; multi-word commands
     ((null? rema)(print sretrieve:help))
     ((>= (length rema) 2)
      (apply sretrieve:process-action configdat (car rema)(cdr rema)))
     (else (debug:print 0 "ERROR: Unrecognised command. Try \"sretrieve help\"")))))

(main)










>
>
>
954
955
956
957
958
959
960
961
962
963
     ;; multi-word commands
     ((null? rema)(print sretrieve:help))
     ((>= (length rema) 2)
      (apply sretrieve:process-action configdat (car rema)(cdr rema)))
     (else (debug:print 0 "ERROR: Unrecognised command. Try \"sretrieve help\"")))))

(main)