1 # $1: dest path
 2 #
 3 # return value:
 4 # 0: not exist
 5 # 1: file exist
 6 # 2: dir exist but empty
 7 # 3: dir exist and not empty
 8 bsttool_query_path()
 9 {
10     local dst=$1
11     local res=$(bst_tool ls "$dst")
12     local line=$(echo "$res" | sed -n '2p')
13     local str=${line/"total size"//}
14     local download="/tmp/bst.tmp"
15     local ret=0
16     if [ "$str" != "$line" ]; then 
17         # normal return
18         ret=$(echo "$line" | awk '{print $NF}')
19         if [ $ret -eq 0 ]; then 
20             ret=2
21         elif [ $ret -gt 1 ]; then 
22             ret=3 # a dir
23         else # size=1
24             # note here we can NOT using type file/dir to determine, 
25             # for example, there is a path /tmp/tmp pointed to a file
26             # when we ls /tmp, it report one item named 'tmp' with type file,
27             # but actually it is a dir.
28             # so here we use another way to determine, that is, download it, 
29             # you can NOT download a dir.
30             echo "dir or file exist, try download to check"
31             rm "$download" > /dev/null 2>&1 # otherwise bst_tool will fail
32             line=$(bst_tool get "$dst" "$download" | sed -n '2p')
33             str=${line/"download success"//}
34             if [ "$str" != "$line" ]; then # string replace changed: download success
35                 # is a file
36                 ret=1
37             else 
38                 # is a dir
39                 ret=3
40             fi
41         fi
42     else
43         # server response error, no such path
44         ret=0
45     fi
46 
47     return $ret
48 }

 1 # $1: dest path
 2 #
 3 # return value
 4 # 0: user allow and delete existing remote file
 5 # 1: user disallow and do nothing
 6 bsttool_duplicate_warning()
 7 {
 8     local resp
 9     echo -n "dest file exist, do you want to cover it? (n/y) "
10     read resp
11     case "$resp" in 
12         "y"|"Y")
13             # remove remote reource first
14             bst_tool delete "$1"
15             return 0
16             ;;
17         *)
18             return 1
19             ;;
20     esac
21 }
22 
23 # $1: local file path
24 # $2: remote file path (optional)
25 bsttool_replace ()
26 {
27     if [ $# -lt 1 ]; then 
28         echo "Usage: bstput src [dst]"
29         return 1
30     fi
31 
32     local src=$1
33     local dst="/"
34     if [ $# -gt 1 ]; then 
35         dst=$2
36     fi
37 
38     # first check dest existence
39     local resp
40     local name=${src##*/} # remove path part, want name only
41     if [ -z "$name" ]; then 
42         name="$src"
43     fi
44 
45     local path=$dst/$name
46     bsttool_query_path "$dst"
47     local res=$?
48     case $res in
49         0)
50             # not exist
51             echo "dir/file not exist, start uploading"
52             ;;
53         1)
54             # file exist
55             bsttool_duplicate_warning "$dst"
56             if [ $? -ne 0 ]; then 
57                 return 1
58             fi
59             ;;
60         2)
61             # empty dir exist
62             echo "dir exist but empty, start uploading"
63             ;;
64         3)
65             # dir exist and not empty
66             # try file again
67             echo "dir exist, try file"
68             bsttool_query_path "$path"
69             res=$?
70             if [ $res -eq 1 ]; then 
71                 # dest file exist, warning...
72                 bsttool_duplicate_warning "$path"
73                 if [ $? -ne 0 ]; then 
74                     return 1
75                 fi
76             fi
77             ;;
78         *)
79             echo "should not reach here !!"
80             return 1
81             ;;
82     esac
83 
84     bst_tool put "$src" "$dst"
85     return 0
86 }

# alias
alias bstput=bsttool_replace
alias bstget='bst_tool get'
alias bstdel='bst_tool delete'
alias bstls='bst_tool ls'

source ~/.bash_bst

 1 # $1: local path
 2 #
 3 # return value
 4 # 0: user allow and delete existing local file
 5 # 1: user disallow and do nothing
 6 bsttool_existence_warning()
 7 {
 8     local resp
 9     echo -n "local file exist, do you want to cover it? (n/y) "
10     read resp
11     case "$resp" in 
12         "y"|"Y")
13             # remove local reource first
14             rm "$1"
15             return 0
16             ;;
17         *)
18             return 1
19             ;;
20     esac
21 }
22 
23 
24 # $1: remote file path
25 # $2: local file path (optional)
26 bsttool_fetch ()
27 {
28     if [ $# -lt 1 ]; then 
29         echo "Usage: bstget remote [local]"
30         return 1
31     fi
32 
33     local remote=$1
34     local name=${remote##*/} # remove path part, want name only
35     if [ -z "$name" ]; then 
36         name="$remote"
37     fi
38 
39     local local="./"
40     if [ $# -gt 1 ]; then 
41         local=$2
42     fi
43 
44     # check dest existence
45     if [ -e "$local" ]; then 
46         if [ -d "$local" ]; then 
47             # for dir, test sub file existence
48             if [ -e "$local/$name" -a ! -d "$local/$name" ]; then 
49                 # dest file exist, warning...
50                 bsttool_existence_warning "$local/$name"
51                 if [ $? -ne 0 ]; then 
52                     return 1
53                 fi
54             fi
55         else 
56         #elif [ -f "$local" ]; then
57             # dest file exist, warning...
58             bsttool_existence_warning "$local"
59             if [ $? -ne 0 ]; then 
60                 return 1
61             fi
62         fi
63     fi
64 
65     bst_tool get "$remote" "$local"
66     return 0
67 }

alias bstget=bsttool_fetch

 1 # $1: remote path
 2 # $2: depth
 3 bsttool_list_recur ()
 4 {
 5     local remote=$1
 6     local depth=$(($2+1))
 7     local res=$(bst_tool ls "$remote")
 8     local line=$(echo "$res" | sed -n '2p')
 9     local str=${line/"total size"//}
10     local ret=0
11     if [ "$str" != "$line" ]; then 
12         # normal return
13         ret=$(echo "$line" | awk '{print $NF}')
14     else
15         # server response error
16         ret=0
17     fi
18 
19     # 6: skip header
20     local max=$(($ret + 6 - 1))
21     local fid=""
22     local size="" # has K/M/G postfix
23     local type=""
24     local name=""
25     echo "$remote [$depth-$ret]:"
26     for n in $(seq 6 $max)
27     do
28         # why sed -n "$np" don't work ?
29         line=$(echo "$res" | sed -n "$n"'p')
30         # row: fid, user, size, timestamp, type, name
31         # if size is empty (for dir, all spaces), using 0 instead
32         # do awk twice to remove redundant space
33         #
34         # add quotation for assignment is important, otherwise we will get errors on bracket (etc) in value
35         eval $(echo "$line" | awk -F'│' '{printf ("%s\t%s\t%s\t%s\n", $2, match($4, /[a-zA-Z0-9.]+/)?$4:"0", $6, $7)}' | awk '{printf ("fid=\"%s\";size=\"%s\";type=\"%s\";name=\"%s\";\n", $1,$2,$3,$4)}')
36         if [ "$type" == "dir" ]; then 
37             # do for sub dirs
38             local path=""
39             if [ ${remote:0-1} == "/" ]; then 
40                 path="$remote$name"
41             else
42                 path="$remote/$name"
43             fi
44 
45             bsttool_list_recur "$path" $depth
46         else
47             echo -e "$fid:\t$size,\t$name"
48         fi
49     done
50 
51     return 0
52 }
53 
54 # $1: remote path
55 bsttool_list_all ()
56 {
57     local remote="/"
58     if [ $# -gt 0 ]; then 
59         remote=$1
60     fi
61 
62     bsttool_list_recur "$remote" 0 
63 }

alias bstrls=bsttool_list_all

  1 # global variable
  2 # we can NOT pass these as recursive function parameters, 
  3 # as shell function can NOT pass modification back to callee.
  4 #
  5 # cover flag (1:no once 2:no all 3:yes once 4:yes all)
  6 g_cover_flag=1
  7 
  8 # return value
  9 # 0: invalid input
 10 # 1: no once
 11 # 2: no all
 12 # 3: yes once
 13 # 4: yes all
 14 bsttool_duplicate_dir_warning()
 15 {
 16     local resp
 17     echo -n "dest file/dir exist, do you want to cover it? (n/N/y/Y) "
 18     read resp
 19     case "$resp" in 
 20         "n")
 21             return 1
 22             ;;
 23         "N")
 24             return 2
 25             ;;
 26         "y")
 27             return 3
 28             ;;
 29         "Y")
 30             return 4
 31             ;;
 32         *)
 33             return 0
 34             ;;
 35     esac
 36 }
 37 
 38 # $1: remote path
 39 # $2: local path (optional)
 40 bsttool_get_all ()
 41 {
 42     if [ $# -lt 1 ]; then 
 43         echo "Usage: bstrget remote [local]"
 44         return 1
 45     fi
 46 
 47     local remote=$1
 48     local local=""
 49     local type=0
 50     g_cover_flag=1 # no once
 51     local dirold=""
 52     if [ $# -gt 1 ]; then 
 53         local=$2
 54     else
 55         # get name part
 56         local tmp=""
 57         if [ ${remote:0-1} == "/" ]; then 
 58             # cut tailing '/'
 59             tmp="${remote:0:$((${#remote}-1))}"
 60         else
 61             tmp="$remote"
 62         fi
 63 
 64         local="${tmp##*/}"
 65     fi
 66 
 67     if [ -z "$local" -o "$local" == "/" ]; then 
 68         # default name
 69         local="setup"
 70     fi
 71 
 72     bsttool_query_path "$remote"
 73     type=$?
 74     case $type in
 75         0)
 76             # not exist
 77             echo "remote not exist"
 78             return 1
 79             ;;
 80         1)
 81             # file exist
 82             # do nothing, leave file to bsttool_get_recur
 83             ;;
 84         2)
 85             # empty dir exist
 86             echo "nothing to download"
 87             return 1
 88             ;;
 89         3)
 90             # dir exist
 91             if [ ! -d "$local" ]; then 
 92                 mkdir -p "$local"
 93             else
 94                 bsttool_duplicate_dir_warning
 95                 g_cover_flag=$?
 96                 # invalid or no (all)
 97                 if [ $g_cover_flag -eq 0 -o $g_cover_flag -eq 1 -o $g_cover_flag -eq 2 ]; then
 98                     return 1
 99                 fi
100             fi
101 
102             dirold="$PWD"
103             cd "$local"
104             ;;
105         *)
106             echo "should not reach here !"
107             return 1
108             ;;
109     esac
110 111     bsttool_get_recur "$remote" "$local" 0
112     if [ ! -z "$dirold" ]; then 
113         cd "$dirold"
114     fi
115     return 0
116 }

 1 # $1: remote path
 2 # $2: local path
 3 # $3: depth
 4 bsttool_get_recur ()
 5 {
 6     local remote=$1
 7     local local=$2
 8     local depth=$(($3+1))
 9 
10     local res=$(bst_tool ls "$remote")
11     local line=$(echo "$res" | sed -n '2p')
12     local str=${line/"total size"//}
13     local ret=0
14     if [ "$str" != "$line" ]; then 
15         # normal return
16         ret=$(echo "$line" | awk '{print $NF}')
17     else
18         # server response error
19         ret=0
20     fi
21 
22     # 6: skip header
23     local max=$(($ret + 6 - 1))
24     local fid=""
25     local size="" # has K/M/G postfix
26     local type=""
27     local name=""
28     local path=""
29     local filesize=0
30     echo "$remote [$depth-$ret]:"
31     for n in $(seq 6 $max)
32     do
33         # why sed -n "$np" don't work ?
34         line=$(echo "$res" | sed -n "$n"'p')
35         # row: fid, user, size, timestamp, type, name
36         # if size is empty (for dir, all spaces), using 0 instead
37         # do awk twice to remove redundant space
38         eval $(echo "$line" | awk -F'│' '{printf ("%s\t%s\t%s\t%s\n", $2, match($4, /[a-zA-Z0-9.]+/)?$4:"0", $6, $7)}' | awk '{printf ("fid=%s;size=%s;type=%s;name=%s;\n", $1,$2,$3,$4)}')
39         if [ "$type" == "dir" ]; then 
40             if [ -d "$name" ]; then 
41                 echo "dir exist: $name"
42                 if [ $g_cover_flag -eq 1 -o $g_cover_flag -eq 3 ]; then 
43                     bsttool_duplicate_dir_warning
44                     g_cover_flag=$?
45                 fi
46 
47                 # do NOT rm dirs even if user select cover it !
48                 # just merge files into it ..
49                 #
50                 # if not yes once or all, then nothing to do
51                 if [ $g_cover_flag -ne 3 -a $g_cover_flag -ne 4 ]; then 
52                     echo "nothing to do"
53                     continue
54                 fi
55             else
56                 mkdir "$name"
57             fi
58 
59             cd "$name"
60             # do for sub dirs
61             if [ ${remote:0-1} == "/" ]; then 
62                 path="$remote$name"
63             else
64                 path="$remote/$name"
65             fi
66 
67             bsttool_get_recur "$path" "$name" $depth
68             cd ..
69         else
70             echo -e "$fid:\t$size,\t$name"
71             if [ -f "$name" ]; then 
72                 echo "file exist: $name"
73                 if [ $g_cover_flag -eq 1 -o $g_cover_flag -eq 3 ]; then 
74                     bsttool_duplicate_dir_warning
75                     g_cover_flag=$?
76                 fi
77 
78                 if [ $g_cover_flag -eq 3 -o $g_cover_flag -eq 4 ]; then 
79                     echo "remove old file before download"
80                     rm "$name"
81                 else
82                     # no need to download
83                     echo "nothing to do"
84                     continue 
85                 fi
86             fi
87 
88             if [ ${remote:0-1} == "/" ]; then 
89                 path="$remote$name"
90             else
91                 path="$remote/$name"
92             fi
93 
94             bst_tool get "$path" "$name"
95         fi
96     done
97 
98     return 0
99 }

 1 # global variable
 2 # we can NOT pass these as recursive function parameters, 
 3 # as shell function can NOT pass modification back to callee.
 4 #
 5 # cover flag (1:no once 2:no all 3:yes once 4:yes all)
 6 g_cover_flag=1
 7 # size limit flag (1:no once 2:no all 3:yes once 4:yes all)
 8 g_limit_flag=1
 9 g_size_limit=1 # MB
10 
11 # return value
12 # 0: invalid input
13 # 1: no once
14 # 2: no all
15 # 3: yes once
16 # 4: yes all
17 bsttool_huge_file_warning()
18 {
19     local resp
20     echo -n "do you want to continue download this large file? (n/N/y/Y) "
21     read resp
22     case "$resp" in 
23         "n")
24             return 1
25             ;;
26         "N")
27             return 2
28             ;;
29         "y")
30             return 3
31             ;;
32         "Y")
33             return 4
34             ;;
35         *)
36             return 0
37             ;;
38     esac
39 }
40 
41 # $1: remote path
42 # $2: local path (optional)
43 bsttool_get_all ()
44 {
45     ……
46     g_limit_flag=1 # no once
47     g_size_limit=$((1024*1024)) # in bytes
48     bsttool_get_recur "$remote" "$local" 0
49     if [ ! -z "$dirold" ]; then 
50         cd "$dirold"
51     fi
52 }

 1             echo -e "$fid:\t$size,\t$name"
 2             # contains KB?
 3             local kfactor=${size/"KB"//}
 4             if [ "$kfactor" != "$size" ]; then
 5                 kfactor=1024
 6             else
 7                 kfactor=1
 8             fi
 9 
10             # contains MB?
11             local mfactor=${size/"MB"//}
12             if [ "$mfactor" != "$size" ]; then
13                 mfactor=$((1024*1024))
14             else
15                 mfactor=1
16             fi
17 
18             # contains GB?
19             local gfactor=${size/"GB"//}
20             if [ "$gfactor" != "$size" ]; then
21                 gfactor=$((1024*1024*1024))
22             else
23                 gfactor=1
24             fi
25 
26             # should we support TB?
27             filesize=$(awk -v val="$size" -v k="$kfactor" -v m="$mfactor" -v g="$gfactor" 'BEGIN{ printf ("%d", strtonum(val)*k*m*g) }')
28             if [ $filesize -gt $g_size_limit ]; then 
29                 echo "file too huge: $g_limit_flag"
30                 if [ $g_limit_flag -eq 1 -o $g_limit_flag -eq 3 ]; then
31                     bsttool_huge_file_warning
32                     g_limit_flag=$?
33                 fi
34             
35                 # if not yes once or all, then nothing to do
36                 if [ $g_limit_flag -ne 3 -a $g_limit_flag -ne 4 ]; then 
37                     # no need to download
38                     echo "nothing to do"
39                     continue
40                 fi
41             fi

alias bstrget=bsttool_get_all

$ ls -lhR ollvm
ollvm:
total 3.0M
drwxrwxr-x 2 yunh yunh 4.0K May 24 20:49 4
drwxrwxr-x 2 yunh yunh 4.0K May 24 20:49 8
drwxrwxr-x 2 yunh yunh 4.0K May 24 20:49 81
-rw-rw-r-- 1 yunh yunh  15K May 24 20:49 bash_bst.txt
-rw-rw-r-- 1 yunh yunh  409 May 24 20:49 build_cmake.sh
-rw-rw-r-- 1 yunh yunh  289 May 24 20:48 build_glibc.sh
drwxrwxr-x 2 yunh yunh 4.0K May 24 20:49 common
-rw-rw-r-- 1 yunh yunh  820 May 24 20:49 create_user.sh
-rw-rw-r-- 1 yunh yunh 2.9M May 24 20:49 global-6.6.5.tar.gz
-rw-rw-r-- 1 yunh yunh  535 May 24 20:49 init_bst.sh
drwxrwxr-x 2 yunh yunh 4.0K May 24 20:49 tutor

ollvm/4:
total 8.0K
-rw-rw-r-- 1 yunh yunh  320 May 24 20:49 build_ollvm4.sh
-rw-rw-r-- 1 yunh yunh 1.4K May 24 20:49 copy_ollvm4.sh

ollvm/8:
total 8.0K
-rw-rw-r-- 1 yunh yunh  531 May 24 20:49 build_ollvm8.sh
-rw-rw-r-- 1 yunh yunh 1.7K May 24 20:49 copy_ollvm8.sh

ollvm/81:
total 56K
-rw-rw-r-- 1 yunh yunh 1.9K May 24 20:49 build_llvm81.sh
-rw-rw-r-- 1 yunh yunh 2.4K May 24 20:49 copy_ollvm81.sh
-rw-rw-r-- 1 yunh yunh  45K May 24 20:49 PassManagerBuilder.cpp

ollvm/common:
total 24K
-rw-rw-r-- 1 yunh yunh  181 May 24 20:49 build_p2psrc.sh
-rw-rw-r-- 1 yunh yunh  899 May 24 20:49 clone_netdisk.sh
-rw-rw-r-- 1 yunh yunh  748 May 24 20:49 clone_p2psrc.sh
-rw-rw-r-- 1 yunh yunh 3.5K May 24 20:49 compare_instructions.sh
-rw-rw-r-- 1 yunh yunh  401 May 24 20:49 download_ndk.sh
-rw-rw-r-- 1 yunh yunh 1.3K May 24 20:49 inject_so.sh

ollvm/tutor:
total 292K
-rw-rw-r-- 1 yunh yunh 1.4K May 24 20:49 build_ollvm.sh
-rw-rw-r-- 1 yunh yunh  819 May 24 20:49 build_skeleton.sh
-rw-rw-r-- 1 yunh yunh 249K May 24 20:49 llvm-pass-tutorial-dev.zip
-rw-rw-r-- 1 yunh yunh  25K May 24 20:49 llvm-pass-tutorial.tar.gz
-rw-rw-r-- 1 yunh yunh  310 May 24 20:49 test.c

 1         # normal return
 2         ret=$(echo "$line" | awk '{print $NF}')
 3         if [ $ret -eq 0 ]; then 
 4             ret=2
 5         elif [ $ret -gt 1 ]; then 
 6             # a dir
 7             #ret=3 
 8             # at this point, we are NOT sure there is no file with same name exist, 
 9             # as bst_tool ls list directory with preference than file, 
10             # so here we need a download try...
11             echo "dir exist, try download to check existence of file with same name"
12             rm "$download" > /dev/null 2>&1 # otherwise bst_tool will fail
13             line=$(bst_tool get "$remote" "$download" | sed -n '2p')
14             str=${line/"download success"//}
15             if [ "$str" != "$line" ]; then # string replace changed: download success
16                 # has same name file
17                 ret=1
18             else 
19                 # only dir
20                 ret=3
21             fi
22         else # size=1
23             # note here we can NOT using type file/dir to determine, 
24             # for example, there is a path /tmp/tmp pointed to a file
25             # when we ls /tmp, it report one item named 'tmp' with type file,
26             # but actually it is a dir.
27             # so here we use another way to determine, that is, download it, 
28             # you can NOT download a dir.
29             echo "dir or file exist, try download to check"
30             rm "$download" > /dev/null 2>&1 # otherwise bst_tool will fail
31             line=$(bst_tool get "$remote" "$download" | sed -n '2p')
32             str=${line/"download success"//}
33             if [ "$str" != "$line" ]; then # string replace changed: download success
34                 # is a file
35                 ret=1
36             else 
37                 # is a dir
38                 ret=3
39             fi
40         fi

 1         # normal return
 2         ret=$(echo "$line" | awk '{print $NF}')
 3         if [ $ret -eq 0 ]; then 
 4             ret=2
 5         elif [ $ret -gt 1 ]; then 
 6             # a dir
 7             ret=3 
 8             # at this point, we are NOT sure there is no file with same name exist, 
 9             # as bst_tool ls list directory with preference than file, 
10             # so here we need a download try...
11             echo "dir exist, try download to check existence of file with same name"
12         else # size=1
13             ret=3
14             # note here we can NOT using type file/dir to determine, 
15             # for example, there is a path /tmp/tmp pointed to a file
16             # when we ls /tmp, it report one item named 'tmp' with type file,
17             # but actually it is a dir.
18             # so here we use another way to determine, that is, download it, 
19             # you can NOT download a dir.
20             echo "dir or file exist, try download to check"
21         fi
22 
23         if [ $ret -eq 3 ]; then 
24             rm "$download" > /dev/null 2>&1 # otherwise bst_tool will fail
25             line=$(bst_tool get "$remote" "$download" | sed -n '2p')
26             str=${line/"download success"//}
27             if [ "$str" != "$line" ]; then # string replace changed: download success
28                 # is a file
29                 ret=1
30             else 
31                 # is a dir
32                 ret=3
33             fi
34         fi