#!/bin/ksh # # ver 1.0, mwang@mindspring.com, January 1998 # * First internet "release" with basic functionality. # ver 2.0, mwang@mindspring.com, August 1998 # * Added recursive put/get. # ver 2.1, mwang@mindspring.com, January 1999 # * Fixed the bug in trap statement pointed out by lars.g.lindberg@seab.se # and vincent.bienfait@seab.se and with their suggestions. # * Fixed a problem with -i option with /absolute/path. # * changed j=${i#./#/} to l=${i#./}, j=${l#/} for compatibility. # * tighten up the code. # ver 2.11, mwang@mindspring.com, March 2000 # * Fixed a typo: ${0%niftpget}niftp -t => ${0%niftpput}niftp -t # ver 2.12, mwang@mindspring.com, June 2001 # * Added -k (ftp_command) to send arbitrary ftp command, eg quote site ... # * Added checking for mainframe FTP server message: # "250 Transfer completed successfully" # "normal" FTP server has the message: # "226 .*Transfer complete." OPATH=$PATH PATH=$(PATH=/usr/bin:/bin:/usr/sbin:/sbin getconf PATH) (( $# == 0 )) || [[ $1 = "help=y" ]] && { (typeset -x PATH=$OPATH; whence perl >/dev/null) || { print -r -- "It requires perl in your PATH to view the man page." exit 1 } ME=$(basename $0) typeset TEMPDIR until [[ -n $TEMPDIR ]] && mkdir -p $TEMPDIR; do TEMPDIR=/tmp/$ME.$$.$RANDOM done trap "rm -rf $TEMPDIR" EXIT INT sed -n "/^## /s:^## ::p" $0 > $TEMPDIR/$ME.pod (typeset -x PATH=$OPATH; perldoc $TEMPDIR/$ME.pod) exit 100 } [[ $0 = *"niftpget" ]] && exec ${0%niftpget}niftp -g "$@" [[ $0 = *"niftpput" ]] && exec ${0%niftpput}niftp -t "$@" while getopts :hf:gtbas:u:p:d:r:c:n:yw:l:o:q:x:i:z:j:v:ek: c do case $c in f) f_opt=1; f_arg=$OPTARG;; g) g_opt=1;; t) t_opt=1;; b) b_opt=1;; a) a_opt=1;; s) s_opt=1; s_arg=$OPTARG;; u) u_opt=1; u_arg=$OPTARG;; p) p_opt=1; p_arg=$OPTARG;; d) d_opt=1; d_arg=$OPTARG;; r) r_opt=1; r_arg=$OPTARG;; c) c_opt=1; c_arg=$OPTARG;; n) n_opt=1; n_arg=$OPTARG;; y) y_opt=1;; w) w_opt=1; w_arg=$OPTARG;; l) l_opt=1; l_arg=$OPTARG;; o) o_opt=1; o_arg=$OPTARG;; q) q_opt=1; q_arg=$OPTARG;; x) x_opt=1; x_arg=$OPTARG;; i) i_opt=1; i_arg=$OPTARG;; z) z_opt=1; z_arg=$OPTARG;; j) j_opt=1; j_arg=$OPTARG;; v) v_opt=1; v_arg=$OPTARG;; e) e_opt=1;; k) k_opt=1; k_arg=$OPTARG;; ?) exec $0;; esac done shift $(($OPTIND - 1)) _ARG="$@" RUN_USER=$(id -un) [[ -f ~$RUN_USER/.niftp ]] && FTPCFG=~$RUN_USER/.niftp [[ $f_opt = 1 ]] && FTPCFG=$f_arg [[ -n "$FTPCFG" && -r "$FTPCFG" ]] && . $FTPCFG (( m = 0 )) [[ $k_opt = 1 || -n "$FTP_COMMAND" ]] && (( m += 64 )) # 1000000 [[ $g_opt = 1 || $GET_OR_PUT = "mget" ]] && (( m += 32 )) # 0100000 [[ $t_opt = 1 || $GET_OR_PUT = "mput" ]] && (( m += 16 )) # 0010000 [[ $x_opt = 1 || -n "$FTP_GET_FILES" || $i_opt = 1 || -n "$FTP_GETR_DIRS" ]] && (( m += 08 )) # 0001000 [[ $z_opt = 1 || -n "$FTP_PUT_FILES" ]] && (( m += 04 )) # 0000100 [[ $j_opt = 1 || -n "$FTP_JUNK_FILES" ]] && (( m += 02 )) # 0000010 [[ $v_opt = 1 || -n "$FTP_VIEW_FILES" ]] && (( m += 01 )) # 0000001 (( (m&48) == 48 )) && { print -r -- "-g and -t cannot both set."; exit 102; } (( (m&40) == 40 )) && { print -r -- "-g and -x/-i cannot both set."; exit 102; } (( (m&36) == 36 )) && { print -r -- "-g and -z cannot both set."; exit 102; } (( (m&24) == 24 )) && { print -r -- "-t and -x/-i cannot both set."; exit 102; } (( (m&20) == 20 )) && { print -r -- "-t and -z cannot both set."; exit 102; } (( m == 0 )) && { print -r -- "There is nothing to do." exit 103 } [[ $g_opt = 1 ]] && GET_OR_PUT=mget [[ $t_opt = 1 ]] && GET_OR_PUT=mput [[ $b_opt = 1 && $a_opt = 1 ]] && { print -r -- "-b and -a cannot both set." exit 104 } [[ $b_opt = 1 ]] && BIN_OR_ASC=binary [[ $a_opt = 1 ]] && BIN_OR_ASC=ascii [[ $s_opt = 1 ]] && FTP_SERVER=$s_arg [[ -z $FTP_SERVER ]] && { print -r -- "you must specify ftp_server option." exit 105 } [[ $u_opt = 1 ]] && FTP_USER=$u_arg : ${FTP_USER:=ftp} [[ $p_opt = 1 ]] && FTP_PASSWD=$p_arg [[ -z $FTP_PASSWD ]] && { if [[ $FTP_USER = "ftp" ]]; then FTP_PASSWD="-" else print -r -- "you must specify ftp server passwd." exit 106 fi } [[ $d_opt = 1 ]] && TARGET_DIR=$d_arg [[ $r_opt = 1 ]] && SOURCE_DIR=$r_arg [[ -n "$_ARG" ]] && { if [[ "$_ARG" = ftp://*([! ]) ]]; then FTP_URL="$_ARG" i=${FTP_URL#ftp://} FTP_SERVER=${i%%/*} j=${i#*/} TARGET_DIR=${j%/*} [[ $TARGET_DIR = $j ]] && TARGET_DIR= FTP_FILES=${j##*/} else FTP_FILES="$_ARG" [[ "$FTP_FILES" = "-" ]] && { [[ -z "$FTP_INPUTS" ]] && { FTP_INPUTS=$(cat) } FTP_FILES="$FTP_INPUTS" } fi } [[ -n "$GET_OR_PUT" && -z "$FTP_FILES" ]] && { print -r -- "you must specify files to get or to put." exit 107 } [[ $x_opt = 1 ]] && FTP_GET_FILES=$x_arg [[ "$FTP_GET_FILES" = "-" ]] && { [[ -z "$FTP_INPUTS" ]] && { FTP_INPUTS=$(cat) } FTP_GET_FILES="$FTP_INPUTS" } [[ $i_opt = 1 ]] && FTP_GETR_DIRS=$i_arg [[ "$FTP_GETR_DIRS" = "-" ]] && { [[ -z "$FTP_INPUTS" ]] && { FTP_INPUTS=$(cat) } FTP_GETR_DIRS="$FTP_INPUTS" } [[ $z_opt = 1 ]] && FTP_PUT_FILES=$z_arg [[ "$FTP_PUT_FILES" = "-" ]] && { [[ -z "$FTP_INPUTS" ]] && { FTP_INPUTS=$(cat) } FTP_PUT_FILES="$FTP_INPUTS" } [[ $v_opt = 1 ]] && FTP_VIEW_FILES=$v_arg [[ "$FTP_VIEW_FILES" = "-" ]] && { [[ -z "$FTP_INPUTS" ]] && { FTP_INPUTS=$(cat) } FTP_VIEW_FILES="$FTP_INPUTS" } [[ $j_opt = 1 ]] && FTP_JUNK_FILES=$j_arg [[ "$FTP_JUNK_FILES" = "-" ]] && { [[ -z "$FTP_INPUTS" ]] && { FTP_INPUTS=$(cat) } FTP_JUNK_FILES="$FTP_INPUTS" } [[ $k_opt = 1 ]] && FTP_COMMAND=$k_arg [[ "$FTP_COMMAND" = "-" ]] && { [[ -z "$FTP_INPUTS" ]] && { FTP_INPUTS=$(cat) } FTP_COMMAND="$FTP_INPUTS" } [[ $w_opt = 1 ]] && FIREWALL_SERVER=$w_arg [[ $l_opt = 1 ]] && FIREWALL_USER=$l_arg [[ $o_opt = 1 ]] && FIREWALL_PASSWD=$o_arg [[ $q_opt = 1 ]] && FIREWALL_ACCOUNT=$q_arg [[ $y_opt = 1 ]] && FIREWALL_ACCESS="yes" [[ $c_opt = 1 ]] && FTP_CNT_MAX=$c_arg : ${FTP_CNT_MAX:=5} [[ $n_opt = 1 ]] && FTP_SECOND=$n_arg : ${FTP_SECOND:=3600} if [[ $FIREWALL_ACCESS = "yes" ]]; then [[ -z $FIREWALL_SERVER ]] && { print -r -- "you must specify firewall server."; exit 108; } [[ -z $FIREWALL_USER ]] && { print -r -- "you must specify firewall login."; exit 109; } [[ -z $FIREWALL_SERVER ]] && { print -r -- "you must specify firewall passwd."; exit 110; } SERVER=$FIREWALL_SERVER FTP_COMMANDS=$( print -r -- "user $FIREWALL_USER $FIREWALL_PASSWD $FIREWALL_ACCOUNT" print -r -- "quote site $FTP_SERVER") else SERVER=$FTP_SERVER FTP_COMMANDS= fi FTP_COMMANDS=$([[ -n "$$FTP_COMMANDS" ]] && print -r -- "$FTP_COMMANDS" print -r -- "user $FTP_USER $FTP_PASSWD" [[ -n "$TARGET_DIR" ]] && print -r -- "cd $TARGET_DIR" [[ -n "$SOURCE_DIR" ]] && print -r -- "lcd $SOURCE_DIR" [[ -n "$BIN_OR_ASC" ]] && print -r -- "$BIN_OR_ASC" print -r -- "prompt" ) tmpa=/tmp/.niftpa-$$ tmpb=/tmp/.niftpb-$$ trap "set +o noglob; rm -f ${tmpa}-[0-9]* ${tmpb}-[0-9]*; exit" 0 1 2 3 set -o noglob rm -rf $tmpa-0 rm -rf $tmpa-1 rm -rf $tmpa-2 [[ -n "$FTP_COMMAND" ]] && print -- "$FTP_COMMAND" >> $tmpa-0 # "\n" in "delete file\ndir" ok. for m in FTP_FILES FTP_GET_FILES FTP_PUT_FILES do [[ $m = FTP_FILES ]] && mgp=$GET_OR_PUT [[ $m = FTP_GET_FILES ]] && mgp=mget [[ $m = FTP_PUT_FILES ]] && mgp=mput gp=${mgp#m} eval n="\$$m" [[ -z "$n" ]] && continue for i in $n do l="${i#./}" j="${l#/}" k=${j%/*} f=${j##*/} if [[ $j = $k ]]; then p=; else p=$k/; fi [[ -n "$f" ]] && { if [[ "$f" = *"*"* ]]; then print -r -- "$mgp $p$f" >> $tmpa-0 else print -r -- "$gp $i $p$f" >> $tmpa-0 fi } until [[ $j = $k ]] do [[ $mgp = "mget" ]] && print -r -- "!mkdir $k" >> $tmpa-1 [[ $mgp = "mput" ]] && print -r -- "mkdir $k" >> $tmpa-1 j=$k k=${j%/*} done done done [[ -f $tmpa-1 ]] && sort -u -o $tmpa-1 $tmpa-1 [[ -f $tmpa-0 ]] && cat $tmpa-0 >> $tmpa-1 rm -f $tmpa-0 [[ -n "$FTP_JUNK_FILES" ]] && { for i in $FTP_JUNK_FILES do j="${i%/}" if [[ "$i" = "$j" ]]; then print -r -- "mdelete $j" >> $tmpa-1 else print -r -- "rmdir $j" >> $tmpa-0 fi done [[ -f $tmpa-0 ]] && sort -ur $tmpa-0 >> $tmpa-1 rm -f $tmpa-0 } [[ -n "$FTP_VIEW_FILES" ]] && { for i in $FTP_VIEW_FILES do if [[ "$i" = "." ]]; then print -r -- "ls -alR" else print -r -- "dir $i" >> $tmpa-1 fi done } [[ $e_opt = 1 ]] && { print -r -- GET_OR_PUT: $GET_OR_PUT print -r -- BIN_OR_ASC: $BIN_OR_ASC print -r -- FTP_URL: $FTP_URL print -r -- FTP_SERVER: $FTP_SERVER print -r -- FTP_USER: $FTP_USER print -r -- FTP_PASSWD: $FTP_PASSWD print -r -- TARGET_DIR: $TARGET_DIR print -r -- FTP_FILES: $FTP_FILES print -r -- FTP_PUT_FILES: $FTP_PUT_FILES print -r -- FTP_GET_FILES: $FTP_GET_FILES print -r -- FTP_VIEW_FILES: $FTP_VIEW_FILES print -r -- FTP_JUNK_FILES: $FTP_JUNK_FILES print -r -- SOURCE_DIR: $SOURCE_DIR print -r -- FIREWALL_ACCESS: $FIREWALL_ACCESS print -r -- FIREWALL_SERVER: $FIREWALL_SERVER print -r -- FIREWALL_USER: $FIREWALL_USER print -r -- FIREWALL_PASSWD: $FIREWALL_PASSWD print -r -- FTP_CNT_MAX: $FTP_CNT_MAX print -r -- FTP_SECOND: $FTP_SECOND print -r -- SERVER: $SERVER print -r -- FTP_COMMANDS: print -r -- "$FTP_COMMANDS" [[ -f $tmpa-1 ]] && cat $tmpa-1 exit 150 } (( FTP_CNT = 0 )) (( NOK = 0 )) while (( FTP_CNT < FTP_CNT_MAX )) do (( FTP_CNT += 1 )) if [[ -f $tmpa-1 ]]; then cp $tmpa-1 $tmpa-0 else > $tmpa-0 fi { print -r -- "niftp starts (trial $FTP_CNT)." if [[ $FIREWALL_ACCESS = "yes" ]]; then print -r -- "connecting $FTP_SERVER (via firewall ${FIREWALL_SERVER})." else print -r -- "connecting $FTP_SERVER." fi } | tee -a $tmpa-2 { print -r -- "$FTP_COMMANDS" cat $tmpa-0 if [[ -n "$FTP_GETR_DIRS" ]]; then NP=0 j= for i in $FTP_GETR_DIRS do (( NP += 1 )) rm -f $tmpb-$NP [[ -z "$j" ]] && print -r -- "pwd" if [[ -n "${SOURCE_DIR}" ]]; then mkdir -p "${SOURCE_DIR%/}/${i#./}" else mkdir -p "./${i#./}" fi if [[ "$i" = "." ]]; then k= l= else print -r -- "cd $i" k=${i%/}/ l=${k#/} fi print -r -- "mls -aR $tmpb-$NP" print -r -- "!touch $tmpb-0" while [[ ! -f $tmpb-0 ]] do sleep 3 done [[ -z "$j" ]] && { j=$(tail -10 $tmpa-2 | sed -n "s:^257 \"\(.*\)\" is current directory.*\$:\1:p") } print -r -- "cd $j" ${0%niftp}ls-aR2find -d - $tmpb-$NP | sed "/^\.$/d; s:^\./::; s:^:!mkdir $l:" ${0%niftp}ls-aR2find -f - $tmpb-$NP | sed "s:^\./::; s:^\(.*\)\$:get $k\1 $l\1:" rm -f $tmpb-$NP rm -f $tmpb-0 done fi | tee -a $tmpa-0 print -r -- "quit" } | ftp -nv $SERVER 2>&1 | tee -a $tmpa-2 print -r -- "niftp stops (trial $FTP_CNT)." | tee -a $tmpa-2 NOK=$(sed -n "/^niftp starts (trial $FTP_CNT)/,\$p" $tmpa-2 | egrep -c "^226 .*Transfer complete|^250 Transfer completed successfully|^250 DELE command successful") if [[ -n "$FTP_GETR_DIRS" ]]; then set $FTP_GETR_DIRS; NP=$# else NP=0 fi (( NOK -= NP )) # each DIR has one extra dir counted, as shown below. # 150 ASCII data connection for /bin/ls (127.0.0.1,56724) (0 bytes). # 226 ASCII Transfer complete. NUM=$(egrep -c "^m?put|^m?get|^m?delete|^dir" $tmpa-0) print -r -- "Finished $NOK/$NUM operations (GET/PUT/DELE/DIR)." if (( NOK >= NUM || FTP_CNT_MAX == 1 )); then (( FTP_CNT = FTP_CNT_MAX )) else sleep $FTP_SECOND fi done if (( NOK >= NUM )); then exit 0 else exit 200 fi ## =head1 NAME ## ## niftp, niftpget, niftpput: non-interactive recursive ftp ksh script. ## ## =head1 SYNOPSIS ## ## B I I<[files]> ## ## B I I<[files]> ## ## B I I<[files]> ## ## =head1 DESCRIPTION ## ## I GET/PUT/DELETE/DIR files on ftp server non-interactively. It ## supports firewall, recursive GET/PUT, standard input, list of pathnames, ## file glob (wildcard *). ## ## I counts the number of operations requested and the number of ## operations successfully finished. It returns 0 upon successful completion ## and has an option for retrial in case of error. ## ## =head2 OPTIONS ## ## The options can be provided either by command line or configuration ## file. The command line options overwrite the configuration file, and ## FTP_URL option overwrites other options. The configuration file options ## are listed after the command line options below. ## ## =item B<-f> ## ## Specify the configuration file, default $HOME/.niftp. ## ## =item B ## ## Display this man page. ## ## =item B<-g, GET_OR_PUT=I> ## ## To get files. You can also use -x option, -g option is kept for ## compatibility with earlier versions. ## ## =item B<-t, GET_OR_PUT=I> ## ## To put files. You can also use -z option, -t option is kept for ## compatibility with earlier versions. -g and -t can not be used ## together, but -x and -z can. ## ## =item B<-b, BIN_OR_ASC=I> ## ## To get/put files under binary mode. This is the default. ## ## =item B<-a, BIN_OR_ASC=I> ## ## To get/put files under ascii mode. ## ## =item B<-s I, FTP_SERVER=I> ## ## Specify the ftp server. ## ## =item B<-u , FTP_USER=I> ## ## Specify the ftp user. The default is I. ## ## =item B<-p I, FTP_PASSWD=I> ## ## Specify the ftp password. The default is "-" if the ftp user is ## "ftp". ## ## =item B<-d I, TARGET_DIR=I> ## ## Change the current directory to on the remote machine. ## ## =item B<-r I, SOURCE_DIR=I> ## ## Change the current directory to on the local machine. ## ## =item B<-y, FIREWALL_ACCESS=I> ## ## To access the ftp server via firewall if -y is used or ## FIREWALL_ACCESS is set to "yes". Otherwise access the ftp server ## directly, which is the default. ## ## =item B<-w I, FIREWALL_SERVER=I> ## ## Specify the firewall server. ## ## =item B<-l I, FIREWALL_USER=I> ## ## Specify the firewall user. ## ## =item B<-o I, FIREWALL_PASSWD=I> ## ## Specify the firewall password. ## ## =item B<-q I, FIREWALL_ACCOUNT=I> ## ## Specify the firewall account number. Some firewalls requires this. ## ## =item B<-n I, FTP_SECOND=I> ## ## Specify the trial interval. niftp counts how many operations ## you requested (NUM), and how many operations finished successfully ## (NOK). If the NOK is less than NUM it is considered as a failure and ## will retry. The default is 3600 seconds, i.e. 1 hour. ## ## =item B<-c I, FTP_CNT_MAX=I> ## ## Specify the number of trials. The default is 5. ## ## =item B<-x I, FTP_GET_FILES=I> ## ## =item B<-z I, FTP_PUT_FILES=I> ## ## =item B<-j I, FTP_JUNK_FILES=I> ## ## =item B<-v I, FTP_VIEW_FILES=I> ## ## Specify the files to GET/PUT/DELETE/DIR. If it is a dash "-", the ## files are from the standard input. Directory is denoted by the "/" ## at the end of the pathname. When you use '*' in filename, specify ## working directory with -d or -r option and use relative pathname. ## ## =item B, FTP_FILES=I> ## ## Specify the files OR URL to GET(-g) or PUT(-t). Besides the URL ## option it is the same as -x and -z, and is kept for compatibility ## with early versions. ## ## =item B<-i, FTP_GETR_DIRS=I> ## ## GET files under recursively. ## ## =item B<-e> ## ## Debug mode. To check various parameters only. It does not connect to ## the ftp server. ## ## =item B<-k, FTP_COMMAND=I> ## ## Issue specified ftp commands. If it is a dash "-", the commands are from ## the the standard input. ## ## =head1 EXAMPLES ## ## $HOME/.niftp: ## FIREWALL_SERVER= ## FIREWALL_USER= ## FIREWALL_PASSWD= ## FTP_SERVER= ## FTP_USER= ## FTP_PASSWD= ## ## =item B ## ## GET all files matching ghost*.tar.gz under the directory ## pub/gnu from the ftp server prep.ai.mit.edu via firewall. ## ## =item B ## ## Put all files under unix-prog (including unix-prog) to ## ftp server specified in $HOME/.niftp file with the same ## directory structure. ## ## =item B -p I -d ftp -v .> ## ## Connecting ftp server ftp.mindspring.com with I and I ## and view all files under current directory. ## ## =item B -p I -d ftp -j I> ## ## Connecting ftp server ftp.mindspring.com with I and I ## and delete I. ## ## =item B ## ## Download all files recursively under pub/multimedia/chinese-music ## on sunsite.unc.edu via firewall. ## ## The options -x(get), -z(put), -v(listing), -j(delete) can be combined. ## ## =head1 FILES ## ## niftp, niftpget, niftpput: these files should be hardlinked. ## ls-aR2find ## ## =head1 SEE ALSO ## ## L, L, L, I ## ## =head1 CAVEATS ## ## Symbolic links to files are treated as seperate regular files. Symbolic ## links to directories are errored out. Recursive GET only works on servers ## who understand "mls -aR". ## ## =head1 AUTHOR ## ## This is free software. You may copy or redistribute it under the same ## terms as Perl itself. ## ## If you modify it, please send a copy of your modification to me. ## ## Copyright (C) Michael Wang, mwang@mindspring.com ## 1998-2001