dump a ftp website files

dumpftp.sh

#!/usr/bin/env bash

# dumpftp.sh ---- download ftp website files
#
# author: Leslie Zhu <[email protected]>
# date: 2014/11/11
#
# Get each href element in each html page, if it's a plain file, then download it;
# else if it's a dir, then get its href element and download them.
#
# Note: it need `curl` command to download files, so ensure `curl` installed.
#
# Usage:
#     $ bash dumpftp.sh      # use default ftp url, you can update it
#     
#     $ bash dumpftp.sh URL  # download the URL ftp website
#     



function fetch_ftp(){           # get each href element in each html page
    local url=$1
    local files=""

    files=$(curl -s $url|grep href|awk -F"href=" '{print $2}'|awk -F">" '{print $1}'|tr -d "\"")

    for filename in ${files};
    do
        case $filename in
            [^?/]*)
                echo $filename;;
        esac
    done
}

function dump_ftp() {           # doadload ftp dir/files
    local website=$1
    local dumpurl=""

    for url in $(fetch_ftp $website);
    do
        dumpurl=${website}"/"${url}
        echo download $dumpurl
        
        case $url in 
            */)
                mkdir $url
                cd $url
                dump_ftp ${dumpurl}
                cd -
                ;;
            *)
                curl -s ${dumpurl} -O
                ;;
        esac
    done
}


#################################
# main point
#################################

# you can update it
default_url="http://example-ftp.com"

# run

if [ $# -eq 0 ]; then
    eval dump_ftp ${default_url}
    exit 0
elif [ $# -eq 1 ]; then
    eval dump_ftp $1
    exit 0
else
    echo "Usage: $0  URL  # download the ftp url files"
    exit -1
fi