files-from-url.sh

#!/usr/bin/env bash

# Downloads multiple files available from an HTML URL, typically the web listing of a directory, 
# as it is rendered by many web servers. 
# ==> Requires lftp
#
set -e

# Example: 
#   ./files-from-url.sh https://knetminer.com/downloads/agri-schemas/gxa/v202110/ gxa
# Downloads all the files in this version into the local gxa/ dir (creates it if necessary)
#
url="$1"
out="$2"

lftp="set ssl:verify-certificate no;"

lftp="$lftp mirror -v --continue '$url' '$out'"
lftp="$lftp;exit"
lftp -e "$lftp"