migrate_minio_to_webdav.sh

#!/usr/bin/env bash
#
# migrate_minio_to_webdav.sh
# ---------------------------------------------------------------------------
# rclone copy of an entire minio (S3) bucket to a WebDAV server. No DB changes.
#
# With django-storages (QFC_IS_LEGACY: false) every object path is already the
# path the WebDAV backend expects, so it's a 1:1 object copy. Covers all
# prefixes (projects/, account/, ...). rclone creates parent collections.
# Idempotent: copy never deletes and skips same-size objects, so it's re-runnable.
#
# Works against the bundled webdav container or an external WebDAV server.
# Back up the DB before touching production.
#
# USE AT YOUR OWN RISK
# AUTHOR: toni.schoenbuchner[@]csgis.de
# ---------------------------------------------------------------------------

set -euo pipefail

############################################################################
# 1) CONFIGURATION - adjust these values to your instance
############################################################################

# --- Source: minio (S3) ---------------------------------------------------
# These values come from your STORAGES["default"]["OPTIONS"].
MINIO_ENDPOINT="${MINIO_ENDPOINT:-http://localhost:8009}"   # minio S3 API, reachable from host
MINIO_ACCESS_KEY="${MINIO_ACCESS_KEY:-minioadmin}"
MINIO_SECRET_KEY="${MINIO_SECRET_KEY:-CHANGE_ME}"           # the secret_key from STORAGES
MINIO_BUCKET="${MINIO_BUCKET:-qfieldcloud-local}"
MINIO_REGION="${MINIO_REGION:-}"                            # region_name from STORAGES, usually empty for minio

# --- Target: WebDAV -------------------------------------------------------
# The URL where the WebDAV server is reachable. For the migration run from the
# host we need a reachable address. If the webdav container has no port
# mapping, expose it temporarily or run this script from a
# container in the same Docker network and use http://webdav:80/.
# For an EXTERNAL WebDAV server, set this to its public URL.
WEBDAV_URL="${WEBDAV_URL:-http://localhost:8020}"          # e.g. host mapping onto the webdav container
WEBDAV_USER="${WEBDAV_USER:-qfield}"                       # = WEBDAV_USERNAME
WEBDAV_PASS="${WEBDAV_PASS:-CHANGE_ME}"                    # = WEBDAV_PASSWORD

# --- rclone tuning --------------------------------------------------------
TRANSFERS="${TRANSFERS:-8}"           # parallel transfers
CHECKERS="${CHECKERS:-16}"            # parallel existence/hash checks
LOGFILE="${LOGFILE:-./qfc_migrate_$(date +%Y%m%d_%H%M%S).log}"

############################################################################
# 2) Is rclone available?
############################################################################

if ! command -v rclone >/dev/null 2>&1; then
  echo "ERROR: rclone is not installed."
  echo "Install (Debian/Ubuntu):  sudo apt-get install -y rclone"
  exit 1
fi

############################################################################
# 3) Temporary rclone configuration (no need to write to ~/.config)
############################################################################

RCLONE_CONF="$(mktemp)"
trap 'rm -f "$RCLONE_CONF"' EXIT

cat > "$RCLONE_CONF" <<EOF
[src]
type = s3
provider = Minio
access_key_id = ${MINIO_ACCESS_KEY}
secret_access_key = ${MINIO_SECRET_KEY}
endpoint = ${MINIO_ENDPOINT}
region = ${MINIO_REGION}
force_path_style = true

[dst]
type = webdav
url = ${WEBDAV_URL}
vendor = other
user = ${WEBDAV_USER}
pass = __OBSCURED__
EOF

# rclone expects the WebDAV password to be "obscured".
OBSCURED_PASS="$(rclone obscure "${WEBDAV_PASS}")"
sed -i "s|__OBSCURED__|${OBSCURED_PASS}|" "$RCLONE_CONF"

RCLONE=(rclone --config "$RCLONE_CONF" --log-file "$LOGFILE" --log-level INFO)

############################################################################
# 4) Subcommands
############################################################################

usage() {
  cat <<USAGE
QFieldCloud minio -> WebDAV migration

rclone copy of a minio (S3) bucket to a WebDAV server. No DB changes.
Idempotent: never deletes at the target, skips same-size objects, re-runnable.

Usage:  $0 <command>

Commands:
  check       Test connection to source and target; show object count/size of the source
  dry-run     Show what would be copied, without writing anything
  copy        Perform the actual copy (idempotent, re-runnable)
  verify      Compare source and target (size + existence of all objects)
  list-src    List all source objects with size
  list-dst    List all target objects with size
  help        Show this help (also: -h, --help)

Recommended order:  check -> dry-run -> copy -> verify

Configuration is read from environment variables (with sensible defaults).
Override them inline or via 'export'. Use SINGLE quotes for values containing
special characters such as & \$ *  (otherwise the shell mangles them):

  export MINIO_SECRET_KEY='your_minio_secret'
  export WEBDAV_URL='http://localhost:8020'
  export WEBDAV_USER='qfc_webdav_user'
  export WEBDAV_PASS='your_webdav_password'

Source (minio / S3):
  MINIO_ENDPOINT     S3 API endpoint            (default: http://localhost:8009)
  MINIO_ACCESS_KEY   access key                 (default: minioadmin)
  MINIO_SECRET_KEY   secret key                 (default: CHANGE_ME)
  MINIO_BUCKET       bucket name                (default: qfieldcloud-local)
  MINIO_REGION       region, usually empty      (default: empty)

Target (WebDAV - local container or external server):
  WEBDAV_URL         WebDAV base URL            (default: http://localhost:8020)
  WEBDAV_USER        WebDAV username            (default: qfield)
  WEBDAV_PASS        WebDAV password            (default: CHANGE_ME)

Tuning:
  TRANSFERS          parallel transfers         (default: 8)
  CHECKERS           parallel checks            (default: 16)
  LOGFILE            log file path              (default: ./qfc_migrate_<timestamp>.log)

Current log file: $LOGFILE
USAGE
}

cmd_check() {
  echo ">> Testing source (minio bucket: ${MINIO_BUCKET}) ..."
  "${RCLONE[@]}" lsd "src:${MINIO_BUCKET}" >/dev/null
  echo "   OK. Source overview:"
  "${RCLONE[@]}" size "src:${MINIO_BUCKET}"
  echo
  echo ">> Testing target (WebDAV: ${WEBDAV_URL}) ..."
  # lsd on root; an empty target is fine
  "${RCLONE[@]}" lsd "dst:" >/dev/null && echo "   OK. WebDAV reachable and authenticated."
}

cmd_dry_run() {
  echo ">> DRY-RUN: minio:${MINIO_BUCKET} -> WebDAV (NOTHING is written)"
  "${RCLONE[@]}" copy "src:${MINIO_BUCKET}" "dst:" \
    --transfers "$TRANSFERS" --checkers "$CHECKERS" \
    --dry-run --progress
}

cmd_copy() {
  echo ">> COPY: minio:${MINIO_BUCKET} -> WebDAV"
  echo "   Idempotent: objects that already exist with the same size are skipped."
  # --no-check-dest would be faster, but we want idempotency/resume:
  # rclone copy skips objects that exist at the target with the same size.
  "${RCLONE[@]}" copy "src:${MINIO_BUCKET}" "dst:" \
    --transfers "$TRANSFERS" --checkers "$CHECKERS" \
    --progress --stats 5s
  echo "   Done. Details in the log: $LOGFILE"
}

cmd_verify() {
  echo ">> VERIFY: comparing source and target ..."
  # WebDAV usually provides no hashes comparable to S3, so we compare size/
  # existence instead of --checksum. --one-way: every source object must exist
  # at the target.
  if "${RCLONE[@]}" check "src:${MINIO_BUCKET}" "dst:" \
        --one-way --size-only --checkers "$CHECKERS"; then
    echo "   SUCCESS: All source objects exist at the target with matching size."
  else
    echo "   WARNING: Differences found. See output above and log: $LOGFILE"
    echo "   (Re-running 'copy' is safe and will fetch any missing objects.)"
    return 1
  fi
}

cmd_list_src() { "${RCLONE[@]}" ls "src:${MINIO_BUCKET}"; }
cmd_list_dst() { "${RCLONE[@]}" ls "dst:"; }

############################################################################
# 5) Dispatch
############################################################################

case "${1:-}" in
  check)            cmd_check ;;
  dry-run)          cmd_dry_run ;;
  copy)             cmd_copy ;;
  verify)           cmd_verify ;;
  list-src)         cmd_list_src ;;
  list-dst)         cmd_list_dst ;;
  help|-h|--help)   usage ;;
  *)                usage; exit 1 ;;
esac