Proton Drive CLI Sync Script with Exclude List and Sync State

gistfile1.txt

#!/bin/bash

# Proton Drive Incremental Sync
# Syncs ~/ProtonSync → /my-files/Sync

set -uo pipefail

DRIVE_BIN="/usr/local/bin/proton-drive"
LOCAL_DIR="$HOME/ProtonSync"
DRIVE_BASE="/my-files/Sync"
SYNC_LOG="${HOME}/scripts/proton_drive_sync.log"
STATE_FILE="${HOME}/scripts/proton_drive_sync_state"

EXISTING_DIRS=""

# Directories/files to exclude from sync (patterns match against full relative path)
EXCLUDE_PATTERNS=(
    ".git"
    ".svn"
    ".idea"
    "vscode"
    "node_modules"
    "vendor"
    "venv"
    "*.log"
    "__pycache__"
    "*.swp"
    "*~"
    "*.o"
    "*.class"
    "*.pyc"
    ".cache"
    "cache/"
    "/cache/"
    "cache2/"
    "Cache/"
    "/Cache/"
    "CacheStorage/"
    "/CacheStorage/"
    "ScriptCache/"
    "Code Cache/"
    "CachedData/"
    "/dev/*"
    "logs/"
    "/logs/"
    "/proc/*"
    "/sys/*"
    "/tmp/*"
    "/run/*"
    "/var/log/*"
    "/mnt/*"
    "/media/*"
    "swapfile"
    "Steam"
    "Games"
    "GOG Games"
    "Thumbnails/"
    "/Thumbnails/"
    "lost+found"
    "Downloads"
    "Videos"
    "VirtualBox VMs"
    ".ecryptfs"
    ".cinnamon"
    ".local"
    ".themes"
    ".linuxmint"
    ".npm"
    ".nvm"
    ".steam"
    ".var"
    "retrodeck"
    "*.log"
)

truncate -s 0 "$SYNC_LOG"

log() {
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "$SYNC_LOG"
}

# ---- Pattern Matching ---------------------------------------------------

should_exclude() {
    local rel_path="$1"
    
    # Also extract just the basename and directory components
    local dir="$(dirname "$rel_path")"
    local base="$(basename "$rel_path")"
    
    for pattern in "${EXCLUDE_PATTERNS[@]}"; do
        case "$rel_path" in
            $pattern) return 0 ;;           # Exact path match
            */$pattern*) return 0 ;;        # Path contains it
            *) ;;                           # Try glob patterns below
        esac
        
        case "$base" in
            $pattern) return 0 ;;           # Filename matches wildcard
        esac
        
        case "$dir" in
            *"$pattern"*) return 0 ;;       # Directory name contains it
        esac
    done
    
    return 1  # Don't exclude
}

# ---- Remote folder management -------------------------------------------

ensure_remote_dir() {
    local remote_dir="$1"

    [[ " $EXISTING_DIRS " == *" $remote_dir "* ]] && return 0

    if [[ "$remote_dir" == "$DRIVE_BASE" ]]; then
        EXISTING_DIRS+=" $remote_dir "
        return 0
    fi

    if "$DRIVE_BIN" filesystem list "$remote_dir" </dev/null >/dev/null 2>&1; then
        EXISTING_DIRS+=" $remote_dir "
        return 0
    fi

    # Split into parent path and folder name for create-folder
    local parent_dir folder_name
    parent_dir="$(dirname "$remote_dir")"
    folder_name="$(basename "$remote_dir")"

    # Recurse: ensure parent exists first
    ensure_remote_dir "$parent_dir"

    log "CREATING DIR: $remote_dir"
    "$DRIVE_BIN" filesystem create-folder "$parent_dir" "$folder_name" </dev/null 2>&1 | tee -a "$SYNC_LOG" || true

    if "$DRIVE_BIN" filesystem list "$remote_dir" </dev/null >/dev/null 2>&1; then
        EXISTING_DIRS+=" $remote_dir "
        log "DIR OK: $remote_dir"
    else
        log "WARNING: Could not create $remote_dir — uploads may fail"
    fi
}

# ---- Remove state entries for folders that will be cleaned --------------------

purge_trash_folders() {
    declare -A live_dirs=()
    while IFS= read -r -d '' d; do
        rel="${d#"${LOCAL_DIR}/"}"
        live_dirs["$rel"]=1
    done < <(find -L "$LOCAL_DIR" -mindepth 1 -maxdepth 1 -type d -print0)

    tmp_state="$(mktemp)"
    purged=0

    while IFS='|' read -r rel_path stored_mtime; do
        [[ -z "$rel_path" ]] && continue

        # ⬇⬇⬇ PROTECT EXCLUDED PATTERNS IN STATE PURGE
        if should_exclude "$rel_path"; then
            echo "${rel_path}|${stored_mtime}" >> "$tmp_state"
            continue
        fi
        # ⬆⬆⬆

        top_folder="${rel_path%%/*}"
        
        if [[ -n "${live_dirs[$top_folder]:-}" ]]; then
            echo "${rel_path}|${stored_mtime}" >> "$tmp_state"
        else
            purged=$((purged + 1))
        fi
    done < "$STATE_FILE" 2>/dev/null || true

    mv "$tmp_state" "$STATE_FILE" 2>/dev/null || true
    log "--- Purged $purged stale state entries ---"
}

# ---- Full cleanup: compare actual remote contents to local ---------------

full_cleanup() {
    log "--- Full cleanup: comparing remote to local ---"

    # Build sets of local top-level files and folders
    declare -A local_files=()
    declare -A local_dirs=()

    while IFS= read -r -d '' f; do
        rel="${f#"${LOCAL_DIR}/"}"
        if [[ -d "$f" ]]; then
            local_dirs["$rel"]=1
        else
            local_files["$rel"]=1
        fi
    done < <(find -L "$LOCAL_DIR" -mindepth 1 -maxdepth 1 -print0)

    deleted=0

    # List remote top-level items only
    while IFS= read -r line; do
        [[ -z "$line" ]] && continue

        name="$(echo "$line" | sed -E 's/.*[0-9]{2}:[0-9]{2} [0-9-]+ //')"
        [[ -z "$name" ]] && continue

        # ⬇⬇⬇ ADD THIS CHECK FOR TOP-LEVEL ITEMS
        if should_exclude "$name"; then
            log "KEEP EXCLUDED: $name (would not be in local anyway)"
            continue
        fi
        # ⬆⬆⬆

        if [[ "$line" == 🗂️* ]]; then
            if [[ -z "${local_dirs[$name]:-}" ]]; then
                log "DELETE DIR: $name (not found locally)"
                if "$DRIVE_BIN" filesystem trash "${DRIVE_BASE}/${name}" </dev/null 2>&1 | tee -a "$SYNC_LOG"; then
                    log "  TRASHED"
                    deleted=$((deleted + 1))
                else
                    log "  FAILED"
                fi
            fi
        elif [[ "$line" == 📄* ]]; then
            if [[ -z "${local_files[$name]:-}" ]]; then
                log "DELETE FILE: $name (not found locally)"
                if "$DRIVE_BIN" filesystem trash "${DRIVE_BASE}/${name}" </dev/null 2>&1 | tee -a "$SYNC_LOG"; then
                    log "  TRASHED"
                    deleted=$((deleted + 1))
                else
                    log "  FAILED"
                fi
            fi
        fi
    done < <("$DRIVE_BIN" filesystem list "$DRIVE_BASE" </dev/null 2>/dev/null)

    log "--- Cleanup complete: $deleted items trashed ---"
}

# ---- State tracking -----------------------------------------------------

get_stored_mtime() {
    awk -F'|' -v k="$1" '$1==k {print $2; exit}' "$STATE_FILE" 2>/dev/null
}

set_stored_mtime() {
    local key="$1" mtime="$2"
    local tmp
    tmp="$(mktemp)"
    [[ -f "$STATE_FILE" ]] && awk -F'|' -v k="$key" '$1!=k' "$STATE_FILE" > "$tmp"
    echo "${key}|${mtime}" >> "$tmp"
    mv "$tmp" "$STATE_FILE"
}

# ---- Main ---------------------------------------------------------------

log "=== Starting incremental sync ==="

if ! "$DRIVE_BIN" filesystem list "$DRIVE_BASE" </dev/null >/dev/null 2>&1; then
    log "ERROR: Cannot access $DRIVE_BASE"
    log "Run: $DRIVE_BIN auth login"
    exit 1
fi
EXISTING_DIRS+=" $DRIVE_BASE "

touch "$STATE_FILE" 2>/dev/null

uploaded=0
skipped=0
failed=0
file_count=0
excluded_count=0

while IFS= read -r -d '' local_file; do
    file_count=$((file_count + 1))
    rel_path="${local_file#${LOCAL_DIR}/}"
    local_mtime="$(stat -c '%Y' "$local_file" 2>/dev/null || stat -f '%m' "$local_file")"

    # Skip excluded paths early
    if should_exclude "$rel_path"; then
        log "SKIP (excluded): $rel_path"
        excluded_count=$((excluded_count + 1))
        continue
    fi

    remote_subpath="$(dirname "$rel_path")"
    if [[ "$remote_subpath" == "." ]]; then
        remote_dir="$DRIVE_BASE"
    else
        remote_dir="${DRIVE_BASE}/${remote_subpath}"
    fi

    log "PROCESSING [$file_count]: $rel_path"

    ensure_remote_dir "$remote_dir"

    stored_mtime="$(get_stored_mtime "$rel_path")"

    if [[ -n "$stored_mtime" && "$local_mtime" == "$stored_mtime" ]]; then
        log "  SKIP (unchanged)"
        skipped=$((skipped + 1))
        continue
    fi

    log "  UPLOAD → $remote_dir"

    if "$DRIVE_BIN" filesystem upload -c replace "$local_file" "$remote_dir" </dev/null; then
        set_stored_mtime "$rel_path" "$local_mtime"
        log "  OK"
        uploaded=$((uploaded + 1))
    else
        log "  FAILED"
        failed=$((failed + 1))
    fi

    sleep 0.5

done < <(find -L "$LOCAL_DIR" -type f -print0)

purge_trash_folders
full_cleanup

log "=== Complete: $uploaded uploaded, $skipped skipped, $excluded_count excluded, $failed failed ($file_count total) ==="