scp-via-ssh-shell Useful when you have only a minimal setup booted like an emergency initramfs with no proper scp/sftp support. Allows recursively copying files via "raw" ssh from a remote to your local system. On the remote host, just ls, cat & find are required for everything to work.

scp-via-ssh.fish

#!/usr/bin/env fish

# Check if all required arguments are provided
set min_args 3
set max_args 4
if test (count $argv) -lt $min_args; or test (count $argv) -gt $max_args
    echo "Usage: "(status filename)" <user@host> <remote_dir> <local_dir> [--dry-run]"
    exit 1
end

# Parse arguments
set remote_host $argv[1]
set remote_dir $argv[2]
set local_dir $argv[3]
set dry_run 0
if test (count $argv) -eq 4; and test $argv[4] = "--dry-run"
    set dry_run 1
    echo "Dry run mode - no files will be copied"
    echo "======================="
end

# Check if pv is installed (only if not in dry-run mode)
if test $dry_run -eq 0; and not command -q pv
    echo "Error: pv (pipe viewer) is not installed. Please install it first."
    exit 1
end

# Function to escape special characters in filenames
function escape_filename
    echo $argv[1] | string escape
end

# Function to convert rwx permissions to octal
function rwx_to_octal
    set perm $argv[1]
    set octal 0
    
    # Owner
    if string match -q "*r*" $perm[1-3]; set octal (math $octal + 400); end
    if string match -q "*w*" $perm[1-3]; set octal (math $octal + 200); end
    if string match -q "*x*" $perm[1-3]; set octal (math $octal + 100); end
    
    # Group
    if string match -q "*r*" $perm[4-6]; set octal (math $octal + 40); end
    if string match -q "*w*" $perm[4-6]; set octal (math $octal + 20); end
    if string match -q "*x*" $perm[4-6]; set octal (math $octal + 10); end
    
    # Others
    if string match -q "*r*" $perm[7-9]; set octal (math $octal + 4); end
    if string match -q "*w*" $perm[7-9]; set octal (math $octal + 2); end
    if string match -q "*x*" $perm[7-9]; set octal (math $octal + 1); end
    
    echo $octal
end

# Function to format file size
function format_size
    set size $argv[1]
    if test $size -lt 1024
        echo $size"B"
    else if test $size -lt 1048576
        printf "%.1fKB\n" (math $size / 1024)
    else if test $size -lt 1073741824
        printf "%.1fMB\n" (math $size / 1048576)
    else
        printf "%.1fGB\n" (math $size / 1073741824)
    end
end

# Get list of all files in remote directory
set remote_files_list (mktemp)

ssh $remote_host "find $remote_dir -type f" 2>/dev/null > $remote_files_list

if test $status -ne 0
    echo "Error: Could not connect to remote host or access directory"
    exit 1
end

# Count total files
set total_files (cat $remote_files_list | count)
set current_file 0

if test $dry_run -eq 1
    echo "Would copy $total_files files to $local_dir"
    echo "Details of files to be copied:"
    echo "======================="
end

# we can' t use a for loop here
# because there might be millions of files, which breaks fish's substiton limit
# therefore we use raw unix commands to index and iterate through the file so no variable overflows
while true;

    # Increment file counter
    set current_file (math $current_file + 1)

    if test $current_file -gt $total_files; break; end

    set remote_file (head -n $current_file $remote_files_list | tail -1)

    # ITERATION BODY BEGIN ====================================================

    # Get relative path by removing remote_dir prefix
    set relative_path (string replace -r "^$remote_dir/?" "" $remote_file)
    
    # Create target directory structure
    set local_file "$local_dir/$relative_path"
    
    # Get file metadata using ls -ln for numeric user/group IDs and consistent format
    set file_meta (ssh $remote_host "ls -ln \"$remote_file\"" 2>/dev/null | awk '{print $1","$5}')
    if test $status -eq 0
        # Split metadata into permissions and size
        echo $file_meta | read -a -d , meta_parts
        set perms $meta_parts[1]
        set file_size $meta_parts[2]
        
        if test $dry_run -eq 1
            # Format permissions for display
            set perms_no_type (echo $perms | string replace -r '^-' '')
            set octal_perms (rwx_to_octal $perms_no_type)
            set formatted_size (format_size $file_size)
            printf "[%s] %s (%s) -> %s\n" $octal_perms $formatted_size $relative_path $local_file
        else
            echo "Copying $current_file/$total_files: $relative_path"
            
            mkdir -p (dirname $local_file)
            
            # Copy file content with progress indication
            set escaped_remote_file (escape_filename $remote_file)
            if test -n "$file_size"
                if test 1048576 -lt $file_size 
                    # show progress for files > 1MB
                    ssh $remote_host "cat \"$escaped_remote_file\"" 2>/dev/null | pv -pter --size $file_size > "$local_file"
                else
                    # Progress is unnecessary for small files
                    ssh $remote_host "cat \"$escaped_remote_file\"" 2>/dev/null > "$local_file"
                end
            else
                # Fallback if we couldn't get the file size
                ssh $remote_host "cat \"$escaped_remote_file\"" 2>/dev/null > "$local_file"
            end
            
            if test $status -ne 0
                echo "Error: Failed to copy $relative_path"
                continue
            end
            
            # Set permissions
            if test -n "$perms"
                set perms_no_type (echo $perms | string replace -r '^-' '')
                set octal_perms (rwx_to_octal $perms_no_type)
                chmod $octal_perms "$local_file"
            end
        end
    else
        echo "Error: Could not access $relative_path"
    end

    # ITERATION BODY END ======================================================
end

if test $dry_run -eq 1
    echo "======================="
    echo "Dry run complete - no files were copied"
else
    echo "Copy operation completed"
end

zREADME.md

Example Usage & Output

$ ./scp-via-ssh.fish root@myhost /root/test_root ./destdir --dry-run
Dry run mode - no files will be copied
=======================
Would copy 6 files to ./destdir
Details of files to be copied:
=======================
[600] 0B (sub_dir_2/sub_sub_dir_3/test_3) -> ./destdir/sub_dir_2/sub_sub_dir_3/test_3
[600] 745,5MB (sub_dir_1/sub_sub_dir_3/biggest.file) -> ./destdir/sub_dir_1/sub_sub_dir_3/biggest.file
[600] 47B (sub_dir_1/sub_sub_dir_2/test_bigger_2) -> ./destdir/sub_dir_1/sub_sub_dir_2/test_bigger_2
[600] 8B (sub_dir_1/sub_sub_dir_2/test_file_3) -> ./destdir/sub_dir_1/sub_sub_dir_2/test_file_3
[600] 8B (sub_dir_1/sub_sub_dir_2/test_file_2) -> ./destdir/sub_dir_1/sub_sub_dir_2/test_file_2
[600] 8B (sub_dir_1/sub_sub_dir_2/test_file_1) -> ./destdir/sub_dir_1/sub_sub_dir_2/test_file_1
=======================
Dry run complete - no files were copied

$ ./scp-via-ssh.fish root@myhost /root/test_root ./destdir                                              
Copying 1/6: sub_dir_2/sub_sub_dir_3/test_3
Copying 2/6: sub_dir_1/sub_sub_dir_3/biggest.file
0:00:03 [5,86MiB/s] [==========================================================================>] 100%
Copying 3/6: sub_dir_1/sub_sub_dir_2/test_bigger_2
Copying 4/6: sub_dir_1/sub_sub_dir_2/test_file_3
Copying 5/6: sub_dir_1/sub_sub_dir_2/test_file_2
Copying 6/6: sub_dir_1/sub_sub_dir_2/test_file_1
Copy operation completed