libssh 0.11.2 aio read + proxyjump stall issue

.gitignore

libssh
build

README.md

A minimal snippet to reproduce sftp_aio_wait_read stall issue.

# clone this repo and move into it
git clone https://gist.github.com/upa/936eb750743fda2b611f4d0262a9a62c jump-sftp-read
cd jump-sftp-read

# Fetch libssh
git clone https://git.libssh.org/projects/libssh.git

# Build the test code
mkdir build && cd build
cmake .. -DCMAKE_BUILD_TYPE=Debug
make -j 10

# Create a file to copy
dd if=/dev/random of=/tmp/test.img bs=1M count=100

# How to run the code
./jump_sftp_read
Usage: ./jump_sftp_read -h HOST -u USER -r REMOTE_FILE [-j JUMP_HOST] [-c CONCURRENT]
  -h HOST         Remote host
  -u USER         Username
  -r REMOTE_FILE  Remote file path to read
  -j JUMP_HOST    Jump host (optional)
  -c CONCURRENT   # of concurrent AIO read requests (default: 32)

# Copy without proxyjump
./jump_sftp_read -u $(whoami) -h localhost -r /tmp/test.img

# Copy with proxyjump (this will stall)
./jump_sftp_read -u $(whoami) -h localhost -r /tmp/test.img -j localhost

# Copy with proxyjump without concurrency
./jump_sftp_read -u $(whoami) -h localhost -r /tmp/test.img -j localhost -c 1

Concurrent async read with proxyjump causes stalling at poll.

i1 ~/w/j/build > gdb --args ./jump_sftp_read -u $(whoami) -h localhost -r /tmp/test.img -j localhost
GNU gdb (Ubuntu 15.0.50.20240403-0ubuntu1) 15.0.50.20240403-git
Copyright (C) 2024 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.
Type "show copying" and "show warranty" for details.
This GDB was configured as "x86_64-linux-gnu".
Type "show configuration" for configuration details.
For bug reporting instructions, please see:
<https://www.gnu.org/software/gdb/bugs/>.
Find the GDB manual and other documentation resources online at:
    <http://www.gnu.org/software/gdb/documentation/>.

For help, type "help".
Type "apropos word" to search for commands related to "word"...
Reading symbols from ./jump_sftp_read...
(gdb) run
Starting program: /home/upa/work/jump-sftp-read/build/jump_sftp_read -u upa -h localhost -r /tmp/test.img -j localhost
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
Using jump host: localhost
Connecting to upa@localhost...
[New Thread 0x7ffff74006c0 (LWP 2657907)]
Authentication with public key successful
Creating SFTP session...
Opening remote file: /tmp/test.img
Creating local file: test.img
Copying file with 32 concurrent requests...
File size: 104857600 bytes
Progress: 1048576/104857600 bytes (1.0%)^C
Thread 1 "jump_sftp_read" received signal SIGINT, Interrupt.
0x00007ffff771b4cd in __GI___poll (fds=0x5555555c8910, nfds=1, timeout=-1)
    at ../sysdeps/unix/sysv/linux/poll.c:29
warning: 29     ../sysdeps/unix/sysv/linux/poll.c: No such file or directory
(gdb) bt
#0  0x00007ffff771b4cd in __GI___poll (fds=0x5555555c8910, nfds=1, timeout=-1)
    at ../sysdeps/unix/sysv/linux/poll.c:29
#1  0x00005555555a4ec7 in ssh_poll_ctx_dopoll ()
#2  0x000055555557d429 in ssh_handle_packets ()
#3  0x000055555557d69d in ssh_handle_packets_termination ()
#4  0x00005555555a1abf in ssh_channel_read_timeout ()
#5  0x0000555555598f7c in sftp_packet_read ()
#6  0x0000555555599d74 in sftp_read_and_dispatch ()
#7  0x000055555559a081 in sftp_recv_response_msg ()
#8  0x000055555559a5ca in sftp_aio_wait_read ()
#9  0x000055555555dd77 in main (argc=9, argv=0x7fffffffe9c8)
    at /home/upa/work/jump-sftp-read/jump_sftp_read.c:206

CMakeLists.txt

cmake_minimum_required(VERSION 3.12.0)

project(jump-sftp-read VERSION 1.0.0 LANGUAGES C)

set(CMAKE_C_STANDARD 99)
set(CMAKE_C_STANDARD_REQUIRED ON)

include(ExternalProject)

# Set installation prefix
set(CMAKE_INSTALL_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/install")

# Build libssh
set(LIBSSH_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/libssh")
set(LIBSSH_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/libssh-build")

ExternalProject_Add(libssh_external
    SOURCE_DIR "${LIBSSH_SOURCE_DIR}"
    BINARY_DIR "${LIBSSH_BINARY_DIR}"
    CMAKE_ARGS
        -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}
        -DCMAKE_BUILD_TYPE=Release
        -DBUILD_SHARED_LIBS=OFF
        -DWITH_EXAMPLES=OFF
        -DWITH_TESTING=OFF
        -DWITH_SFTP=ON
        -DWITH_SERVER=OFF
        -DWITH_GSSAPI=OFF
        -DWITH_ZLIB=ON
    BUILD_ALWAYS 0
    INSTALL_COMMAND ${CMAKE_COMMAND} --build . --target install
)

# Find required libraries
find_package(Threads REQUIRED)
find_package(OpenSSL REQUIRED)
find_package(ZLIB REQUIRED)

# Create executable
add_executable(jump_sftp_read jump_sftp_read.c)

add_dependencies(jump_sftp_read libssh_external)

target_include_directories(jump_sftp_read PRIVATE 
    ${CMAKE_INSTALL_PREFIX}/include
    ${LIBSSH_SOURCE_DIR}/include
)

target_link_directories(jump_sftp_read PRIVATE 
    ${CMAKE_INSTALL_PREFIX}/lib
)

target_link_libraries(jump_sftp_read PRIVATE
    ssh
    OpenSSL::SSL 
    OpenSSL::Crypto
    ZLIB::ZLIB
    Threads::Threads
)

# Platform-specific libraries
if(APPLE)
    find_library(CORE_FOUNDATION_FRAMEWORK CoreFoundation)
    find_library(SECURITY_FRAMEWORK Security)
    if(CORE_FOUNDATION_FRAMEWORK AND SECURITY_FRAMEWORK)
        target_link_libraries(jump_sftp_read PRIVATE 
            ${CORE_FOUNDATION_FRAMEWORK}
            ${SECURITY_FRAMEWORK}
        )
    endif()
elseif(UNIX)
    target_link_libraries(jump_sftp_read PRIVATE dl)
endif()

jump_sftp_read.c

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <fcntl.h>
#include <errno.h>

#include <libssh/libssh.h>
#include <libssh/sftp.h>
#include <libssh/config_parser.h>

#define BUF_SIZE 8192

#define min(a, b) ((a < b) ? a : b)

static void usage(const char *prog) {
	fprintf(stderr,
		"Usage: %s -h HOST -u USER -r REMOTE_FILE "
		"[-j JUMP_HOST] [-c CONCURRENT]\n", prog);
	fprintf(stderr,
		"  -h HOST         Remote host\n"
		"  -u USER         Username\n"
		"  -r REMOTE_FILE  Remote file path to read\n"
		"  -j JUMP_HOST    Jump host (optional)\n"
		"  -c CONCURRENT   # of concurrent AIO read requests (default: 32)\n");
	exit(1);
}

int main(int argc, char **argv) {
	ssh_session session = NULL;
	sftp_session sftp = NULL;
	sftp_file remote_file = NULL;
	int local_fd = 0;
    
	char *host = NULL;
	char *user = NULL;
	char *jump_host = NULL;
	char *remote_path = NULL;
	char *local_filename = NULL;
	int nr_conc = 32; // Default number of concurrent requests
	int opt;
	int rc;
    
	// Parse command line arguments
	while ((opt = getopt(argc, argv, "h:u:j:r:c:")) != -1) {
		switch (opt) {
		case 'h':
			host = optarg;
			break;
		case 'u':
			user = optarg;
			break;
		case 'j':
			jump_host = optarg;
			break;
		case 'r':
			remote_path = optarg;
			break;
		case 'c':
			nr_conc = atoi(optarg);
			break;
		default:
			usage(argv[0]);
		}
	}
    
	if (!host || !user || !remote_path) {
		usage(argv[0]);
	}
    
	// Extract filename from remote path
	local_filename = strrchr(remote_path, '/');
	if (local_filename)
		local_filename++; // Skip the '/'
	else
		local_filename = remote_path;
    
	// Create SSH session
	ssh_init();

	session = ssh_new();
	if (!session) {
		fprintf(stderr, "Failed to create SSH session\n");
		return 1;
	}
    
	// Set basic SSH options
	rc = ssh_options_set(session, SSH_OPTIONS_HOST, host);
	if (rc != SSH_OK) {
		fprintf(stderr, "Failed to set hostname: %s\n", ssh_get_error(session));
		return 1;
	}
    
	rc = ssh_options_set(session, SSH_OPTIONS_USER, user);
	if (rc != SSH_OK) {
		fprintf(stderr, "Failed to set username: %s\n", ssh_get_error(session));
		return 1;
	}
    
	// Set proxy jump if specified
	if (jump_host) {
		rc = ssh_config_parse_proxy_jump(session, jump_host, true);
		if (rc != SSH_OK) {
			fprintf(stderr, "Failed to set proxy jump: %s\n",
				ssh_get_error(session));
			return 1;
		}
		printf("Using jump host: %s\n", jump_host);
	} else
		printf("Direct connection (no jump host)\n");

    
	// Connect to SSH server
	printf("Connecting to %s@%s...\n", user, host);
	rc = ssh_connect(session);
	if (rc != SSH_OK) {
		fprintf(stderr, "Failed to connect: %s\n", ssh_get_error(session));
		return 1;
	}
    
	// Authenticate (try public key first, then password)
	rc = ssh_userauth_publickey_auto(session, NULL, NULL);
	if (rc == SSH_AUTH_SUCCESS) {
		printf("Authentication with public key successful\n");
	} else {
		char *password = getpass("Password: ");
		rc = ssh_userauth_password(session, NULL, password);
		if (rc != SSH_AUTH_SUCCESS) {
			fprintf(stderr, "Authentication failed: %s\n",
				ssh_get_error(session));
			return 1;
		}
		printf("Authentication with password successful\n");
	}
    
	// Create SFTP session
	printf("Creating SFTP session...\n");
	sftp = sftp_new(session);
	if (!sftp) {
		fprintf(stderr, "Failed to create SFTP session: %s\n",
			ssh_get_error(session));
		return 1;
	}
    
	rc = sftp_init(sftp);
	if (rc != SSH_OK) {
		fprintf(stderr, "Failed to initialize SFTP: %s\n",
			ssh_get_error(session));
		return 1;
	}
    
	// Open remote file
	printf("Opening remote file: %s\n", remote_path);
	remote_file = sftp_open(sftp, remote_path, O_RDONLY, 0);
	if (!remote_file) {
		fprintf(stderr, "Failed to open remote file %s: %s\n",
			remote_path, ssh_get_error(session));
		return 1;
	}
    
	// Open local file for writing
	printf("Creating local file: %s\n", local_filename);
	local_fd = open(local_filename, O_RDWR | O_CREAT, 0644);
	if (local_fd < 0) {
		fprintf(stderr, "Failed to create local file %s: %s\n",
			local_filename, strerror(errno));
		return 1;
	}
    
	// Copy file content using asynchronous SFTP operations
	printf("Copying file with %d concurrent requests...\n", nr_conc);
    
	// Get file size to determine how much data to read
	sftp_attributes attrs = sftp_fstat(remote_file);
	if (!attrs) {
		fprintf(stderr, "Failed to get remote file attributes: %s\n",
			ssh_get_error(session));
		return 1;
	}
	ssize_t file_size = attrs->size;
	printf("File size: %llu bytes\n", (unsigned long long)file_size);
    

	// Prepare requests
	sftp_aio reqs[nr_conc];
	char buf[BUF_SIZE];

	ssize_t remaind, thrown, written, len;

	remaind = thrown = file_size;
	written = 0;
    
	// Main loop for async read
	while (written < file_size) {
		for (int i = 0; i < nr_conc && thrown > 0; i++) {
			len = min(thrown, BUF_SIZE);
			len = sftp_aio_begin_read(remote_file, len, &reqs[i]);
			if (len == SSH_ERROR) {
				fprintf(stderr, "sftp_aio_begin_read");
				return 1;
			}
			thrown -= len;
		}

		for (int i = 0; i < nr_conc && remaind > 0; i++) {
			len = sftp_aio_wait_read(&reqs[i], buf, sizeof(buf));
			if (len == SSH_ERROR) {
				fprintf(stderr, "sftp_aio_wait_read");
				return 1;
			}

			len = write(local_fd, buf, len);
			if (len < 0) {
				fprintf(stderr, "Error: write: %s", strerror(errno));
				return 1;
			}
			written += len;

			printf("\rProgress: %llu/%llu bytes (%.1f%%)", 
			       (unsigned long long)written, 
			       (unsigned long long)file_size,
			       (double)written / file_size * 100.0);
			fflush(stdout);
		}
	}
    
	printf("\nSuccessfully copied %llu bytes to %s\n", 
	       (unsigned long long)written, local_filename);
    
	return 0;
}