Last active
August 6, 2026 13:16
-
-
Save rapando/08b4f930189e95f1172f6a4512679ff0 to your computer and use it in GitHub Desktop.
Download in rust
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| use indicatif::{ProgressBar, ProgressStyle}; | |
| use reqwest::blocking::{get, Client}; | |
| use std::{ | |
| error, | |
| fs::File, | |
| io::{Seek, SeekFrom, Write}, | |
| sync::{mpsc, Arc, Mutex}, | |
| thread, | |
| time::Instant, | |
| }; | |
| const DOWNLOAD_URL: &str = "https://www.stats.govt.nz/assets/Uploads/Annual-balance-sheets/Annual-balance-sheets-2023-provisional/Download-data/accumulation-accounts-2008-2023-provisional.csv"; | |
| const FILE_NAME: &str = "downloads/file.csv"; | |
| const FILE_NAME_2: &str = "downloads/file2.csv"; | |
| const CHUNK_SIZE: u64 = 1024 * 8; // each chunk is 8 KB | |
| fn main() { | |
| println!("LINEAR:: starting linear download"); | |
| match linear_download() { | |
| Ok(()) => { | |
| println!("LINEAR:: finished linear download successfully"); | |
| } | |
| Err(e) => { | |
| println!("LINEAR:: failed to download file because {e}"); | |
| } | |
| } | |
| println!("CHUNKED"); | |
| match concurrent_download() { | |
| Ok(()) => { | |
| println!("download finished"); | |
| } | |
| Err(e) => { | |
| println!("CONCURRENT: failed to download file because {e}"); | |
| } | |
| }; | |
| match download_with_progress_bar() { | |
| Ok(()) => { | |
| println!("finished download"); | |
| } | |
| Err(e) => { | |
| println!("failed to download with progress bar: {e}"); | |
| } | |
| }; | |
| } | |
| fn linear_download() -> Result<(), Box<dyn error::Error>> { | |
| let now = Instant::now(); | |
| let response = get(DOWNLOAD_URL)?; | |
| let content = response.bytes()?; | |
| let mut downloaded_file = File::create(FILE_NAME)?; | |
| downloaded_file.write_all(&content)?; | |
| let duration = now.elapsed(); | |
| println!("Downloaded file in {duration:?}"); | |
| Ok(()) | |
| } | |
| fn concurrent_download() -> Result<(), Box<dyn error::Error>> { | |
| let now = Instant::now(); | |
| let client = Client::new(); | |
| let response = client.head(DOWNLOAD_URL).send()?; | |
| let total_size = response | |
| .headers() | |
| .get("content-length") | |
| .ok_or("Content-Length header missing")? | |
| .to_str()? | |
| .parse::<u64>()?; | |
| println!("file size: {total_size}"); | |
| let file = File::create(FILE_NAME_2)?; | |
| file.set_len(total_size)?; | |
| // Arc for sharing the file handle | |
| let file = Arc::new(Mutex::new(file)); | |
| // create threads for each chunk | |
| let mut handles = vec![]; | |
| for start in (0..total_size).step_by(CHUNK_SIZE as usize) { | |
| let end = (start + CHUNK_SIZE - 1).min(total_size - 1); | |
| let client = client.clone(); | |
| let file = Arc::clone(&file); | |
| let url = DOWNLOAD_URL.to_string(); | |
| let handle = thread::spawn(move || { | |
| // println!("Downloading range: {start}-{end}"); | |
| let response = client | |
| .get(&url) | |
| .header("Range", format!("bytes={start}-{end}")) | |
| .send() | |
| .map_err(|e| format!("Request failed: {e}"))? | |
| .bytes() | |
| .map_err(|e| format!("Failed to read bytes because {e}"))?; | |
| let mut file = file.lock().map_err(|e| format!("Mutex lock failed: {e}"))?; | |
| file.seek(SeekFrom::Start(start)) | |
| .map_err(|e| format!("Seek failed: {e}"))?; | |
| file.write_all(&response) | |
| .map_err(|e| format!("Write failed: {e}"))?; | |
| Ok::<(), String>(()) | |
| }); | |
| handles.push(handle); | |
| } | |
| let duration = now.elapsed(); | |
| println!("Downloads in chunks took: {duration:?}"); | |
| Ok(()) | |
| } | |
| fn download_with_progress_bar() -> Result<(), String> { | |
| let client = Client::new(); | |
| let vlc_url = "https://get.videolan.org/vlc/3.0.21/macosx/vlc-3.0.21-intel64.dmg"; | |
| let destination = "downloads/vlc.dmg"; | |
| let total_size = client | |
| .head(vlc_url) | |
| .send() | |
| .map_err(|e| format!("failed to send head request: {e}"))? | |
| .headers() | |
| .get(reqwest::header::CONTENT_LENGTH) | |
| .ok_or("Content-length header missing")? | |
| .to_str() | |
| .map_err(|e| format!("invalid content-length header: {e}"))? | |
| .parse::<u64>() | |
| .map_err(|e| format!("failed to parse content-length: {e}"))?; | |
| let file = File::create(destination).map_err(|e| format!("failed to create file because {e}"))?; | |
| file.set_len(total_size).map_err(|e| format!("failed to set file size because: {e}"))?; | |
| let file = Arc::new(Mutex::new(file)); | |
| let progress_bar = ProgressBar::new(total_size); | |
| progress_bar.set_style( | |
| ProgressStyle::default_bar() | |
| .template("{spinner:.green} [{elapsed_precise}] [{bar:40.cyan/blue}] {bytes}/{total_bytes} ({bytes_per_sec}) {msg}") | |
| .unwrap() | |
| .progress_chars("#>-"), | |
| ); | |
| let (sender, receiver) = mpsc::channel::<u64>(); | |
| let chunk_size: u64 = 1024 * 1024; // 1MB chunks | |
| // spawn threads for downloading chunks | |
| for start in (0..total_size).step_by(chunk_size as usize) { | |
| let end = (start + chunk_size - 1).min(total_size - 1); | |
| let client = client.clone(); | |
| let file = Arc::clone(&file); | |
| let sender = sender.clone(); | |
| let url = vlc_url.to_string(); | |
| thread::spawn(move || { | |
| match client.get(&url).header("Range", format!("bytes={start}-{end}")) | |
| .send() | |
| .and_then(|res| res.bytes()) { | |
| Ok(response) => { | |
| let mut file = file.lock().expect("failed to lock file"); | |
| file.seek(SeekFrom::Start(start)).expect("seek failed"); | |
| file.write_all(&response).expect("write failed"); | |
| sender.send(response.len() as u64).expect("failed to send progress"); | |
| } | |
| Err(e) => { | |
| eprintln!("failed to download chunk: {e}"); | |
| } | |
| } | |
| }); | |
| } | |
| for downloaded in receiver { | |
| progress_bar.inc(downloaded); | |
| } | |
| progress_bar.finish_with_message("Download complete"); | |
| Ok(()) | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment