diff --git a/src/main.rs b/src/main.rs index 0a557ad..5cb0c58 100644 --- a/src/main.rs +++ b/src/main.rs @@ -362,10 +362,12 @@ async fn download_history(config:DownloadHistoryConfig)->AResult<()>{ let asset_id_string=config.asset_id.to_string(); + //limit concurrent downloads + let mut join_set=tokio::task::JoinSet::new(); + //poll paged list of all asset versions let mut cursor:Option=None; let mut asset_list=Vec::new(); - let mut join_handles=Vec::new(); loop{ let mut page=download_page(&client,config.cookie.as_str(),config.asset_id,cursor).await?; let mut cancel_paging=false; @@ -375,11 +377,14 @@ async fn download_history(config:DownloadHistoryConfig)->AResult<()>{ cancel_paging=true; continue;//don't trust roblox returned order } + while CONCURRENT_REQUESTS<=join_set.len(){ + join_set.join_next().await.unwrap()??; + } let client=client.clone(); let cookie=config.cookie.clone(); let asset_id_str=asset_id_string.clone(); let output_folder=config.output_folder.clone(); - join_handles.push(tokio::spawn(async move{ + join_set.spawn(async move{ let resp=download_asset_version(&client,cookie.as_str(),asset_id_str.as_str(),version_number.to_string().as_str()).await?; let contents=match maybe_gzip_decode(std::io::Cursor::new(resp.bytes().await?))?{ ReaderType::GZip(readable)=>read_readable(readable)?, @@ -392,7 +397,7 @@ async fn download_history(config:DownloadHistoryConfig)->AResult<()>{ tokio::fs::write(path,contents).await?; Ok::<_,anyhow::Error>(()) - })); + }); } if page.nextPageCursor.is_none()||cancel_paging{ for asset_version in page.data.into_iter(){ @@ -413,8 +418,8 @@ async fn download_history(config:DownloadHistoryConfig)->AResult<()>{ path.set_file_name("versions.json"); tokio::fs::write(path,serde_json::to_string(&asset_list)?).await?; - for join_handle in join_handles{ - join_handle.await??; + while let Some(result)=join_set.join_next().await{ + result??; } Ok(())