From eb20f8056d6956723ae644cf4431a25ca1a6b418 Mon Sep 17 00:00:00 2001 From: Quaternions Date: Fri, 12 Jan 2024 16:19:01 -0800 Subject: [PATCH] make download_history faster --- src/main.rs | 123 +++++++++++++++++++++++++++------------------------- 1 file changed, 65 insertions(+), 58 deletions(-) diff --git a/src/main.rs b/src/main.rs index ffd1e57..691fb5e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -288,36 +288,36 @@ async fn download_list(cookie:String,asset_id_file_map:AssetIDFileMap)->AResult< Ok(()) } +async fn download_page(client:&reqwest::Client,cookie:&str,asset_id:AssetID,cursor:Option)->AResult{ + let mut url=reqwest::Url::parse(format!("https://develop.roblox.com/v1/assets/{}/saved-versions",asset_id).as_str())?; + //url borrow scope + { + let mut query=url.query_pairs_mut();//borrow here + //query.append_pair("sortOrder","Asc"); + //query.append_pair("limit","100"); + //query.append_pair("count","100"); + match cursor.as_deref(){ + Some(next_page)=>{query.append_pair("cursor",next_page);} + None=>(), + } + } + println!("page url={}",url); + let resp=client.get(url) + .header("Cookie",cookie) + .send().await?; + Ok(resp.json::().await?) +} + async fn get_version_history(client:&reqwest::Client,cookie:&str,asset_id:AssetID)->AResult>{ let mut cursor:Option=None; let mut asset_list=Vec::new(); loop{ - let mut url=reqwest::Url::parse(format!("https://develop.roblox.com/v1/assets/{}/saved-versions",asset_id).as_str())?; - //url borrow scope - { - let mut query=url.query_pairs_mut();//borrow here - //query.append_pair("sortOrder","Asc"); - //query.append_pair("limit","100"); - //query.append_pair("count","100"); - match cursor.as_deref(){ - Some(next_page)=>{query.append_pair("cursor",next_page);} - None=>(), - } - } - println!("page url={}",url); - let resp=client.get(url) - .header("Cookie",cookie) - .send().await?; - match resp.json::().await{ - Ok(mut page)=>{ - asset_list.append(&mut page.data); - if page.nextPageCursor.is_none(){ - break; - } - cursor=page.nextPageCursor; - }, - Err(e)=>panic!("error: {}",e), + let mut page=download_page(client,cookie,asset_id,cursor).await?; + asset_list.append(&mut page.data); + if page.nextPageCursor.is_none(){ + break; } + cursor=page.nextPageCursor; } asset_list.sort_by(|a,b|a.assetVersionNumber.cmp(&b.assetVersionNumber)); Ok(asset_list) @@ -356,45 +356,52 @@ struct DownloadHistoryConfig{ async fn download_history(config:DownloadHistoryConfig)->AResult<()>{ let client=reqwest::Client::new(); + let asset_id_string=config.asset_id.to_string(); + //poll paged list of all asset versions - let asset_list=get_version_history(&client,&config.cookie.as_str(),config.asset_id).await?; + let mut cursor:Option=None; + let mut asset_list=Vec::new(); + let mut join_handles=Vec::new(); + loop{ + let mut page=download_page(&client,config.cookie.as_str(),config.asset_id,cursor).await?; + for asset_version in &page.data{ + let version_number=asset_version.assetVersionNumber; + let client=client.clone(); + let cookie=config.cookie.clone(); + let asset_id_str=asset_id_string.clone(); + let output_folder=config.output_folder.clone(); + join_handles.push(tokio::spawn(async move{ + let resp=download_asset_version(&client,cookie.as_str(),asset_id_str.as_str(),version_number.to_string().as_str()).await?; + let contents=match maybe_gzip_decode(std::io::Cursor::new(resp.bytes().await?))?{ + ReaderType::GZip(readable)=>read_readable(readable)?, + ReaderType::Raw(readable)=>read_readable(readable)?, + }; + + let mut path=output_folder; + path.set_file_name(format!("{}_v{}.rbxl",config.asset_id,version_number)); + + tokio::fs::write(path,contents).await?; + + Ok::<_,anyhow::Error>(()) + })); + } + asset_list.append(&mut page.data); + if page.nextPageCursor.is_none(){ + break; + } + cursor=page.nextPageCursor; + } + + asset_list.sort_by(|a,b|a.assetVersionNumber.cmp(&b.assetVersionNumber)); + let mut path=config.output_folder.clone(); path.set_file_name("versions.json"); tokio::fs::write(path,serde_json::to_string(&asset_list)?).await?; - //download all versions - let asset_id_string=config.asset_id.to_string(); - futures::stream::iter(asset_list) - .map(|asset_version|{ - let client=&client; - let cookie=config.cookie.as_str(); - let asset_id_str=asset_id_string.as_str(); - let output_folder=config.output_folder.clone(); - async move{ - let resp=download_asset_version(client,cookie,asset_id_str,asset_version.assetVersionNumber.to_string().as_str()).await?; - let contents=match maybe_gzip_decode(std::io::Cursor::new(resp.bytes().await?))?{ - ReaderType::GZip(readable)=>read_readable(readable)?, - ReaderType::Raw(readable)=>read_readable(readable)?, - }; + for join_handle in join_handles{ + join_handle.await??; + } - let mut path=output_folder; - path.set_file_name(format!("{}_v{}.rbxl",config.asset_id,asset_version.assetVersionNumber)); - - Ok((path,contents)) - } - }) - .buffer_unordered(CONCURRENT_REQUESTS) - .for_each(|b:AResult<_>|async{ - match b{ - Ok((dest,data))=>{ - match tokio::fs::write(dest,data).await{ - Err(e)=>eprintln!("fs error: {}",e), - _=>(), - } - }, - Err(e)=>eprintln!("dl error: {}",e), - } - }).await; Ok(()) }