From 40c10d546862c866466bdc7d26159cfdfc89a2c4 Mon Sep 17 00:00:00 2001
From: Quaternions <krakow20@gmail.com>
Date: Fri, 12 Jan 2024 16:19:01 -0800
Subject: [PATCH] make download_history faster
---
src/main.rs | 121 +++++++++++++++++++++++++++-------------------------
1 file changed, 63 insertions(+), 58 deletions(-)
diff --git a/src/main.rs b/src/main.rs
index ffd1e57..17d556c 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -288,36 +288,36 @@ async fn download_list(cookie:String,asset_id_file_map:AssetIDFileMap)->AResult<
Ok(())
}
+async fn download_page(client:&reqwest::Client,cookie:&str,asset_id:AssetID,cursor:Option<String>)->AResult<VersionPage>{
+ let mut url=reqwest::Url::parse(format!("https://develop.roblox.com/v1/assets/{}/saved-versions",asset_id).as_str())?;
+ //url borrow scope
+ {
+ let mut query=url.query_pairs_mut();//borrow here
+ //query.append_pair("sortOrder","Asc");
+ //query.append_pair("limit","100");
+ //query.append_pair("count","100");
+ match cursor.as_deref(){
+ Some(next_page)=>{query.append_pair("cursor",next_page);}
+ None=>(),
+ }
+ }
+ println!("page url={}",url);
+ let resp=client.get(url)
+ .header("Cookie",cookie)
+ .send().await?;
+ Ok(resp.json::<VersionPage>().await?)
+}
+
async fn get_version_history(client:&reqwest::Client,cookie:&str,asset_id:AssetID)->AResult<Vec<AssetVersion>>{
let mut cursor:Option<String>=None;
let mut asset_list=Vec::new();
loop{
- let mut url=reqwest::Url::parse(format!("https://develop.roblox.com/v1/assets/{}/saved-versions",asset_id).as_str())?;
- //url borrow scope
- {
- let mut query=url.query_pairs_mut();//borrow here
- //query.append_pair("sortOrder","Asc");
- //query.append_pair("limit","100");
- //query.append_pair("count","100");
- match cursor.as_deref(){
- Some(next_page)=>{query.append_pair("cursor",next_page);}
- None=>(),
- }
- }
- println!("page url={}",url);
- let resp=client.get(url)
- .header("Cookie",cookie)
- .send().await?;
- match resp.json::<VersionPage>().await{
- Ok(mut page)=>{
- asset_list.append(&mut page.data);
- if page.nextPageCursor.is_none(){
- break;
- }
- cursor=page.nextPageCursor;
- },
- Err(e)=>panic!("error: {}",e),
+ let mut page=download_page(client,cookie,asset_id,cursor).await?;
+ asset_list.append(&mut page.data);
+ if page.nextPageCursor.is_none(){
+ break;
}
+ cursor=page.nextPageCursor;
}
asset_list.sort_by(|a,b|a.assetVersionNumber.cmp(&b.assetVersionNumber));
Ok(asset_list)
@@ -356,45 +356,50 @@ struct DownloadHistoryConfig{
async fn download_history(config:DownloadHistoryConfig)->AResult<()>{
let client=reqwest::Client::new();
+ let asset_id_string=config.asset_id.to_string();
+
//poll paged list of all asset versions
- let asset_list=get_version_history(&client,&config.cookie.as_str(),config.asset_id).await?;
+ let mut cursor:Option<String>=None;
+ let mut asset_list=Vec::new();
+ let mut join_handles=Vec::new();
+ loop{
+ let mut page=download_page(&client,config.cookie.as_str(),config.asset_id,cursor).await?;
+ for asset_version in &page.data{
+ let version_number=asset_version.assetVersionNumber;
+ let client=client.clone();
+ let cookie=config.cookie.clone();
+ let asset_id_str=asset_id_string.clone();
+ let output_folder=config.output_folder.clone();
+ join_handles.push(tokio::spawn(async move{
+ let resp=download_asset_version(&client,cookie.as_str(),asset_id_str.as_str(),version_number.to_string().as_str()).await?;
+ let contents=match maybe_gzip_decode(std::io::Cursor::new(resp.bytes().await?))?{
+ ReaderType::GZip(readable)=>read_readable(readable)?,
+ ReaderType::Raw(readable)=>read_readable(readable)?,
+ };
+
+ let mut path=output_folder;
+ path.set_file_name(format!("{}_v{}.rbxl",config.asset_id,version_number));
+
+ tokio::fs::write(path,contents).await?;
+
+ Ok::<_,anyhow::Error>(())
+ }));
+ }
+ asset_list.append(&mut page.data);
+ if page.nextPageCursor.is_none(){
+ break;
+ }
+ cursor=page.nextPageCursor;
+ }
+
let mut path=config.output_folder.clone();
path.set_file_name("versions.json");
tokio::fs::write(path,serde_json::to_string(&asset_list)?).await?;
- //download all versions
- let asset_id_string=config.asset_id.to_string();
- futures::stream::iter(asset_list)
- .map(|asset_version|{
- let client=&client;
- let cookie=config.cookie.as_str();
- let asset_id_str=asset_id_string.as_str();
- let output_folder=config.output_folder.clone();
- async move{
- let resp=download_asset_version(client,cookie,asset_id_str,asset_version.assetVersionNumber.to_string().as_str()).await?;
- let contents=match maybe_gzip_decode(std::io::Cursor::new(resp.bytes().await?))?{
- ReaderType::GZip(readable)=>read_readable(readable)?,
- ReaderType::Raw(readable)=>read_readable(readable)?,
- };
+ for join_handle in join_handles{
+ join_handle.await??;
+ }
- let mut path=output_folder;
- path.set_file_name(format!("{}_v{}.rbxl",config.asset_id,asset_version.assetVersionNumber));
-
- Ok((path,contents))
- }
- })
- .buffer_unordered(CONCURRENT_REQUESTS)
- .for_each(|b:AResult<_>|async{
- match b{
- Ok((dest,data))=>{
- match tokio::fs::write(dest,data).await{
- Err(e)=>eprintln!("fs error: {}",e),
- _=>(),
- }
- },
- Err(e)=>eprintln!("dl error: {}",e),
- }
- }).await;
Ok(())
}