StrafesNET/asset-tool
3
Fork 1
Code Issues Pull Requests Packages Projects Releases 4 Wiki Activity

make download_history faster

Browse Source
This commit is contained in:
Quaternions 2024-01-12 16:19:01 -08:00
parent b3f5dd6c4d
commit eb20f8056d
1 changed files with 65 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

79
src/main.rs
View File

@ -288,10 +288,7 @@ async fn download_list(cookie:String,asset_id_file_map:AssetIDFileMap)->AResult<
Ok(())
}
async fn get_version_history(client:&reqwest::Client,cookie:&str,asset_id:AssetID)->AResult<Vec<AssetVersion>>{
let mut cursor:Option<String>=None;
let mut asset_list=Vec::new();
loop{
async fn download_page(client:&reqwest::Client,cookie:&str,asset_id:AssetID,cursor:Option<String>)->AResult<VersionPage>{
let mut url=reqwest::Url::parse(format!("https://develop.roblox.com/v1/assets/{}/saved-versions",asset_id).as_str())?;
//url borrow scope
{
@ -308,16 +305,19 @@ async fn get_version_history(client:&reqwest::Client,cookie:&str,asset_id:AssetI
let resp=client.get(url)
.header("Cookie",cookie)
.send().await?;
match resp.json::<VersionPage>().await{
Ok(mut page)=>{
Ok(resp.json::<VersionPage>().await?)
}
async fn get_version_history(client:&reqwest::Client,cookie:&str,asset_id:AssetID)->AResult<Vec<AssetVersion>>{
let mut cursor:Option<String>=None;
let mut asset_list=Vec::new();
loop{
let mut page=download_page(client,cookie,asset_id,cursor).await?;
asset_list.append(&mut page.data);
if page.nextPageCursor.is_none(){
break;
}
cursor=page.nextPageCursor;
},
Err(e)=>panic!("error: {}",e),
}
}
asset_list.sort_by(|a,b|a.assetVersionNumber.cmp(&b.assetVersionNumber));
Ok(asset_list)
@ -356,45 +356,52 @@ struct DownloadHistoryConfig{
async fn download_history(config:DownloadHistoryConfig)->AResult<()>{
let client=reqwest::Client::new();
//poll paged list of all asset versions
let asset_list=get_version_history(&client,&config.cookie.as_str(),config.asset_id).await?;
let mut path=config.output_folder.clone();
path.set_file_name("versions.json");
tokio::fs::write(path,serde_json::to_string(&asset_list)?).await?;
//download all versions
let asset_id_string=config.asset_id.to_string();
futures::stream::iter(asset_list)
.map(|asset_version|{
let client=&client;
let cookie=config.cookie.as_str();
let asset_id_str=asset_id_string.as_str();
//poll paged list of all asset versions
let mut cursor:Option<String>=None;
let mut asset_list=Vec::new();
let mut join_handles=Vec::new();
loop{
let mut page=download_page(&client,config.cookie.as_str(),config.asset_id,cursor).await?;
for asset_version in &page.data{
let version_number=asset_version.assetVersionNumber;
let client=client.clone();
let cookie=config.cookie.clone();
let asset_id_str=asset_id_string.clone();
let output_folder=config.output_folder.clone();
async move{
let resp=download_asset_version(client,cookie,asset_id_str,asset_version.assetVersionNumber.to_string().as_str()).await?;
join_handles.push(tokio::spawn(async move{
let resp=download_asset_version(&client,cookie.as_str(),asset_id_str.as_str(),version_number.to_string().as_str()).await?;
let contents=match maybe_gzip_decode(std::io::Cursor::new(resp.bytes().await?))?{
ReaderType::GZip(readable)=>read_readable(readable)?,
ReaderType::Raw(readable)=>read_readable(readable)?,
};
let mut path=output_folder;
path.set_file_name(format!("{}_v{}.rbxl",config.asset_id,asset_version.assetVersionNumber));
path.set_file_name(format!("{}_v{}.rbxl",config.asset_id,version_number));
Ok((path,contents))
tokio::fs::write(path,contents).await?;
Ok::<_,anyhow::Error>(())
}));
}
})
.buffer_unordered(CONCURRENT_REQUESTS)
.for_each(|b:AResult<_>|async{
match b{
Ok((dest,data))=>{
match tokio::fs::write(dest,data).await{
Err(e)=>eprintln!("fs error: {}",e),
_=>(),
asset_list.append(&mut page.data);
if page.nextPageCursor.is_none(){
break;
}
},
Err(e)=>eprintln!("dl error: {}",e),
cursor=page.nextPageCursor;
}
}).await;
asset_list.sort_by(|a,b|a.assetVersionNumber.cmp(&b.assetVersionNumber));
let mut path=config.output_folder.clone();
path.set_file_name("versions.json");
tokio::fs::write(path,serde_json::to_string(&asset_list)?).await?;
for join_handle in join_handles{
join_handle.await??;
}
Ok(())
}