make download_history faster

src/main.rs

@@ -288,36 +288,36 @@ async fn download_list(cookie:String,asset_id_file_map:AssetIDFileMap)->AResult<
 	Ok(())
 }
 
+async fn download_page(client:&reqwest::Client,cookie:&str,asset_id:AssetID,cursor:Option<String>)->AResult<VersionPage>{
+	let mut url=reqwest::Url::parse(format!("https://develop.roblox.com/v1/assets/{}/saved-versions",asset_id).as_str())?;
+	//url borrow scope
+	{
+		let mut query=url.query_pairs_mut();//borrow here
+		//query.append_pair("sortOrder","Asc");
+		//query.append_pair("limit","100");
+		//query.append_pair("count","100");
+		match cursor.as_deref(){
+			Some(next_page)=>{query.append_pair("cursor",next_page);}
+			None=>(),
+		}
+	}
+	println!("page url={}",url);
+	let resp=client.get(url)
+	.header("Cookie",cookie)
+	.send().await?;
+	Ok(resp.json::<VersionPage>().await?)
+}
+
 async fn get_version_history(client:&reqwest::Client,cookie:&str,asset_id:AssetID)->AResult<Vec<AssetVersion>>{
 	let mut cursor:Option<String>=None;
 	let mut asset_list=Vec::new();
 	loop{
-		let mut url=reqwest::Url::parse(format!("https://develop.roblox.com/v1/assets/{}/saved-versions",asset_id).as_str())?;
+		let mut page=download_page(client,cookie,asset_id,cursor).await?;
-		//url borrow scope
+		asset_list.append(&mut page.data);
-		{
+		if page.nextPageCursor.is_none(){
-			let mut query=url.query_pairs_mut();//borrow here
+			break;
-			//query.append_pair("sortOrder","Asc");
-			//query.append_pair("limit","100");
-			//query.append_pair("count","100");
-			match cursor.as_deref(){
-				Some(next_page)=>{query.append_pair("cursor",next_page);}
-				None=>(),
-			}
-		}
-		println!("page url={}",url);
-		let resp=client.get(url)
-		.header("Cookie",cookie)
-		.send().await?;
-		match resp.json::<VersionPage>().await{
-			Ok(mut page)=>{
-				asset_list.append(&mut page.data);
-				if page.nextPageCursor.is_none(){
-					break;
-				}
-				cursor=page.nextPageCursor;
-			},
-			Err(e)=>panic!("error: {}",e),
 		}
+		cursor=page.nextPageCursor;
 	}
 	asset_list.sort_by(|a,b|a.assetVersionNumber.cmp(&b.assetVersionNumber));
 	Ok(asset_list)
@@ -356,45 +356,52 @@ struct DownloadHistoryConfig{
 async fn download_history(config:DownloadHistoryConfig)->AResult<()>{
 	let client=reqwest::Client::new();
 
+	let asset_id_string=config.asset_id.to_string();
+
 	//poll paged list of all asset versions
-	let asset_list=get_version_history(&client,&config.cookie.as_str(),config.asset_id).await?;
+	let mut cursor:Option<String>=None;
+	let mut asset_list=Vec::new();
+	let mut join_handles=Vec::new();
+	loop{
+		let mut page=download_page(&client,config.cookie.as_str(),config.asset_id,cursor).await?;
+		for asset_version in &page.data{
+			let version_number=asset_version.assetVersionNumber;
+			let client=client.clone();
+			let cookie=config.cookie.clone();
+			let asset_id_str=asset_id_string.clone();
+			let output_folder=config.output_folder.clone();
+			join_handles.push(tokio::spawn(async move{
+				let resp=download_asset_version(&client,cookie.as_str(),asset_id_str.as_str(),version_number.to_string().as_str()).await?;
+				let contents=match maybe_gzip_decode(std::io::Cursor::new(resp.bytes().await?))?{
+					ReaderType::GZip(readable)=>read_readable(readable)?,
+					ReaderType::Raw(readable)=>read_readable(readable)?,
+				};
+
+				let mut path=output_folder;
+				path.set_file_name(format!("{}_v{}.rbxl",config.asset_id,version_number));
+
+				tokio::fs::write(path,contents).await?;
+
+				Ok::<_,anyhow::Error>(())
+			}));
+		}
+		asset_list.append(&mut page.data);
+		if page.nextPageCursor.is_none(){
+			break;
+		}
+		cursor=page.nextPageCursor;
+	}
+
+	asset_list.sort_by(|a,b|a.assetVersionNumber.cmp(&b.assetVersionNumber));
+
 	let mut path=config.output_folder.clone();
 	path.set_file_name("versions.json");
 	tokio::fs::write(path,serde_json::to_string(&asset_list)?).await?;
 
-	//download all versions
+	for join_handle in join_handles{
-	let asset_id_string=config.asset_id.to_string();
+		join_handle.await??;
-	futures::stream::iter(asset_list)
+	}
-	.map(|asset_version|{
-		let client=&client;
-		let cookie=config.cookie.as_str();
-		let asset_id_str=asset_id_string.as_str();
-		let output_folder=config.output_folder.clone();
-		async move{
-			let resp=download_asset_version(client,cookie,asset_id_str,asset_version.assetVersionNumber.to_string().as_str()).await?;
-			let contents=match maybe_gzip_decode(std::io::Cursor::new(resp.bytes().await?))?{
-				ReaderType::GZip(readable)=>read_readable(readable)?,
-				ReaderType::Raw(readable)=>read_readable(readable)?,
-			};
 
-			let mut path=output_folder;
-			path.set_file_name(format!("{}_v{}.rbxl",config.asset_id,asset_version.assetVersionNumber));
-
-			Ok((path,contents))
-		}
-	})
-	.buffer_unordered(CONCURRENT_REQUESTS)
-	.for_each(|b:AResult<_>|async{
-		match b{
-			Ok((dest,data))=>{
-				match tokio::fs::write(dest,data).await{
-					Err(e)=>eprintln!("fs error: {}",e),
-					_=>(),
-				}
-			},
-			Err(e)=>eprintln!("dl error: {}",e),
-		}
-	}).await;
 	Ok(())
 }