add --continue feature for download history

src/main.rs

@@ -25,8 +25,13 @@ struct Cli{
 	cookie_env:Option<String>,
 	#[arg(long)]
 	cookie_file:Option<std::path::PathBuf>,
+	//TODO: read the versions.json file instead of doing this
 	#[arg(long)]
 	start_version:Option<u64>,
+	#[arg(long)]
+	end_version:Option<u64>,
+	#[arg(long)]
+	r#continue:bool,
 
 	//decompile options
 	#[arg(long)]
@@ -136,6 +141,8 @@ async fn main()->AResult<()>{
 
 	match cli.command{
 		Commands::DownloadHistory=>download_history(DownloadHistoryConfig{
+			continue_from_versions:cli.r#continue,
+			end_version:cli.end_version,
 			start_version:cli.start_version.unwrap_or(0),
 			output_folder:cli.output.unwrap(),
 			cookie:cookie.unwrap(),
@@ -351,13 +358,54 @@ async fn download_asset_version(client:&reqwest::Client,cookie:&str,asset_id_str
 }
 
 struct DownloadHistoryConfig{
+	continue_from_versions:bool,
+	end_version:Option<u64>,
 	start_version:u64,
 	output_folder:std::path::PathBuf,
 	cookie:String,
 	asset_id:AssetID,
 }
 
-async fn download_history(config:DownloadHistoryConfig)->AResult<()>{
+async fn download_history(mut config:DownloadHistoryConfig)->AResult<()>{
+	let mut asset_list:Vec<AssetVersion>=Vec::new();
+	if config.end_version.is_none()&&config.continue_from_versions{
+		//load prexisting versions list
+		let mut versions_path=config.output_folder.clone();
+		versions_path.push("versions.json");
+		match std::fs::File::open(versions_path){
+			Ok(versions_file)=>asset_list.append(&mut serde_json::from_reader(versions_file)?),
+			Err(e)=>match e.kind(){
+				std::io::ErrorKind::NotFound=>return Err(anyhow::Error::msg("Cannot continue from versions.json - file does not exist")),
+				_=>Err(e)?,
+			}
+		}
+		//write down which versions are contained
+		let mut reference_hashmap=std::collections::HashSet::new();
+		for asset_version in &asset_list{
+			reference_hashmap.insert(asset_version.assetVersionNumber);
+		}
+		//find the highest number
+		match asset_list.iter().map(|asset_version|asset_version.assetVersionNumber).max(){
+			Some(max)=>{
+				//count down contiguously until a number is missing
+				for i in (1..=max).rev(){
+					if !reference_hashmap.contains(&i){
+						//that is end_version
+						config.end_version=Some(i);
+						break;
+					}
+				}
+				//if all versions are contained, set start_version to the max + 1
+				if config.end_version.is_none(){
+					config.start_version=max+1;
+				}
+			},
+			None=>Err(anyhow::Error::msg("Cannot continue from versions.json - there are no previous versions"))?,
+		}
+		//clear the asset list, I can't tell if roblox provides a way to start on a specific page
+		//so it just starts from the beginning every time
+		asset_list.clear();
+	}
 	let client=reqwest::Client::new();
 
 	let asset_id_string=config.asset_id.to_string();
@@ -367,12 +415,21 @@ async fn download_history(config:DownloadHistoryConfig)->AResult<()>{
 
 	//poll paged list of all asset versions
 	let mut cursor:Option<String>=None;
-	let mut asset_list=Vec::new();
 	loop{
 		let mut page=download_page(&client,config.cookie.as_str(),config.asset_id,cursor).await?;
+		let client=&client;
+		let cookie=config.cookie.clone();
+		let asset_id_str=asset_id_string.clone();
+		let output_folder=config.output_folder.clone();
+		let data=&page.data;
+		let join_set=&mut join_set;
+		let error_catcher=||async move{
 		let mut cancel_paging=false;
-		for asset_version in &page.data{
+		for asset_version in data{
 			let version_number=asset_version.assetVersionNumber;
+			if config.end_version.is_some_and(|v|v<version_number){
+				continue;
+			}
 			if version_number<config.start_version{
 				cancel_paging=true;
 				continue;//don't trust roblox returned order
@@ -381,9 +438,9 @@ async fn download_history(config:DownloadHistoryConfig)->AResult<()>{
 				join_set.join_next().await.unwrap()??;
 			}
 			let client=client.clone();
-			let cookie=config.cookie.clone();
-			let asset_id_str=asset_id_string.clone();
-			let output_folder=config.output_folder.clone();
+			let cookie=cookie.clone();
+			let asset_id_str=asset_id_str.clone();
+			let output_folder=output_folder.clone();
 			join_set.spawn(async move{
 				let resp=download_asset_version(&client,cookie.as_str(),asset_id_str.as_str(),version_number.to_string().as_str()).await?;
 				let contents=match maybe_gzip_decode(std::io::Cursor::new(resp.bytes().await?))?{
@@ -391,7 +448,7 @@ async fn download_history(config:DownloadHistoryConfig)->AResult<()>{
 					ReaderType::Raw(readable)=>read_readable(readable)?,
 				};
 
-				let mut path=output_folder;
+				let mut path=output_folder.clone();
 				path.set_file_name(format!("{}_v{}.rbxl",config.asset_id,version_number));
 
 				tokio::fs::write(path,contents).await?;
@@ -399,6 +456,16 @@ async fn download_history(config:DownloadHistoryConfig)->AResult<()>{
 				Ok::<_,anyhow::Error>(())
 			});
 		}
+		Ok::<_,anyhow::Error>(cancel_paging)
+		};
+		let cancel_paging=match error_catcher().await{
+			Ok(cancel)=>cancel,
+			Err(e)=>{
+				println!("download error: {}",e);
+				//cancel download and write versions
+				true
+			},
+		};
 		if page.nextPageCursor.is_none()||cancel_paging{
 			for asset_version in page.data.into_iter(){
 				if config.start_version<=asset_version.assetVersionNumber{
@@ -908,7 +975,7 @@ struct DecompileHistoryConfig{
 }
 
 async fn decompile_history_into_git(config:DecompileHistoryConfig)->AResult<()>{
-	//poll paged list of all asset versions
+	//use prexisting versions list
 	let mut versions_path=config.input_folder.clone();
 	versions_path.push("versions.json");
 	let asset_list:Vec<AssetVersion>=serde_json::from_reader(std::fs::File::open(versions_path)?)?;