add repair duplicates

Cargo.toml

@@ -9,7 +9,7 @@ edition = "2021"
 clap = { version = "4.4.2", features = ["derive"] }
 futures = "0.3.31"
 siphasher = "1.0.1"
-submissions-api = { version = "0.4.0", registry = "strafesnet" }
+submissions-api = { version = "0.7.2", registry = "strafesnet" }
 tokio = { version = "1.42.0", features = ["fs", "macros", "rt-multi-thread"] }
 
 [profile.release]

src/main.rs

@@ -15,10 +15,26 @@ struct Cli{
 
 #[derive(Subcommand)]
 enum Commands{
+	RepairDuplicates(RepairDuplicatesCommand),
+	RepairPolicies(RepairPoliciesCommand),
 	Review(ReviewCommand),
 	UploadScripts(UploadScriptsCommand),
 }
 
+#[derive(Args)]
+struct RepairDuplicatesCommand{
+	#[arg(long)]
+	session_id_file:PathBuf,
+	#[arg(long)]
+	api_url:String,
+}
+#[derive(Args)]
+struct RepairPoliciesCommand{
+	#[arg(long)]
+	session_id_file:PathBuf,
+	#[arg(long)]
+	api_url:String,
+}
 #[derive(Args)]
 struct ReviewCommand{
 	#[arg(long)]
@@ -38,6 +54,14 @@ struct UploadScriptsCommand{
 async fn main(){
 	let cli=Cli::parse();
 	match cli.command{
+		Commands::RepairDuplicates(command)=>repair_duplicates(RepairDuplicatesConfig{
+			session_id:std::fs::read_to_string(command.session_id_file).unwrap(),
+			api_url:command.api_url,
+		}).await.unwrap(),
+		Commands::RepairPolicies(command)=>repair_policies(RepairPoliciesConfig{
+			session_id:std::fs::read_to_string(command.session_id_file).unwrap(),
+			api_url:command.api_url,
+		}).await.unwrap(),
 		Commands::Review(command)=>review(ReviewConfig{
 			session_id:std::fs::read_to_string(command.session_id_file).unwrap(),
 			api_url:command.api_url,
@@ -54,6 +78,8 @@ enum ScriptActionParseResult{
 	Block,
 	Exit,
 	Delete,
+	Purge,
+	Skip,
 }
 struct ParseScriptActionErr;
 impl std::str::FromStr for ScriptActionParseResult{
@@ -64,6 +90,8 @@ impl std::str::FromStr for ScriptActionParseResult{
 			"block\n"=>Ok(Self::Block),
 			"exit\n"=>Ok(Self::Exit),
 			"delete\n"=>Ok(Self::Delete),
+			"purge\n"=>Ok(Self::Purge),
+			"skip\n"=>Ok(Self::Skip),
 			_=>Err(ParseScriptActionErr),
 		}
 	}
@@ -79,6 +107,7 @@ enum ReviewError{
 	NoScript,
 	WriteCurrent(std::io::Error),
 	ActionIO(std::io::Error),
+	PurgeScript(submissions_api::Error),
 	ReadCurrent(std::io::Error),
 	DeduplicateModified(submissions_api::types::SingleItemError),
 	UploadModified(submissions_api::Error),
@@ -104,7 +133,8 @@ async fn review(config:ReviewConfig)->Result<(),ReviewError>{
 		Policy:Some(submissions_api::types::Policy::None),
 	}).await.map_err(ReviewError::GetPolicies)?;
 
-	for unreviewed_policy in unreviewed_policies{
+	let unreviewed_policy_count=unreviewed_policies.len();
+	for (i,unreviewed_policy) in unreviewed_policies.into_iter().enumerate(){
 		// download source code
 		let script_response=api.get_script_from_hash(submissions_api::types::HashRequest{
 			hash:unreviewed_policy.FromScriptHash.as_str(),
@@ -116,6 +146,10 @@ async fn review(config:ReviewConfig)->Result<(),ReviewError>{
 		//load source into current.lua
 		tokio::fs::write("current.lua",source.as_str()).await.map_err(ReviewError::WriteCurrent)?;
 
+		// print script name
+		println!("policy {}/{unreviewed_policy_count}",i+1);
+		println!("script name: {}",script_response.Name);
+
 		//prompt action in terminal
 		//wait for input
 		let script_action;
@@ -133,6 +167,17 @@ async fn review(config:ReviewConfig)->Result<(),ReviewError>{
 		let mut to_script_id=None;
 		// interpret action
 		let reviewed_policy=match script_action{
+			ScriptActionParseResult::Purge=>{
+				// remove script and policy from the database.
+				let remove_script_fut=api.delete_script(submissions_api::types::GetScriptRequest{
+					ScriptID:script_response.ID,
+				});
+				let remove_script_policy_fut=api.delete_script_policy(submissions_api::types::GetScriptPolicyRequest{
+					ScriptPolicyID:unreviewed_policy.ID,
+				});
+				tokio::try_join!(remove_script_fut,remove_script_policy_fut).map_err(ReviewError::PurgeScript)?;
+				continue;
+			},
 			ScriptActionParseResult::Pass=>{
 				//if current.lua was updated, create an allowed and replace file and set script_action to replace(new_id)
 				let modified_source=tokio::fs::read_to_string("current.lua").await.map_err(ReviewError::ReadCurrent)?;
@@ -140,7 +185,7 @@ async fn review(config:ReviewConfig)->Result<(),ReviewError>{
 					submissions_api::types::Policy::Allowed
 				}else{
 					// compute hash
-					let hash=hash_source(source.as_str());
+					let hash=hash_source(modified_source.as_str());
 
 					// check if modified script already exists
 					let maybe_script_response=api.get_script_from_hash(submissions_api::types::HashRequest{
@@ -153,8 +198,9 @@ async fn review(config:ReviewConfig)->Result<(),ReviewError>{
 						None=>api.create_script(submissions_api::types::CreateScriptRequest{
 							Name:script_response.Name.as_str(),
 							Source:modified_source.as_str(),
-							SubmissionID:Some(script_response.SubmissionID),
+							ResourceType:script_response.ResourceType,
-						}).await.map_err(ReviewError::UploadModified)?.ID
+							ResourceID:Some(script_response.ResourceID),
+						}).await.map_err(ReviewError::UploadModified)?.ScriptID
 					});
 
 					// use replace policy
@@ -164,6 +210,7 @@ async fn review(config:ReviewConfig)->Result<(),ReviewError>{
 			ScriptActionParseResult::Block=>submissions_api::types::Policy::Blocked,
 			ScriptActionParseResult::Exit=>break,
 			ScriptActionParseResult::Delete=>submissions_api::types::Policy::Delete,
+			ScriptActionParseResult::Skip=>continue,
 		};
 
 		// update policy
@@ -173,6 +220,8 @@ async fn review(config:ReviewConfig)->Result<(),ReviewError>{
 			ToScriptID:to_script_id,
 			Policy:Some(reviewed_policy),
 		}).await.map_err(ReviewError::UpdateScriptPolicy)?;
+
+		println!("updated {:?} From: {:?} To: {:?} with policy {reviewed_policy:?}",unreviewed_policy.ID,script_response.ID,to_script_id);
 	}
 
 	Ok(())
@@ -282,8 +331,9 @@ async fn get_or_create_script(api:&submissions_api::external::Context,source:&st
 		None=>api.create_script(submissions_api::types::CreateScriptRequest{
 			Name:"Script",
 			Source:source,
-			SubmissionID:None,
+			ResourceType:submissions_api::types::ResourceType::Unknown,
-		}).await.map_err(GOCError::Other)?.ID
+			ResourceID:None,
+		}).await.map_err(GOCError::Other)?.ScriptID
 	})
 }
 
@@ -299,9 +349,9 @@ async fn check_or_create_script_poicy(
 	match script_policy_result{
 		Some(script_policy_reponse)=>{
 			// check that everything matches the expectation
-			assert!(hash==script_policy_reponse.FromScriptHash);
+			assert_eq!(hash,script_policy_reponse.FromScriptHash);
-			assert!(script_policy.ToScriptID==script_policy_reponse.ToScriptID);
+			assert_eq!(script_policy.ToScriptID,script_policy_reponse.ToScriptID);
-			assert!(script_policy.Policy==script_policy_reponse.Policy);
+			assert_eq!(script_policy.Policy,script_policy_reponse.Policy);
 		},
 		None=>{
 			// create a new policy
@@ -407,3 +457,123 @@ async fn upload_scripts(config:UploadConfig)->Result<(),ScriptUploadError>{
 
 	Ok(())
 }
+
+#[allow(dead_code)]
+#[derive(Debug)]
+enum RepairPoliciesError{
+	Cookie(submissions_api::CookieError),
+	Reqwest(submissions_api::ReqwestError),
+	GetPolicies(submissions_api::Error),
+	GetScripts(submissions_api::types::SingleItemError),
+	UpdateScriptPolicy(submissions_api::Error),
+}
+
+struct RepairPoliciesConfig{
+	session_id:String,
+	api_url:String,
+}
+async fn repair_policies(config:RepairPoliciesConfig)->Result<(),RepairPoliciesError>{
+	let cookie=submissions_api::Cookie::new(&config.session_id).map_err(RepairPoliciesError::Cookie)?;
+	let api=&submissions_api::external::Context::new(config.api_url,cookie).map_err(RepairPoliciesError::Reqwest)?;
+
+	const LIMIT:u32=100;
+	let mut page=1;
+	loop{
+		println!("Downloading page {page}...");
+		let policies=api.get_script_policies(submissions_api::types::GetScriptPoliciesRequest{
+			Page:page,
+			Limit:LIMIT,
+			FromScriptHash:None,
+			ToScriptID:None,
+			Policy:Some(submissions_api::types::Policy::Replace),
+		}).await.map_err(RepairPoliciesError::GetPolicies)?;
+
+		futures::stream::iter(policies.iter().map(Ok)).try_for_each_concurrent(REMOTE_CONCURRENCY,async|policy|{
+			let from_script=api.get_script_from_hash(submissions_api::types::HashRequest{
+				hash:policy.FromScriptHash.as_str(),
+			}).await.map_err(RepairPoliciesError::GetScripts)?;
+
+			if let Some(from_script)=from_script{
+				if policy.ToScriptID==from_script.ID{
+					// invalid policy.  Reset the policy to None
+					api.update_script_policy(submissions_api::types::UpdateScriptPolicyRequest{
+						ID:policy.ID,
+						FromScriptID:None,
+						ToScriptID:None,
+						Policy:Some(submissions_api::types::Policy::None),
+					}).await.map_err(RepairPoliciesError::UpdateScriptPolicy)?;
+					println!("Policy updated! {:?}",policy.ID);
+				}
+			}else{
+				println!("Script did not exist! hash={}",policy.FromScriptHash);
+			}
+			Ok(())
+		}).await?;
+
+		if policies.len()<LIMIT as usize{
+			// We scanned all policies
+			println!("Done!");
+			break;
+		}else{
+			page+=1;
+		}
+	}
+
+	Ok(())
+}
+
+#[allow(dead_code)]
+#[derive(Debug)]
+enum RepairDuplicatesError{
+	Cookie(submissions_api::CookieError),
+	Reqwest(submissions_api::ReqwestError),
+	GetScripts(submissions_api::Error),
+	DeleteScript(submissions_api::Error),
+}
+
+struct RepairDuplicatesConfig{
+	session_id:String,
+	api_url:String,
+}
+async fn repair_duplicates(config:RepairDuplicatesConfig)->Result<(),RepairDuplicatesError>{
+	let cookie=submissions_api::Cookie::new(&config.session_id).map_err(RepairDuplicatesError::Cookie)?;
+	let api=&submissions_api::external::Context::new(config.api_url,cookie).map_err(RepairDuplicatesError::Reqwest)?;
+
+	let mut sources=std::collections::HashSet::new();
+
+	const LIMIT:u32=100;
+	let mut page=1;
+	loop{
+		println!("Downloading page {page}...");
+		let scripts=api.get_scripts(submissions_api::types::GetScriptsRequest{
+			Page:page,
+			Limit:LIMIT,
+			Name:None,
+			Hash:None,
+			Source:None,
+			ResourceType:None,
+			ResourceID:None,
+		}).await.map_err(RepairDuplicatesError::GetScripts)?;
+
+		let done=scripts.len()<LIMIT as usize;
+
+		for script in scripts{
+			if !sources.insert(script.Source){
+				println!("Deleting duplicate script {:?}",script.ID);
+				api.delete_script(submissions_api::types::GetScriptRequest{
+					ScriptID:script.ID,
+				}).await.map_err(RepairDuplicatesError::DeleteScript)?;
+			}
+		}
+
+		if done{
+			// We scanned all policies
+			println!("Done!");
+			break;
+		}else{
+			page+=1;
+		}
+	}
+
+	Ok(())
+}