add repair duplicates

src/main.rs

@@ -15,11 +15,19 @@ struct Cli{
 
 #[derive(Subcommand)]
 enum Commands{
+	RepairDuplicates(RepairDuplicatesCommand),
 	RepairPolicies(RepairPoliciesCommand),
 	Review(ReviewCommand),
 	UploadScripts(UploadScriptsCommand),
 }
 
+#[derive(Args)]
+struct RepairDuplicatesCommand{
+	#[arg(long)]
+	session_id_file:PathBuf,
+	#[arg(long)]
+	api_url:String,
+}
 #[derive(Args)]
 struct RepairPoliciesCommand{
 	#[arg(long)]
@@ -46,7 +54,11 @@ struct UploadScriptsCommand{
 async fn main(){
 	let cli=Cli::parse();
 	match cli.command{
-		Commands::RepairPolicies(command)=>repair(RepairPoliciesConfig{
+		Commands::RepairDuplicates(command)=>repair_duplicates(RepairDuplicatesConfig{
+			session_id:std::fs::read_to_string(command.session_id_file).unwrap(),
+			api_url:command.api_url,
+		}).await.unwrap(),
+		Commands::RepairPolicies(command)=>repair_policies(RepairPoliciesConfig{
 			session_id:std::fs::read_to_string(command.session_id_file).unwrap(),
 			api_url:command.api_url,
 		}).await.unwrap(),
@@ -460,7 +472,7 @@ struct RepairPoliciesConfig{
 	session_id:String,
 	api_url:String,
 }
-async fn repair(config:RepairPoliciesConfig)->Result<(),RepairPoliciesError>{
+async fn repair_policies(config:RepairPoliciesConfig)->Result<(),RepairPoliciesError>{
 	let cookie=submissions_api::Cookie::new(&config.session_id).map_err(RepairPoliciesError::Cookie)?;
 	let api=&submissions_api::external::Context::new(config.api_url,cookie).map_err(RepairPoliciesError::Reqwest)?;
 
@@ -509,3 +521,59 @@ async fn repair(config:RepairPoliciesConfig)->Result<(),RepairPoliciesError>{
 
 	Ok(())
 }
+
+#[allow(dead_code)]
+#[derive(Debug)]
+enum RepairDuplicatesError{
+	Cookie(submissions_api::CookieError),
+	Reqwest(submissions_api::ReqwestError),
+	GetScripts(submissions_api::Error),
+	DeleteScript(submissions_api::Error),
+}
+
+struct RepairDuplicatesConfig{
+	session_id:String,
+	api_url:String,
+}
+async fn repair_duplicates(config:RepairDuplicatesConfig)->Result<(),RepairDuplicatesError>{
+	let cookie=submissions_api::Cookie::new(&config.session_id).map_err(RepairDuplicatesError::Cookie)?;
+	let api=&submissions_api::external::Context::new(config.api_url,cookie).map_err(RepairDuplicatesError::Reqwest)?;
+
+	let mut sources=std::collections::HashSet::new();
+
+	const LIMIT:u32=100;
+	let mut page=1;
+	loop{
+		println!("Downloading page {page}...");
+		let scripts=api.get_scripts(submissions_api::types::GetScriptsRequest{
+			Page:page,
+			Limit:LIMIT,
+			Name:None,
+			Hash:None,
+			Source:None,
+			ResourceType:None,
+			ResourceID:None,
+		}).await.map_err(RepairDuplicatesError::GetScripts)?;
+
+		let done=scripts.len()<LIMIT as usize;
+
+		for script in scripts{
+			if !sources.insert(script.Source){
+				println!("Deleting duplicate script {:?}",script.ID);
+				api.delete_script(submissions_api::types::GetScriptRequest{
+					ScriptID:script.ID,
+				}).await.map_err(RepairDuplicatesError::DeleteScript)?;
+			}
+		}
+
+		if done{
+			// We scanned all policies
+			println!("Done!");
+			break;
+		}else{
+			page+=1;
+		}
+	}
+
+	Ok(())
+}