From f27386acfd4b7178fa4d5e19593f7fcac7ed799f Mon Sep 17 00:00:00 2001 From: Quaternions Date: Sat, 5 Jul 2025 02:17:16 -0700 Subject: [PATCH 1/8] update deps --- Cargo.lock | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 96b7f35..a3f9026 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -135,9 +135,9 @@ checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" [[package]] name = "cc" -version = "1.2.27" +version = "1.2.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d487aa071b5f64da6f19a3e848e3578944b726ee5a4854b82172f02aa876bfdc" +checksum = "4ad45f4f74e4e20eaa392913b7b33a7091c87e59628f4dd27888205ad888843c" dependencies = [ "shlex", ] @@ -705,6 +705,17 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "io-uring" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b86e202f00093dcba4275d4636b93ef9dd75d025ae560d2521b45ea28ab49013" +dependencies = [ + "bitflags", + "cfg-if", + "libc", +] + [[package]] name = "ipnet" version = "2.11.0" @@ -987,9 +998,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.12.21" +version = "0.12.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c8cea6b35bcceb099f30173754403d2eba0a5dc18cea3630fccd88251909288" +checksum = "cbc931937e6ca3a06e3b6c0aa7841849b160a90351d6ab467a8b9b9959767531" dependencies = [ "base64", "bytes", @@ -1250,9 +1261,9 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "submissions-api" -version = "0.8.1" +version = "0.8.2" source = "sparse+https://git.itzana.me/api/packages/strafesnet/cargo/" -checksum = "a08deea49e9e34f2f2f23219f4a565681b4c1ae46f8012496d9a8fe10897efd3" +checksum = "ced7fba3c9851f2ee97e4be939a544b92794c234ab090b76e8f2cef540f5c92f" dependencies = [ "chrono", "reqwest", @@ -1345,15 +1356,17 @@ dependencies = [ [[package]] name = "tokio" -version = "1.45.1" +version = "1.46.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75ef51a33ef1da925cea3e4eb122833cb377c61439ca401b770f54902b806779" +checksum = "0cc3a2344dafbe23a245241fe8b09735b521110d30fcefbbd5feb1797ca35d17" dependencies = [ "backtrace", "bytes", + "io-uring", "libc", "mio", "pin-project-lite", + "slab", "socket2", "tokio-macros", "windows-sys 0.52.0", -- 2.49.1 From d31bf7da2476e1447157748373e64c48a90d81be Mon Sep 17 00:00:00 2001 From: Quaternions Date: Sat, 5 Jul 2025 02:40:59 -0700 Subject: [PATCH 2/8] create missing policies --- src/main.rs | 63 ++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 60 insertions(+), 3 deletions(-) diff --git a/src/main.rs b/src/main.rs index ed6cc91..f3f9601 100644 --- a/src/main.rs +++ b/src/main.rs @@ -487,6 +487,7 @@ enum RepairError{ UpdateScriptPolicy(submissions_api::Error), DeleteScript(submissions_api::Error), DeleteScriptPolicy(submissions_api::Error), + CreateScriptPolicy(submissions_api::Error), } struct RepairConfig{ session_id:String, @@ -557,7 +558,8 @@ async fn repair(config:RepairConfig)->Result<(),RepairError>{ download_policies(api), )?; - let mut script_id_from_hash=std::collections::HashMap::new(); + let mut script_from_id=std::collections::HashMap::new(); + let mut script_from_hash=std::collections::HashMap::new(); let mut unique_sources=std::collections::HashSet::new(); let mut duplicate_scripts=Vec::new(); @@ -570,7 +572,8 @@ async fn repair(config:RepairConfig)->Result<(),RepairError>{ ScriptID:script.ID, }); }else{ - script_id_from_hash.insert(script.Hash.as_str(),script); + script_from_id.insert(script.ID,script); + script_from_hash.insert(script.Hash.as_str(),script); } } @@ -598,9 +601,12 @@ async fn repair(config:RepairConfig)->Result<(),RepairError>{ let mut update_policies=Vec::new(); let mut policies_not_unique=Vec::new(); let mut policies_missing_from_script=Vec::new(); + let mut check_policy_exists=Vec::new(); + + let mut policy_from_hash=std::collections::HashMap::new(); for policy in &policies{ - let from_script=script_id_from_hash.get(policy.FromScriptHash.as_str()); + let from_script=script_from_hash.get(policy.FromScriptHash.as_str()); if let Some(&from_script)=from_script{ if policy.ToScriptID==from_script.ID{ @@ -619,6 +625,14 @@ async fn repair(config:RepairConfig)->Result<(),RepairError>{ policies_not_unique.push(submissions_api::types::GetScriptPolicyRequest{ ScriptPolicyID:policy.ID, }); + }else{ + policy_from_hash.insert(policy.FromScriptHash.as_str(),policy); + // if policy is replace, but destination script is not allowed + if let Some(&to_script)=script_from_id.get(&policy.ToScriptID){ + check_policy_exists.push((policy,to_script)); + }else{ + println!("ToScript does not exist! {:?} {:?} DOING NOTHING",policy.ToScriptID,policy.ID); + } } } }else{ @@ -629,6 +643,31 @@ async fn repair(config:RepairConfig)->Result<(),RepairError>{ } } + let mut create_allow_policy=Vec::new(); + for (policy,to_script) in check_policy_exists{ + if let Some(&allow_policy)=policy_from_hash.get(to_script.Hash.as_str()){ + if allow_policy.Policy!=submissions_api::types::Policy::Allowed{ + println!("Policy {:?} ToScript {:?} Policy {:?} was expected to be Allowed, but was {:?}!",policy.ID,to_script.ID,allow_policy.ID,allow_policy.Policy); + } + }else{ + if policy.Policy==submissions_api::types::Policy::Replace{ + println!("Policy {:?} ToScript {:?} has no Allowed policy!",policy.ID,to_script.ID); + create_allow_policy.push(submissions_api::types::CreateScriptPolicyRequest{ + FromScriptID:to_script.ID, + ToScriptID:to_script.ID, + Policy:submissions_api::types::Policy::Allowed, + }); + } + } + } + + // All scripts should have exactly one policy + for script in &scripts{ + if !policy_from_hash.contains_key(script.Hash.as_str()){ + println!("Script {:?} has no policy!",script.ID); + } + } + if !update_policies.is_empty(){ print!("Update {} policies? [y/N]: ",update_policies.len()); std::io::stdout().flush().map_err(RepairError::Io)?; @@ -680,6 +719,24 @@ async fn repair(config:RepairConfig)->Result<(),RepairError>{ api.delete_script_policy(request) }).await.map_err(RepairError::DeleteScriptPolicy)?; } + if !create_allow_policy.is_empty(){ + print!("Create {} missing Allowed policies? [y/N]: ",create_allow_policy.len()); + std::io::stdout().flush().map_err(RepairError::Io)?; + + let mut input=String::new(); + std::io::stdin().read_line(&mut input).map_err(RepairError::Io)?; + match input.trim(){ + "y"|"Y"=>(), + _=>{ + println!("Quitting."); + return Ok(()); + }, + } + futures::stream::iter(create_allow_policy).map(Ok).try_for_each_concurrent(REMOTE_CONCURRENCY,async|request|{ + api.create_script_policy(request).await?; + Ok(()) + }).await.map_err(RepairError::CreateScriptPolicy)?; + } Ok(()) } -- 2.49.1 From 5b9def8ca9d8d069431155c0ac3589ce39c567f8 Mon Sep 17 00:00:00 2001 From: Quaternions Date: Sat, 5 Jul 2025 02:50:33 -0700 Subject: [PATCH 3/8] import policy --- src/main.rs | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/main.rs b/src/main.rs index f3f9601..8b4ab80 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,7 +1,7 @@ use clap::{Args,Parser,Subcommand}; use futures::{StreamExt,TryStreamExt}; use rand::seq::SliceRandom; -use submissions_api::types::{ScriptResponse,ScriptPolicyResponse}; +use submissions_api::types::{Policy,ScriptResponse,ScriptPolicyResponse}; use std::io::Write; use std::path::PathBuf; @@ -134,7 +134,7 @@ async fn review(config:ReviewConfig)->Result<(),ReviewError>{ Limit:100, FromScriptHash:None, ToScriptID:None, - Policy:Some(submissions_api::types::Policy::None), + Policy:Some(Policy::None), }).await.map_err(ReviewError::GetPolicies)?; let unreviewed_policy_count=unreviewed_policies.len(); @@ -186,7 +186,7 @@ async fn review(config:ReviewConfig)->Result<(),ReviewError>{ //if current.lua was updated, create an allowed and replace file and set script_action to replace(new_id) let modified_source=tokio::fs::read_to_string("current.lua").await.map_err(ReviewError::ReadCurrent)?; if modified_source==source{ - submissions_api::types::Policy::Allowed + Policy::Allowed }else{ // compute hash let hash=hash_source(modified_source.as_str()); @@ -213,7 +213,7 @@ async fn review(config:ReviewConfig)->Result<(),ReviewError>{ api.create_script_policy(submissions_api::types::CreateScriptPolicyRequest{ FromScriptID:new_script.ScriptID, ToScriptID:new_script.ScriptID, - Policy:submissions_api::types::Policy::Allowed, + Policy:Policy::Allowed, }).await.map_err(ReviewError::UploadModifiedPolicy)?; new_script.ScriptID @@ -221,12 +221,12 @@ async fn review(config:ReviewConfig)->Result<(),ReviewError>{ }); // use replace policy - submissions_api::types::Policy::Replace + Policy::Replace } }, - ScriptActionParseResult::Block=>submissions_api::types::Policy::Blocked, + ScriptActionParseResult::Block=>Policy::Blocked, ScriptActionParseResult::Exit=>break, - ScriptActionParseResult::Delete=>submissions_api::types::Policy::Delete, + ScriptActionParseResult::Delete=>Policy::Delete, ScriptActionParseResult::Skip=>continue, }; @@ -437,7 +437,7 @@ async fn upload_scripts(config:UploadConfig)->Result<(),ScriptUploadError>{ submissions_api::types::CreateScriptPolicyRequest{ FromScriptID:script_ids[source.as_str()], ToScriptID:script_ids[allowed_map[id].as_str()], - Policy:submissions_api::types::Policy::Replace, + Policy:Policy::Replace, } ).await.map_err(ScriptUploadError::GetOrCreatePolicyReplace) }); @@ -451,7 +451,7 @@ async fn upload_scripts(config:UploadConfig)->Result<(),ScriptUploadError>{ submissions_api::types::CreateScriptPolicyRequest{ FromScriptID:script_ids[source.as_str()], ToScriptID:script_ids[source.as_str()], - Policy:submissions_api::types::Policy::Allowed, + Policy:Policy::Allowed, } ).await.map_err(ScriptUploadError::GetOrCreatePolicyAllowed) }); @@ -465,7 +465,7 @@ async fn upload_scripts(config:UploadConfig)->Result<(),ScriptUploadError>{ submissions_api::types::CreateScriptPolicyRequest{ FromScriptID:script_ids[source.as_str()], ToScriptID:script_ids[source.as_str()], - Policy:submissions_api::types::Policy::Blocked, + Policy:Policy::Blocked, } ).await.map_err(ScriptUploadError::GetOrCreatePolicyBlocked) }); @@ -533,7 +533,7 @@ async fn download_policies(api:&submissions_api::external::Context)->ResultResult<(),RepairError>{ ID:policy.ID, FromScriptID:None, ToScriptID:None, - Policy:Some(submissions_api::types::Policy::None), + Policy:Some(Policy::None), }); }else{ // if not unique @@ -646,16 +646,16 @@ async fn repair(config:RepairConfig)->Result<(),RepairError>{ let mut create_allow_policy=Vec::new(); for (policy,to_script) in check_policy_exists{ if let Some(&allow_policy)=policy_from_hash.get(to_script.Hash.as_str()){ - if allow_policy.Policy!=submissions_api::types::Policy::Allowed{ + if allow_policy.Policy!=Policy::Allowed{ println!("Policy {:?} ToScript {:?} Policy {:?} was expected to be Allowed, but was {:?}!",policy.ID,to_script.ID,allow_policy.ID,allow_policy.Policy); } }else{ - if policy.Policy==submissions_api::types::Policy::Replace{ + if policy.Policy==Policy::Replace{ println!("Policy {:?} ToScript {:?} has no Allowed policy!",policy.ID,to_script.ID); create_allow_policy.push(submissions_api::types::CreateScriptPolicyRequest{ FromScriptID:to_script.ID, ToScriptID:to_script.ID, - Policy:submissions_api::types::Policy::Allowed, + Policy:Policy::Allowed, }); } } -- 2.49.1 From af21785cb5cad15064e501e64e16a001beb17022 Mon Sep 17 00:00:00 2001 From: Quaternions Date: Sat, 5 Jul 2025 02:58:08 -0700 Subject: [PATCH 4/8] scan all policies --- src/main.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main.rs b/src/main.rs index 8b4ab80..aa14bf9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -533,7 +533,7 @@ async fn download_policies(api:&submissions_api::external::Context)->ResultResult<(),RepairError>{ let from_script=script_from_hash.get(policy.FromScriptHash.as_str()); if let Some(&from_script)=from_script{ - if policy.ToScriptID==from_script.ID{ + if policy.Policy==Policy::Replace&&policy.ToScriptID==from_script.ID{ // invalid policy. Reset the policy to None println!("Invalid policy {:?}, queueing update...",policy.ID); update_policies.push(submissions_api::types::UpdateScriptPolicyRequest{ -- 2.49.1 From a0235cb1b98427cff9015617fd17e0a8be2b6ccb Mon Sep 17 00:00:00 2001 From: Quaternions Date: Sat, 5 Jul 2025 03:05:26 -0700 Subject: [PATCH 5/8] display duplicate --- src/main.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main.rs b/src/main.rs index aa14bf9..3b59fa7 100644 --- a/src/main.rs +++ b/src/main.rs @@ -596,7 +596,7 @@ async fn repair(config:RepairConfig)->Result<(),RepairError>{ }).await.map_err(RepairError::DeleteScript)?; } - let mut unique_policies=std::collections::HashSet::new(); + let mut unique_policies=std::collections::HashMap::new(); let mut update_policies=Vec::new(); let mut policies_not_unique=Vec::new(); @@ -620,8 +620,8 @@ async fn repair(config:RepairConfig)->Result<(),RepairError>{ }); }else{ // if not unique - if !unique_policies.insert(policy.FromScriptHash.as_str()){ - println!("Policy is not unique! hash={} id={:?}",policy.FromScriptHash,policy.ID); + if let Some(old_policy)=unique_policies.insert(policy.FromScriptHash.as_str(),policy){ + println!("Policy is not unique! hash={} {:?} {:?}",policy.FromScriptHash,policy.ID,old_policy.ID); policies_not_unique.push(submissions_api::types::GetScriptPolicyRequest{ ScriptPolicyID:policy.ID, }); -- 2.49.1 From 4192dc20b923bc545f32a7e3d431340e5a556075 Mon Sep 17 00:00:00 2001 From: Quaternions Date: Sat, 5 Jul 2025 03:10:23 -0700 Subject: [PATCH 6/8] consolidate collections --- src/main.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/main.rs b/src/main.rs index 3b59fa7..50f3698 100644 --- a/src/main.rs +++ b/src/main.rs @@ -596,15 +596,13 @@ async fn repair(config:RepairConfig)->Result<(),RepairError>{ }).await.map_err(RepairError::DeleteScript)?; } - let mut unique_policies=std::collections::HashMap::new(); + let mut policy_from_hash=std::collections::HashMap::new(); let mut update_policies=Vec::new(); let mut policies_not_unique=Vec::new(); let mut policies_missing_from_script=Vec::new(); let mut check_policy_exists=Vec::new(); - let mut policy_from_hash=std::collections::HashMap::new(); - for policy in &policies{ let from_script=script_from_hash.get(policy.FromScriptHash.as_str()); @@ -620,13 +618,12 @@ async fn repair(config:RepairConfig)->Result<(),RepairError>{ }); }else{ // if not unique - if let Some(old_policy)=unique_policies.insert(policy.FromScriptHash.as_str(),policy){ + if let Some(old_policy)=policy_from_hash.insert(policy.FromScriptHash.as_str(),policy){ println!("Policy is not unique! hash={} {:?} {:?}",policy.FromScriptHash,policy.ID,old_policy.ID); policies_not_unique.push(submissions_api::types::GetScriptPolicyRequest{ ScriptPolicyID:policy.ID, }); }else{ - policy_from_hash.insert(policy.FromScriptHash.as_str(),policy); // if policy is replace, but destination script is not allowed if let Some(&to_script)=script_from_id.get(&policy.ToScriptID){ check_policy_exists.push((policy,to_script)); -- 2.49.1 From 2141af46d31b242037d2e5e790d85c19447520f8 Mon Sep 17 00:00:00 2001 From: Quaternions Date: Sat, 5 Jul 2025 03:10:44 -0700 Subject: [PATCH 7/8] only replace expects allowed --- src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index 50f3698..35b443c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -643,7 +643,7 @@ async fn repair(config:RepairConfig)->Result<(),RepairError>{ let mut create_allow_policy=Vec::new(); for (policy,to_script) in check_policy_exists{ if let Some(&allow_policy)=policy_from_hash.get(to_script.Hash.as_str()){ - if allow_policy.Policy!=Policy::Allowed{ + if policy.Policy==Policy::Replace&&allow_policy.Policy!=Policy::Allowed{ println!("Policy {:?} ToScript {:?} Policy {:?} was expected to be Allowed, but was {:?}!",policy.ID,to_script.ID,allow_policy.ID,allow_policy.Policy); } }else{ -- 2.49.1 From d609804d3346472c672ea7688bbd532a6c5a1cc1 Mon Sep 17 00:00:00 2001 From: Quaternions Date: Sat, 5 Jul 2025 03:17:44 -0700 Subject: [PATCH 8/8] deduplicate create missing allowed policies --- src/main.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/main.rs b/src/main.rs index 35b443c..d9f0a84 100644 --- a/src/main.rs +++ b/src/main.rs @@ -640,7 +640,7 @@ async fn repair(config:RepairConfig)->Result<(),RepairError>{ } } - let mut create_allow_policy=Vec::new(); + let mut create_allow_policy=std::collections::HashSet::new(); for (policy,to_script) in check_policy_exists{ if let Some(&allow_policy)=policy_from_hash.get(to_script.Hash.as_str()){ if policy.Policy==Policy::Replace&&allow_policy.Policy!=Policy::Allowed{ @@ -649,11 +649,7 @@ async fn repair(config:RepairConfig)->Result<(),RepairError>{ }else{ if policy.Policy==Policy::Replace{ println!("Policy {:?} ToScript {:?} has no Allowed policy!",policy.ID,to_script.ID); - create_allow_policy.push(submissions_api::types::CreateScriptPolicyRequest{ - FromScriptID:to_script.ID, - ToScriptID:to_script.ID, - Policy:Policy::Allowed, - }); + create_allow_policy.insert(to_script.ID); } } } @@ -729,8 +725,12 @@ async fn repair(config:RepairConfig)->Result<(),RepairError>{ return Ok(()); }, } - futures::stream::iter(create_allow_policy).map(Ok).try_for_each_concurrent(REMOTE_CONCURRENCY,async|request|{ - api.create_script_policy(request).await?; + futures::stream::iter(create_allow_policy).map(Ok).try_for_each_concurrent(REMOTE_CONCURRENCY,async|script_id|{ + api.create_script_policy(submissions_api::types::CreateScriptPolicyRequest{ + FromScriptID:script_id, + ToScriptID:script_id, + Policy:Policy::Allowed, + }).await?; Ok(()) }).await.map_err(RepairError::CreateScriptPolicy)?; } -- 2.49.1