remove mapfixer + asset-tool functions

This commit is contained in:
Quaternions 2024-03-08 10:01:57 -08:00
parent c2d0a4487c
commit c1ddcdb0c5

View File

@ -14,20 +14,13 @@ struct Cli {
#[derive(Subcommand)] #[derive(Subcommand)]
enum Commands { enum Commands {
Download(MapList),
DownloadTextures(PathBufList), DownloadTextures(PathBufList),
ExtractTextures(PathBufList), ExtractTextures(PathBufList),
ConvertTextures, ConvertTextures,
VPKContents, VPKContents,
BSPContents, BSPContents,
DownloadMeshes(PathBufList), DownloadMeshes(PathBufList),
Extract(PathBufList),
WriteAttributes, WriteAttributes,
Interactive,
Replace,
Scan,
UnzipAll,
Upload,
} }
#[derive(Args)] #[derive(Args)]
@ -35,28 +28,16 @@ struct PathBufList {
paths:Vec<PathBuf> paths:Vec<PathBuf>
} }
#[derive(Args)]
struct MapList {
maps: Vec<u64>,
}
fn main() -> AResult<()> { fn main() -> AResult<()> {
let cli = Cli::parse(); let cli = Cli::parse();
match cli.command { match cli.command {
Commands::Download(map_list)=>download(map_list.maps),
Commands::DownloadTextures(pathlist)=>download_textures(pathlist.paths), Commands::DownloadTextures(pathlist)=>download_textures(pathlist.paths),
Commands::ExtractTextures(pathlist)=>extract_textures(vec![cli.path.unwrap()],pathlist.paths), Commands::ExtractTextures(pathlist)=>extract_textures(vec![cli.path.unwrap()],pathlist.paths),
Commands::VPKContents=>vpk_contents(cli.path.unwrap()), Commands::VPKContents=>vpk_contents(cli.path.unwrap()),
Commands::BSPContents=>bsp_contents(cli.path.unwrap()), Commands::BSPContents=>bsp_contents(cli.path.unwrap()),
Commands::ConvertTextures=>convert_textures(), Commands::ConvertTextures=>convert_textures(),
Commands::DownloadMeshes(pathlist)=>download_meshes(pathlist.paths), Commands::DownloadMeshes(pathlist)=>download_meshes(pathlist.paths),
Commands::Extract(pathlist)=>extract(pathlist.paths),
Commands::WriteAttributes=>write_attributes(), Commands::WriteAttributes=>write_attributes(),
Commands::Interactive=>interactive(),
Commands::Replace=>replace(),
Commands::Scan=>scan(),
Commands::UnzipAll=>unzip_all(),
Commands::Upload=>upload(),
} }
} }
@ -92,40 +73,7 @@ fn recursive_collect_regex(objects: &mut std::vec::Vec<rbx_dom_weak::types::Ref>
} }
} }
} }
fn get_full_name(dom:&rbx_dom_weak::WeakDom,instance:&rbx_dom_weak::Instance) -> String{
let mut full_name=instance.name.clone();
let mut pref=instance.parent();
while let Some(parent)=dom.get_by_ref(pref){
full_name.insert(0, '.');
full_name.insert_str(0, &parent.name);
pref=parent.parent();
}
full_name
}
//download
//download list of maps to maps/unprocessed
//scan (scripts)
//iter maps/unprocessed
//passing moves to maps/verified
//failing moves to maps/blocked
//replace (edits & deletions)
//iter maps/blocked
//replace scripts and put in maps/unprocessed
//upload
//iter maps/verified
//interactively print DisplayName/Creator and ask for target upload ids
//interactive
//iter maps/unprocessed
//for each unique script, load it into the file current.lua and have it open in sublime text
//I can edit the file and it will edit it in place
//I pass/fail(with comment)/allow each script
fn get_script_refs(dom:&rbx_dom_weak::WeakDom) -> Vec<rbx_dom_weak::types::Ref>{
let mut scripts = std::vec::Vec::new();
recursive_collect_superclass(&mut scripts, dom, dom.root(),"LuaSourceContainer");
scripts
}
fn get_button_refs(dom:&rbx_dom_weak::WeakDom) -> Vec<rbx_dom_weak::types::Ref>{ fn get_button_refs(dom:&rbx_dom_weak::WeakDom) -> Vec<rbx_dom_weak::types::Ref>{
let mut buttons = std::vec::Vec::new(); let mut buttons = std::vec::Vec::new();
recursive_collect_regex(&mut buttons, dom, dom.root(),lazy_regex::regex!(r"Button(\d+)$")); recursive_collect_regex(&mut buttons, dom, dom.root(),lazy_regex::regex!(r"Button(\d+)$"));
@ -197,109 +145,6 @@ fn get_dom<R:Read+Seek>(input:&mut R)->AResult<rbx_dom_weak::WeakDom>{
} }
} }
fn get_id() -> AResult<u32>{
match std::fs::read_to_string("id"){
Ok(id_file)=>Ok(id_file.parse::<u32>()?),
Err(e) => match e.kind() {
std::io::ErrorKind::NotFound => Ok(0),//implicitly take on id=0
_ => Err(e)?,
}
}
}
fn get_set_from_file(file:&str) -> AResult<std::collections::HashSet<String>>{
let mut set=std::collections::HashSet::<String>::new();
for entry in std::fs::read_dir(file)? {
set.insert(std::fs::read_to_string(entry?.path())?);
}
Ok(set)
}
fn get_allowed_set() -> AResult<std::collections::HashSet<String>>{
get_set_from_file("scripts/allowed")
}
fn get_blocked() -> AResult<std::collections::HashSet<String>>{
get_set_from_file("scripts/blocked")
}
fn get_allowed_map() -> AResult<std::collections::HashMap::<u32,String>>{
let mut allowed_map = std::collections::HashMap::<u32,String>::new();
for entry in std::fs::read_dir("scripts/allowed")? {
let entry=entry?;
allowed_map.insert(entry.path().file_stem().unwrap().to_str().unwrap().parse::<u32>()?,std::fs::read_to_string(entry.path())?);
}
Ok(allowed_map)
}
fn get_replace_map() -> AResult<std::collections::HashMap::<String,u32>>{
let mut replace = std::collections::HashMap::<String,u32>::new();
for entry in std::fs::read_dir("scripts/replace")? {
let entry=entry?;
replace.insert(std::fs::read_to_string(entry.path())?,entry.path().file_stem().unwrap().to_str().unwrap().parse::<u32>()?);
}
Ok(replace)
}
fn check_source_illegal_keywords(source:&String)->bool{
source.find("getfenv").is_some()||source.find("require").is_some()
}
fn find_first_child_class<'a>(dom:&'a rbx_dom_weak::WeakDom,instance:&'a rbx_dom_weak::Instance,name:&'a str,class:&'a str) -> Option<&'a rbx_dom_weak::Instance> {
for &referent in instance.children() {
if let Some(c) = dom.get_by_ref(referent) {
if c.name==name&&class_is_a(c.class.as_str(),class) {
return Some(c);
}
}
}
None
}
fn get_mapinfo(dom:&rbx_dom_weak::WeakDom) -> AResult<(String,String,String)>{
let workspace_children=dom.root().children();
if workspace_children.len()!=1{
return Err(anyhow::Error::msg("there can only be one model"));
}
if let Some(model_instance) = dom.get_by_ref(workspace_children[0]) {
if let (Some(creator),Some(displayname))=(find_first_child_class(dom, model_instance, "Creator", "StringValue"),find_first_child_class(dom, model_instance, "DisplayName", "StringValue")){
if let (
Some(rbx_dom_weak::types::Variant::String(creator_string)),
Some(rbx_dom_weak::types::Variant::String(displayname_string))
)=(
creator.properties.get("Value"),
displayname.properties.get("Value")
){
return Ok((model_instance.name.clone(),creator_string.clone(),displayname_string.clone()));
}
}
}
return Err(anyhow::Error::msg("no stuff in map"));
}
fn download(map_list: Vec<u64>) -> AResult<()>{
let header=format!("Cookie: .ROBLOSECURITY={}",std::env::var("RBXCOOKIE")?);
let shared_args=&[
"-q",
"--header",
header.as_str(),
"-O",
];
let processes_result:Result<Vec<_>, _>=map_list.iter().map(|map_id|{
std::process::Command::new("wget")
.args(shared_args)
.arg(format!("maps/unprocessed/{}.rbxm",map_id))
.arg(format!("https://assetdelivery.roblox.com/v1/asset/?ID={}",map_id))
.spawn()
}).collect();
//naively wait for all because idk how to make an async progress bar lmao
for child in processes_result?{
let output=child.wait_with_output()?;
println!("map exit_success:{}",output.status.success());
}
Ok(())
}
struct RobloxAssetId(u64); struct RobloxAssetId(u64);
struct RobloxAssetIdParseErr; struct RobloxAssetIdParseErr;
impl std::str::FromStr for RobloxAssetId { impl std::str::FromStr for RobloxAssetId {
@ -524,489 +369,6 @@ fn convert_textures() -> AResult<()>{
Ok(()) Ok(())
} }
enum Scan{
Passed,
Blocked,
Flagged,
}
fn scan() -> AResult<()>{
let mut id = get_id()?;
//Construct allowed scripts
let allowed_set = get_allowed_set()?;
let mut blocked = get_blocked()?;
for entry in std::fs::read_dir("maps/unprocessed")? {
let file_thing=entry?;
let mut input = std::io::BufReader::new(std::fs::File::open(file_thing.path())?);
let dom = get_dom(&mut input)?;
let script_refs = get_script_refs(&dom);
//check scribb
let mut fail_count=0;
let mut fail_type=Scan::Passed;
for &script_ref in script_refs.iter() {
if let Some(script)=dom.get_by_ref(script_ref){
if let Some(rbx_dom_weak::types::Variant::String(s)) = script.properties.get("Source") {
//flag keywords and instantly fail
if check_source_illegal_keywords(s){
println!("{:?} - flagged.",file_thing.file_name());
fail_type=Scan::Flagged;
break;
}
if allowed_set.contains(s) {
continue;
}else{
fail_type=Scan::Blocked;//no need to check for Flagged, it breaks the loop.
fail_count+=1;
if !blocked.contains(s) {
blocked.insert(s.clone());//all fixed! just clone!
std::fs::write(format!("scripts/blocked/{}.lua",id),s)?;
id+=1;
}
}
}else{
panic!("FATAL: failed to get source for {:?}",file_thing.file_name());
}
}else{
panic!("FATAL: failed to get_by_ref {:?}",script_ref);
}
}
let mut dest=match fail_type {
Scan::Passed => PathBuf::from("maps/processed"),
Scan::Blocked => {
println!("{:?} - {} {} not allowed.",file_thing.file_name(),fail_count,if fail_count==1 {"script"}else{"scripts"});
PathBuf::from("maps/blocked")
}
Scan::Flagged => PathBuf::from("maps/flagged")
};
dest.push(file_thing.file_name());
std::fs::rename(file_thing.path(), dest)?;
}
std::fs::write("id",id.to_string())?;
Ok(())
}
fn extract(paths: Vec<PathBuf>) -> AResult<()>{
let mut id = 0;
//Construct allowed scripts
let mut script_set = std::collections::HashSet::<String>::new();
for path in paths {
let file_name=path.file_name();
let mut input = std::io::BufReader::new(std::fs::File::open(&path)?);
let dom = get_dom(&mut input)?;
let script_refs = get_script_refs(&dom);
//extract scribb
for &script_ref in script_refs.iter() {
if let Some(script)=dom.get_by_ref(script_ref){
if let Some(rbx_dom_weak::types::Variant::String(s)) = script.properties.get("Source") {
if script_set.contains(s) {
continue;
}else{
script_set.insert(s.clone());
std::fs::write(format!("scripts/extracted/{:?}_{}_{}.lua",file_name,id,script.name),s)?;
id+=1;
}
}else{
panic!("FATAL: failed to get source for {:?}",file_name);
}
}else{
panic!("FATAL: failed to get_by_ref {:?}",script_ref);
}
}
}
println!("extracted {} {}",id,if id==1 {"script"}else{"scripts"});
Ok(())
}
fn replace() -> AResult<()>{
let allowed_map=get_allowed_map()?;
let replace_map=get_replace_map()?;
for entry in std::fs::read_dir("maps/blocked")? {
let file_thing=entry?;
let mut input = std::io::BufReader::new(std::fs::File::open(file_thing.path())?);
let mut dom = get_dom(&mut input)?;
let script_refs = get_script_refs(&dom);
//check scribb
let mut any_failed=false;
for &script_ref in script_refs.iter() {
if let Some(script)=dom.get_by_ref(script_ref){
if let Some(rbx_dom_weak::types::Variant::String(source)) = script.properties.get("Source") {
if let (Some(replace_id),Some(replace_script))=(replace_map.get(source),dom.get_by_ref_mut(script.referent())) {
println!("replace {}",replace_id);
//replace the source
if let Some(replace_source)=allowed_map.get(replace_id){
replace_script.properties.insert("Source".to_string(), rbx_dom_weak::types::Variant::String(replace_source.clone()));
}else{
println!("failed to get replacement source {}",replace_id);
any_failed=true;
}
}else{
println!("failed to failed to get replace_id and replace_script");
any_failed=true;
}
}else{
panic!("FATAL: failed to get source for {:?}",file_thing.file_name());
}
}else{
panic!("FATAL: failed to get_by_ref {:?}",script_ref);
}
}
if any_failed {
println!("One or more scripts failed to replace.");
}else{
let mut dest=PathBuf::from("maps/unprocessed");
dest.push(file_thing.file_name());
let output = std::io::BufWriter::new(std::fs::File::open(dest)?);
//write workspace:GetChildren()[1]
let workspace_children=dom.root().children();
if workspace_children.len()!=1{
return Err(anyhow::Error::msg("there can only be one model"));
}
rbx_binary::to_writer(output, &dom, &[workspace_children[0]])?;
}
}
Ok(())
}
enum UploadAction {
Upload(u64),
Skip,
New,
Delete,
}
struct ParseUploadActionErr;
impl std::str::FromStr for UploadAction {
type Err=ParseUploadActionErr;
fn from_str(s: &str) -> Result<Self, Self::Err>{
if s=="skip\n"{
Ok(Self::Skip)
}else if s=="new\n"{
Ok(Self::New)
}else if s=="delete\n"{
Ok(Self::Delete)
}else if let Ok(asset_id)=s[..s.len()-1].parse::<u64>(){
Ok(Self::Upload(asset_id))
}else{
Err(ParseUploadActionErr)
}
}
}
fn upload() -> AResult<()>{
//interactive prompt per upload:
for entry in std::fs::read_dir("maps/passed")? {
let file_thing=entry?;
println!("map file: {:?}",file_thing.file_name());
let mut input = std::io::BufReader::new(std::fs::File::open(file_thing.path())?);
let dom = get_dom(&mut input)?;
let (modelname,creator,displayname) = get_mapinfo(&dom)?;
//Creator: [auto fill creator]
//DisplayName: [auto fill DisplayName]
//id: ["New" for blank because of my double enter key]
print!("Model name: {}\nCreator: {}\nDisplayName: {}\nAction or Upload Asset Id: ",modelname,creator,displayname);
std::io::Write::flush(&mut std::io::stdout())?;
let upload_action;
loop{
let mut upload_action_string = String::new();
std::io::stdin().read_line(&mut upload_action_string)?;
if let Ok(parsed_upload_action)=upload_action_string.parse::<UploadAction>(){
upload_action=parsed_upload_action;
break;
}else{
print!("Action or Upload Asset Id: ");
std::io::Write::flush(&mut std::io::stdout())?;
}
}
match upload_action {
UploadAction::Upload(asset_id) => {
let status=std::process::Command::new("../rbxcompiler-linux-amd64")
.arg("--compile=false")
.arg("--group=6980477")
.arg(format!("--asset={}",asset_id))
.arg(format!("--input={}",file_thing.path().into_os_string().into_string().unwrap()))
.status()?;
match status.code() {
Some(0)=>{
//move file
let mut dest=PathBuf::from("maps/uploaded");
dest.push(file_thing.file_name());
std::fs::rename(file_thing.path(), dest)?;
}
Some(code)=>println!("upload failed! code={}",code),
None => println!("no status code!"),
}
}
UploadAction::Skip => continue,
UploadAction::New => {
let output=std::process::Command::new("../rbxcompiler-linux-amd64")
.arg("--compile=false")
.arg("--group=6980477")
.arg("--new-asset=true")
.arg(format!("--input={}",file_thing.path().into_os_string().into_string().unwrap()))
.output()?;
match output.status.code() {
Some(0)=>{
//print output
println!("{}", std::str::from_utf8(output.stdout.as_slice())?);
//move file
let mut dest=PathBuf::from("maps/uploaded");
dest.push(file_thing.file_name());
std::fs::rename(file_thing.path(), dest)?;
}
Some(code)=>println!("upload failed! code={}",code),
None => println!("no status code!"),
}
}
UploadAction::Delete => std::fs::remove_file(file_thing.path())?,
}
}
Ok(())
}
enum Interactive{
Passed,
Blocked,
Flagged,
}
enum ScriptAction {
Pass,
Replace(u32),
Flag,
Block,
Delete,
}
enum ScriptActionParseResult {
Pass,
Block,
Exit,
Delete,
}
struct ParseScriptActionErr;
impl std::str::FromStr for ScriptActionParseResult {
type Err=ParseScriptActionErr;
fn from_str(s: &str) -> Result<Self, Self::Err>{
if s=="pass\n"||s=="1\n"{
Ok(Self::Pass)
}else if s=="block\n"{
Ok(Self::Block)
}else if s=="exit\n"{
Ok(Self::Exit)
}else if s=="delete\n"{
Ok(Self::Delete)
}else{
Err(ParseScriptActionErr)
}
}
}
fn interactive() -> AResult<()>{
let mut id=get_id()?;
//Construct allowed scripts
let mut allowed_set=get_allowed_set()?;
let mut allowed_map=get_allowed_map()?;
let mut replace_map=get_replace_map()?;
let mut blocked = get_blocked()?;
'map_loop: for entry in std::fs::read_dir("maps/unprocessed")? {
let file_thing=entry?;
println!("processing map={:?}",file_thing.file_name());
let mut input = std::io::BufReader::new(std::fs::File::open(file_thing.path())?);
let mut dom = get_dom(&mut input)?;
let script_refs = get_script_refs(&dom);
//check scribb
let mut script_count=0;
let mut replace_count=0;
let mut block_count=0;
let mut fail_type=Interactive::Passed;
for &script_ref in script_refs.iter() {
if let Some(script)=dom.get_by_ref(script_ref){
if let Some(rbx_dom_weak::types::Variant::String(source)) = script.properties.get("Source") {
script_count+=1;
let source_action=if check_source_illegal_keywords(source) {
ScriptAction::Flag//script triggers flagging -> Flag
} else if blocked.contains(source) {
ScriptAction::Block//script is blocked -> Block
} else if allowed_set.contains(source) {
ScriptAction::Pass//script is allowed -> Pass
}else if let Some(replace_id)=replace_map.get(source) {
ScriptAction::Replace(*replace_id)
}else{
//interactive logic goes here
print!("unresolved source location={}\naction: ",get_full_name(&dom, script));
std::io::Write::flush(&mut std::io::stdout())?;
//load source into current.lua
std::fs::write("current.lua",source)?;
//prompt action in terminal
//wait for input
let script_action;
loop{
let mut action_string = String::new();
std::io::stdin().read_line(&mut action_string)?;
if let Ok(parsed_script_action)=action_string.parse::<ScriptActionParseResult>(){
script_action=parsed_script_action;
break;
}else{
print!("action: ");
std::io::Write::flush(&mut std::io::stdout())?;
}
}
//update allowed/replace/blocked
match script_action{
ScriptActionParseResult::Pass => {
//if current.lua was updated, create an allowed and replace file and set script_action to replace(new_id)
let modified_source=std::fs::read_to_string("current.lua")?;
if &modified_source==source{
//it's always new.
//insert allowed_set
allowed_set.insert(modified_source.clone());
//insert allowed_map
allowed_map.insert(id,modified_source.clone());
//write allowed/id.lua
std::fs::write(format!("scripts/allowed/{}.lua",id),modified_source)?;
id+=1;
ScriptAction::Pass
}else{
//insert allowed_set
allowed_set.insert(modified_source.clone());
//insert allowed_map
allowed_map.insert(id,modified_source.clone());
//insert replace_map
replace_map.insert(source.clone(),id);//this cannot be reached if it already exists
//write allowed/id.lua
std::fs::write(format!("scripts/allowed/{}.lua",id),modified_source)?;
//write replace/id.lua
std::fs::write(format!("scripts/replace/{}.lua",id),source)?;
let ret=ScriptAction::Replace(id);
id+=1;
ret
}
},
ScriptActionParseResult::Block => {
blocked.insert(source.clone());
std::fs::write(format!("scripts/blocked/{}.lua",id),source)?;
id+=1;
ScriptAction::Block
},
ScriptActionParseResult::Exit => break 'map_loop,
ScriptActionParseResult::Delete => ScriptAction::Delete,
}
};
let location=get_full_name(&dom, script);
match source_action{
ScriptAction::Pass => println!("passed source location={}",location),
ScriptAction::Replace(replace_id)=>{
//replace the source
if let (Some(replace_source),Some(replace_script))=(allowed_map.get(&replace_id),dom.get_by_ref_mut(script.referent())){
replace_count+=1;
println!("replaced source id={} location={}",replace_id,location);
replace_script.properties.insert("Source".to_string(), rbx_dom_weak::types::Variant::String(replace_source.clone()));
}else{
panic!("failed to get replacement source id={} location={}",replace_id,location);
}
},
ScriptAction::Delete => {
println!("deleted source location={}",location);
replace_count+=1;//trigger a new file generation
dom.destroy(script.referent());
},
ScriptAction::Flag => {
println!("flagged source location={}",location);
fail_type=Interactive::Flagged;
},
ScriptAction::Block => {
block_count+=1;
println!("blocked source location={}",location);
match fail_type{
Interactive::Passed => fail_type=Interactive::Blocked,
_=>(),
}
},
}
}else{
panic!("FATAL: failed to get source for {:?}",file_thing.file_name());
}
}else{
panic!("FATAL: failed to get_by_ref {:?}",script_ref);
}
}
let mut dest=match fail_type{
Interactive::Passed => {
println!("map={:?} passed with {} {}",file_thing.file_name(),script_count,if script_count==1 {"script"}else{"scripts"});
if replace_count==0{
PathBuf::from("maps/passed")
}else{
//create new file
println!("{} {} replaced - generating new file...",replace_count,if replace_count==1 {"script was"}else{"scripts were"});
let mut dest=PathBuf::from("maps/passed");
dest.push(file_thing.file_name());
let output = std::io::BufWriter::new(std::fs::File::create(dest)?);
//write workspace:GetChildren()[1]
let workspace_children=dom.root().children();
if workspace_children.len()!=1{
return Err(anyhow::Error::msg("there can only be one model"));
}
rbx_binary::to_writer(output, &dom, &[workspace_children[0]])?;
//move original to processed folder
PathBuf::from("maps/unaltered")
}
},//write map into maps/processed
Interactive::Blocked => {
println!("map={:?} blocked with {}/{} {} blocked",file_thing.file_name(),block_count,script_count,if script_count==1 {"script"}else{"scripts"});
PathBuf::from("maps/blocked")
},//write map into maps/blocked
Interactive::Flagged => {
println!("map={:?} flagged",file_thing.file_name());
PathBuf::from("maps/flagged")
},//write map into maps/flagged
};
dest.push(file_thing.file_name());
std::fs::rename(file_thing.path(), dest)?;
}
std::fs::write("id",id.to_string())?;
Ok(())
}
fn unzip_all()->AResult<()>{
for entry in std::fs::read_dir("maps/unprocessed")? {
let file_thing=entry?;
println!("processing map={:?}",file_thing.file_name());
let mut input = std::io::BufReader::new(std::fs::File::open(file_thing.path())?);
match maybe_gzip_decode(&mut input){
Ok(ReaderType::GZip(mut readable)) => {
//gzip
let mut extracted:Vec<u8>=Vec::new();
//read the entire thing to the end so that I can clone the data and write a png to processed images
readable.read_to_end(&mut extracted)?;
//write extracted
let mut dest=PathBuf::from("maps/unzipped");
dest.push(file_thing.file_name());
std::fs::write(dest, &mut extracted)?;
//delete ugly gzip file
std::fs::remove_file(file_thing.path())?;
},
Ok(ReaderType::Raw(_)) => (),
Err(e) => Err(e)?,
}
}
Ok(())
}
fn write_attributes() -> AResult<()>{ fn write_attributes() -> AResult<()>{
for entry in std::fs::read_dir("maps/unprocessed")? { for entry in std::fs::read_dir("maps/unprocessed")? {
let file_thing=entry?; let file_thing=entry?;