use std::{io::{Read, Seek}, path::PathBuf}; use clap::{Args, Parser, Subcommand}; use anyhow::Result as AResult; #[derive(Parser)] #[command(author, version, about, long_about = None)] #[command(propagate_version = true)] struct Cli { #[arg(long)] path:Option, #[command(subcommand)] command: Commands, } #[derive(Subcommand)] enum Commands { Download(MapList), DownloadTextures(PathBufList), ExtractTextures(PathBufList), ConvertTextures, VPKContents, BSPContents, DownloadMeshes(PathBufList), Extract(PathBufList), WriteAttributes, Interactive, Replace, Scan, UnzipAll, Upload, } #[derive(Args)] struct PathBufList { paths:Vec } #[derive(Args)] struct MapList { maps: Vec, } fn main() -> AResult<()> { let cli = Cli::parse(); match cli.command { Commands::Download(map_list)=>download(map_list.maps), Commands::DownloadTextures(pathlist)=>download_textures(pathlist.paths), Commands::ExtractTextures(pathlist)=>extract_textures(vec![cli.path.unwrap()],pathlist.paths), Commands::VPKContents=>vpk_contents(cli.path.unwrap()), Commands::BSPContents=>bsp_contents(cli.path.unwrap()), Commands::ConvertTextures=>convert_textures(), Commands::DownloadMeshes(pathlist)=>download_meshes(pathlist.paths), Commands::Extract(pathlist)=>extract(pathlist.paths), Commands::WriteAttributes=>write_attributes(), Commands::Interactive=>interactive(), Commands::Replace=>replace(), Commands::Scan=>scan(), Commands::UnzipAll=>unzip_all(), Commands::Upload=>upload(), } } fn class_is_a(class: &str, superclass: &str) -> bool { if class==superclass { return true } let class_descriptor=rbx_reflection_database::get().classes.get(class); if let Some(descriptor) = &class_descriptor { if let Some(class_super) = &descriptor.superclass { return class_is_a(&class_super, superclass) } } false } fn recursive_collect_superclass(objects: &mut std::vec::Vec,dom: &rbx_dom_weak::WeakDom, instance: &rbx_dom_weak::Instance, superclass: &str){ for &referent in instance.children() { if let Some(c) = dom.get_by_ref(referent) { if class_is_a(c.class.as_str(), superclass) { objects.push(c.referent());//copy ref } recursive_collect_superclass(objects,dom,c,superclass); } } } fn recursive_collect_regex(objects: &mut std::vec::Vec,dom: &rbx_dom_weak::WeakDom, instance: &rbx_dom_weak::Instance, regex: &lazy_regex::Lazy){ for &referent in instance.children() { if let Some(c) = dom.get_by_ref(referent) { if regex.captures(c.name.as_str()).is_some(){ objects.push(c.referent());//copy ref } recursive_collect_regex(objects,dom,c,regex); } } } fn get_full_name(dom:&rbx_dom_weak::WeakDom,instance:&rbx_dom_weak::Instance) -> String{ let mut full_name=instance.name.clone(); let mut pref=instance.parent(); while let Some(parent)=dom.get_by_ref(pref){ full_name.insert(0, '.'); full_name.insert_str(0, &parent.name); pref=parent.parent(); } full_name } //download //download list of maps to maps/unprocessed //scan (scripts) //iter maps/unprocessed //passing moves to maps/verified //failing moves to maps/blocked //replace (edits & deletions) //iter maps/blocked //replace scripts and put in maps/unprocessed //upload //iter maps/verified //interactively print DisplayName/Creator and ask for target upload ids //interactive //iter maps/unprocessed //for each unique script, load it into the file current.lua and have it open in sublime text //I can edit the file and it will edit it in place //I pass/fail(with comment)/allow each script fn get_script_refs(dom:&rbx_dom_weak::WeakDom) -> Vec{ let mut scripts = std::vec::Vec::new(); recursive_collect_superclass(&mut scripts, dom, dom.root(),"LuaSourceContainer"); scripts } fn get_button_refs(dom:&rbx_dom_weak::WeakDom) -> Vec{ let mut buttons = std::vec::Vec::new(); recursive_collect_regex(&mut buttons, dom, dom.root(),lazy_regex::regex!(r"Button(\d+)$")); buttons } fn get_texture_refs(dom:&rbx_dom_weak::WeakDom) -> Vec{ let mut objects = std::vec::Vec::new(); recursive_collect_superclass(&mut objects, dom, dom.root(),"Decal"); //get ids //clear vec //next class objects } fn get_mesh_refs(dom:&rbx_dom_weak::WeakDom) -> Vec{ let mut objects = std::vec::Vec::new(); recursive_collect_superclass(&mut objects, dom, dom.root(),"FileMesh"); recursive_collect_superclass(&mut objects, dom, dom.root(),"MeshPart"); //get ids //clear vec //next class objects } enum ReaderType<'a, R:Read+Seek>{ GZip(flate2::read::GzDecoder<&'a mut R>), Raw(&'a mut R), } fn maybe_gzip_decode(input:&mut R)->AResult>{ let mut first_2=[0u8;2]; if let (Ok(()),Ok(()))=(std::io::Read::read_exact(input, &mut first_2),std::io::Seek::rewind(input)){ match &first_2{ b"\x1f\x8b"=>Ok(ReaderType::GZip(flate2::read::GzDecoder::new(input))), _=>Ok(ReaderType::Raw(input)), } }else{ Err(anyhow::Error::msg("failed to peek")) } } fn load_dom(input:&mut R)->AResult{ let mut first_8=[0u8;8]; if let (Ok(()),Ok(()))=(std::io::Read::read_exact(input, &mut first_8),std::io::Seek::rewind(input)){ match &first_8[0..4]{ b"{ match &first_8[4..8]{ b"lox!"=>rbx_binary::from_reader(input).map_err(anyhow::Error::msg), b"lox "=>rbx_xml::from_reader(input,rbx_xml::DecodeOptions::default()).map_err(anyhow::Error::msg), other=>Err(anyhow::Error::msg(format!("Unknown Roblox file type {:?}",other))), } }, _=>Err(anyhow::Error::msg("unsupported file type")), } }else{ Err(anyhow::Error::msg("peek failed")) } } fn get_dom(input:&mut R)->AResult{ match maybe_gzip_decode(input){ Ok(ReaderType::GZip(mut readable)) => { //gzip let mut extracted:Vec=Vec::new(); readable.read_to_end(&mut extracted)?; Ok(load_dom(&mut std::io::Cursor::new(extracted))?) }, Ok(ReaderType::Raw(readable)) => Ok(load_dom(readable)?), Err(e) => Err(e)?, } } fn get_id() -> AResult{ match std::fs::read_to_string("id"){ Ok(id_file)=>Ok(id_file.parse::()?), Err(e) => match e.kind() { std::io::ErrorKind::NotFound => Ok(0),//implicitly take on id=0 _ => Err(e)?, } } } fn get_set_from_file(file:&str) -> AResult>{ let mut set=std::collections::HashSet::::new(); for entry in std::fs::read_dir(file)? { set.insert(std::fs::read_to_string(entry?.path())?); } Ok(set) } fn get_allowed_set() -> AResult>{ get_set_from_file("scripts/allowed") } fn get_blocked() -> AResult>{ get_set_from_file("scripts/blocked") } fn get_allowed_map() -> AResult>{ let mut allowed_map = std::collections::HashMap::::new(); for entry in std::fs::read_dir("scripts/allowed")? { let entry=entry?; allowed_map.insert(entry.path().file_stem().unwrap().to_str().unwrap().parse::()?,std::fs::read_to_string(entry.path())?); } Ok(allowed_map) } fn get_replace_map() -> AResult>{ let mut replace = std::collections::HashMap::::new(); for entry in std::fs::read_dir("scripts/replace")? { let entry=entry?; replace.insert(std::fs::read_to_string(entry.path())?,entry.path().file_stem().unwrap().to_str().unwrap().parse::()?); } Ok(replace) } fn check_source_illegal_keywords(source:&String)->bool{ source.find("getfenv").is_some()||source.find("require").is_some() } fn find_first_child_class<'a>(dom:&'a rbx_dom_weak::WeakDom,instance:&'a rbx_dom_weak::Instance,name:&'a str,class:&'a str) -> Option<&'a rbx_dom_weak::Instance> { for &referent in instance.children() { if let Some(c) = dom.get_by_ref(referent) { if c.name==name&&class_is_a(c.class.as_str(),class) { return Some(c); } } } None } fn get_mapinfo(dom:&rbx_dom_weak::WeakDom) -> AResult<(String,String,String)>{ let workspace_children=dom.root().children(); if workspace_children.len()!=1{ return Err(anyhow::Error::msg("there can only be one model")); } if let Some(model_instance) = dom.get_by_ref(workspace_children[0]) { if let (Some(creator),Some(displayname))=(find_first_child_class(dom, model_instance, "Creator", "StringValue"),find_first_child_class(dom, model_instance, "DisplayName", "StringValue")){ if let ( Some(rbx_dom_weak::types::Variant::String(creator_string)), Some(rbx_dom_weak::types::Variant::String(displayname_string)) )=( creator.properties.get("Value"), displayname.properties.get("Value") ){ return Ok((model_instance.name.clone(),creator_string.clone(),displayname_string.clone())); } } } return Err(anyhow::Error::msg("no stuff in map")); } fn download(map_list: Vec) -> AResult<()>{ let header=format!("Cookie: .ROBLOSECURITY={}",std::env::var("RBXCOOKIE")?); let shared_args=&[ "-q", "--header", header.as_str(), "-O", ]; let processes_result:Result, _>=map_list.iter().map(|map_id|{ std::process::Command::new("wget") .args(shared_args) .arg(format!("maps/unprocessed/{}.rbxm",map_id)) .arg(format!("https://assetdelivery.roblox.com/v1/asset/?ID={}",map_id)) .spawn() }).collect(); //naively wait for all because idk how to make an async progress bar lmao for child in processes_result?{ let output=child.wait_with_output()?; println!("map exit_success:{}",output.status.success()); } Ok(()) } struct RobloxAssetId(u64); struct RobloxAssetIdParseErr; impl std::str::FromStr for RobloxAssetId { type Err=RobloxAssetIdParseErr; fn from_str(s: &str) -> Result{ let regman=lazy_regex::regex!(r"(\d+)$"); if let Some(captures) = regman.captures(s) { if captures.len()==2{//captures[0] is all captures concatenated, and then each individual capture if let Ok(id) = captures[0].parse::() { return Ok(Self(id)); } } } Err(RobloxAssetIdParseErr) } } /* The ones I'm interested in: Beam.Texture Decal.Texture FileMesh.MeshId FileMesh.TextureId MaterialVariant.ColorMap MaterialVariant.MetalnessMap MaterialVariant.NormalMap MaterialVariant.RoughnessMap MeshPart.MeshId MeshPart.TextureID ParticleEmitter.Texture Sky.MoonTextureId Sky.SkyboxBk Sky.SkyboxDn Sky.SkyboxFt Sky.SkyboxLf Sky.SkyboxRt Sky.SkyboxUp Sky.SunTextureId SurfaceAppearance.ColorMap SurfaceAppearance.MetalnessMap SurfaceAppearance.NormalMap SurfaceAppearance.RoughnessMap SurfaceAppearance.TexturePack */ fn download_textures(paths: Vec) -> AResult<()>{ println!("download_textures paths:{:?}",paths); let header=format!("Cookie: .ROBLOSECURITY={}",std::env::var("RBXCOOKIE")?); let shared_args=&[ "-q", "--header", header.as_str(), "-O", ]; let mut texture_list=std::collections::HashSet::new(); for path in paths { let mut input = std::io::BufReader::new(std::fs::File::open(path.clone())?); match get_dom(&mut input){ Ok(dom)=>{ let object_refs = get_texture_refs(&dom); for &object_ref in object_refs.iter() { if let Some(object)=dom.get_by_ref(object_ref){ if let Some(rbx_dom_weak::types::Variant::Content(content)) = object.properties.get("Texture") { println!("Texture content:{:?}",content); if let Ok(asset_id)=content.clone().into_string().parse::(){ texture_list.insert(asset_id.0); } } } } }, Err(e)=>println!("error loading map {:?}: {:?}",path.file_name(),e), } } println!("Texture list:{:?}",texture_list); let processes_result:Result, _>=texture_list.iter().map(|asset_id|{ std::process::Command::new("wget") .args(shared_args) .arg(format!("textures/unprocessed/{}",asset_id)) .arg(format!("https://assetdelivery.roblox.com/v1/asset/?ID={}",asset_id)) .spawn() }).collect(); //naively wait for all because idk how to make an async progress bar lmao for child in processes_result?{ let output=child.wait_with_output()?; println!("texture exit_success:{}",output.status.success()); } Ok(()) } fn download_meshes(paths: Vec) -> AResult<()>{ println!("download_meshes paths:{:?}",paths); let header=format!("Cookie: .ROBLOSECURITY={}",std::env::var("RBXCOOKIE")?); let shared_args=&[ "-q", "--header", header.as_str(), "-O", ]; let mut mesh_list=std::collections::HashSet::new(); for path in paths { let mut input = std::io::BufReader::new(std::fs::File::open(path.clone())?); match get_dom(&mut input){ Ok(dom)=>{ let object_refs = get_mesh_refs(&dom); for &object_ref in object_refs.iter() { if let Some(object)=dom.get_by_ref(object_ref){ if let Some(rbx_dom_weak::types::Variant::Content(content)) = object.properties.get("MeshId") { println!("Mesh content:{:?}",content); if let Ok(asset_id)=content.clone().into_string().parse::(){ mesh_list.insert(asset_id.0); } } } } }, Err(e)=>println!("error loading map {:?}: {:?}",path.file_name(),e), } } println!("Mesh list:{:?}",mesh_list); let processes_result:Result, _>=mesh_list.iter().map(|asset_id|{ std::process::Command::new("wget") .args(shared_args) .arg(format!("meshes/unprocessed/{}",asset_id)) .arg(format!("https://assetdelivery.roblox.com/v1/asset/?ID={}",asset_id)) .spawn() }).collect(); //naively wait for all because idk how to make an async progress bar lmao for child in processes_result?{ let output=child.wait_with_output()?; println!("Mesh exit_success:{}",output.status.success()); } Ok(()) } fn load_image(input:&mut R)->AResult{ let mut fourcc=[0u8;4]; input.read_exact(&mut fourcc)?; input.rewind()?; match &fourcc{ b"\x89PNG"=>Ok(image::load(input,image::ImageFormat::Png)?), b"\xFF\xD8\xFF\xE0"=>Ok(image::load(input,image::ImageFormat::Jpeg)?),//JFIF b"Err(anyhow::Error::msg("Roblox xml garbage is not supported yet")), other=>Err(anyhow::Error::msg(format!("Unknown texture format {:?}",other))), } } fn convert(file_thing:std::fs::DirEntry) -> AResult<()>{ let mut input = std::io::BufReader::new(std::fs::File::open(file_thing.path())?); let mut extracted_input=None; let image=match maybe_gzip_decode(&mut input){ Ok(ReaderType::GZip(mut readable)) => { //gzip let mut extracted:Vec=Vec::new(); //read the entire thing to the end so that I can clone the data and write a png to processed images readable.read_to_end(&mut extracted)?; extracted_input=Some(extracted.clone()); load_image(&mut std::io::Cursor::new(extracted)) }, Ok(ReaderType::Raw(readable)) => load_image(readable), Err(e) => Err(e)?, }?.to_rgba8();//this sets a=255, arcane is actually supposed to look like that let format=if image.width()%4!=0||image.height()%4!=0{ image_dds::ImageFormat::R8G8B8A8Srgb }else{ image_dds::ImageFormat::BC7Srgb }; //this fails if the image dimensions are not a multiple of 4 let dds = image_dds::dds_from_image( &image, format, image_dds::Quality::Slow, image_dds::Mipmaps::GeneratedAutomatic, )?; //write dds let mut dest=PathBuf::from("textures/dds"); dest.push(file_thing.file_name()); dest.set_extension("dds"); let mut writer = std::io::BufWriter::new(std::fs::File::create(dest)?); dds.write(&mut writer)?; if let Some(mut extracted)=extracted_input{ //write extracted to processed let mut dest=PathBuf::from("textures/processed"); dest.push(file_thing.file_name()); std::fs::write(dest, &mut extracted)?; //delete ugly gzip file std::fs::remove_file(file_thing.path())?; }else{ //move file to processed let mut dest=PathBuf::from("textures/processed"); dest.push(file_thing.file_name()); std::fs::rename(file_thing.path(), dest)?; } Ok(()) } fn convert_textures() -> AResult<()>{ let start = std::time::Instant::now(); let mut threads=Vec::new(); for entry in std::fs::read_dir("textures/unprocessed")? { let file_thing=entry?; threads.push(std::thread::spawn(move ||{ let file_name=format!("{:?}",file_thing); let result=convert(file_thing); if let Err(e)=result{ println!("error processing file:{:?} error message:{:?}",file_name,e); } })); } let mut i=0; let n_threads=threads.len(); for thread in threads{ i+=1; if let Err(e)=thread.join(){ println!("thread error: {:?}",e); }else{ println!("{}/{}",i,n_threads); } } println!("{:?}", start.elapsed()); Ok(()) } enum Scan{ Passed, Blocked, Flagged, } fn scan() -> AResult<()>{ let mut id = get_id()?; //Construct allowed scripts let allowed_set = get_allowed_set()?; let mut blocked = get_blocked()?; for entry in std::fs::read_dir("maps/unprocessed")? { let file_thing=entry?; let mut input = std::io::BufReader::new(std::fs::File::open(file_thing.path())?); let dom = get_dom(&mut input)?; let script_refs = get_script_refs(&dom); //check scribb let mut fail_count=0; let mut fail_type=Scan::Passed; for &script_ref in script_refs.iter() { if let Some(script)=dom.get_by_ref(script_ref){ if let Some(rbx_dom_weak::types::Variant::String(s)) = script.properties.get("Source") { //flag keywords and instantly fail if check_source_illegal_keywords(s){ println!("{:?} - flagged.",file_thing.file_name()); fail_type=Scan::Flagged; break; } if allowed_set.contains(s) { continue; }else{ fail_type=Scan::Blocked;//no need to check for Flagged, it breaks the loop. fail_count+=1; if !blocked.contains(s) { blocked.insert(s.clone());//all fixed! just clone! std::fs::write(format!("scripts/blocked/{}.lua",id),s)?; id+=1; } } }else{ panic!("FATAL: failed to get source for {:?}",file_thing.file_name()); } }else{ panic!("FATAL: failed to get_by_ref {:?}",script_ref); } } let mut dest=match fail_type { Scan::Passed => PathBuf::from("maps/processed"), Scan::Blocked => { println!("{:?} - {} {} not allowed.",file_thing.file_name(),fail_count,if fail_count==1 {"script"}else{"scripts"}); PathBuf::from("maps/blocked") } Scan::Flagged => PathBuf::from("maps/flagged") }; dest.push(file_thing.file_name()); std::fs::rename(file_thing.path(), dest)?; } std::fs::write("id",id.to_string())?; Ok(()) } fn extract(paths: Vec) -> AResult<()>{ let mut id = 0; //Construct allowed scripts let mut script_set = std::collections::HashSet::::new(); for path in paths { let file_name=path.file_name(); let mut input = std::io::BufReader::new(std::fs::File::open(&path)?); let dom = get_dom(&mut input)?; let script_refs = get_script_refs(&dom); //extract scribb for &script_ref in script_refs.iter() { if let Some(script)=dom.get_by_ref(script_ref){ if let Some(rbx_dom_weak::types::Variant::String(s)) = script.properties.get("Source") { if script_set.contains(s) { continue; }else{ script_set.insert(s.clone()); std::fs::write(format!("scripts/extracted/{:?}_{}_{}.lua",file_name,id,script.name),s)?; id+=1; } }else{ panic!("FATAL: failed to get source for {:?}",file_name); } }else{ panic!("FATAL: failed to get_by_ref {:?}",script_ref); } } } println!("extracted {} {}",id,if id==1 {"script"}else{"scripts"}); Ok(()) } fn replace() -> AResult<()>{ let allowed_map=get_allowed_map()?; let replace_map=get_replace_map()?; for entry in std::fs::read_dir("maps/blocked")? { let file_thing=entry?; let mut input = std::io::BufReader::new(std::fs::File::open(file_thing.path())?); let mut dom = get_dom(&mut input)?; let script_refs = get_script_refs(&dom); //check scribb let mut any_failed=false; for &script_ref in script_refs.iter() { if let Some(script)=dom.get_by_ref(script_ref){ if let Some(rbx_dom_weak::types::Variant::String(source)) = script.properties.get("Source") { if let (Some(replace_id),Some(replace_script))=(replace_map.get(source),dom.get_by_ref_mut(script.referent())) { println!("replace {}",replace_id); //replace the source if let Some(replace_source)=allowed_map.get(replace_id){ replace_script.properties.insert("Source".to_string(), rbx_dom_weak::types::Variant::String(replace_source.clone())); }else{ println!("failed to get replacement source {}",replace_id); any_failed=true; } }else{ println!("failed to failed to get replace_id and replace_script"); any_failed=true; } }else{ panic!("FATAL: failed to get source for {:?}",file_thing.file_name()); } }else{ panic!("FATAL: failed to get_by_ref {:?}",script_ref); } } if any_failed { println!("One or more scripts failed to replace."); }else{ let mut dest=PathBuf::from("maps/unprocessed"); dest.push(file_thing.file_name()); let output = std::io::BufWriter::new(std::fs::File::open(dest)?); //write workspace:GetChildren()[1] let workspace_children=dom.root().children(); if workspace_children.len()!=1{ return Err(anyhow::Error::msg("there can only be one model")); } rbx_binary::to_writer(output, &dom, &[workspace_children[0]])?; } } Ok(()) } enum UploadAction { Upload(u64), Skip, New, Delete, } struct ParseUploadActionErr; impl std::str::FromStr for UploadAction { type Err=ParseUploadActionErr; fn from_str(s: &str) -> Result{ if s=="skip\n"{ Ok(Self::Skip) }else if s=="new\n"{ Ok(Self::New) }else if s=="delete\n"{ Ok(Self::Delete) }else if let Ok(asset_id)=s[..s.len()-1].parse::(){ Ok(Self::Upload(asset_id)) }else{ Err(ParseUploadActionErr) } } } fn upload() -> AResult<()>{ //interactive prompt per upload: for entry in std::fs::read_dir("maps/passed")? { let file_thing=entry?; println!("map file: {:?}",file_thing.file_name()); let mut input = std::io::BufReader::new(std::fs::File::open(file_thing.path())?); let dom = get_dom(&mut input)?; let (modelname,creator,displayname) = get_mapinfo(&dom)?; //Creator: [auto fill creator] //DisplayName: [auto fill DisplayName] //id: ["New" for blank because of my double enter key] print!("Model name: {}\nCreator: {}\nDisplayName: {}\nAction or Upload Asset Id: ",modelname,creator,displayname); std::io::Write::flush(&mut std::io::stdout())?; let upload_action; loop{ let mut upload_action_string = String::new(); std::io::stdin().read_line(&mut upload_action_string)?; if let Ok(parsed_upload_action)=upload_action_string.parse::(){ upload_action=parsed_upload_action; break; }else{ print!("Action or Upload Asset Id: "); std::io::Write::flush(&mut std::io::stdout())?; } } match upload_action { UploadAction::Upload(asset_id) => { let status=std::process::Command::new("../rbxcompiler-linux-amd64") .arg("--compile=false") .arg("--group=6980477") .arg(format!("--asset={}",asset_id)) .arg(format!("--input={}",file_thing.path().into_os_string().into_string().unwrap())) .status()?; match status.code() { Some(0)=>{ //move file let mut dest=PathBuf::from("maps/uploaded"); dest.push(file_thing.file_name()); std::fs::rename(file_thing.path(), dest)?; } Some(code)=>println!("upload failed! code={}",code), None => println!("no status code!"), } } UploadAction::Skip => continue, UploadAction::New => { let output=std::process::Command::new("../rbxcompiler-linux-amd64") .arg("--compile=false") .arg("--group=6980477") .arg("--new-asset=true") .arg(format!("--input={}",file_thing.path().into_os_string().into_string().unwrap())) .output()?; match output.status.code() { Some(0)=>{ //print output println!("{}", std::str::from_utf8(output.stdout.as_slice())?); //move file let mut dest=PathBuf::from("maps/uploaded"); dest.push(file_thing.file_name()); std::fs::rename(file_thing.path(), dest)?; } Some(code)=>println!("upload failed! code={}",code), None => println!("no status code!"), } } UploadAction::Delete => std::fs::remove_file(file_thing.path())?, } } Ok(()) } enum Interactive{ Passed, Blocked, Flagged, } enum ScriptAction { Pass, Replace(u32), Flag, Block, Delete, } enum ScriptActionParseResult { Pass, Block, Exit, Delete, } struct ParseScriptActionErr; impl std::str::FromStr for ScriptActionParseResult { type Err=ParseScriptActionErr; fn from_str(s: &str) -> Result{ if s=="pass\n"||s=="1\n"{ Ok(Self::Pass) }else if s=="block\n"{ Ok(Self::Block) }else if s=="exit\n"{ Ok(Self::Exit) }else if s=="delete\n"{ Ok(Self::Delete) }else{ Err(ParseScriptActionErr) } } } fn interactive() -> AResult<()>{ let mut id=get_id()?; //Construct allowed scripts let mut allowed_set=get_allowed_set()?; let mut allowed_map=get_allowed_map()?; let mut replace_map=get_replace_map()?; let mut blocked = get_blocked()?; 'map_loop: for entry in std::fs::read_dir("maps/unprocessed")? { let file_thing=entry?; println!("processing map={:?}",file_thing.file_name()); let mut input = std::io::BufReader::new(std::fs::File::open(file_thing.path())?); let mut dom = get_dom(&mut input)?; let script_refs = get_script_refs(&dom); //check scribb let mut script_count=0; let mut replace_count=0; let mut block_count=0; let mut fail_type=Interactive::Passed; for &script_ref in script_refs.iter() { if let Some(script)=dom.get_by_ref(script_ref){ if let Some(rbx_dom_weak::types::Variant::String(source)) = script.properties.get("Source") { script_count+=1; let source_action=if check_source_illegal_keywords(source) { ScriptAction::Flag//script triggers flagging -> Flag } else if blocked.contains(source) { ScriptAction::Block//script is blocked -> Block } else if allowed_set.contains(source) { ScriptAction::Pass//script is allowed -> Pass }else if let Some(replace_id)=replace_map.get(source) { ScriptAction::Replace(*replace_id) }else{ //interactive logic goes here print!("unresolved source location={}\naction: ",get_full_name(&dom, script)); std::io::Write::flush(&mut std::io::stdout())?; //load source into current.lua std::fs::write("current.lua",source)?; //prompt action in terminal //wait for input let script_action; loop{ let mut action_string = String::new(); std::io::stdin().read_line(&mut action_string)?; if let Ok(parsed_script_action)=action_string.parse::(){ script_action=parsed_script_action; break; }else{ print!("action: "); std::io::Write::flush(&mut std::io::stdout())?; } } //update allowed/replace/blocked match script_action{ ScriptActionParseResult::Pass => { //if current.lua was updated, create an allowed and replace file and set script_action to replace(new_id) let modified_source=std::fs::read_to_string("current.lua")?; if &modified_source==source{ //it's always new. //insert allowed_set allowed_set.insert(modified_source.clone()); //insert allowed_map allowed_map.insert(id,modified_source.clone()); //write allowed/id.lua std::fs::write(format!("scripts/allowed/{}.lua",id),modified_source)?; id+=1; ScriptAction::Pass }else{ //insert allowed_set allowed_set.insert(modified_source.clone()); //insert allowed_map allowed_map.insert(id,modified_source.clone()); //insert replace_map replace_map.insert(source.clone(),id);//this cannot be reached if it already exists //write allowed/id.lua std::fs::write(format!("scripts/allowed/{}.lua",id),modified_source)?; //write replace/id.lua std::fs::write(format!("scripts/replace/{}.lua",id),source)?; let ret=ScriptAction::Replace(id); id+=1; ret } }, ScriptActionParseResult::Block => { blocked.insert(source.clone()); std::fs::write(format!("scripts/blocked/{}.lua",id),source)?; id+=1; ScriptAction::Block }, ScriptActionParseResult::Exit => break 'map_loop, ScriptActionParseResult::Delete => ScriptAction::Delete, } }; let location=get_full_name(&dom, script); match source_action{ ScriptAction::Pass => println!("passed source location={}",location), ScriptAction::Replace(replace_id)=>{ //replace the source if let (Some(replace_source),Some(replace_script))=(allowed_map.get(&replace_id),dom.get_by_ref_mut(script.referent())){ replace_count+=1; println!("replaced source id={} location={}",replace_id,location); replace_script.properties.insert("Source".to_string(), rbx_dom_weak::types::Variant::String(replace_source.clone())); }else{ panic!("failed to get replacement source id={} location={}",replace_id,location); } }, ScriptAction::Delete => { println!("deleted source location={}",location); replace_count+=1;//trigger a new file generation dom.destroy(script.referent()); }, ScriptAction::Flag => { println!("flagged source location={}",location); fail_type=Interactive::Flagged; }, ScriptAction::Block => { block_count+=1; println!("blocked source location={}",location); match fail_type{ Interactive::Passed => fail_type=Interactive::Blocked, _=>(), } }, } }else{ panic!("FATAL: failed to get source for {:?}",file_thing.file_name()); } }else{ panic!("FATAL: failed to get_by_ref {:?}",script_ref); } } let mut dest=match fail_type{ Interactive::Passed => { println!("map={:?} passed with {} {}",file_thing.file_name(),script_count,if script_count==1 {"script"}else{"scripts"}); if replace_count==0{ PathBuf::from("maps/passed") }else{ //create new file println!("{} {} replaced - generating new file...",replace_count,if replace_count==1 {"script was"}else{"scripts were"}); let mut dest=PathBuf::from("maps/passed"); dest.push(file_thing.file_name()); let output = std::io::BufWriter::new(std::fs::File::create(dest)?); //write workspace:GetChildren()[1] let workspace_children=dom.root().children(); if workspace_children.len()!=1{ return Err(anyhow::Error::msg("there can only be one model")); } rbx_binary::to_writer(output, &dom, &[workspace_children[0]])?; //move original to processed folder PathBuf::from("maps/unaltered") } },//write map into maps/processed Interactive::Blocked => { println!("map={:?} blocked with {}/{} {} blocked",file_thing.file_name(),block_count,script_count,if script_count==1 {"script"}else{"scripts"}); PathBuf::from("maps/blocked") },//write map into maps/blocked Interactive::Flagged => { println!("map={:?} flagged",file_thing.file_name()); PathBuf::from("maps/flagged") },//write map into maps/flagged }; dest.push(file_thing.file_name()); std::fs::rename(file_thing.path(), dest)?; } std::fs::write("id",id.to_string())?; Ok(()) } fn unzip_all()->AResult<()>{ for entry in std::fs::read_dir("maps/unprocessed")? { let file_thing=entry?; println!("processing map={:?}",file_thing.file_name()); let mut input = std::io::BufReader::new(std::fs::File::open(file_thing.path())?); match maybe_gzip_decode(&mut input){ Ok(ReaderType::GZip(mut readable)) => { //gzip let mut extracted:Vec=Vec::new(); //read the entire thing to the end so that I can clone the data and write a png to processed images readable.read_to_end(&mut extracted)?; //write extracted let mut dest=PathBuf::from("maps/unzipped"); dest.push(file_thing.file_name()); std::fs::write(dest, &mut extracted)?; //delete ugly gzip file std::fs::remove_file(file_thing.path())?; }, Ok(ReaderType::Raw(_)) => (), Err(e) => Err(e)?, } } Ok(()) } fn write_attributes() -> AResult<()>{ for entry in std::fs::read_dir("maps/unprocessed")? { let file_thing=entry?; println!("processing map={:?}",file_thing.file_name()); let mut input = std::io::BufReader::new(std::fs::File::open(file_thing.path())?); let mut dom = get_dom(&mut input)?; let button_refs = get_button_refs(&dom); for &button_ref in &button_refs { if let Some(button)=dom.get_by_ref_mut(button_ref){ match button.properties.get_mut("Attributes"){ Some(rbx_dom_weak::types::Variant::Attributes(attributes))=>{ println!("Appending Ref={} to existing attributes for {}",button_ref,button.name); attributes.insert("Ref".to_string(),rbx_dom_weak::types::Variant::String(button_ref.to_string())); }, None=>{ println!("Creating new attributes with Ref={} for {}",button_ref,button.name); let mut attributes=rbx_dom_weak::types::Attributes::new(); attributes.insert("Ref".to_string(),rbx_dom_weak::types::Variant::String(button_ref.to_string())); button.properties.insert("Attributes".to_string(),rbx_dom_weak::types::Variant::Attributes(attributes)); } _=>unreachable!("Fetching attributes did not return attributes."), } } } let mut dest={ let mut dest=PathBuf::from("maps/attributes"); dest.push(file_thing.file_name()); let output = std::io::BufWriter::new(std::fs::File::create(dest)?); //write workspace:GetChildren()[1] let workspace_children=dom.root().children(); if workspace_children.len()!=1{ return Err(anyhow::Error::msg("there can only be one model")); } rbx_binary::to_writer(output, &dom, &[workspace_children[0]])?; //move original to processed folder PathBuf::from("maps/unaltered") }; dest.push(file_thing.file_name()); std::fs::rename(file_thing.path(), dest)?; } Ok(()) } enum VMTContent{ VMT(String), VTF(String), Patch(vmt_parser::material::PatchMaterial), Unsupported,//don't want to deal with whatever vmt variant Unresolved,//could not locate a texture because of vmt content } impl VMTContent{ fn vtf(opt:Option)->Self{ match opt{ Some(s)=>Self::VTF(s), None=>Self::Unresolved, } } } fn get_some_texture(material:vmt_parser::material::Material)->AResult{ //just grab some texture from somewhere for now Ok(match material{ vmt_parser::material::Material::LightMappedGeneric(mat)=>VMTContent::vtf(Some(mat.base_texture)), vmt_parser::material::Material::VertexLitGeneric(mat)=>VMTContent::vtf(mat.base_texture.or(mat.decal_texture)),//this just dies if there is none vmt_parser::material::Material::VertexLitGenericDx6(mat)=>VMTContent::vtf(mat.base_texture.or(mat.decal_texture)), vmt_parser::material::Material::UnlitGeneric(mat)=>VMTContent::vtf(mat.base_texture), vmt_parser::material::Material::UnlitTwoTexture(mat)=>VMTContent::vtf(mat.base_texture), vmt_parser::material::Material::Water(mat)=>VMTContent::vtf(mat.base_texture), vmt_parser::material::Material::WorldVertexTransition(mat)=>VMTContent::vtf(Some(mat.base_texture)), vmt_parser::material::Material::EyeRefract(mat)=>VMTContent::vtf(Some(mat.cornea_texture)), vmt_parser::material::Material::SubRect(mat)=>VMTContent::VMT(mat.material),//recursive vmt_parser::material::Material::Sprite(mat)=>VMTContent::vtf(Some(mat.base_texture)), vmt_parser::material::Material::SpriteCard(mat)=>VMTContent::vtf(mat.base_texture), vmt_parser::material::Material::Cable(mat)=>VMTContent::vtf(Some(mat.base_texture)), vmt_parser::material::Material::Refract(mat)=>VMTContent::vtf(mat.base_texture), vmt_parser::material::Material::Modulate(mat)=>VMTContent::vtf(Some(mat.base_texture)), vmt_parser::material::Material::DecalModulate(mat)=>VMTContent::vtf(Some(mat.base_texture)), vmt_parser::material::Material::Sky(mat)=>VMTContent::vtf(Some(mat.base_texture)), vmt_parser::material::Material::Replacements(_mat)=>VMTContent::Unsupported, vmt_parser::material::Material::Patch(mat)=>VMTContent::Patch(mat), _=>return Err(anyhow::Error::msg("vmt failed to parse")), }) } fn get_vmtAResult>>>(find_stuff:&F,search_name:String)->AResult{ if let Some(stuff)=find_stuff(search_name)?{ //decode vmt and then write let stuff=String::from_utf8(stuff)?; let material=vmt_parser::from_str(stuff.as_str())?; println!("vmt material={:?}",material); return Ok(material); } Err(anyhow::Error::msg("vmt not found")) } fn recursive_vmt_loaderAResult>>>(find_stuff:&F,material:vmt_parser::material::Material)->AResult>>{ match get_some_texture(material)?{ VMTContent::VMT(s)=>recursive_vmt_loader(find_stuff,get_vmt(find_stuff,s)?), VMTContent::VTF(s)=>{ let mut texture_file_name=PathBuf::from("materials"); texture_file_name.push(s); texture_file_name.set_extension("vtf"); find_stuff(texture_file_name.into_os_string().into_string().unwrap()) }, VMTContent::Patch(mat)=>recursive_vmt_loader(find_stuff, mat.resolve(|search_name|{ match find_stuff(search_name.to_string())?{ Some(bytes)=>Ok(String::from_utf8(bytes)?), None=>Err(anyhow::Error::msg("could not find vmt")), } })? ), VMTContent::Unsupported=>{println!("Unsupported vmt");Ok(None)},//print and move on VMTContent::Unresolved=>{println!("Unresolved vmt");Ok(None)}, } } fn extract_textures(paths:Vec,vpk_paths:Vec)->AResult<()>{ let vpk_list:Vec=vpk_paths.into_iter().map(|vpk_path|vpk::VPK::read(&vpk_path).expect("vpk file does not exist")).collect(); for path in paths{ let mut deduplicate=std::collections::HashSet::new(); let bsp=vbsp::Bsp::read(std::fs::read(path)?.as_ref())?; for texture in bsp.textures(){ deduplicate.insert(PathBuf::from(texture.name())); } //dedupe prop models let mut model_dedupe=std::collections::HashSet::new(); for prop in bsp.static_props(){ model_dedupe.insert(prop.model()); } //grab texture names from props for model_name in model_dedupe{ //.mdl, .vvd, .dx90.vtx let mut path=PathBuf::from(model_name); let file_name=PathBuf::from(path.file_stem().unwrap()); path.pop(); path.push(file_name); let mut vvd_path=path.clone(); let mut vtx_path=path.clone(); vvd_path.set_extension("vvd"); vtx_path.set_extension("dx90.vtx"); match (bsp.pack.get(model_name),bsp.pack.get(vvd_path.as_os_str().to_str().unwrap()),bsp.pack.get(vtx_path.as_os_str().to_str().unwrap())){ (Ok(Some(mdl_file)),Ok(Some(vvd_file)),Ok(Some(vtx_file)))=>{ match (vmdl::mdl::Mdl::read(mdl_file.as_ref()),vmdl::vvd::Vvd::read(vvd_file.as_ref()),vmdl::vtx::Vtx::read(vtx_file.as_ref())){ (Ok(mdl),Ok(vvd),Ok(vtx))=>{ let model=vmdl::Model::from_parts(mdl,vtx,vvd); for texture in model.textures(){ for search_path in &texture.search_paths{ let mut path=PathBuf::from(search_path.as_str()); path.push(texture.name.as_str()); deduplicate.insert(path); } } }, _=>println!("model_name={} error",model_name), } }, _=>println!("no model name={}",model_name), } } let pack=&bsp.pack; let vpk_list=&vpk_list; std::thread::scope(move|s|{ let mut thread_handles=Vec::new(); for texture_name in deduplicate{ let mut found_texture=false; //LMAO imagine having to write type names let write_image=|mut stuff,write_file_name|{ let image=vtf::from_bytes(&mut stuff)?.highres_image.decode(0)?.to_rgba8(); let format=if image.width()%4!=0||image.height()%4!=0{ image_dds::ImageFormat::R8G8B8A8Srgb }else{ image_dds::ImageFormat::BC7Srgb }; //this fails if the image dimensions are not a multiple of 4 let dds = image_dds::dds_from_image( &image, format, image_dds::Quality::Slow, image_dds::Mipmaps::GeneratedAutomatic, )?; //write dds let mut dest=PathBuf::from("textures/dds"); dest.push(write_file_name); dest.set_extension("dds"); std::fs::create_dir_all(dest.parent().unwrap())?; let mut writer = std::io::BufWriter::new(std::fs::File::create(dest)?); dds.write(&mut writer)?; Ok::<(),anyhow::Error>(()) }; let find_stuff=|search_file_name:String|{ println!("search_file_name={}",search_file_name); match pack.get(search_file_name.as_str())?{ Some(file)=>return Ok(Some(file)), _=>(), } //search pak list for vpk_index in vpk_list{ if let Some(vpk_entry)=vpk_index.tree.get(search_file_name.as_str()){ return Ok(Some(match vpk_entry.get()?{ std::borrow::Cow::Borrowed(bytes)=>bytes.to_vec(), std::borrow::Cow::Owned(bytes)=>bytes, })); } } Ok::>,anyhow::Error>(None) }; let loader=|texture_name:String|{ let mut texture_file_name=PathBuf::from("materials"); //lower case let texture_file_name_lowercase=texture_name.to_lowercase(); texture_file_name.push(texture_file_name_lowercase.clone()); //remove stem and search for both vtf and vmt files let stem=PathBuf::from(texture_file_name.file_stem().unwrap()); texture_file_name.pop(); texture_file_name.push(stem); //somehow search for both files let mut texture_file_name_vmt=texture_file_name.clone(); texture_file_name.set_extension("vtf"); texture_file_name_vmt.set_extension("vmt"); if let Some(stuff)=find_stuff(texture_file_name.to_string_lossy().to_string())?{ return Ok(Some(stuff)) } recursive_vmt_loader(&find_stuff,get_vmt(&find_stuff,texture_file_name_vmt.to_string_lossy().to_string())?) }; if let Some(stuff)=loader(texture_name.to_string_lossy().to_string())?{ found_texture=true; let texture_name=texture_name.clone(); thread_handles.push(s.spawn(move||write_image(stuff,texture_name))); } if !found_texture{ println!("no data"); } } for thread in thread_handles{ match thread.join(){ Ok(Err(e))=>println!("write error: {:?}",e), Err(e)=>println!("thread error: {:?}",e), Ok(_)=>(), } } Ok::<(),anyhow::Error>(()) })? } Ok(()) } fn vpk_contents(vpk_path:PathBuf)->AResult<()>{ let vpk_index=vpk::VPK::read(&vpk_path)?; for (label,entry) in vpk_index.tree.into_iter(){ println!("vpk label={} entry={:?}",label,entry); } Ok(()) } fn bsp_contents(path:PathBuf)->AResult<()>{ let bsp=vbsp::Bsp::read(std::fs::read(path)?.as_ref())?; for file_name in bsp.pack.into_zip().into_inner().unwrap().file_names(){ println!("file_name={:?}",file_name); } Ok(()) }