use std::{io::{Read, Seek}, path::PathBuf}; use clap::{Args, Parser, Subcommand}; use anyhow::Result as AResult; #[derive(Parser)] #[command(author, version, about, long_about = None)] #[command(propagate_version = true)] struct Cli { #[command(subcommand)] command: Commands, } #[derive(Subcommand)] enum Commands { DownloadTextures(DownloadTexturesSubcommand), ExtractTextures(ExtractTexturesSubcommand), ConvertTextures(ConvertTexturesSubcommand), VPKContents(VPKContentsSubcommand), BSPContents(BSPContentsSubcommand), DownloadMeshes(DownloadMeshesSubcommand), WriteAttributes(WriteAttributesSubcommand), } #[derive(Args)] struct DownloadTexturesSubcommand { roblox_files:Vec } #[derive(Args)] struct ExtractTexturesSubcommand { bsp_file:PathBuf, vpk_dir_files:Vec } #[derive(Args)] struct ConvertTexturesSubcommand { } #[derive(Args)] struct VPKContentsSubcommand { input_file:PathBuf, } #[derive(Args)] struct BSPContentsSubcommand { input_file:PathBuf, } #[derive(Args)] struct DownloadMeshesSubcommand { roblox_files:Vec } #[derive(Args)] struct WriteAttributesSubcommand { } fn main() -> AResult<()> { let cli = Cli::parse(); match cli.command { Commands::DownloadTextures(subcommand)=>download_textures(subcommand.roblox_files), Commands::ExtractTextures(subcommand)=>extract_textures(vec![subcommand.bsp_file],subcommand.vpk_dir_files), Commands::VPKContents(subcommand)=>vpk_contents(subcommand.input_file), Commands::BSPContents(subcommand)=>bsp_contents(subcommand.input_file), Commands::ConvertTextures(_subcommand)=>convert_textures(), Commands::DownloadMeshes(subcommand)=>download_meshes(subcommand.roblox_files), Commands::WriteAttributes(_subcommand)=>write_attributes(), } } fn class_is_a(class: &str, superclass: &str) -> bool { if class==superclass { return true } let class_descriptor=rbx_reflection_database::get().classes.get(class); if let Some(descriptor) = &class_descriptor { if let Some(class_super) = &descriptor.superclass { return class_is_a(&class_super, superclass) } } false } fn recursive_collect_superclass(objects: &mut std::vec::Vec,dom: &rbx_dom_weak::WeakDom, instance: &rbx_dom_weak::Instance, superclass: &str){ for &referent in instance.children() { if let Some(c) = dom.get_by_ref(referent) { if class_is_a(c.class.as_str(), superclass) { objects.push(c.referent());//copy ref } recursive_collect_superclass(objects,dom,c,superclass); } } } fn recursive_collect_regex(objects: &mut std::vec::Vec,dom: &rbx_dom_weak::WeakDom, instance: &rbx_dom_weak::Instance, regex: &lazy_regex::Lazy){ for &referent in instance.children() { if let Some(c) = dom.get_by_ref(referent) { if regex.captures(c.name.as_str()).is_some(){ objects.push(c.referent());//copy ref } recursive_collect_regex(objects,dom,c,regex); } } } fn get_button_refs(dom:&rbx_dom_weak::WeakDom) -> Vec{ let mut buttons = std::vec::Vec::new(); recursive_collect_regex(&mut buttons, dom, dom.root(),lazy_regex::regex!(r"Button(\d+)$")); buttons } fn get_texture_refs(dom:&rbx_dom_weak::WeakDom) -> Vec{ let mut objects = std::vec::Vec::new(); recursive_collect_superclass(&mut objects, dom, dom.root(),"Decal"); //get ids //clear vec //next class objects } fn get_mesh_refs(dom:&rbx_dom_weak::WeakDom) -> Vec{ let mut objects = std::vec::Vec::new(); recursive_collect_superclass(&mut objects, dom, dom.root(),"FileMesh"); recursive_collect_superclass(&mut objects, dom, dom.root(),"MeshPart"); //get ids //clear vec //next class objects } enum ReaderType<'a, R:Read+Seek>{ GZip(flate2::read::GzDecoder<&'a mut R>), Raw(&'a mut R), } fn maybe_gzip_decode(input:&mut R)->AResult>{ let mut first_2=[0u8;2]; if let (Ok(()),Ok(()))=(std::io::Read::read_exact(input, &mut first_2),std::io::Seek::rewind(input)){ match &first_2{ b"\x1f\x8b"=>Ok(ReaderType::GZip(flate2::read::GzDecoder::new(input))), _=>Ok(ReaderType::Raw(input)), } }else{ Err(anyhow::Error::msg("failed to peek")) } } fn load_dom(input:&mut R)->AResult{ let mut first_8=[0u8;8]; if let (Ok(()),Ok(()))=(std::io::Read::read_exact(input, &mut first_8),std::io::Seek::rewind(input)){ match &first_8[0..4]{ b"{ match &first_8[4..8]{ b"lox!"=>rbx_binary::from_reader(input).map_err(anyhow::Error::msg), b"lox "=>rbx_xml::from_reader(input,rbx_xml::DecodeOptions::default()).map_err(anyhow::Error::msg), other=>Err(anyhow::Error::msg(format!("Unknown Roblox file type {:?}",other))), } }, _=>Err(anyhow::Error::msg("unsupported file type")), } }else{ Err(anyhow::Error::msg("peek failed")) } } fn get_dom(input:&mut R)->AResult{ match maybe_gzip_decode(input){ Ok(ReaderType::GZip(mut readable)) => { //gzip let mut extracted:Vec=Vec::new(); readable.read_to_end(&mut extracted)?; Ok(load_dom(&mut std::io::Cursor::new(extracted))?) }, Ok(ReaderType::Raw(readable)) => Ok(load_dom(readable)?), Err(e) => Err(e)?, } } struct RobloxAssetId(u64); struct RobloxAssetIdParseErr; impl std::str::FromStr for RobloxAssetId { type Err=RobloxAssetIdParseErr; fn from_str(s: &str) -> Result{ let regman=lazy_regex::regex!(r"(\d+)$"); if let Some(captures) = regman.captures(s) { if captures.len()==2{//captures[0] is all captures concatenated, and then each individual capture if let Ok(id) = captures[0].parse::() { return Ok(Self(id)); } } } Err(RobloxAssetIdParseErr) } } /* The ones I'm interested in: Beam.Texture Decal.Texture FileMesh.MeshId FileMesh.TextureId MaterialVariant.ColorMap MaterialVariant.MetalnessMap MaterialVariant.NormalMap MaterialVariant.RoughnessMap MeshPart.MeshId MeshPart.TextureID ParticleEmitter.Texture Sky.MoonTextureId Sky.SkyboxBk Sky.SkyboxDn Sky.SkyboxFt Sky.SkyboxLf Sky.SkyboxRt Sky.SkyboxUp Sky.SunTextureId SurfaceAppearance.ColorMap SurfaceAppearance.MetalnessMap SurfaceAppearance.NormalMap SurfaceAppearance.RoughnessMap SurfaceAppearance.TexturePack */ fn download_textures(paths: Vec) -> AResult<()>{ println!("download_textures paths:{:?}",paths); let header=format!("Cookie: .ROBLOSECURITY={}",std::env::var("RBXCOOKIE")?); let shared_args=&[ "-q", "--header", header.as_str(), "-O", ]; let mut texture_list=std::collections::HashSet::new(); for path in paths { let mut input = std::io::BufReader::new(std::fs::File::open(path.clone())?); match get_dom(&mut input){ Ok(dom)=>{ let object_refs = get_texture_refs(&dom); for &object_ref in object_refs.iter() { if let Some(object)=dom.get_by_ref(object_ref){ if let Some(rbx_dom_weak::types::Variant::Content(content)) = object.properties.get("Texture") { println!("Texture content:{:?}",content); if let Ok(asset_id)=content.clone().into_string().parse::(){ texture_list.insert(asset_id.0); } } } } }, Err(e)=>println!("error loading map {:?}: {:?}",path.file_name(),e), } } println!("Texture list:{:?}",texture_list); let processes_result:Result, _>=texture_list.iter().map(|asset_id|{ std::process::Command::new("wget") .args(shared_args) .arg(format!("textures/unprocessed/{}",asset_id)) .arg(format!("https://assetdelivery.roblox.com/v1/asset/?ID={}",asset_id)) .spawn() }).collect(); //naively wait for all because idk how to make an async progress bar lmao for child in processes_result?{ let output=child.wait_with_output()?; println!("texture exit_success:{}",output.status.success()); } Ok(()) } fn download_meshes(paths: Vec) -> AResult<()>{ println!("download_meshes paths:{:?}",paths); let header=format!("Cookie: .ROBLOSECURITY={}",std::env::var("RBXCOOKIE")?); let shared_args=&[ "-q", "--header", header.as_str(), "-O", ]; let mut mesh_list=std::collections::HashSet::new(); for path in paths { let mut input = std::io::BufReader::new(std::fs::File::open(path.clone())?); match get_dom(&mut input){ Ok(dom)=>{ let object_refs = get_mesh_refs(&dom); for &object_ref in object_refs.iter() { if let Some(object)=dom.get_by_ref(object_ref){ if let Some(rbx_dom_weak::types::Variant::Content(content)) = object.properties.get("MeshId") { println!("Mesh content:{:?}",content); if let Ok(asset_id)=content.clone().into_string().parse::(){ mesh_list.insert(asset_id.0); } } } } }, Err(e)=>println!("error loading map {:?}: {:?}",path.file_name(),e), } } println!("Mesh list:{:?}",mesh_list); let processes_result:Result, _>=mesh_list.iter().map(|asset_id|{ std::process::Command::new("wget") .args(shared_args) .arg(format!("meshes/unprocessed/{}",asset_id)) .arg(format!("https://assetdelivery.roblox.com/v1/asset/?ID={}",asset_id)) .spawn() }).collect(); //naively wait for all because idk how to make an async progress bar lmao for child in processes_result?{ let output=child.wait_with_output()?; println!("Mesh exit_success:{}",output.status.success()); } Ok(()) } fn load_image(input:&mut R)->AResult{ let mut fourcc=[0u8;4]; input.read_exact(&mut fourcc)?; input.rewind()?; match &fourcc{ b"\x89PNG"=>Ok(image::load(input,image::ImageFormat::Png)?), b"\xFF\xD8\xFF\xE0"=>Ok(image::load(input,image::ImageFormat::Jpeg)?),//JFIF b"Err(anyhow::Error::msg("Roblox xml garbage is not supported yet")), other=>Err(anyhow::Error::msg(format!("Unknown texture format {:?}",other))), } } fn convert(file_thing:std::fs::DirEntry) -> AResult<()>{ let mut input = std::io::BufReader::new(std::fs::File::open(file_thing.path())?); let mut extracted_input=None; let image=match maybe_gzip_decode(&mut input){ Ok(ReaderType::GZip(mut readable)) => { //gzip let mut extracted:Vec=Vec::new(); //read the entire thing to the end so that I can clone the data and write a png to processed images readable.read_to_end(&mut extracted)?; extracted_input=Some(extracted.clone()); load_image(&mut std::io::Cursor::new(extracted)) }, Ok(ReaderType::Raw(readable)) => load_image(readable), Err(e) => Err(e)?, }?.to_rgba8();//this sets a=255, arcane is actually supposed to look like that let format=if image.width()%4!=0||image.height()%4!=0{ image_dds::ImageFormat::R8G8B8A8Srgb }else{ image_dds::ImageFormat::BC7Srgb }; //this fails if the image dimensions are not a multiple of 4 let dds = image_dds::dds_from_image( &image, format, image_dds::Quality::Slow, image_dds::Mipmaps::GeneratedAutomatic, )?; //write dds let mut dest=PathBuf::from("textures/dds"); dest.push(file_thing.file_name()); dest.set_extension("dds"); let mut writer = std::io::BufWriter::new(std::fs::File::create(dest)?); dds.write(&mut writer)?; if let Some(mut extracted)=extracted_input{ //write extracted to processed let mut dest=PathBuf::from("textures/processed"); dest.push(file_thing.file_name()); std::fs::write(dest, &mut extracted)?; //delete ugly gzip file std::fs::remove_file(file_thing.path())?; }else{ //move file to processed let mut dest=PathBuf::from("textures/processed"); dest.push(file_thing.file_name()); std::fs::rename(file_thing.path(), dest)?; } Ok(()) } fn convert_textures() -> AResult<()>{ let start = std::time::Instant::now(); let mut threads=Vec::new(); for entry in std::fs::read_dir("textures/unprocessed")? { let file_thing=entry?; threads.push(std::thread::spawn(move ||{ let file_name=format!("{:?}",file_thing); let result=convert(file_thing); if let Err(e)=result{ println!("error processing file:{:?} error message:{:?}",file_name,e); } })); } let mut i=0; let n_threads=threads.len(); for thread in threads{ i+=1; if let Err(e)=thread.join(){ println!("thread error: {:?}",e); }else{ println!("{}/{}",i,n_threads); } } println!("{:?}", start.elapsed()); Ok(()) } fn write_attributes() -> AResult<()>{ for entry in std::fs::read_dir("maps/unprocessed")? { let file_thing=entry?; println!("processing map={:?}",file_thing.file_name()); let mut input = std::io::BufReader::new(std::fs::File::open(file_thing.path())?); let mut dom = get_dom(&mut input)?; let button_refs = get_button_refs(&dom); for &button_ref in &button_refs { if let Some(button)=dom.get_by_ref_mut(button_ref){ match button.properties.get_mut("Attributes"){ Some(rbx_dom_weak::types::Variant::Attributes(attributes))=>{ println!("Appending Ref={} to existing attributes for {}",button_ref,button.name); attributes.insert("Ref".to_string(),rbx_dom_weak::types::Variant::String(button_ref.to_string())); }, None=>{ println!("Creating new attributes with Ref={} for {}",button_ref,button.name); let mut attributes=rbx_dom_weak::types::Attributes::new(); attributes.insert("Ref".to_string(),rbx_dom_weak::types::Variant::String(button_ref.to_string())); button.properties.insert("Attributes".to_string(),rbx_dom_weak::types::Variant::Attributes(attributes)); } _=>unreachable!("Fetching attributes did not return attributes."), } } } let mut dest={ let mut dest=PathBuf::from("maps/attributes"); dest.push(file_thing.file_name()); let output = std::io::BufWriter::new(std::fs::File::create(dest)?); //write workspace:GetChildren()[1] let workspace_children=dom.root().children(); if workspace_children.len()!=1{ return Err(anyhow::Error::msg("there can only be one model")); } rbx_binary::to_writer(output, &dom, &[workspace_children[0]])?; //move original to processed folder PathBuf::from("maps/unaltered") }; dest.push(file_thing.file_name()); std::fs::rename(file_thing.path(), dest)?; } Ok(()) } enum VMTContent{ VMT(String), VTF(String), Patch(vmt_parser::material::PatchMaterial), Unsupported,//don't want to deal with whatever vmt variant Unresolved,//could not locate a texture because of vmt content } impl VMTContent{ fn vtf(opt:Option)->Self{ match opt{ Some(s)=>Self::VTF(s), None=>Self::Unresolved, } } } fn get_some_texture(material:vmt_parser::material::Material)->AResult{ //just grab some texture from somewhere for now Ok(match material{ vmt_parser::material::Material::LightMappedGeneric(mat)=>VMTContent::vtf(Some(mat.base_texture)), vmt_parser::material::Material::VertexLitGeneric(mat)=>VMTContent::vtf(mat.base_texture.or(mat.decal_texture)),//this just dies if there is none vmt_parser::material::Material::VertexLitGenericDx6(mat)=>VMTContent::vtf(mat.base_texture.or(mat.decal_texture)), vmt_parser::material::Material::UnlitGeneric(mat)=>VMTContent::vtf(mat.base_texture), vmt_parser::material::Material::UnlitTwoTexture(mat)=>VMTContent::vtf(mat.base_texture), vmt_parser::material::Material::Water(mat)=>VMTContent::vtf(mat.base_texture), vmt_parser::material::Material::WorldVertexTransition(mat)=>VMTContent::vtf(Some(mat.base_texture)), vmt_parser::material::Material::EyeRefract(mat)=>VMTContent::vtf(Some(mat.cornea_texture)), vmt_parser::material::Material::SubRect(mat)=>VMTContent::VMT(mat.material),//recursive vmt_parser::material::Material::Sprite(mat)=>VMTContent::vtf(Some(mat.base_texture)), vmt_parser::material::Material::SpriteCard(mat)=>VMTContent::vtf(mat.base_texture), vmt_parser::material::Material::Cable(mat)=>VMTContent::vtf(Some(mat.base_texture)), vmt_parser::material::Material::Refract(mat)=>VMTContent::vtf(mat.base_texture), vmt_parser::material::Material::Modulate(mat)=>VMTContent::vtf(Some(mat.base_texture)), vmt_parser::material::Material::DecalModulate(mat)=>VMTContent::vtf(Some(mat.base_texture)), vmt_parser::material::Material::Sky(mat)=>VMTContent::vtf(Some(mat.base_texture)), vmt_parser::material::Material::Replacements(_mat)=>VMTContent::Unsupported, vmt_parser::material::Material::Patch(mat)=>VMTContent::Patch(mat), _=>return Err(anyhow::Error::msg("vmt failed to parse")), }) } fn get_vmtAResult>>>(find_stuff:&F,search_name:String)->AResult{ if let Some(stuff)=find_stuff(search_name)?{ //decode vmt and then write let stuff=String::from_utf8(stuff)?; let material=vmt_parser::from_str(stuff.as_str())?; println!("vmt material={:?}",material); return Ok(material); } Err(anyhow::Error::msg("vmt not found")) } fn recursive_vmt_loaderAResult>>>(find_stuff:&F,material:vmt_parser::material::Material)->AResult>>{ match get_some_texture(material)?{ VMTContent::VMT(s)=>recursive_vmt_loader(find_stuff,get_vmt(find_stuff,s)?), VMTContent::VTF(s)=>{ let mut texture_file_name=PathBuf::from("materials"); texture_file_name.push(s); texture_file_name.set_extension("vtf"); find_stuff(texture_file_name.into_os_string().into_string().unwrap()) }, VMTContent::Patch(mat)=>recursive_vmt_loader(find_stuff, mat.resolve(|search_name|{ match find_stuff(search_name.to_string())?{ Some(bytes)=>Ok(String::from_utf8(bytes)?), None=>Err(anyhow::Error::msg("could not find vmt")), } })? ), VMTContent::Unsupported=>{println!("Unsupported vmt");Ok(None)},//print and move on VMTContent::Unresolved=>{println!("Unresolved vmt");Ok(None)}, } } fn extract_textures(paths:Vec,vpk_paths:Vec)->AResult<()>{ let vpk_list:Vec=vpk_paths.into_iter().map(|vpk_path|vpk::VPK::read(&vpk_path).expect("vpk file does not exist")).collect(); for path in paths{ let mut deduplicate=std::collections::HashSet::new(); let bsp=vbsp::Bsp::read(std::fs::read(path)?.as_ref())?; for texture in bsp.textures(){ deduplicate.insert(PathBuf::from(texture.name())); } //dedupe prop models let mut model_dedupe=std::collections::HashSet::new(); for prop in bsp.static_props(){ model_dedupe.insert(prop.model()); } //grab texture names from props for model_name in model_dedupe{ //.mdl, .vvd, .dx90.vtx let mut path=PathBuf::from(model_name); let file_name=PathBuf::from(path.file_stem().unwrap()); path.pop(); path.push(file_name); let mut vvd_path=path.clone(); let mut vtx_path=path.clone(); vvd_path.set_extension("vvd"); vtx_path.set_extension("dx90.vtx"); match (bsp.pack.get(model_name),bsp.pack.get(vvd_path.as_os_str().to_str().unwrap()),bsp.pack.get(vtx_path.as_os_str().to_str().unwrap())){ (Ok(Some(mdl_file)),Ok(Some(vvd_file)),Ok(Some(vtx_file)))=>{ match (vmdl::mdl::Mdl::read(mdl_file.as_ref()),vmdl::vvd::Vvd::read(vvd_file.as_ref()),vmdl::vtx::Vtx::read(vtx_file.as_ref())){ (Ok(mdl),Ok(vvd),Ok(vtx))=>{ let model=vmdl::Model::from_parts(mdl,vtx,vvd); for texture in model.textures(){ for search_path in &texture.search_paths{ let mut path=PathBuf::from(search_path.as_str()); path.push(texture.name.as_str()); deduplicate.insert(path); } } }, _=>println!("model_name={} error",model_name), } }, _=>println!("no model name={}",model_name), } } let pack=&bsp.pack; let vpk_list=&vpk_list; std::thread::scope(move|s|{ let mut thread_handles=Vec::new(); for texture_name in deduplicate{ let mut found_texture=false; //LMAO imagine having to write type names let write_image=|mut stuff,write_file_name|{ let image=vtf::from_bytes(&mut stuff)?.highres_image.decode(0)?.to_rgba8(); let format=if image.width()%4!=0||image.height()%4!=0{ image_dds::ImageFormat::R8G8B8A8Srgb }else{ image_dds::ImageFormat::BC7Srgb }; //this fails if the image dimensions are not a multiple of 4 let dds = image_dds::dds_from_image( &image, format, image_dds::Quality::Slow, image_dds::Mipmaps::GeneratedAutomatic, )?; //write dds let mut dest=PathBuf::from("textures/dds"); dest.push(write_file_name); dest.set_extension("dds"); std::fs::create_dir_all(dest.parent().unwrap())?; let mut writer = std::io::BufWriter::new(std::fs::File::create(dest)?); dds.write(&mut writer)?; Ok::<(),anyhow::Error>(()) }; let find_stuff=|search_file_name:String|{ println!("search_file_name={}",search_file_name); match pack.get(search_file_name.as_str())?{ Some(file)=>return Ok(Some(file)), _=>(), } //search pak list for vpk_index in vpk_list{ if let Some(vpk_entry)=vpk_index.tree.get(search_file_name.as_str()){ return Ok(Some(match vpk_entry.get()?{ std::borrow::Cow::Borrowed(bytes)=>bytes.to_vec(), std::borrow::Cow::Owned(bytes)=>bytes, })); } } Ok::>,anyhow::Error>(None) }; let loader=|texture_name:String|{ let mut texture_file_name=PathBuf::from("materials"); //lower case let texture_file_name_lowercase=texture_name.to_lowercase(); texture_file_name.push(texture_file_name_lowercase.clone()); //remove stem and search for both vtf and vmt files let stem=PathBuf::from(texture_file_name.file_stem().unwrap()); texture_file_name.pop(); texture_file_name.push(stem); //somehow search for both files let mut texture_file_name_vmt=texture_file_name.clone(); texture_file_name.set_extension("vtf"); texture_file_name_vmt.set_extension("vmt"); if let Some(stuff)=find_stuff(texture_file_name.to_string_lossy().to_string())?{ return Ok(Some(stuff)) } recursive_vmt_loader(&find_stuff,get_vmt(&find_stuff,texture_file_name_vmt.to_string_lossy().to_string())?) }; if let Some(stuff)=loader(texture_name.to_string_lossy().to_string())?{ found_texture=true; let texture_name=texture_name.clone(); thread_handles.push(s.spawn(move||write_image(stuff,texture_name))); } if !found_texture{ println!("no data"); } } for thread in thread_handles{ match thread.join(){ Ok(Err(e))=>println!("write error: {:?}",e), Err(e)=>println!("thread error: {:?}",e), Ok(_)=>(), } } Ok::<(),anyhow::Error>(()) })? } Ok(()) } fn vpk_contents(vpk_path:PathBuf)->AResult<()>{ let vpk_index=vpk::VPK::read(&vpk_path)?; for (label,entry) in vpk_index.tree.into_iter(){ println!("vpk label={} entry={:?}",label,entry); } Ok(()) } fn bsp_contents(path:PathBuf)->AResult<()>{ let bsp=vbsp::Bsp::read(std::fs::read(path)?.as_ref())?; for file_name in bsp.pack.into_zip().into_inner().unwrap().file_names(){ println!("file_name={:?}",file_name); } Ok(()) }