diff --git a/src/common.rs b/src/common.rs new file mode 100644 index 0000000..95744c9 --- /dev/null +++ b/src/common.rs @@ -0,0 +1,75 @@ +use std::path::PathBuf; +use std::io::{Read,Seek}; +use anyhow::Result as AResult; + +fn load_image<R:Read+Seek+std::io::BufRead>(input:&mut R)->AResult<image::DynamicImage>{ + let mut fourcc=[0u8;4]; + input.read_exact(&mut fourcc)?; + input.rewind()?; + match &fourcc{ + b"\x89PNG"=>Ok(image::load(input,image::ImageFormat::Png)?), + b"\xFF\xD8\xFF\xE0"=>Ok(image::load(input,image::ImageFormat::Jpeg)?),//JFIF + b"<rob"=>Err(anyhow::Error::msg("Roblox xml garbage is not supported yet")), + other=>Err(anyhow::Error::msg(format!("Unknown texture format {:?}",other))), + } +} + +fn convert(file_thing:std::fs::DirEntry) -> AResult<()>{ + let mut input = std::io::BufReader::new(std::fs::File::open(file_thing.path())?); + + let image=load_image(&mut input)?.to_rgba8();//this sets a=255, arcane is actually supposed to look like that + + let format=if image.width()%4!=0||image.height()%4!=0{ + image_dds::ImageFormat::Rgba8UnormSrgb + }else{ + image_dds::ImageFormat::BC7RgbaUnormSrgb + }; + //this fails if the image dimensions are not a multiple of 4 + let dds = image_dds::dds_from_image( + &image, + format, + image_dds::Quality::Slow, + image_dds::Mipmaps::GeneratedAutomatic, + )?; + + //write dds + let mut dest=PathBuf::from("textures"); + dest.push(file_thing.file_name()); + dest.set_extension("dds"); + let mut writer = std::io::BufWriter::new(std::fs::File::create(dest)?); + dds.write(&mut writer)?; + + //move file to processed + let mut dest=PathBuf::from("textures/processed"); + dest.push(file_thing.file_name()); + std::fs::rename(file_thing.path(), dest)?; + Ok(()) +} +pub fn convert_textures() -> AResult<()>{ + std::fs::create_dir_all("textures/unprocessed")?; + std::fs::create_dir_all("textures/processed")?; + let start = std::time::Instant::now(); + let mut threads=Vec::new(); + for entry in std::fs::read_dir("textures/unprocessed")? { + let file_thing=entry?; + threads.push(std::thread::spawn(move ||{ + let file_name=format!("{:?}",file_thing); + let result=convert(file_thing); + if let Err(e)=result{ + println!("error processing file:{:?} error message:{:?}",file_name,e); + } + })); + } + let mut i=0; + let n_threads=threads.len(); + for thread in threads{ + i+=1; + if let Err(e)=thread.join(){ + println!("thread error: {:?}",e); + }else{ + println!("{}/{}",i,n_threads); + } + } + println!("{:?}", start.elapsed()); + Ok(()) +} diff --git a/src/main.rs b/src/main.rs index 927f1d7..e54d7b1 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,8 +1,9 @@ -use std::{collections::HashSet,io::{Read,Seek},path::PathBuf}; -use clap::{Args,Parser,Subcommand}; +mod common; +mod roblox; +mod source; + +use clap::{Parser,Subcommand}; use anyhow::Result as AResult; -use rbx_dom_weak::Instance; -use strafesnet_deferred_loader::rbxassetid::RobloxAssetId; #[derive(Parser)] #[command(author, version, about, long_about = None)] @@ -13,714 +14,19 @@ struct Cli { } #[derive(Subcommand)] -enum Commands { - RobloxToSNF(RobloxToSNFSubcommand), - SourceToSNF(SourceToSNFSubcommand), - DownloadTextures(DownloadTexturesSubcommand), - ExtractTextures(ExtractTexturesSubcommand), - ConvertTextures(ConvertTexturesSubcommand), - VPKContents(VPKContentsSubcommand), - BSPContents(BSPContentsSubcommand), - DownloadMeshes(DownloadMeshesSubcommand), -} - -#[derive(Args)] -struct RobloxToSNFSubcommand { - #[arg(long)] - output_folder:PathBuf, - #[arg(required=true)] - input_files:Vec<PathBuf>, -} -#[derive(Args)] -struct SourceToSNFSubcommand { - #[arg(long)] - output_folder:PathBuf, - #[arg(required=true)] - input_files:Vec<PathBuf>, -} -#[derive(Args)] -struct DownloadTexturesSubcommand { - #[arg(long,required=true)] - roblox_files:Vec<PathBuf> -} -#[derive(Args)] -struct ExtractTexturesSubcommand { - #[arg(long)] - bsp_file:PathBuf, - #[arg(long)] - vpk_dir_files:Vec<PathBuf> -} -#[derive(Args)] -struct ConvertTexturesSubcommand { -} -#[derive(Args)] -struct VPKContentsSubcommand { - #[arg(long)] - input_file:PathBuf, -} -#[derive(Args)] -struct BSPContentsSubcommand { - #[arg(long)] - input_file:PathBuf, -} -#[derive(Args)] -struct DownloadMeshesSubcommand { - #[arg(long,required=true)] - roblox_files:Vec<PathBuf> +enum Commands{ + #[command(flatten)] + Roblox(roblox::Commands), + #[command(flatten)] + Source(source::Commands), + ConvertTextures, } fn main() -> AResult<()> { let cli = Cli::parse(); - match cli.command { - Commands::RobloxToSNF(subcommand)=>roblox_to_snf(subcommand.input_files,subcommand.output_folder), - Commands::SourceToSNF(subcommand)=>source_to_snf(subcommand.input_files,subcommand.output_folder), - Commands::DownloadTextures(subcommand)=>download_textures(subcommand.roblox_files), - Commands::ExtractTextures(subcommand)=>extract_textures(vec![subcommand.bsp_file],subcommand.vpk_dir_files), - Commands::VPKContents(subcommand)=>vpk_contents(subcommand.input_file), - Commands::BSPContents(subcommand)=>bsp_contents(subcommand.input_file), - Commands::ConvertTextures(_subcommand)=>convert_textures(), - Commands::DownloadMeshes(subcommand)=>download_meshes(subcommand.roblox_files), + match cli.command{ + Commands::Roblox(commands)=>commands.run(), + Commands::Source(commands)=>commands.run(), + Commands::ConvertTextures=>common::convert_textures(), } } - -enum ReaderType<'a, R:Read+Seek>{ - GZip(flate2::read::GzDecoder<&'a mut R>), - Raw(&'a mut R), -} - -fn maybe_gzip_decode<R:Read+Seek>(input:&mut R)->AResult<ReaderType<R>>{ - let mut first_2=[0u8;2]; - if let (Ok(()),Ok(()))=(std::io::Read::read_exact(input, &mut first_2),std::io::Seek::rewind(input)){ - match &first_2{ - b"\x1f\x8b"=>Ok(ReaderType::GZip(flate2::read::GzDecoder::new(input))), - _=>Ok(ReaderType::Raw(input)), - } - }else{ - Err(anyhow::Error::msg("failed to peek")) - } -} - -fn load_dom<R:Read+Seek>(input:&mut R)->AResult<rbx_dom_weak::WeakDom>{ - let mut first_8=[0u8;8]; - if let (Ok(()),Ok(()))=(std::io::Read::read_exact(input, &mut first_8),std::io::Seek::rewind(input)){ - match &first_8[0..4]{ - b"<rob"=>{ - match &first_8[4..8]{ - b"lox!"=>rbx_binary::from_reader(input).map_err(anyhow::Error::msg), - b"lox "=>rbx_xml::from_reader(input,rbx_xml::DecodeOptions::default()).map_err(anyhow::Error::msg), - other=>Err(anyhow::Error::msg(format!("Unknown Roblox file type {:?}",other))), - } - }, - _=>Err(anyhow::Error::msg("unsupported file type")), - } - }else{ - Err(anyhow::Error::msg("peek failed")) - } -} - -/* The ones I'm interested in: -Beam.Texture -Decal.Texture -FileMesh.MeshId -FileMesh.TextureId -MaterialVariant.ColorMap -MaterialVariant.MetalnessMap -MaterialVariant.NormalMap -MaterialVariant.RoughnessMap -MeshPart.MeshId -MeshPart.TextureID -ParticleEmitter.Texture -Sky.MoonTextureId -Sky.SkyboxBk -Sky.SkyboxDn -Sky.SkyboxFt -Sky.SkyboxLf -Sky.SkyboxRt -Sky.SkyboxUp -Sky.SunTextureId -SurfaceAppearance.ColorMap -SurfaceAppearance.MetalnessMap -SurfaceAppearance.NormalMap -SurfaceAppearance.RoughnessMap -SurfaceAppearance.TexturePack -*/ -fn accumulate_content_id(content_list:&mut HashSet<u64>,object:&Instance,property:&str){ - if let Some(rbx_dom_weak::types::Variant::Content(content))=object.properties.get(property){ - if let Ok(asset_id)=AsRef::<str>::as_ref(content).parse::<RobloxAssetId>(){ - content_list.insert(asset_id.0); - }else{ - println!("Content failed to parse into AssetID: {:?}",content); - } - }else{ - println!("property={} does not exist for class={}",object.class.as_str(),property); - } -} -fn download_textures(paths:Vec<PathBuf>)->AResult<()>{ - println!("Reading files, this could take a hot minute..."); - let mut texture_list=HashSet::new(); - for path in paths{ - let file=match std::fs::File::open(path.as_path()){ - Ok(file)=>file, - Err(e)=>{ - println!("file error {e}"); - continue; - } - }; - let mut input=std::io::BufReader::new(file); - match load_dom(&mut input){ - Ok(dom)=>{ - for object in dom.into_raw().1.into_values(){ - match object.class.as_str(){ - "Beam"=>accumulate_content_id(&mut texture_list,&object,"Texture"), - "Decal"=>accumulate_content_id(&mut texture_list,&object,"Texture"), - "Texture"=>accumulate_content_id(&mut texture_list,&object,"Texture"), - "FileMesh"=>accumulate_content_id(&mut texture_list,&object,"TextureId"), - "MeshPart"=>accumulate_content_id(&mut texture_list,&object,"TextureID"), - "ParticleEmitter"=>accumulate_content_id(&mut texture_list,&object,"Texture"), - "Sky"=>{ - accumulate_content_id(&mut texture_list,&object,"MoonTextureId"); - accumulate_content_id(&mut texture_list,&object,"SkyboxBk"); - accumulate_content_id(&mut texture_list,&object,"SkyboxDn"); - accumulate_content_id(&mut texture_list,&object,"SkyboxFt"); - accumulate_content_id(&mut texture_list,&object,"SkyboxLf"); - accumulate_content_id(&mut texture_list,&object,"SkyboxRt"); - accumulate_content_id(&mut texture_list,&object,"SkyboxUp"); - accumulate_content_id(&mut texture_list,&object,"SunTextureId"); - }, - _=>(), - } - } - }, - Err(e)=>println!("error loading map {:?}: {:?}",path.file_name(),e), - } - } - let texture_list_string=texture_list.into_iter().map(|id|id.to_string()).collect::<Vec<String>>(); - println!("Texture list:{:?}",texture_list_string.join(" ")); - std::fs::create_dir_all("textures/unprocessed")?; - let output=std::process::Command::new("asset-tool") - .args(["download","--cookie-literal","","--output-folder","textures/unprocessed/"]) - .args(texture_list_string) - .spawn()? - .wait_with_output()?; - println!("Asset tool exit_success:{}",output.status.success()); - Ok(()) -} -fn download_meshes(paths:Vec<PathBuf>)->AResult<()>{ - println!("Reading files, this could take a hot minute..."); - let mut mesh_list=HashSet::new(); - for path in paths{ - let file=match std::fs::File::open(path.as_path()){ - Ok(file)=>file, - Err(e)=>{ - println!("file error {e}"); - continue; - } - }; - let mut input=std::io::BufReader::new(file); - match load_dom(&mut input){ - Ok(dom)=>{ - for object in dom.into_raw().1.into_values(){ - match object.class.as_str(){ - "MeshPart"=>accumulate_content_id(&mut mesh_list,&object,"MeshId"), - "SpecialMesh"=>accumulate_content_id(&mut mesh_list,&object,"MeshId"), - _=>(), - } - } - }, - Err(e)=>println!("error loading map {:?}: {:?}",path.file_name(),e), - } - } - let mesh_list_string=mesh_list.into_iter().map(|id|id.to_string()).collect::<Vec<String>>(); - println!("Mesh list:{:?}",mesh_list_string.join(" ")); - std::fs::create_dir_all("meshes/")?; - let output=std::process::Command::new("asset-tool") - .args(["download","--cookie-literal","","--output-folder","meshes/"]) - .args(mesh_list_string) - .spawn()? - .wait_with_output()?; - println!("Asset tool exit_success:{}",output.status.success()); - Ok(()) -} - -fn load_image<R:Read+Seek+std::io::BufRead>(input:&mut R)->AResult<image::DynamicImage>{ - let mut fourcc=[0u8;4]; - input.read_exact(&mut fourcc)?; - input.rewind()?; - match &fourcc{ - b"\x89PNG"=>Ok(image::load(input,image::ImageFormat::Png)?), - b"\xFF\xD8\xFF\xE0"=>Ok(image::load(input,image::ImageFormat::Jpeg)?),//JFIF - b"<rob"=>Err(anyhow::Error::msg("Roblox xml garbage is not supported yet")), - other=>Err(anyhow::Error::msg(format!("Unknown texture format {:?}",other))), - } -} - -fn convert(file_thing:std::fs::DirEntry) -> AResult<()>{ - let mut input = std::io::BufReader::new(std::fs::File::open(file_thing.path())?); - - let mut extracted_input=None; - let image=match maybe_gzip_decode(&mut input){ - Ok(ReaderType::GZip(mut readable)) => { - //gzip - let mut extracted:Vec<u8>=Vec::new(); - //read the entire thing to the end so that I can clone the data and write a png to processed images - readable.read_to_end(&mut extracted)?; - extracted_input=Some(extracted.clone()); - load_image(&mut std::io::Cursor::new(extracted)) - }, - Ok(ReaderType::Raw(readable)) => load_image(readable), - Err(e) => Err(e)?, - }?.to_rgba8();//this sets a=255, arcane is actually supposed to look like that - - let format=if image.width()%4!=0||image.height()%4!=0{ - image_dds::ImageFormat::Rgba8UnormSrgb - }else{ - image_dds::ImageFormat::BC7RgbaUnormSrgb - }; - //this fails if the image dimensions are not a multiple of 4 - let dds = image_dds::dds_from_image( - &image, - format, - image_dds::Quality::Slow, - image_dds::Mipmaps::GeneratedAutomatic, - )?; - - //write dds - let mut dest=PathBuf::from("textures"); - dest.push(file_thing.file_name()); - dest.set_extension("dds"); - let mut writer = std::io::BufWriter::new(std::fs::File::create(dest)?); - dds.write(&mut writer)?; - - if let Some(mut extracted)=extracted_input{ - //write extracted to processed - let mut dest=PathBuf::from("textures/processed"); - dest.push(file_thing.file_name()); - std::fs::write(dest, &mut extracted)?; - //delete ugly gzip file - std::fs::remove_file(file_thing.path())?; - }else{ - //move file to processed - let mut dest=PathBuf::from("textures/processed"); - dest.push(file_thing.file_name()); - std::fs::rename(file_thing.path(), dest)?; - } - Ok(()) -} -fn convert_textures() -> AResult<()>{ - std::fs::create_dir_all("textures/unprocessed")?; - std::fs::create_dir_all("textures/processed")?; - let start = std::time::Instant::now(); - let mut threads=Vec::new(); - for entry in std::fs::read_dir("textures/unprocessed")? { - let file_thing=entry?; - threads.push(std::thread::spawn(move ||{ - let file_name=format!("{:?}",file_thing); - let result=convert(file_thing); - if let Err(e)=result{ - println!("error processing file:{:?} error message:{:?}",file_name,e); - } - })); - } - let mut i=0; - let n_threads=threads.len(); - for thread in threads{ - i+=1; - if let Err(e)=thread.join(){ - println!("thread error: {:?}",e); - }else{ - println!("{}/{}",i,n_threads); - } - } - println!("{:?}", start.elapsed()); - Ok(()) -} - -enum VMTContent{ - VMT(String), - VTF(String), - Patch(vmt_parser::material::PatchMaterial), - Unsupported,//don't want to deal with whatever vmt variant - Unresolved,//could not locate a texture because of vmt content -} -impl VMTContent{ - fn vtf(opt:Option<String>)->Self{ - match opt{ - Some(s)=>Self::VTF(s), - None=>Self::Unresolved, - } - } -} - -fn get_some_texture(material:vmt_parser::material::Material)->AResult<VMTContent>{ - //just grab some texture from somewhere for now - Ok(match material{ - vmt_parser::material::Material::LightMappedGeneric(mat)=>VMTContent::vtf(Some(mat.base_texture)), - vmt_parser::material::Material::VertexLitGeneric(mat)=>VMTContent::vtf(mat.base_texture.or(mat.decal_texture)),//this just dies if there is none - vmt_parser::material::Material::VertexLitGenericDx6(mat)=>VMTContent::vtf(mat.base_texture.or(mat.decal_texture)), - vmt_parser::material::Material::UnlitGeneric(mat)=>VMTContent::vtf(mat.base_texture), - vmt_parser::material::Material::UnlitTwoTexture(mat)=>VMTContent::vtf(mat.base_texture), - vmt_parser::material::Material::Water(mat)=>VMTContent::vtf(mat.base_texture), - vmt_parser::material::Material::WorldVertexTransition(mat)=>VMTContent::vtf(Some(mat.base_texture)), - vmt_parser::material::Material::EyeRefract(mat)=>VMTContent::vtf(Some(mat.cornea_texture)), - vmt_parser::material::Material::SubRect(mat)=>VMTContent::VMT(mat.material),//recursive - vmt_parser::material::Material::Sprite(mat)=>VMTContent::vtf(Some(mat.base_texture)), - vmt_parser::material::Material::SpriteCard(mat)=>VMTContent::vtf(mat.base_texture), - vmt_parser::material::Material::Cable(mat)=>VMTContent::vtf(Some(mat.base_texture)), - vmt_parser::material::Material::Refract(mat)=>VMTContent::vtf(mat.base_texture), - vmt_parser::material::Material::Modulate(mat)=>VMTContent::vtf(Some(mat.base_texture)), - vmt_parser::material::Material::DecalModulate(mat)=>VMTContent::vtf(Some(mat.base_texture)), - vmt_parser::material::Material::Sky(mat)=>VMTContent::vtf(Some(mat.base_texture)), - vmt_parser::material::Material::Replacements(_mat)=>VMTContent::Unsupported, - vmt_parser::material::Material::Patch(mat)=>VMTContent::Patch(mat), - _=>return Err(anyhow::Error::msg("vmt failed to parse")), - }) -} - -fn get_vmt<F:Fn(String)->AResult<Option<Vec<u8>>>>(find_stuff:&F,search_name:String)->AResult<vmt_parser::material::Material>{ - if let Some(stuff)=find_stuff(search_name)?{ - //decode vmt and then write - let stuff=String::from_utf8(stuff)?; - let material=vmt_parser::from_str(stuff.as_str())?; - println!("vmt material={:?}",material); - return Ok(material); - } - Err(anyhow::Error::msg("vmt not found")) -} - -fn recursive_vmt_loader<F:Fn(String)->AResult<Option<Vec<u8>>>>(find_stuff:&F,material:vmt_parser::material::Material)->AResult<Option<Vec<u8>>>{ - match get_some_texture(material)?{ - VMTContent::VMT(s)=>recursive_vmt_loader(find_stuff,get_vmt(find_stuff,s)?), - VMTContent::VTF(s)=>{ - let mut texture_file_name=PathBuf::from("materials"); - texture_file_name.push(s); - texture_file_name.set_extension("vtf"); - find_stuff(texture_file_name.into_os_string().into_string().unwrap()) - }, - VMTContent::Patch(mat)=>recursive_vmt_loader(find_stuff, - mat.resolve(|search_name|{ - match find_stuff(search_name.to_string())?{ - Some(bytes)=>Ok(String::from_utf8(bytes)?), - None=>Err(anyhow::Error::msg("could not find vmt")), - } - })? - ), - VMTContent::Unsupported=>{println!("Unsupported vmt");Ok(None)},//print and move on - VMTContent::Unresolved=>{println!("Unresolved vmt");Ok(None)}, - } -} - -fn extract_textures(paths:Vec<PathBuf>,vpk_paths:Vec<PathBuf>)->AResult<()>{ - std::fs::create_dir_all("textures")?; - let vpk_list:Vec<vpk::VPK>=vpk_paths.into_iter().map(|vpk_path|vpk::VPK::read(&vpk_path).expect("vpk file does not exist")).collect(); - for path in paths{ - let mut deduplicate=std::collections::HashSet::new(); - let bsp=vbsp::Bsp::read(std::fs::read(path)?.as_ref())?; - for texture in bsp.textures(){ - deduplicate.insert(PathBuf::from(texture.name())); - } - //dedupe prop models - let mut model_dedupe=std::collections::HashSet::new(); - for prop in bsp.static_props(){ - model_dedupe.insert(prop.model()); - } - - //grab texture names from props - for model_name in model_dedupe{ - //.mdl, .vvd, .dx90.vtx - let mut path=PathBuf::from(model_name); - let file_name=PathBuf::from(path.file_stem().unwrap()); - path.pop(); - path.push(file_name); - let mut vvd_path=path.clone(); - let mut vtx_path=path.clone(); - vvd_path.set_extension("vvd"); - vtx_path.set_extension("dx90.vtx"); - match (bsp.pack.get(model_name),bsp.pack.get(vvd_path.as_os_str().to_str().unwrap()),bsp.pack.get(vtx_path.as_os_str().to_str().unwrap())){ - (Ok(Some(mdl_file)),Ok(Some(vvd_file)),Ok(Some(vtx_file)))=>{ - match (vmdl::mdl::Mdl::read(mdl_file.as_ref()),vmdl::vvd::Vvd::read(vvd_file.as_ref()),vmdl::vtx::Vtx::read(vtx_file.as_ref())){ - (Ok(mdl),Ok(vvd),Ok(vtx))=>{ - let model=vmdl::Model::from_parts(mdl,vtx,vvd); - for texture in model.textures(){ - for search_path in &texture.search_paths{ - let mut path=PathBuf::from(search_path.as_str()); - path.push(texture.name.as_str()); - deduplicate.insert(path); - } - } - }, - _=>println!("model_name={} error",model_name), - } - }, - _=>println!("no model name={}",model_name), - } - } - - let pack=&bsp.pack; - let vpk_list=&vpk_list; - std::thread::scope(move|s|{ - let mut thread_handles=Vec::new(); - for texture_name in deduplicate{ - let mut found_texture=false; - //LMAO imagine having to write type names - let write_image=|mut stuff,write_file_name|{ - let image=vtf::from_bytes(&mut stuff)?.highres_image.decode(0)?.to_rgba8(); - - let format=if image.width()%4!=0||image.height()%4!=0{ - image_dds::ImageFormat::Rgba8UnormSrgb - }else{ - image_dds::ImageFormat::BC7RgbaUnormSrgb - }; - //this fails if the image dimensions are not a multiple of 4 - let dds = image_dds::dds_from_image( - &image, - format, - image_dds::Quality::Slow, - image_dds::Mipmaps::GeneratedAutomatic, - )?; - - //write dds - let mut dest=PathBuf::from("textures"); - dest.push(write_file_name); - dest.set_extension("dds"); - std::fs::create_dir_all(dest.parent().unwrap())?; - let mut writer = std::io::BufWriter::new(std::fs::File::create(dest)?); - dds.write(&mut writer)?; - Ok::<(),anyhow::Error>(()) - }; - let find_stuff=|search_file_name:String|{ - println!("search_file_name={}",search_file_name); - match pack.get(search_file_name.as_str())?{ - Some(file)=>return Ok(Some(file)), - _=>(), - } - //search pak list - for vpk_index in vpk_list{ - if let Some(vpk_entry)=vpk_index.tree.get(search_file_name.as_str()){ - return Ok(Some(match vpk_entry.get()?{ - std::borrow::Cow::Borrowed(bytes)=>bytes.to_vec(), - std::borrow::Cow::Owned(bytes)=>bytes, - })); - } - } - Ok::<Option<Vec<u8>>,anyhow::Error>(None) - }; - let loader=|texture_name:String|{ - let mut texture_file_name=PathBuf::from("materials"); - //lower case - let texture_file_name_lowercase=texture_name.to_lowercase(); - texture_file_name.push(texture_file_name_lowercase.clone()); - //remove stem and search for both vtf and vmt files - let stem=PathBuf::from(texture_file_name.file_stem().unwrap()); - texture_file_name.pop(); - texture_file_name.push(stem); - //somehow search for both files - let mut texture_file_name_vmt=texture_file_name.clone(); - texture_file_name.set_extension("vtf"); - texture_file_name_vmt.set_extension("vmt"); - if let Some(stuff)=find_stuff(texture_file_name.to_string_lossy().to_string())?{ - return Ok(Some(stuff)) - } - recursive_vmt_loader(&find_stuff,get_vmt(&find_stuff,texture_file_name_vmt.to_string_lossy().to_string())?) - }; - if let Some(stuff)=loader(texture_name.to_string_lossy().to_string())?{ - found_texture=true; - let texture_name=texture_name.clone(); - thread_handles.push(s.spawn(move||write_image(stuff,texture_name))); - } - if !found_texture{ - println!("no data"); - } - } - for thread in thread_handles{ - match thread.join(){ - Ok(Err(e))=>println!("write error: {:?}",e), - Err(e)=>println!("thread error: {:?}",e), - Ok(_)=>(), - } - } - Ok::<(),anyhow::Error>(()) - })? - } - Ok(()) -} - -fn vpk_contents(vpk_path:PathBuf)->AResult<()>{ - let vpk_index=vpk::VPK::read(&vpk_path)?; - for (label,entry) in vpk_index.tree.into_iter(){ - println!("vpk label={} entry={:?}",label,entry); - } - Ok(()) -} - -fn bsp_contents(path:PathBuf)->AResult<()>{ - let bsp=vbsp::Bsp::read(std::fs::read(path)?.as_ref())?; - for file_name in bsp.pack.into_zip().into_inner().unwrap().file_names(){ - println!("file_name={:?}",file_name); - } - Ok(()) -} - -#[derive(Debug)] -#[allow(dead_code)] -enum ConvertError{ - IO(std::io::Error), - SNFMap(strafesnet_snf::map::Error), - RbxLoader(strafesnet_rbx_loader::ReadError), - BspLoader(strafesnet_bsp_loader::ReadError), -} -impl std::fmt::Display for ConvertError{ - fn fmt(&self,f:&mut std::fmt::Formatter<'_>)->std::fmt::Result{ - write!(f,"{self:?}") - } -} -impl std::error::Error for ConvertError{} - -type MapThread=std::thread::JoinHandle<Result<(),ConvertError>>; - -fn roblox_to_snf(pathlist:Vec<std::path::PathBuf>,output_folder:PathBuf)->AResult<()>{ - let n_paths=pathlist.len(); - let start = std::time::Instant::now(); - let mut threads:std::collections::VecDeque<MapThread>=std::collections::VecDeque::new(); - let mut i=0; - let mut join_thread=|thread:MapThread|{ - i+=1; - if let Err(e)=thread.join(){ - println!("thread error: {:?}",e); - }else{ - println!("{}/{}",i,n_paths); - } - }; - for path in pathlist{ - if 32<=threads.len(){ - join_thread(threads.pop_front().unwrap()); - } - let output_folder=output_folder.clone(); - threads.push_back(std::thread::spawn(move ||{ - let model=strafesnet_rbx_loader::read( - std::fs::File::open(path.as_path()) - .map_err(ConvertError::IO)? - ).map_err(ConvertError::RbxLoader)?; - - let mut place=model.into_place(); - place.run_scripts(); - - let mut loader=strafesnet_deferred_loader::roblox_legacy(); - - let (texture_loader,mesh_loader)=loader.get_inner_mut(); - - let map_step1=strafesnet_rbx_loader::convert( - &place, - |name|texture_loader.acquire_render_config_id(name), - |name|mesh_loader.acquire_mesh_id(name), - ); - - let meshpart_meshes=mesh_loader.load_meshes().map_err(ConvertError::IO)?; - - let map_step2=map_step1.add_meshpart_meshes_and_calculate_attributes( - meshpart_meshes.into_iter().map(|(mesh_id,loader_model)| - (mesh_id,strafesnet_rbx_loader::data::RobloxMeshBytes::new(loader_model.get())) - ) - ); - - let (textures,render_configs)=loader.into_render_configs().map_err(ConvertError::IO)?.consume(); - - let map=map_step2.add_render_configs_and_textures( - render_configs.into_iter(), - textures.into_iter().map(|(texture_id,texture)| - (texture_id,match texture{ - strafesnet_deferred_loader::texture::Texture::ImageDDS(data)=>data, - }) - ) - ); - - let mut dest=output_folder.clone(); - dest.push(path.file_stem().unwrap()); - dest.set_extension("snfm"); - let file=std::fs::File::create(dest).map_err(ConvertError::IO)?; - - strafesnet_snf::map::write_map(file,map).map_err(ConvertError::SNFMap)?; - Ok(()) - })); - } - - for thread in threads{ - join_thread(thread); - } - println!("{:?}", start.elapsed()); - Ok(()) -} - -fn source_to_snf(pathlist:Vec<std::path::PathBuf>,output_folder:PathBuf)->AResult<()>{ - let n_paths=pathlist.len(); - let start = std::time::Instant::now(); - let mut threads:std::collections::VecDeque<MapThread>=std::collections::VecDeque::new(); - let mut i=0; - let mut join_thread=|thread:MapThread|{ - i+=1; - if let Err(e)=thread.join(){ - println!("thread error: {:?}",e); - }else{ - println!("{}/{}",i,n_paths); - } - }; - for path in pathlist{ - if 32<=threads.len(){ - join_thread(threads.pop_front().unwrap()); - } - let output_folder=output_folder.clone(); - threads.push_back(std::thread::spawn(move ||{ - let bsp=strafesnet_bsp_loader::read( - std::fs::File::open(path.as_path()) - .map_err(ConvertError::IO)? - ).map_err(ConvertError::BspLoader)?; - let mut loader=strafesnet_deferred_loader::source_legacy(); - - let (texture_loader,mesh_loader)=loader.get_inner_mut(); - - let map_step1=strafesnet_bsp_loader::convert( - &bsp, - |name|texture_loader.acquire_render_config_id(name), - |name|mesh_loader.acquire_mesh_id(name), - ); - - let prop_meshes=mesh_loader.load_meshes(&bsp.as_ref()); - - let map_step2=map_step1.add_prop_meshes( - //the type conflagulator 9000 - prop_meshes.into_iter().map(|(mesh_id,loader_model)| - (mesh_id,strafesnet_bsp_loader::data::ModelData{ - mdl:strafesnet_bsp_loader::data::MdlData::new(loader_model.mdl.get()), - vtx:strafesnet_bsp_loader::data::VtxData::new(loader_model.vtx.get()), - vvd:strafesnet_bsp_loader::data::VvdData::new(loader_model.vvd.get()), - }) - ), - |name|texture_loader.acquire_render_config_id(name), - ); - - let (textures,render_configs)=loader.into_render_configs().map_err(ConvertError::IO)?.consume(); - - let map=map_step2.add_render_configs_and_textures( - render_configs.into_iter(), - textures.into_iter().map(|(texture_id,texture)| - (texture_id,match texture{ - strafesnet_deferred_loader::texture::Texture::ImageDDS(data)=>data, - }) - ), - ); - - let mut dest=output_folder.clone(); - dest.push(path.file_stem().unwrap()); - dest.set_extension("snfm"); - let file=std::fs::File::create(dest).map_err(ConvertError::IO)?; - - strafesnet_snf::map::write_map(file,map).map_err(ConvertError::SNFMap)?; - Ok(()) - })); - } - - for thread in threads{ - join_thread(thread); - } - println!("{:?}", start.elapsed()); - Ok(()) -} diff --git a/src/roblox.rs b/src/roblox.rs new file mode 100644 index 0000000..53a7a5b --- /dev/null +++ b/src/roblox.rs @@ -0,0 +1,273 @@ +use std::path::PathBuf; +use std::io::{Read,Seek}; +use std::collections::HashSet; +use clap::{Args,Subcommand}; +use anyhow::Result as AResult; +use rbx_dom_weak::Instance; +use strafesnet_deferred_loader::rbxassetid::RobloxAssetId; + +#[derive(Subcommand)] +pub enum Commands{ + RobloxToSNF(RobloxToSNFSubcommand), + DownloadTextures(DownloadTexturesSubcommand), + DownloadMeshes(DownloadMeshesSubcommand), +} + +#[derive(Args)] +pub struct RobloxToSNFSubcommand { + #[arg(long)] + output_folder:PathBuf, + #[arg(required=true)] + input_files:Vec<PathBuf>, +} +#[derive(Args)] +pub struct DownloadTexturesSubcommand { + #[arg(long,required=true)] + roblox_files:Vec<PathBuf> +} +#[derive(Args)] +pub struct DownloadMeshesSubcommand { + #[arg(long,required=true)] + roblox_files:Vec<PathBuf> +} + +impl Commands{ + pub fn run(self)->AResult<()>{ + match self{ + Commands::RobloxToSNF(subcommand)=>roblox_to_snf(subcommand.input_files,subcommand.output_folder), + Commands::DownloadTextures(subcommand)=>download_textures(subcommand.roblox_files), + Commands::DownloadMeshes(subcommand)=>download_meshes(subcommand.roblox_files), + } + } +} + +fn load_dom<R:Read+Seek>(input:&mut R)->AResult<rbx_dom_weak::WeakDom>{ + let mut first_8=[0u8;8]; + if let (Ok(()),Ok(()))=(std::io::Read::read_exact(input, &mut first_8),std::io::Seek::rewind(input)){ + match &first_8[0..4]{ + b"<rob"=>{ + match &first_8[4..8]{ + b"lox!"=>rbx_binary::from_reader(input).map_err(anyhow::Error::msg), + b"lox "=>rbx_xml::from_reader(input,rbx_xml::DecodeOptions::default()).map_err(anyhow::Error::msg), + other=>Err(anyhow::Error::msg(format!("Unknown Roblox file type {:?}",other))), + } + }, + _=>Err(anyhow::Error::msg("unsupported file type")), + } + }else{ + Err(anyhow::Error::msg("peek failed")) + } +} + +/* The ones I'm interested in: +Beam.Texture +Decal.Texture +FileMesh.MeshId +FileMesh.TextureId +MaterialVariant.ColorMap +MaterialVariant.MetalnessMap +MaterialVariant.NormalMap +MaterialVariant.RoughnessMap +MeshPart.MeshId +MeshPart.TextureID +ParticleEmitter.Texture +Sky.MoonTextureId +Sky.SkyboxBk +Sky.SkyboxDn +Sky.SkyboxFt +Sky.SkyboxLf +Sky.SkyboxRt +Sky.SkyboxUp +Sky.SunTextureId +SurfaceAppearance.ColorMap +SurfaceAppearance.MetalnessMap +SurfaceAppearance.NormalMap +SurfaceAppearance.RoughnessMap +SurfaceAppearance.TexturePack +*/ +fn accumulate_content_id(content_list:&mut HashSet<u64>,object:&Instance,property:&str){ + if let Some(rbx_dom_weak::types::Variant::Content(content))=object.properties.get(property){ + if let Ok(asset_id)=AsRef::<str>::as_ref(content).parse::<RobloxAssetId>(){ + content_list.insert(asset_id.0); + }else{ + println!("Content failed to parse into AssetID: {:?}",content); + } + }else{ + println!("property={} does not exist for class={}",object.class.as_str(),property); + } +} +fn download_textures(paths:Vec<PathBuf>)->AResult<()>{ + println!("Reading files, this could take a hot minute..."); + let mut texture_list=HashSet::new(); + for path in paths{ + let file=match std::fs::File::open(path.as_path()){ + Ok(file)=>file, + Err(e)=>{ + println!("file error {e}"); + continue; + } + }; + let mut input=std::io::BufReader::new(file); + match load_dom(&mut input){ + Ok(dom)=>{ + for object in dom.into_raw().1.into_values(){ + match object.class.as_str(){ + "Beam"=>accumulate_content_id(&mut texture_list,&object,"Texture"), + "Decal"=>accumulate_content_id(&mut texture_list,&object,"Texture"), + "Texture"=>accumulate_content_id(&mut texture_list,&object,"Texture"), + "FileMesh"=>accumulate_content_id(&mut texture_list,&object,"TextureId"), + "MeshPart"=>accumulate_content_id(&mut texture_list,&object,"TextureID"), + "ParticleEmitter"=>accumulate_content_id(&mut texture_list,&object,"Texture"), + "Sky"=>{ + accumulate_content_id(&mut texture_list,&object,"MoonTextureId"); + accumulate_content_id(&mut texture_list,&object,"SkyboxBk"); + accumulate_content_id(&mut texture_list,&object,"SkyboxDn"); + accumulate_content_id(&mut texture_list,&object,"SkyboxFt"); + accumulate_content_id(&mut texture_list,&object,"SkyboxLf"); + accumulate_content_id(&mut texture_list,&object,"SkyboxRt"); + accumulate_content_id(&mut texture_list,&object,"SkyboxUp"); + accumulate_content_id(&mut texture_list,&object,"SunTextureId"); + }, + _=>(), + } + } + }, + Err(e)=>println!("error loading map {:?}: {:?}",path.file_name(),e), + } + } + let texture_list_string=texture_list.into_iter().map(|id|id.to_string()).collect::<Vec<String>>(); + println!("Texture list:{:?}",texture_list_string.join(" ")); + std::fs::create_dir_all("textures/unprocessed")?; + let output=std::process::Command::new("asset-tool") + .args(["download","--cookie-literal","","--output-folder","textures/unprocessed/"]) + .args(texture_list_string) + .spawn()? + .wait_with_output()?; + println!("Asset tool exit_success:{}",output.status.success()); + Ok(()) +} +fn download_meshes(paths:Vec<PathBuf>)->AResult<()>{ + println!("Reading files, this could take a hot minute..."); + let mut mesh_list=HashSet::new(); + for path in paths{ + let file=match std::fs::File::open(path.as_path()){ + Ok(file)=>file, + Err(e)=>{ + println!("file error {e}"); + continue; + } + }; + let mut input=std::io::BufReader::new(file); + match load_dom(&mut input){ + Ok(dom)=>{ + for object in dom.into_raw().1.into_values(){ + match object.class.as_str(){ + "MeshPart"=>accumulate_content_id(&mut mesh_list,&object,"MeshId"), + "SpecialMesh"=>accumulate_content_id(&mut mesh_list,&object,"MeshId"), + _=>(), + } + } + }, + Err(e)=>println!("error loading map {:?}: {:?}",path.file_name(),e), + } + } + let mesh_list_string=mesh_list.into_iter().map(|id|id.to_string()).collect::<Vec<String>>(); + println!("Mesh list:{:?}",mesh_list_string.join(" ")); + std::fs::create_dir_all("meshes/")?; + let output=std::process::Command::new("asset-tool") + .args(["download","--cookie-literal","","--output-folder","meshes/"]) + .args(mesh_list_string) + .spawn()? + .wait_with_output()?; + println!("Asset tool exit_success:{}",output.status.success()); + Ok(()) +} + +#[derive(Debug)] +#[allow(dead_code)] +enum ConvertError{ + IO(std::io::Error), + SNFMap(strafesnet_snf::map::Error), + RbxLoader(strafesnet_rbx_loader::ReadError), +} +impl std::fmt::Display for ConvertError{ + fn fmt(&self,f:&mut std::fmt::Formatter<'_>)->std::fmt::Result{ + write!(f,"{self:?}") + } +} +impl std::error::Error for ConvertError{} + +type MapThread=std::thread::JoinHandle<Result<(),ConvertError>>; + +fn roblox_to_snf(pathlist:Vec<std::path::PathBuf>,output_folder:PathBuf)->AResult<()>{ + let n_paths=pathlist.len(); + let start = std::time::Instant::now(); + let mut threads:std::collections::VecDeque<MapThread>=std::collections::VecDeque::new(); + let mut i=0; + let mut join_thread=|thread:MapThread|{ + i+=1; + if let Err(e)=thread.join(){ + println!("thread error: {:?}",e); + }else{ + println!("{}/{}",i,n_paths); + } + }; + for path in pathlist{ + if 32<=threads.len(){ + join_thread(threads.pop_front().unwrap()); + } + let output_folder=output_folder.clone(); + threads.push_back(std::thread::spawn(move ||{ + let model=strafesnet_rbx_loader::read( + std::fs::File::open(path.as_path()) + .map_err(ConvertError::IO)? + ).map_err(ConvertError::RbxLoader)?; + + let mut place=model.into_place(); + place.run_scripts(); + + let mut loader=strafesnet_deferred_loader::roblox_legacy(); + + let (texture_loader,mesh_loader)=loader.get_inner_mut(); + + let map_step1=strafesnet_rbx_loader::convert( + &place, + |name|texture_loader.acquire_render_config_id(name), + |name|mesh_loader.acquire_mesh_id(name), + ); + + let meshpart_meshes=mesh_loader.load_meshes().map_err(ConvertError::IO)?; + + let map_step2=map_step1.add_meshpart_meshes_and_calculate_attributes( + meshpart_meshes.into_iter().map(|(mesh_id,loader_model)| + (mesh_id,strafesnet_rbx_loader::data::RobloxMeshBytes::new(loader_model.get())) + ) + ); + + let (textures,render_configs)=loader.into_render_configs().map_err(ConvertError::IO)?.consume(); + + let map=map_step2.add_render_configs_and_textures( + render_configs.into_iter(), + textures.into_iter().map(|(texture_id,texture)| + (texture_id,match texture{ + strafesnet_deferred_loader::texture::Texture::ImageDDS(data)=>data, + }) + ) + ); + + let mut dest=output_folder.clone(); + dest.push(path.file_stem().unwrap()); + dest.set_extension("snfm"); + let file=std::fs::File::create(dest).map_err(ConvertError::IO)?; + + strafesnet_snf::map::write_map(file,map).map_err(ConvertError::SNFMap)?; + Ok(()) + })); + } + + for thread in threads{ + join_thread(thread); + } + println!("{:?}", start.elapsed()); + Ok(()) +} diff --git a/src/source.rs b/src/source.rs new file mode 100644 index 0000000..1d0f803 --- /dev/null +++ b/src/source.rs @@ -0,0 +1,364 @@ +use std::path::PathBuf; +use clap::{Args,Subcommand}; +use anyhow::Result as AResult; + +#[derive(Subcommand)] +pub enum Commands{ + SourceToSNF(SourceToSNFSubcommand), + ExtractTextures(ExtractTexturesSubcommand), + VPKContents(VPKContentsSubcommand), + BSPContents(BSPContentsSubcommand), +} + +#[derive(Args)] +pub struct SourceToSNFSubcommand { + #[arg(long)] + output_folder:PathBuf, + #[arg(required=true)] + input_files:Vec<PathBuf>, +} +#[derive(Args)] +pub struct ExtractTexturesSubcommand { + #[arg(long)] + bsp_file:PathBuf, + #[arg(long)] + vpk_dir_files:Vec<PathBuf> +} +#[derive(Args)] +pub struct VPKContentsSubcommand { + #[arg(long)] + input_file:PathBuf, +} +#[derive(Args)] +pub struct BSPContentsSubcommand { + #[arg(long)] + input_file:PathBuf, +} + +impl Commands{ + pub fn run(self)->AResult<()>{ + match self{ + Commands::SourceToSNF(subcommand)=>source_to_snf(subcommand.input_files,subcommand.output_folder), + Commands::ExtractTextures(subcommand)=>extract_textures(vec![subcommand.bsp_file],subcommand.vpk_dir_files), + Commands::VPKContents(subcommand)=>vpk_contents(subcommand.input_file), + Commands::BSPContents(subcommand)=>bsp_contents(subcommand.input_file), + } + } +} + + +enum VMTContent{ + VMT(String), + VTF(String), + Patch(vmt_parser::material::PatchMaterial), + Unsupported,//don't want to deal with whatever vmt variant + Unresolved,//could not locate a texture because of vmt content +} +impl VMTContent{ + fn vtf(opt:Option<String>)->Self{ + match opt{ + Some(s)=>Self::VTF(s), + None=>Self::Unresolved, + } + } +} + +fn get_some_texture(material:vmt_parser::material::Material)->AResult<VMTContent>{ + //just grab some texture from somewhere for now + Ok(match material{ + vmt_parser::material::Material::LightMappedGeneric(mat)=>VMTContent::vtf(Some(mat.base_texture)), + vmt_parser::material::Material::VertexLitGeneric(mat)=>VMTContent::vtf(mat.base_texture.or(mat.decal_texture)),//this just dies if there is none + vmt_parser::material::Material::VertexLitGenericDx6(mat)=>VMTContent::vtf(mat.base_texture.or(mat.decal_texture)), + vmt_parser::material::Material::UnlitGeneric(mat)=>VMTContent::vtf(mat.base_texture), + vmt_parser::material::Material::UnlitTwoTexture(mat)=>VMTContent::vtf(mat.base_texture), + vmt_parser::material::Material::Water(mat)=>VMTContent::vtf(mat.base_texture), + vmt_parser::material::Material::WorldVertexTransition(mat)=>VMTContent::vtf(Some(mat.base_texture)), + vmt_parser::material::Material::EyeRefract(mat)=>VMTContent::vtf(Some(mat.cornea_texture)), + vmt_parser::material::Material::SubRect(mat)=>VMTContent::VMT(mat.material),//recursive + vmt_parser::material::Material::Sprite(mat)=>VMTContent::vtf(Some(mat.base_texture)), + vmt_parser::material::Material::SpriteCard(mat)=>VMTContent::vtf(mat.base_texture), + vmt_parser::material::Material::Cable(mat)=>VMTContent::vtf(Some(mat.base_texture)), + vmt_parser::material::Material::Refract(mat)=>VMTContent::vtf(mat.base_texture), + vmt_parser::material::Material::Modulate(mat)=>VMTContent::vtf(Some(mat.base_texture)), + vmt_parser::material::Material::DecalModulate(mat)=>VMTContent::vtf(Some(mat.base_texture)), + vmt_parser::material::Material::Sky(mat)=>VMTContent::vtf(Some(mat.base_texture)), + vmt_parser::material::Material::Replacements(_mat)=>VMTContent::Unsupported, + vmt_parser::material::Material::Patch(mat)=>VMTContent::Patch(mat), + _=>return Err(anyhow::Error::msg("vmt failed to parse")), + }) +} + +fn get_vmt<F:Fn(String)->AResult<Option<Vec<u8>>>>(find_stuff:&F,search_name:String)->AResult<vmt_parser::material::Material>{ + if let Some(stuff)=find_stuff(search_name)?{ + //decode vmt and then write + let stuff=String::from_utf8(stuff)?; + let material=vmt_parser::from_str(stuff.as_str())?; + println!("vmt material={:?}",material); + return Ok(material); + } + Err(anyhow::Error::msg("vmt not found")) +} + +fn recursive_vmt_loader<F:Fn(String)->AResult<Option<Vec<u8>>>>(find_stuff:&F,material:vmt_parser::material::Material)->AResult<Option<Vec<u8>>>{ + match get_some_texture(material)?{ + VMTContent::VMT(s)=>recursive_vmt_loader(find_stuff,get_vmt(find_stuff,s)?), + VMTContent::VTF(s)=>{ + let mut texture_file_name=PathBuf::from("materials"); + texture_file_name.push(s); + texture_file_name.set_extension("vtf"); + find_stuff(texture_file_name.into_os_string().into_string().unwrap()) + }, + VMTContent::Patch(mat)=>recursive_vmt_loader(find_stuff, + mat.resolve(|search_name|{ + match find_stuff(search_name.to_string())?{ + Some(bytes)=>Ok(String::from_utf8(bytes)?), + None=>Err(anyhow::Error::msg("could not find vmt")), + } + })? + ), + VMTContent::Unsupported=>{println!("Unsupported vmt");Ok(None)},//print and move on + VMTContent::Unresolved=>{println!("Unresolved vmt");Ok(None)}, + } +} + +fn extract_textures(paths:Vec<PathBuf>,vpk_paths:Vec<PathBuf>)->AResult<()>{ + std::fs::create_dir_all("textures")?; + let vpk_list:Vec<vpk::VPK>=vpk_paths.into_iter().map(|vpk_path|vpk::VPK::read(&vpk_path).expect("vpk file does not exist")).collect(); + for path in paths{ + let mut deduplicate=std::collections::HashSet::new(); + let bsp=vbsp::Bsp::read(std::fs::read(path)?.as_ref())?; + for texture in bsp.textures(){ + deduplicate.insert(PathBuf::from(texture.name())); + } + //dedupe prop models + let mut model_dedupe=std::collections::HashSet::new(); + for prop in bsp.static_props(){ + model_dedupe.insert(prop.model()); + } + + //grab texture names from props + for model_name in model_dedupe{ + //.mdl, .vvd, .dx90.vtx + let mut path=PathBuf::from(model_name); + let file_name=PathBuf::from(path.file_stem().unwrap()); + path.pop(); + path.push(file_name); + let mut vvd_path=path.clone(); + let mut vtx_path=path.clone(); + vvd_path.set_extension("vvd"); + vtx_path.set_extension("dx90.vtx"); + match (bsp.pack.get(model_name),bsp.pack.get(vvd_path.as_os_str().to_str().unwrap()),bsp.pack.get(vtx_path.as_os_str().to_str().unwrap())){ + (Ok(Some(mdl_file)),Ok(Some(vvd_file)),Ok(Some(vtx_file)))=>{ + match (vmdl::mdl::Mdl::read(mdl_file.as_ref()),vmdl::vvd::Vvd::read(vvd_file.as_ref()),vmdl::vtx::Vtx::read(vtx_file.as_ref())){ + (Ok(mdl),Ok(vvd),Ok(vtx))=>{ + let model=vmdl::Model::from_parts(mdl,vtx,vvd); + for texture in model.textures(){ + for search_path in &texture.search_paths{ + let mut path=PathBuf::from(search_path.as_str()); + path.push(texture.name.as_str()); + deduplicate.insert(path); + } + } + }, + _=>println!("model_name={} error",model_name), + } + }, + _=>println!("no model name={}",model_name), + } + } + + let pack=&bsp.pack; + let vpk_list=&vpk_list; + std::thread::scope(move|s|{ + let mut thread_handles=Vec::new(); + for texture_name in deduplicate{ + let mut found_texture=false; + //LMAO imagine having to write type names + let write_image=|mut stuff,write_file_name|{ + let image=vtf::from_bytes(&mut stuff)?.highres_image.decode(0)?.to_rgba8(); + + let format=if image.width()%4!=0||image.height()%4!=0{ + image_dds::ImageFormat::Rgba8UnormSrgb + }else{ + image_dds::ImageFormat::BC7RgbaUnormSrgb + }; + //this fails if the image dimensions are not a multiple of 4 + let dds = image_dds::dds_from_image( + &image, + format, + image_dds::Quality::Slow, + image_dds::Mipmaps::GeneratedAutomatic, + )?; + + //write dds + let mut dest=PathBuf::from("textures"); + dest.push(write_file_name); + dest.set_extension("dds"); + std::fs::create_dir_all(dest.parent().unwrap())?; + let mut writer = std::io::BufWriter::new(std::fs::File::create(dest)?); + dds.write(&mut writer)?; + Ok::<(),anyhow::Error>(()) + }; + let find_stuff=|search_file_name:String|{ + println!("search_file_name={}",search_file_name); + match pack.get(search_file_name.as_str())?{ + Some(file)=>return Ok(Some(file)), + _=>(), + } + //search pak list + for vpk_index in vpk_list{ + if let Some(vpk_entry)=vpk_index.tree.get(search_file_name.as_str()){ + return Ok(Some(match vpk_entry.get()?{ + std::borrow::Cow::Borrowed(bytes)=>bytes.to_vec(), + std::borrow::Cow::Owned(bytes)=>bytes, + })); + } + } + Ok::<Option<Vec<u8>>,anyhow::Error>(None) + }; + let loader=|texture_name:String|{ + let mut texture_file_name=PathBuf::from("materials"); + //lower case + let texture_file_name_lowercase=texture_name.to_lowercase(); + texture_file_name.push(texture_file_name_lowercase.clone()); + //remove stem and search for both vtf and vmt files + let stem=PathBuf::from(texture_file_name.file_stem().unwrap()); + texture_file_name.pop(); + texture_file_name.push(stem); + //somehow search for both files + let mut texture_file_name_vmt=texture_file_name.clone(); + texture_file_name.set_extension("vtf"); + texture_file_name_vmt.set_extension("vmt"); + if let Some(stuff)=find_stuff(texture_file_name.to_string_lossy().to_string())?{ + return Ok(Some(stuff)) + } + recursive_vmt_loader(&find_stuff,get_vmt(&find_stuff,texture_file_name_vmt.to_string_lossy().to_string())?) + }; + if let Some(stuff)=loader(texture_name.to_string_lossy().to_string())?{ + found_texture=true; + let texture_name=texture_name.clone(); + thread_handles.push(s.spawn(move||write_image(stuff,texture_name))); + } + if !found_texture{ + println!("no data"); + } + } + for thread in thread_handles{ + match thread.join(){ + Ok(Err(e))=>println!("write error: {:?}",e), + Err(e)=>println!("thread error: {:?}",e), + Ok(_)=>(), + } + } + Ok::<(),anyhow::Error>(()) + })? + } + Ok(()) +} + +fn vpk_contents(vpk_path:PathBuf)->AResult<()>{ + let vpk_index=vpk::VPK::read(&vpk_path)?; + for (label,entry) in vpk_index.tree.into_iter(){ + println!("vpk label={} entry={:?}",label,entry); + } + Ok(()) +} + +fn bsp_contents(path:PathBuf)->AResult<()>{ + let bsp=vbsp::Bsp::read(std::fs::read(path)?.as_ref())?; + for file_name in bsp.pack.into_zip().into_inner().unwrap().file_names(){ + println!("file_name={:?}",file_name); + } + Ok(()) +} + +#[derive(Debug)] +#[allow(dead_code)] +enum ConvertError{ + IO(std::io::Error), + SNFMap(strafesnet_snf::map::Error), + BspLoader(strafesnet_bsp_loader::ReadError), +} +impl std::fmt::Display for ConvertError{ + fn fmt(&self,f:&mut std::fmt::Formatter<'_>)->std::fmt::Result{ + write!(f,"{self:?}") + } +} +impl std::error::Error for ConvertError{} + +type MapThread=std::thread::JoinHandle<Result<(),ConvertError>>; + +fn source_to_snf(pathlist:Vec<std::path::PathBuf>,output_folder:PathBuf)->AResult<()>{ + let n_paths=pathlist.len(); + let start = std::time::Instant::now(); + let mut threads:std::collections::VecDeque<MapThread>=std::collections::VecDeque::new(); + let mut i=0; + let mut join_thread=|thread:MapThread|{ + i+=1; + if let Err(e)=thread.join(){ + println!("thread error: {:?}",e); + }else{ + println!("{}/{}",i,n_paths); + } + }; + for path in pathlist{ + if 32<=threads.len(){ + join_thread(threads.pop_front().unwrap()); + } + let output_folder=output_folder.clone(); + threads.push_back(std::thread::spawn(move ||{ + let bsp=strafesnet_bsp_loader::read( + std::fs::File::open(path.as_path()) + .map_err(ConvertError::IO)? + ).map_err(ConvertError::BspLoader)?; + let mut loader=strafesnet_deferred_loader::source_legacy(); + + let (texture_loader,mesh_loader)=loader.get_inner_mut(); + + let map_step1=strafesnet_bsp_loader::convert( + &bsp, + |name|texture_loader.acquire_render_config_id(name), + |name|mesh_loader.acquire_mesh_id(name), + ); + + let prop_meshes=mesh_loader.load_meshes(&bsp.as_ref()); + + let map_step2=map_step1.add_prop_meshes( + //the type conflagulator 9000 + prop_meshes.into_iter().map(|(mesh_id,loader_model)| + (mesh_id,strafesnet_bsp_loader::data::ModelData{ + mdl:strafesnet_bsp_loader::data::MdlData::new(loader_model.mdl.get()), + vtx:strafesnet_bsp_loader::data::VtxData::new(loader_model.vtx.get()), + vvd:strafesnet_bsp_loader::data::VvdData::new(loader_model.vvd.get()), + }) + ), + |name|texture_loader.acquire_render_config_id(name), + ); + + let (textures,render_configs)=loader.into_render_configs().map_err(ConvertError::IO)?.consume(); + + let map=map_step2.add_render_configs_and_textures( + render_configs.into_iter(), + textures.into_iter().map(|(texture_id,texture)| + (texture_id,match texture{ + strafesnet_deferred_loader::texture::Texture::ImageDDS(data)=>data, + }) + ), + ); + + let mut dest=output_folder.clone(); + dest.push(path.file_stem().unwrap()); + dest.set_extension("snfm"); + let file=std::fs::File::create(dest).map_err(ConvertError::IO)?; + + strafesnet_snf::map::write_map(file,map).map_err(ConvertError::SNFMap)?; + Ok(()) + })); + } + + for thread in threads{ + join_thread(thread); + } + println!("{:?}", start.elapsed()); + Ok(()) +}