diff --git a/src/roblox.rs b/src/roblox.rs
index e43a11e..da2c0d1 100644
--- a/src/roblox.rs
+++ b/src/roblox.rs
@@ -241,33 +241,16 @@ async fn download_retry(stats:&mut Stats,context:&rbx_asset::cookie::CookieConte
}
}
}
-#[allow(unused)]
-#[derive(Debug)]
+#[derive(Debug,thiserror::Error)]
enum ConvertTextureError{
- Io(std::io::Error),
- Image(image::ImageError),
- DDS(image_dds::CreateDdsError),
- DDSWrite(image_dds::ddsfile::Error),
-}
-impl From<std::io::Error> for ConvertTextureError{
- fn from(value:std::io::Error)->Self{
- Self::Io(value)
- }
-}
-impl From<image::ImageError> for ConvertTextureError{
- fn from(value:image::ImageError)->Self{
- Self::Image(value)
- }
-}
-impl From<image_dds::CreateDdsError> for ConvertTextureError{
- fn from(value:image_dds::CreateDdsError)->Self{
- Self::DDS(value)
- }
-}
-impl From<image_dds::ddsfile::Error> for ConvertTextureError{
- fn from(value:image_dds::ddsfile::Error)->Self{
- Self::DDSWrite(value)
- }
+ #[error("Io error {0:?}")]
+ Io(#[from]std::io::Error),
+ #[error("Image error {0:?}")]
+ Image(#[from]image::ImageError),
+ #[error("DDS create error {0:?}")]
+ DDS(#[from]image_dds::CreateDdsError),
+ #[error("DDS write error {0:?}")]
+ DDSWrite(#[from]image_dds::ddsfile::Error),
}
async fn convert_texture(asset_id:RobloxAssetId,download_result:DownloadResult)->Result<(),ConvertTextureError>{
let data=match download_result{
diff --git a/src/source.rs b/src/source.rs
index 95cad8f..cebda5f 100644
--- a/src/source.rs
+++ b/src/source.rs
@@ -1,7 +1,10 @@
use std::path::{Path,PathBuf};
+use std::borrow::Cow;
use clap::{Args,Subcommand};
use anyhow::Result as AResult;
-use strafesnet_deferred_loader::deferred_loader::LoadFailureMode;
+use futures::StreamExt;
+use strafesnet_deferred_loader::loader::Loader;
+use strafesnet_deferred_loader::deferred_loader::{LoadFailureMode,MeshDeferredLoader,RenderConfigDeferredLoader};
#[derive(Subcommand)]
pub enum Commands{
@@ -19,11 +22,11 @@ pub struct SourceToSNFSubcommand {
input_files:Vec<PathBuf>,
}
#[derive(Args)]
-pub struct ExtractTexturesSubcommand {
+pub struct ExtractTexturesSubcommand{
+ #[arg(required=true)]
+ bsp_files:Vec<PathBuf>,
#[arg(long)]
- bsp_file:PathBuf,
- #[arg(long)]
- vpk_dir_files:Vec<PathBuf>
+ vpk_dir_file:Vec<PathBuf>,
}
#[derive(Args)]
pub struct VPKContentsSubcommand {
@@ -39,8 +42,8 @@ pub struct BSPContentsSubcommand {
impl Commands{
pub async fn run(self)->AResult<()>{
match self{
- Commands::ExtractTextures(subcommand)=>extract_textures(vec![subcommand.bsp_file],subcommand.vpk_dir_files),
Commands::SourceToSNF(subcommand)=>source_to_snf(subcommand.input_files,subcommand.output_folder).await,
+ Commands::ExtractTextures(subcommand)=>extract_textures(subcommand.bsp_files,subcommand.vpk_dir_file).await,
Commands::VPKContents(subcommand)=>vpk_contents(subcommand.input_file),
Commands::BSPContents(subcommand)=>bsp_contents(subcommand.input_file),
}
@@ -121,139 +124,220 @@ fn recursive_vmt_loader<F:Fn(String)->AResult<Option<Vec<u8>>>>(find_stuff:&F,ma
VMTContent::Unresolved=>{println!("Unresolved vmt");Ok(None)},
}
}
-
-fn extract_textures(paths:Vec<PathBuf>,vpk_paths:Vec<PathBuf>)->AResult<()>{
- std::fs::create_dir_all("textures")?;
- let vpk_list:Vec<vpk::VPK>=vpk_paths.into_iter().map(|vpk_path|vpk::VPK::read(&vpk_path).expect("vpk file does not exist")).collect();
- for path in paths{
- let mut deduplicate=std::collections::HashSet::new();
- let bsp=vbsp::Bsp::read(std::fs::read(path)?.as_ref())?;
- for texture in bsp.textures(){
- deduplicate.insert(PathBuf::from(texture.name()));
- }
- //dedupe prop models
- let mut model_dedupe=std::collections::HashSet::new();
- for prop in bsp.static_props(){
- model_dedupe.insert(prop.model());
- }
-
- //grab texture names from props
- for model_name in model_dedupe{
- //.mdl, .vvd, .dx90.vtx
- let mut path=PathBuf::from(model_name);
- let file_name=PathBuf::from(path.file_stem().unwrap());
- path.pop();
- path.push(file_name);
- let mut vvd_path=path.clone();
- let mut vtx_path=path.clone();
- vvd_path.set_extension("vvd");
- vtx_path.set_extension("dx90.vtx");
- match (bsp.pack.get(model_name),bsp.pack.get(vvd_path.as_os_str().to_str().unwrap()),bsp.pack.get(vtx_path.as_os_str().to_str().unwrap())){
- (Ok(Some(mdl_file)),Ok(Some(vvd_file)),Ok(Some(vtx_file)))=>{
- match (vmdl::mdl::Mdl::read(mdl_file.as_ref()),vmdl::vvd::Vvd::read(vvd_file.as_ref()),vmdl::vtx::Vtx::read(vtx_file.as_ref())){
- (Ok(mdl),Ok(vvd),Ok(vtx))=>{
- let model=vmdl::Model::from_parts(mdl,vtx,vvd);
- for texture in model.textures(){
- for search_path in &texture.search_paths{
- let mut path=PathBuf::from(search_path.as_str());
- path.push(texture.name.as_str());
- deduplicate.insert(path);
- }
- }
- },
- _=>println!("model_name={} error",model_name),
- }
- },
- _=>println!("no model name={}",model_name),
- }
- }
-
- let pack=&bsp.pack;
- let vpk_list=&vpk_list;
- std::thread::scope(move|s|{
- let mut thread_handles=Vec::new();
- for texture_name in deduplicate{
- let mut found_texture=false;
- //LMAO imagine having to write type names
- let write_image=|mut stuff,write_file_name|{
- let image=vtf::from_bytes(&mut stuff)?.highres_image.decode(0)?.to_rgba8();
-
- let format=if image.width()%4!=0||image.height()%4!=0{
- image_dds::ImageFormat::Rgba8UnormSrgb
- }else{
- image_dds::ImageFormat::BC7RgbaUnormSrgb
- };
- //this fails if the image dimensions are not a multiple of 4
- let dds = image_dds::dds_from_image(
- &image,
- format,
- image_dds::Quality::Slow,
- image_dds::Mipmaps::GeneratedAutomatic,
- )?;
-
- //write dds
- let mut dest=PathBuf::from("textures");
- dest.push(write_file_name);
- dest.set_extension("dds");
- std::fs::create_dir_all(dest.parent().unwrap())?;
- let mut writer = std::io::BufWriter::new(std::fs::File::create(dest)?);
- dds.write(&mut writer)?;
- Ok::<(),anyhow::Error>(())
- };
- let find_stuff=|search_file_name:String|{
- println!("search_file_name={}",search_file_name);
- match pack.get(search_file_name.as_str())?{
- Some(file)=>return Ok(Some(file)),
- _=>(),
- }
- //search pak list
- for vpk_index in vpk_list{
- if let Some(vpk_entry)=vpk_index.tree.get(search_file_name.as_str()){
- return Ok(Some(match vpk_entry.get()?{
- std::borrow::Cow::Borrowed(bytes)=>bytes.to_vec(),
- std::borrow::Cow::Owned(bytes)=>bytes,
- }));
- }
- }
- Ok::<Option<Vec<u8>>,anyhow::Error>(None)
- };
- let loader=|texture_name:String|{
- let mut texture_file_name=PathBuf::from("materials");
- //lower case
- let texture_file_name_lowercase=texture_name.to_lowercase();
- texture_file_name.push(texture_file_name_lowercase.clone());
- //remove stem and search for both vtf and vmt files
- let stem=PathBuf::from(texture_file_name.file_stem().unwrap());
- texture_file_name.pop();
- texture_file_name.push(stem);
- //somehow search for both files
- let mut texture_file_name_vmt=texture_file_name.clone();
- texture_file_name.set_extension("vtf");
- texture_file_name_vmt.set_extension("vmt");
- if let Some(stuff)=find_stuff(texture_file_name.to_string_lossy().to_string())?{
- return Ok(Some(stuff))
- }
- recursive_vmt_loader(&find_stuff,get_vmt(&find_stuff,texture_file_name_vmt.to_string_lossy().to_string())?)
- };
- if let Some(stuff)=loader(texture_name.to_string_lossy().to_string())?{
- found_texture=true;
- let texture_name=texture_name.clone();
- thread_handles.push(s.spawn(move||write_image(stuff,texture_name)));
- }
- if !found_texture{
- println!("no data");
- }
- }
- for thread in thread_handles{
- match thread.join(){
- Ok(Err(e))=>println!("write error: {:?}",e),
- Err(e)=>println!("thread error: {:?}",e),
- Ok(_)=>(),
- }
- }
- Ok::<(),anyhow::Error>(())
- })?
+#[derive(Debug,thiserror::Error)]
+enum ExtractTextureError{
+ #[error("Io error {0:?}")]
+ Io(#[from]std::io::Error),
+ #[error("Bsp error {0:?}")]
+ Bsp(#[from]vbsp::BspError),
+ #[error("MeshLoad error {0:?}")]
+ MeshLoad(#[from]strafesnet_bsp_loader::loader::MeshError),
+}
+fn find_stuff(){
+ println!("search_file_name={}",search_file_name);
+ match pack.get(search_file_name.as_str())?{
+ Some(file)=>return Ok(Some(file)),
+ _=>(),
}
+ //search pak list
+ for vpk_index in vpk_list{
+ if let Some(vpk_entry)=vpk_index.tree.get(search_file_name.as_str()){
+ return Ok(Some(match vpk_entry.get()?{
+ std::borrow::Cow::Borrowed(bytes)=>bytes.to_vec(),
+ std::borrow::Cow::Owned(bytes)=>bytes,
+ }));
+ }
+ }
+ Ok::<Option<Vec<u8>>,anyhow::Error>(None)
+}
+fn load_texture(){
+ let mut texture_file_name=PathBuf::from("materials");
+ //lower case
+ let texture_file_name_lowercase=texture_name.to_lowercase();
+ texture_file_name.push(texture_file_name_lowercase.clone());
+ //remove stem and search for both vtf and vmt files
+ let stem=PathBuf::from(texture_file_name.file_stem().unwrap());
+ texture_file_name.pop();
+ texture_file_name.push(stem);
+ //somehow search for both files
+ let mut texture_file_name_vmt=texture_file_name.clone();
+ texture_file_name.set_extension("vtf");
+ texture_file_name_vmt.set_extension("vmt");
+ if let Some(stuff)=find_stuff(texture_file_name.to_string_lossy().to_string())?{
+ return Ok(Some(stuff))
+ }
+ recursive_vmt_loader(&find_stuff,get_vmt(&find_stuff,texture_file_name_vmt.to_string_lossy().to_string())?)
+}
+async fn gimme_them_textures(path:PathBuf)->Result<(),ExtractTextureError>{
+ let mut deduplicate=std::collections::HashSet::new();
+ let bsp=vbsp::Bsp::read(tokio::fs::read(path).await?.as_ref())?;
+
+ let mut texture_deferred_loader=RenderConfigDeferredLoader::new();
+ for texture in bsp.textures(){
+ texture_deferred_loader.acquire_render_config_id(Some(Cow::Borrowed(texture.name())));
+ }
+
+ let mut mesh_deferred_loader=MeshDeferredLoader::new();
+ for prop in bsp.static_props(){
+ mesh_deferred_loader.acquire_mesh_id(prop.model());
+ }
+
+ // TODO: include more packs in mesh loader
+ let mut mesh_loader=strafesnet_bsp_loader::loader::MeshLoader::new(&strafesnet_bsp_loader::Bsp::new(bsp),&mut texture_deferred_loader);
+
+ // load models and collect requested textures
+ for &model_path in mesh_deferred_loader.indices(){
+ // TODO: fix loader to return pre-converted mesh
+ let model:vmdl::Model=mesh_loader.load(model_path)?;
+ for texture in model.textures(){
+ for search_path in &texture.search_paths{
+ let mut path=PathBuf::from(search_path.as_str());
+ path.push(texture.name.as_str());
+ let path=path.to_str().unwrap().to_owned();
+ texture_deferred_loader.acquire_render_config_id(Some(Cow::Owned(path)));
+ }
+ }
+ }
+
+ for &texture_path in texture_deferred_loader.indices(){
+ if let Some(texture)=load_texture(texture_path)?{
+ send_texture.send(texture).unwrap();
+ }
+ }
+
+ let pack=&bsp.pack;
+ let vpk_list=&vpk_list;
+ std::thread::scope(move|s|{
+ let mut thread_handles=Vec::new();
+ for texture_name in deduplicate{
+ let mut found_texture=false;
+ //LMAO imagine having to write type names
+ let write_image=|mut stuff,write_file_name|{
+ let image=vtf::from_bytes(&mut stuff)?.highres_image.decode(0)?.to_rgba8();
+
+ let format=if image.width()%4!=0||image.height()%4!=0{
+ image_dds::ImageFormat::Rgba8UnormSrgb
+ }else{
+ image_dds::ImageFormat::BC7RgbaUnormSrgb
+ };
+ //this fails if the image dimensions are not a multiple of 4
+ let dds = image_dds::dds_from_image(
+ &image,
+ format,
+ image_dds::Quality::Slow,
+ image_dds::Mipmaps::GeneratedAutomatic,
+ )?;
+
+ //write dds
+ let mut dest=PathBuf::from("textures");
+ dest.push(write_file_name);
+ dest.set_extension("dds");
+ std::fs::create_dir_all(dest.parent().unwrap())?;
+ let mut writer = std::io::BufWriter::new(std::fs::File::create(dest)?);
+ dds.write(&mut writer)?;
+ Ok::<(),anyhow::Error>(())
+ };
+ let find_stuff=|search_file_name:String|{
+ println!("search_file_name={}",search_file_name);
+ match pack.get(search_file_name.as_str())?{
+ Some(file)=>return Ok(Some(file)),
+ _=>(),
+ }
+ //search pak list
+ for vpk_index in vpk_list{
+ if let Some(vpk_entry)=vpk_index.tree.get(search_file_name.as_str()){
+ return Ok(Some(match vpk_entry.get()?{
+ std::borrow::Cow::Borrowed(bytes)=>bytes.to_vec(),
+ std::borrow::Cow::Owned(bytes)=>bytes,
+ }));
+ }
+ }
+ Ok::<Option<Vec<u8>>,anyhow::Error>(None)
+ };
+ let loader=|texture_name:String|{
+ let mut texture_file_name=PathBuf::from("materials");
+ //lower case
+ let texture_file_name_lowercase=texture_name.to_lowercase();
+ texture_file_name.push(texture_file_name_lowercase.clone());
+ //remove stem and search for both vtf and vmt files
+ let stem=PathBuf::from(texture_file_name.file_stem().unwrap());
+ texture_file_name.pop();
+ texture_file_name.push(stem);
+ //somehow search for both files
+ let mut texture_file_name_vmt=texture_file_name.clone();
+ texture_file_name.set_extension("vtf");
+ texture_file_name_vmt.set_extension("vmt");
+ if let Some(stuff)=find_stuff(texture_file_name.to_string_lossy().to_string())?{
+ return Ok(Some(stuff))
+ }
+ recursive_vmt_loader(&find_stuff,get_vmt(&find_stuff,texture_file_name_vmt.to_string_lossy().to_string())?)
+ };
+ if let Some(stuff)=loader(texture_name.to_string_lossy().to_string())?{
+ found_texture=true;
+ let texture_name=texture_name.clone();
+ thread_handles.push(s.spawn(move||write_image(stuff,texture_name)));
+ }
+ if !found_texture{
+ println!("no data");
+ }
+ }
+ for thread in thread_handles{
+ match thread.join(){
+ Ok(Err(e))=>println!("write error: {:?}",e),
+ Err(e)=>println!("thread error: {:?}",e),
+ Ok(_)=>(),
+ }
+ }
+ Ok::<(),anyhow::Error>(())
+ })?
+ Ok(())
+}
+
+async fn extract_textures(paths:Vec<PathBuf>,vpk_paths:Vec<PathBuf>)->AResult<()>{
+ tokio::try_join!(
+ tokio::fs::create_dir_all("extracted_textures"),
+ tokio::fs::create_dir_all("textures"),
+ tokio::fs::create_dir_all("meshes"),
+ )?;
+ let thread_limit=std::thread::available_parallelism()?.get();
+
+ // load vpk list
+ let vpk_list=futures::stream::iter(vpk_paths).map(|vpk_path|async{
+ // idk why it doesn't want to pass out the errors but this is fatal anyways
+ tokio::task::spawn_blocking(move||vpk::VPK::read(&vpk_path)).await.unwrap().unwrap()
+ })
+ .buffer_unordered(thread_limit)
+ .collect::<Vec<vpk::VPK>>().await;
+
+ // leak vpk_list for static lifetime?
+
+ let (send_texture,mut recv_texture)=tokio::sync::mpsc::channel(thread_limit);
+ let mut it=paths.into_iter();
+ let extract_thread=tokio::spawn(async move{
+ static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
+ SEM.add_permits(thread_limit);
+ while let (Ok(permit),Some(path))=(SEM.acquire().await,it.next()){
+ let send=send_texture.clone();
+ tokio::spawn(async move{
+ let result=gimme_them_textures(path,send).await;
+ drop(permit);
+ result.unwrap();
+ });
+ }
+ });
+
+ // convert images
+ static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
+ SEM.add_permits(thread_limit);
+ while let (Ok(permit),Some(data))=(SEM.acquire().await,recv_texture.recv().await){
+ tokio::spawn(async move{
+ let result=convert_texture(data).await;
+ drop(permit);
+ result.unwrap();
+ });
+ }
+ extract_thread.await??;
+ SEM.acquire_many(thread_limit as u32).await?;
Ok(())
}