v0.7.0 loader rewrite

Cargo.lock

@@ -698,7 +698,9 @@ checksum = "835a3dc7d1ec9e75e2b5fb4ba75396837112d2060b03f7d43bc1897c7f7211da"
 
 [[package]]
 name = "fixed_wide"
-version = "0.1.1"
+version = "0.1.2"
+source = "sparse+https://git.itzana.me/api/packages/strafesnet/cargo/"
+checksum = "a7e01a5b738e313c912fc41c425cf36e10c51647d3fd21d96db3d616344549fa"
 dependencies = [
  "arrayvec",
  "bnum",
@@ -1259,7 +1261,7 @@ dependencies = [
  "image",
  "intel_tex_2",
  "strum",
- "thiserror",
+ "thiserror 1.0.69",
 ]
 
 [[package]]
@@ -1439,6 +1441,8 @@ checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa"
 [[package]]
 name = "linear_ops"
 version = "0.1.0"
+source = "sparse+https://git.itzana.me/api/packages/strafesnet/cargo/"
+checksum = "b2e6977ac24f47086d8a7a2d4ae1c720e86dfdc8407cf5e34c18bfa01053c456"
 dependencies = [
  "fixed_wide",
  "paste",
@@ -1564,7 +1568,7 @@ dependencies = [
 
 [[package]]
 name = "map-tool"
-version = "1.6.0"
+version = "1.7.0"
 dependencies = [
  "anyhow",
  "clap",
@@ -1583,6 +1587,7 @@ dependencies = [
  "strafesnet_deferred_loader",
  "strafesnet_rbx_loader",
  "strafesnet_snf",
+ "thiserror 2.0.11",
  "tokio",
  "vbsp",
  "vmdl",
@@ -1615,7 +1620,7 @@ checksum = "317f146e2eb7021892722af37cf1b971f0a70c8406f487e24952667616192c64"
 dependencies = [
  "cfg-if",
  "miette-derive",
- "thiserror",
+ "thiserror 1.0.69",
  "unicode-width",
 ]
 
@@ -2170,6 +2175,8 @@ dependencies = [
 [[package]]
 name = "ratio_ops"
 version = "0.1.0"
+source = "sparse+https://git.itzana.me/api/packages/strafesnet/cargo/"
+checksum = "01239195d6afe0509e7e3511b716c0540251dfe7ece0a9a5a27116afb766c42c"
 
 [[package]]
 name = "rav1e"
@@ -2201,7 +2208,7 @@ dependencies = [
  "rand_chacha",
  "simd_helpers",
  "system-deps",
- "thiserror",
+ "thiserror 1.0.69",
  "v_frame",
  "wasm-bindgen",
 ]
@@ -2267,7 +2274,7 @@ dependencies = [
  "rbx_dom_weak",
  "rbx_reflection",
  "rbx_reflection_database",
- "thiserror",
+ "thiserror 1.0.69",
 ]
 
 [[package]]
@@ -2298,7 +2305,7 @@ checksum = "c1b43fe592a4ce6fe54eb215fb82735efbb516d2cc045a94e3dc0234ff293620"
 dependencies = [
  "rbx_types",
  "serde",
- "thiserror",
+ "thiserror 1.0.69",
 ]
 
 [[package]]
@@ -2325,7 +2332,7 @@ dependencies = [
  "lazy_static",
  "rand",
  "serde",
- "thiserror",
+ "thiserror 1.0.69",
 ]
 
 [[package]]
@@ -2345,6 +2352,8 @@ dependencies = [
 [[package]]
 name = "rbxassetid"
 version = "0.1.0"
+source = "sparse+https://git.itzana.me/api/packages/strafesnet/cargo/"
+checksum = "e6821fe9eaff54cd142932cb04c612b7599d9b8586973145b7ec1230ae84d184"
 dependencies = [
  "url",
 ]
@@ -2478,6 +2487,8 @@ dependencies = [
 [[package]]
 name = "roblox_emulator"
 version = "0.4.7"
+source = "sparse+https://git.itzana.me/api/packages/strafesnet/cargo/"
+checksum = "7fc98335ce4b8548b725d727c5b32bd0b38274606c48fce6b6e7e5807d94db6b"
 dependencies = [
  "glam",
  "mlua",
@@ -2739,17 +2750,22 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
 [[package]]
 name = "strafesnet_bsp_loader"
 version = "0.3.0"
+source = "sparse+https://git.itzana.me/api/packages/strafesnet/cargo/"
+checksum = "3cc98773f2b98eb708b098946870b769975b63a396b84698b67e3d968029005d"
 dependencies = [
  "glam",
  "strafesnet_common",
  "strafesnet_deferred_loader",
  "vbsp",
  "vmdl",
+ "vpk",
 ]
 
 [[package]]
 name = "strafesnet_common"
 version = "0.6.0"
+source = "sparse+https://git.itzana.me/api/packages/strafesnet/cargo/"
+checksum = "0c1d7a83e1f6b579c6a9b4dc70c92373ab53b938601cd75928dd6795b5ffef21"
 dependencies = [
  "arrayvec",
  "bitflags 2.8.0",
@@ -2763,6 +2779,8 @@ dependencies = [
 [[package]]
 name = "strafesnet_deferred_loader"
 version = "0.5.0"
+source = "sparse+https://git.itzana.me/api/packages/strafesnet/cargo/"
+checksum = "63d5d48e587d5f8bf5385bee3505ed790727fef68de855cf58247a08c5952bef"
 dependencies = [
  "strafesnet_common",
 ]
@@ -2770,6 +2788,8 @@ dependencies = [
 [[package]]
 name = "strafesnet_rbx_loader"
 version = "0.6.0"
+source = "sparse+https://git.itzana.me/api/packages/strafesnet/cargo/"
+checksum = "e4659a49128c8d12b9fbdb289969cae04bfc5c1750d4273897700c5c17730d8a"
 dependencies = [
  "bytemuck",
  "glam",
@@ -2787,7 +2807,9 @@ dependencies = [
 
 [[package]]
 name = "strafesnet_snf"
-version = "0.2.0"
+version = "0.3.0"
+source = "sparse+https://git.itzana.me/api/packages/strafesnet/cargo/"
+checksum = "fd24a22c484ca04213fa44b1d34bfbec385f0d176a2b5829cfa59ba7987b80d5"
 dependencies = [
  "binrw 0.14.1",
  "id",
@@ -2974,7 +2996,16 @@ version = "1.0.69"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52"
 dependencies = [
- "thiserror-impl",
+ "thiserror-impl 1.0.69",
+]
+
+[[package]]
+name = "thiserror"
+version = "2.0.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d452f284b73e6d76dd36758a0c8684b1d5be31f92b89d07fd5822175732206fc"
+dependencies = [
+ "thiserror-impl 2.0.11",
 ]
 
 [[package]]
@@ -2988,6 +3019,17 @@ dependencies = [
  "syn 2.0.96",
 ]
 
+[[package]]
+name = "thiserror-impl"
+version = "2.0.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.96",
+]
+
 [[package]]
 name = "tiff"
 version = "0.9.1"
@@ -3254,7 +3296,7 @@ dependencies = [
  "num_enum",
  "serde",
  "static_assertions",
- "thiserror",
+ "thiserror 1.0.69",
  "vdf-reader",
  "zip-lzma",
 ]
@@ -3275,7 +3317,7 @@ dependencies = [
  "miette",
  "parse-display 0.9.1",
  "serde",
- "thiserror",
+ "thiserror 1.0.69",
 ]
 
 [[package]]
@@ -3304,7 +3346,7 @@ dependencies = [
  "itertools 0.13.0",
  "num_enum",
  "static_assertions",
- "thiserror",
+ "thiserror 1.0.69",
  "tracing",
 ]
 
@@ -3317,7 +3359,7 @@ dependencies = [
  "miette",
  "serde",
  "serde_repr",
- "thiserror",
+ "thiserror 1.0.69",
  "vdf-reader",
 ]
 
@@ -3329,7 +3371,7 @@ checksum = "60ec10e731515f58d5494d472f027d9c6fc8500fcb790ff55751031bcad87b6b"
 dependencies = [
  "ahash",
  "binrw 0.13.3",
- "thiserror",
+ "thiserror 1.0.69",
 ]
 
 [[package]]

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "map-tool"
-version = "1.6.0"
+version = "1.7.0"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@@ -18,10 +18,12 @@ rbx_binary = { version = "0.7.4", registry = "strafesnet" }
 rbx_dom_weak = { version = "2.7.0", registry = "strafesnet" }
 rbx_reflection_database = { version = "0.2.10", registry = "strafesnet" }
 rbx_xml = { version = "0.13.3", registry = "strafesnet" }
-strafesnet_bsp_loader = { version = "0.2.1", registry = "strafesnet" }
+rbxassetid = { version = "0.1.0", registry = "strafesnet" }
-strafesnet_deferred_loader = { version = "0.4.0", features = ["legacy"], registry = "strafesnet" }
+strafesnet_bsp_loader = { version = "0.3.0", registry = "strafesnet" }
-strafesnet_rbx_loader = { version = "0.5.1", registry = "strafesnet" }
+strafesnet_deferred_loader = { version = "0.5.0", registry = "strafesnet" }
-strafesnet_snf = { version = "0.2.0", registry = "strafesnet" }
+strafesnet_rbx_loader = { version = "0.6.0", registry = "strafesnet" }
+strafesnet_snf = { version = "0.3.0", registry = "strafesnet" }
+thiserror = "2.0.11"
 tokio = { version = "1.43.0", features = ["macros", "rt-multi-thread", "fs"] }
 vbsp = "0.6.0"
 vmdl = "0.2.0"

src/common.rs

@@ -1,75 +0,0 @@
-use std::path::PathBuf;
-use std::io::{Read,Seek};
-use anyhow::Result as AResult;
-
-fn load_image<R:Read+Seek+std::io::BufRead>(input:&mut R)->AResult<image::DynamicImage>{
-	let mut fourcc=[0u8;4];
-	input.read_exact(&mut fourcc)?;
-	input.rewind()?;
-	match &fourcc{
-		b"\x89PNG"=>Ok(image::load(input,image::ImageFormat::Png)?),
-		b"\xFF\xD8\xFF\xE0"=>Ok(image::load(input,image::ImageFormat::Jpeg)?),//JFIF
-		b"<rob"=>Err(anyhow::Error::msg("Roblox xml garbage is not supported yet")),
-		other=>Err(anyhow::Error::msg(format!("Unknown texture format {:?}",other))),
-	}
-}
-
-fn convert(file_thing:std::fs::DirEntry) -> AResult<()>{
-	let mut input = std::io::BufReader::new(std::fs::File::open(file_thing.path())?);
-
-	let image=load_image(&mut input)?.to_rgba8();//this sets a=255, arcane is actually supposed to look like that
-
-	let format=if image.width()%4!=0||image.height()%4!=0{
-		image_dds::ImageFormat::Rgba8UnormSrgb
-	}else{
-		image_dds::ImageFormat::BC7RgbaUnormSrgb
-	};
-	//this fails if the image dimensions are not a multiple of 4
-	let dds = image_dds::dds_from_image(
-		&image,
-		format,
-		image_dds::Quality::Slow,
-		image_dds::Mipmaps::GeneratedAutomatic,
-	)?;
-
-	//write dds
-	let mut dest=PathBuf::from("textures");
-	dest.push(file_thing.file_name());
-	dest.set_extension("dds");
-	let mut writer = std::io::BufWriter::new(std::fs::File::create(dest)?);
-	dds.write(&mut writer)?;
-
-	//move file to processed
-	let mut dest=PathBuf::from("textures/processed");
-	dest.push(file_thing.file_name());
-	std::fs::rename(file_thing.path(), dest)?;
-	Ok(())
-}
-pub fn convert_textures() -> AResult<()>{
-	std::fs::create_dir_all("textures/unprocessed")?;
-	std::fs::create_dir_all("textures/processed")?;
-	let start = std::time::Instant::now();
-	let mut threads=Vec::new();
-	for entry in std::fs::read_dir("textures/unprocessed")? {
-		let file_thing=entry?;
-		threads.push(std::thread::spawn(move ||{
-			let file_name=format!("{:?}",file_thing);
-			let result=convert(file_thing);
-			if let Err(e)=result{
-				println!("error processing file:{:?} error message:{:?}",file_name,e);
-			}
-		}));
-	}
-	let mut i=0;
-	let n_threads=threads.len();
-	for thread in threads{
-		i+=1;
-		if let Err(e)=thread.join(){
-			println!("thread error: {:?}",e);
-		}else{
-			println!("{}/{}",i,n_threads);
-		}
-	}
-	println!("{:?}", start.elapsed());
-	Ok(())
-}

src/main.rs

@@ -1,4 +1,3 @@
-mod common;
 mod roblox;
 mod source;
 
@@ -19,7 +18,6 @@ enum Commands{
 	Roblox(roblox::Commands),
 	#[command(flatten)]
 	Source(source::Commands),
-	ConvertTextures,
 }
 
 #[tokio::main]
@@ -28,6 +26,5 @@ async fn main()->AResult<()>{
 	match cli.command{
 		Commands::Roblox(commands)=>commands.run().await,
 		Commands::Source(commands)=>commands.run().await,
-		Commands::ConvertTextures=>common::convert_textures(),
 	}
 }

src/roblox.rs

@@ -241,33 +241,16 @@ async fn download_retry(stats:&mut Stats,context:&rbx_asset::cookie::CookieConte
 		}
 	}
 }
-#[allow(unused)]
+#[derive(Debug,thiserror::Error)]
-#[derive(Debug)]
 enum ConvertTextureError{
-	Io(std::io::Error),
+	#[error("Io error {0:?}")]
-	Image(image::ImageError),
+	Io(#[from]std::io::Error),
-	DDS(image_dds::CreateDdsError),
+	#[error("Image error {0:?}")]
-	DDSWrite(image_dds::ddsfile::Error),
+	Image(#[from]image::ImageError),
-}
+	#[error("DDS create error {0:?}")]
-impl From<std::io::Error> for ConvertTextureError{
+	DDS(#[from]image_dds::CreateDdsError),
-	fn from(value:std::io::Error)->Self{
+	#[error("DDS write error {0:?}")]
-		Self::Io(value)
+	DDSWrite(#[from]image_dds::ddsfile::Error),
-	}
-}
-impl From<image::ImageError> for ConvertTextureError{
-	fn from(value:image::ImageError)->Self{
-		Self::Image(value)
-	}
-}
-impl From<image_dds::CreateDdsError> for ConvertTextureError{
-	fn from(value:image_dds::CreateDdsError)->Self{
-		Self::DDS(value)
-	}
-}
-impl From<image_dds::ddsfile::Error> for ConvertTextureError{
-	fn from(value:image_dds::ddsfile::Error)->Self{
-		Self::DDSWrite(value)
-	}
 }
 async fn convert_texture(asset_id:RobloxAssetId,download_result:DownloadResult)->Result<(),ConvertTextureError>{
 	let data=match download_result{

src/source.rs

@@ -1,7 +1,11 @@
 use std::path::{Path,PathBuf};
+use std::borrow::Cow;
 use clap::{Args,Subcommand};
 use anyhow::Result as AResult;
-use strafesnet_deferred_loader::deferred_loader::LoadFailureMode;
+use futures::StreamExt;
+use strafesnet_bsp_loader::loader::BspFinder;
+use strafesnet_deferred_loader::loader::Loader;
+use strafesnet_deferred_loader::deferred_loader::{LoadFailureMode,MeshDeferredLoader,RenderConfigDeferredLoader};
 
 #[derive(Subcommand)]
 pub enum Commands{
@@ -17,13 +21,15 @@ pub struct SourceToSNFSubcommand {
 	output_folder:PathBuf,
 	#[arg(required=true)]
 	input_files:Vec<PathBuf>,
+	#[arg(long)]
+	vpks:Vec<PathBuf>,
 }
 #[derive(Args)]
-pub struct ExtractTexturesSubcommand {
+pub struct ExtractTexturesSubcommand{
+	#[arg(required=true)]
+	bsp_files:Vec<PathBuf>,
 	#[arg(long)]
-	bsp_file:PathBuf,
+	vpks:Vec<PathBuf>,
-	#[arg(long)]
-	vpk_dir_files:Vec<PathBuf>
 }
 #[derive(Args)]
 pub struct VPKContentsSubcommand {
@@ -39,8 +45,8 @@ pub struct BSPContentsSubcommand {
 impl Commands{
 	pub async fn run(self)->AResult<()>{
 		match self{
-			Commands::ExtractTextures(subcommand)=>extract_textures(vec![subcommand.bsp_file],subcommand.vpk_dir_files),
+			Commands::SourceToSNF(subcommand)=>source_to_snf(subcommand.input_files,subcommand.output_folder,subcommand.vpks).await,
-			Commands::SourceToSNF(subcommand)=>source_to_snf(subcommand.input_files,subcommand.output_folder).await,
+			Commands::ExtractTextures(subcommand)=>extract_textures(subcommand.bsp_files,subcommand.vpks).await,
 			Commands::VPKContents(subcommand)=>vpk_contents(subcommand.input_file),
 			Commands::BSPContents(subcommand)=>bsp_contents(subcommand.input_file),
 		}
@@ -64,9 +70,9 @@ impl VMTContent{
 	}
 }
 
-fn get_some_texture(material:vmt_parser::material::Material)->AResult<VMTContent>{
+fn get_some_texture(material:vmt_parser::material::Material)->VMTContent{
 	//just grab some texture from somewhere for now
-	Ok(match material{
+	match material{
 		vmt_parser::material::Material::LightMappedGeneric(mat)=>VMTContent::vtf(Some(mat.base_texture)),
 		vmt_parser::material::Material::VertexLitGeneric(mat)=>VMTContent::vtf(mat.base_texture.or(mat.decal_texture)),//this just dies if there is none
 		vmt_parser::material::Material::VertexLitGenericDx6(mat)=>VMTContent::vtf(mat.base_texture.or(mat.decal_texture)),
@@ -85,175 +91,258 @@ fn get_some_texture(material:vmt_parser::material::Material)->AResult<VMTContent
 		vmt_parser::material::Material::Sky(mat)=>VMTContent::vtf(Some(mat.base_texture)),
 		vmt_parser::material::Material::Replacements(_mat)=>VMTContent::Unsupported,
 		vmt_parser::material::Material::Patch(mat)=>VMTContent::Patch(mat),
-		_=>return Err(anyhow::Error::msg("vmt failed to parse")),
+		_=>unreachable!(),
-	})
-}
-
-fn get_vmt<F:Fn(String)->AResult<Option<Vec<u8>>>>(find_stuff:&F,search_name:String)->AResult<vmt_parser::material::Material>{
-	if let Some(stuff)=find_stuff(search_name)?{
-		//decode vmt and then write
-		let stuff=String::from_utf8(stuff)?;
-		let material=vmt_parser::from_str(stuff.as_str())?;
-		println!("vmt material={:?}",material);
-		return Ok(material);
 	}
-	Err(anyhow::Error::msg("vmt not found"))
 }
 
-fn recursive_vmt_loader<F:Fn(String)->AResult<Option<Vec<u8>>>>(find_stuff:&F,material:vmt_parser::material::Material)->AResult<Option<Vec<u8>>>{
+#[derive(Debug,thiserror::Error)]
-	match get_some_texture(material)?{
+enum GetVMTError{
-		VMTContent::VMT(s)=>recursive_vmt_loader(find_stuff,get_vmt(find_stuff,s)?),
+	#[error("Bsp error {0:?}")]
+	Bsp(#[from]vbsp::BspError),
+	#[error("Utf8 error {0:?}")]
+	Utf8(#[from]std::str::Utf8Error),
+	#[error("Vdf error {0:?}")]
+	Vdf(#[from]vmt_parser::VdfError),
+	#[error("Vmt not found")]
+	NotFound,
+}
+
+fn get_vmt(finder:BspFinder,search_name:&str)->Result<vmt_parser::material::Material,GetVMTError>{
+	let vmt_data=finder.find(search_name)?.ok_or(GetVMTError::NotFound)?;
+	//decode vmt and then write
+	let vmt_str=core::str::from_utf8(&vmt_data)?;
+	let material=vmt_parser::from_str(vmt_str)?;
+	//println!("vmt material={:?}",material);
+	Ok(material)
+}
+
+#[derive(Debug,thiserror::Error)]
+enum LoadVMTError{
+	#[error("Bsp error {0:?}")]
+	Bsp(#[from]vbsp::BspError),
+	#[error("GetVMT error {0:?}")]
+	GetVMT(#[from]GetVMTError),
+	#[error("FromUtf8 error {0:?}")]
+	FromUtf8(#[from]std::string::FromUtf8Error),
+	#[error("Vdf error {0:?}")]
+	Vdf(#[from]vmt_parser::VdfError),
+	#[error("Vmt unsupported")]
+	Unsupported,
+	#[error("Vmt unresolved")]
+	Unresolved,
+	#[error("Vmt not found")]
+	NotFound,
+}
+fn recursive_vmt_loader<'bsp,'vpk,'a>(finder:BspFinder<'bsp,'vpk>,material:vmt_parser::material::Material)->Result<Option<Cow<'a,[u8]>>,LoadVMTError>
+	where
+		'bsp:'a,
+		'vpk:'a,
+{
+	match get_some_texture(material){
+		VMTContent::VMT(s)=>recursive_vmt_loader(finder,get_vmt(finder,s.as_str())?),
 		VMTContent::VTF(s)=>{
 			let mut texture_file_name=PathBuf::from("materials");
 			texture_file_name.push(s);
 			texture_file_name.set_extension("vtf");
-			find_stuff(texture_file_name.into_os_string().into_string().unwrap())
+			Ok(finder.find(texture_file_name.to_str().unwrap())?)
 		},
-		VMTContent::Patch(mat)=>recursive_vmt_loader(find_stuff,
+		VMTContent::Patch(mat)=>recursive_vmt_loader(finder,
-			mat.resolve(|search_name|{
+			mat.resolve(|search_name|
-				match find_stuff(search_name.to_string())?{
+				match finder.find(search_name)?{
-					Some(bytes)=>Ok(String::from_utf8(bytes)?),
+					Some(bytes)=>Ok(String::from_utf8(bytes.into_owned())?),
-					None=>Err(anyhow::Error::msg("could not find vmt")),
+					None=>Err(LoadVMTError::NotFound),
 				}
-			})?
+			)?
 		),
-		VMTContent::Unsupported=>{println!("Unsupported vmt");Ok(None)},//print and move on
+		VMTContent::Unsupported=>Err(LoadVMTError::Unsupported),
-		VMTContent::Unresolved=>{println!("Unresolved vmt");Ok(None)},
+		VMTContent::Unresolved=>Err(LoadVMTError::Unresolved),
 	}
 }
-
+fn load_texture<'bsp,'vpk,'a>(finder:BspFinder<'bsp,'vpk>,texture_name:&str)->Result<Option<Cow<'a,[u8]>>,LoadVMTError>
-fn extract_textures(paths:Vec<PathBuf>,vpk_paths:Vec<PathBuf>)->AResult<()>{
+	where
-	std::fs::create_dir_all("textures")?;
+		'bsp:'a,
-	let vpk_list:Vec<vpk::VPK>=vpk_paths.into_iter().map(|vpk_path|vpk::VPK::read(&vpk_path).expect("vpk file does not exist")).collect();
+		'vpk:'a,
-	for path in paths{
+{
-		let mut deduplicate=std::collections::HashSet::new();
+	let mut texture_file_name=PathBuf::from("materials");
-		let bsp=vbsp::Bsp::read(std::fs::read(path)?.as_ref())?;
+	//lower case
-		for texture in bsp.textures(){
+	let texture_file_name_lowercase=texture_name.to_lowercase();
-			deduplicate.insert(PathBuf::from(texture.name()));
+	texture_file_name.push(texture_file_name_lowercase.clone());
-		}
+	//remove stem and search for both vtf and vmt files
-		//dedupe prop models
+	let stem=PathBuf::from(texture_file_name.file_stem().unwrap());
-		let mut model_dedupe=std::collections::HashSet::new();
+	texture_file_name.pop();
-		for prop in bsp.static_props(){
+	texture_file_name.push(stem);
-			model_dedupe.insert(prop.model());
+	if let Some(stuff)=finder.find(texture_file_name.to_str().unwrap())?{
-		}
+		return Ok(Some(stuff))
-
-		//grab texture names from props
-		for model_name in model_dedupe{
-			//.mdl, .vvd, .dx90.vtx
-			let mut path=PathBuf::from(model_name);
-			let file_name=PathBuf::from(path.file_stem().unwrap());
-			path.pop();
-			path.push(file_name);
-			let mut vvd_path=path.clone();
-			let mut vtx_path=path.clone();
-			vvd_path.set_extension("vvd");
-			vtx_path.set_extension("dx90.vtx");
-			match (bsp.pack.get(model_name),bsp.pack.get(vvd_path.as_os_str().to_str().unwrap()),bsp.pack.get(vtx_path.as_os_str().to_str().unwrap())){
-				(Ok(Some(mdl_file)),Ok(Some(vvd_file)),Ok(Some(vtx_file)))=>{
-					match (vmdl::mdl::Mdl::read(mdl_file.as_ref()),vmdl::vvd::Vvd::read(vvd_file.as_ref()),vmdl::vtx::Vtx::read(vtx_file.as_ref())){
-						(Ok(mdl),Ok(vvd),Ok(vtx))=>{
-							let model=vmdl::Model::from_parts(mdl,vtx,vvd);
-							for texture in model.textures(){
-								for search_path in &texture.search_paths{
-									let mut path=PathBuf::from(search_path.as_str());
-									path.push(texture.name.as_str());
-									deduplicate.insert(path);
-								}
-							}
-						},
-						_=>println!("model_name={} error",model_name),
-					}
-				},
-				_=>println!("no model name={}",model_name),
-			}
-		}
-
-		let pack=&bsp.pack;
-		let vpk_list=&vpk_list;
-		std::thread::scope(move|s|{
-			let mut thread_handles=Vec::new();
-			for texture_name in deduplicate{
-				let mut found_texture=false;
-				//LMAO imagine having to write type names
-				let write_image=|mut stuff,write_file_name|{
-					let image=vtf::from_bytes(&mut stuff)?.highres_image.decode(0)?.to_rgba8();
-
-					let format=if image.width()%4!=0||image.height()%4!=0{
-						image_dds::ImageFormat::Rgba8UnormSrgb
-					}else{
-						image_dds::ImageFormat::BC7RgbaUnormSrgb
-					};
-					//this fails if the image dimensions are not a multiple of 4
-					let dds = image_dds::dds_from_image(
-						&image,
-						format,
-						image_dds::Quality::Slow,
-						image_dds::Mipmaps::GeneratedAutomatic,
-					)?;
-
-					//write dds
-					let mut dest=PathBuf::from("textures");
-					dest.push(write_file_name);
-					dest.set_extension("dds");
-					std::fs::create_dir_all(dest.parent().unwrap())?;
-					let mut writer = std::io::BufWriter::new(std::fs::File::create(dest)?);
-					dds.write(&mut writer)?;
-					Ok::<(),anyhow::Error>(())
-				};
-				let find_stuff=|search_file_name:String|{
-					println!("search_file_name={}",search_file_name);
-					match pack.get(search_file_name.as_str())?{
-						Some(file)=>return Ok(Some(file)),
-						_=>(),
-					}
-					//search pak list
-					for vpk_index in vpk_list{
-						if let Some(vpk_entry)=vpk_index.tree.get(search_file_name.as_str()){
-							return Ok(Some(match vpk_entry.get()?{
-								std::borrow::Cow::Borrowed(bytes)=>bytes.to_vec(),
-								std::borrow::Cow::Owned(bytes)=>bytes,
-							}));
-						}
-					}
-					Ok::<Option<Vec<u8>>,anyhow::Error>(None)
-				};
-				let loader=|texture_name:String|{
-					let mut texture_file_name=PathBuf::from("materials");
-					//lower case
-					let texture_file_name_lowercase=texture_name.to_lowercase();
-					texture_file_name.push(texture_file_name_lowercase.clone());
-					//remove stem and search for both vtf and vmt files
-					let stem=PathBuf::from(texture_file_name.file_stem().unwrap());
-					texture_file_name.pop();
-					texture_file_name.push(stem);
-					//somehow search for both files
-					let mut texture_file_name_vmt=texture_file_name.clone();
-					texture_file_name.set_extension("vtf");
-					texture_file_name_vmt.set_extension("vmt");
-					if let Some(stuff)=find_stuff(texture_file_name.to_string_lossy().to_string())?{
-						return Ok(Some(stuff))
-					}
-					recursive_vmt_loader(&find_stuff,get_vmt(&find_stuff,texture_file_name_vmt.to_string_lossy().to_string())?)
-				};
-				if let Some(stuff)=loader(texture_name.to_string_lossy().to_string())?{
-					found_texture=true;
-					let texture_name=texture_name.clone();
-					thread_handles.push(s.spawn(move||write_image(stuff,texture_name)));
-				}
-				if !found_texture{
-					println!("no data");
-				}
-			}
-			for thread in thread_handles{
-				match thread.join(){
-					Ok(Err(e))=>println!("write error: {:?}",e),
-					Err(e)=>println!("thread error: {:?}",e),
-					Ok(_)=>(),
-				}
-			}
-			Ok::<(),anyhow::Error>(())
-		})?
 	}
+	//somehow search for both files
+	let mut texture_file_name_vmt=texture_file_name.clone();
+	texture_file_name.set_extension("vtf");
+	texture_file_name_vmt.set_extension("vmt");
+	recursive_vmt_loader(finder,get_vmt(finder,texture_file_name_vmt.to_str().unwrap())?)
+}
+#[derive(Debug,thiserror::Error)]
+enum ExtractTextureError{
+	#[error("Io error {0:?}")]
+	Io(#[from]std::io::Error),
+	#[error("Bsp error {0:?}")]
+	Bsp(#[from]vbsp::BspError),
+	#[error("MeshLoad error {0:?}")]
+	MeshLoad(#[from]strafesnet_bsp_loader::loader::MeshError),
+	#[error("Load VMT error {0:?}")]
+	LoadVMT(#[from]LoadVMTError),
+}
+async fn gimme_them_textures(path:&Path,vpk_list:&[vpk::VPK],send_texture:tokio::sync::mpsc::Sender<(Vec<u8>,String)>)->Result<(),ExtractTextureError>{
+	let bsp=vbsp::Bsp::read(tokio::fs::read(path).await?.as_ref())?;
+	let loader_bsp=strafesnet_bsp_loader::Bsp::new(bsp);
+	let bsp=loader_bsp.as_ref();
+
+	let mut texture_deferred_loader=RenderConfigDeferredLoader::new();
+	for texture in bsp.textures(){
+		texture_deferred_loader.acquire_render_config_id(Some(Cow::Borrowed(texture.name())));
+	}
+
+	let mut mesh_deferred_loader=MeshDeferredLoader::new();
+	for prop in bsp.static_props(){
+		mesh_deferred_loader.acquire_mesh_id(prop.model());
+	}
+
+	let finder=BspFinder{
+		bsp:&loader_bsp,
+		vpks:vpk_list
+	};
+
+	let mut mesh_loader=strafesnet_bsp_loader::loader::ModelLoader::new(finder);
+	// load models and collect requested textures
+	for model_path in mesh_deferred_loader.into_indices(){
+		let model:vmdl::Model=match mesh_loader.load(model_path){
+			Ok(model)=>model,
+			Err(e)=>{
+				println!("Model={model_path} Load model error: {e}");
+				continue;
+			},
+		};
+		for texture in model.textures(){
+			for search_path in &texture.search_paths{
+				let mut path=PathBuf::from(search_path.as_str());
+				path.push(texture.name.as_str());
+				let path=path.to_str().unwrap().to_owned();
+				texture_deferred_loader.acquire_render_config_id(Some(Cow::Owned(path)));
+			}
+		}
+	}
+
+	for texture_path in texture_deferred_loader.into_indices(){
+		match load_texture(finder,&texture_path){
+			Ok(Some(texture))=>send_texture.send(
+				(texture.into_owned(),texture_path.into_owned())
+			).await.unwrap(),
+			Ok(None)=>(),
+			Err(e)=>println!("Texture={texture_path} Load error: {e}"),
+		}
+	}
+
+	Ok(())
+}
+
+
+#[derive(Debug,thiserror::Error)]
+enum ConvertTextureError{
+	#[error("Bsp error {0:?}")]
+	Bsp(#[from]vbsp::BspError),
+	#[error("Vtf error {0:?}")]
+	Vtf(#[from]vtf::Error),
+	#[error("DDS create error {0:?}")]
+	DDS(#[from]image_dds::CreateDdsError),
+	#[error("DDS write error {0:?}")]
+	DDSWrite(#[from]image_dds::ddsfile::Error),
+	#[error("Io error {0:?}")]
+	Io(#[from]std::io::Error),
+}
+
+async fn convert_texture(texture:Vec<u8>,write_file_name:impl AsRef<Path>)->Result<(),ConvertTextureError>{
+	let image=vtf::from_bytes(&texture)?.highres_image.decode(0)?.to_rgba8();
+
+	let format=if image.width()%4!=0||image.height()%4!=0{
+		image_dds::ImageFormat::Rgba8UnormSrgb
+	}else{
+		image_dds::ImageFormat::BC7RgbaUnormSrgb
+	};
+	//this fails if the image dimensions are not a multiple of 4
+	let dds = image_dds::dds_from_image(
+		&image,
+		format,
+		image_dds::Quality::Slow,
+		image_dds::Mipmaps::GeneratedAutomatic,
+	)?;
+
+	//write dds
+	let mut dest=PathBuf::from("textures");
+	dest.push(write_file_name);
+	dest.set_extension("dds");
+	std::fs::create_dir_all(dest.parent().unwrap())?;
+	let mut writer=std::io::BufWriter::new(std::fs::File::create(dest)?);
+	dds.write(&mut writer)?;
+
+	Ok(())
+}
+
+async fn read_vpks(vpk_paths:Vec<PathBuf>,thread_limit:usize)->Vec<vpk::VPK>{
+	futures::stream::iter(vpk_paths).map(|vpk_path|async{
+		// idk why it doesn't want to pass out the errors but this is fatal anyways
+		tokio::task::spawn_blocking(move||vpk::VPK::read(&vpk_path)).await.unwrap().unwrap()
+	})
+	.buffer_unordered(thread_limit)
+	.collect().await
+}
+
+async fn extract_textures(paths:Vec<PathBuf>,vpk_paths:Vec<PathBuf>)->AResult<()>{
+	tokio::try_join!(
+		tokio::fs::create_dir_all("extracted_textures"),
+		tokio::fs::create_dir_all("textures"),
+		tokio::fs::create_dir_all("meshes"),
+	)?;
+	let thread_limit=std::thread::available_parallelism()?.get();
+
+	// load vpk list
+	let vpk_list=read_vpks(vpk_paths,thread_limit).await;
+
+	// leak vpk_list for static lifetime?
+	let vpk_list:&[vpk::VPK]=vpk_list.leak();
+
+	let (send_texture,mut recv_texture)=tokio::sync::mpsc::channel(thread_limit);
+	let mut it=paths.into_iter();
+	let extract_thread=tokio::spawn(async move{
+		static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
+		SEM.add_permits(thread_limit);
+		while let (Ok(permit),Some(path))=(SEM.acquire().await,it.next()){
+			let send=send_texture.clone();
+			tokio::spawn(async move{
+				let result=gimme_them_textures(&path,vpk_list,send).await;
+				drop(permit);
+				match result{
+					Ok(())=>(),
+					Err(e)=>println!("Map={path:?} Decode error: {e:?}"),
+				}
+			});
+		}
+	});
+
+	// convert images
+	static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
+	SEM.add_permits(thread_limit);
+	while let (Ok(permit),Some((data,dest)))=(SEM.acquire().await,recv_texture.recv().await){
+		// TODO: dedup dest?
+		tokio::spawn(async move{
+			let result=convert_texture(data,dest).await;
+			drop(permit);
+			match result{
+				Ok(())=>(),
+				Err(e)=>println!("Convert error: {e:?}"),
+			}
+		});
+	}
+	extract_thread.await?;
+	_=SEM.acquire_many(thread_limit as u32).await?;
 	Ok(())
 }
 
@@ -288,14 +377,14 @@ impl std::fmt::Display for ConvertError{
 }
 impl std::error::Error for ConvertError{}
 
-async fn convert_to_snf(path:&Path,output_folder:PathBuf)->AResult<()>{
+async fn convert_to_snf(path:&Path,vpk_list:&[vpk::VPK],output_folder:PathBuf)->AResult<()>{
 	let entire_file=tokio::fs::read(path).await?;
 
 	let bsp=strafesnet_bsp_loader::read(
 		std::io::Cursor::new(entire_file)
 	).map_err(ConvertError::BspRead)?;
 
-	let map=bsp.to_snf(LoadFailureMode::DefaultToNone).map_err(ConvertError::BspLoad)?;
+	let map=bsp.to_snf(LoadFailureMode::DefaultToNone,vpk_list).map_err(ConvertError::BspLoad)?;
 
 	let mut dest=output_folder;
 	dest.push(path.file_stem().unwrap());
@@ -306,10 +395,17 @@ async fn convert_to_snf(path:&Path,output_folder:PathBuf)->AResult<()>{
 
 	Ok(())
 }
-async fn source_to_snf(paths:Vec<std::path::PathBuf>,output_folder:PathBuf)->AResult<()>{
+async fn source_to_snf(paths:Vec<std::path::PathBuf>,output_folder:PathBuf,vpk_paths:Vec<PathBuf>)->AResult<()>{
 	let start=std::time::Instant::now();
 
 	let thread_limit=std::thread::available_parallelism()?.get();
+
+	// load vpk list
+	let vpk_list=read_vpks(vpk_paths,thread_limit).await;
+
+	// leak vpk_list for static lifetime?
+	let vpk_list:&[vpk::VPK]=vpk_list.leak();
+
 	let mut it=paths.into_iter();
 	static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
 	SEM.add_permits(thread_limit);
@@ -317,7 +413,7 @@ async fn source_to_snf(paths:Vec<std::path::PathBuf>,output_folder:PathBuf)->ARe
 	while let (Ok(permit),Some(path))=(SEM.acquire().await,it.next()){
 		let output_folder=output_folder.clone();
 		tokio::spawn(async move{
-			let result=convert_to_snf(path.as_path(),output_folder).await;
+			let result=convert_to_snf(path.as_path(),vpk_list,output_folder).await;
 			drop(permit);
 			match result{
 				Ok(())=>(),