naive multithread

.cargo/config.toml

@@ -1,2 +0,0 @@
-[registries.strafesnet]
-index = "sparse+https://git.itzana.me/api/packages/strafesnet/cargo/"

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "map-tool"
-version = "1.5.7"
+version = "1.5.0"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@@ -9,22 +9,21 @@ edition = "2021"
 anyhow = "1.0.75"
 clap = { version = "4.4.2", features = ["derive"] }
 flate2 = "1.0.27"
-image = "0.25.2"
-image_dds = "0.6.0"
+image = "0.24.7"
+image_dds = "0.1.1"
 lazy-regex = "3.1.0"
-rbx_binary = { version = "0.7.4", registry = "strafesnet" }
-rbx_dom_weak = { version = "2.7.0", registry = "strafesnet" }
-rbx_reflection_database = { version = "0.2.10", registry = "strafesnet" }
-rbx_xml = { version = "0.13.3", registry = "strafesnet" }
-strafesnet_bsp_loader = { version = "0.2.1", registry = "strafesnet" }
-strafesnet_deferred_loader = { version = "0.4.0", features = ["legacy"], registry = "strafesnet" }
-strafesnet_rbx_loader = { version = "0.5.1", registry = "strafesnet" }
-strafesnet_snf = { version = "0.2.0", registry = "strafesnet" }
-vbsp = "0.6.0"
-vmdl = "0.2.0"
-vmt-parser = "0.2.0"
+rbx_binary = "0.7.1"
+rbx_dom_weak = "2.5.0"
+rbx_reflection_database = "0.2.7"
+rbx_xml = "0.13.1"
+strafesnet_bsp_loader = { git = "https://git.itzana.me/StrafesNET/bsp_loader", rev = "79b8120106e63de237b8ed59c3d3e30db51e7146" }
+strafesnet_rbx_loader = { git = "https://git.itzana.me/StrafesNET/rbx_loader", rev = "0d2c27f60f4a448063edc57d9a23766775fb2cc5" }
+strafesnet_snf = { git = "https://git.itzana.me/StrafesNET/snf", rev = "dea408daeef576cff8ffa61356c89a9d03d95f6b" }
+vbsp = "0.5.0"
+vmdl = "0.1.1"
+vmt-parser = "0.1.1"
 vpk = "0.2.0"
-vtf = "0.3.0"
+vtf = "0.2.1"
 
 #[profile.release]
 #lto = true

src/main.rs

@@ -2,7 +2,6 @@ use std::{collections::HashSet,io::{Read,Seek},path::PathBuf};
 use clap::{Args,Parser,Subcommand};
 use anyhow::Result as AResult;
 use rbx_dom_weak::Instance;
-use strafesnet_deferred_loader::rbxassetid::RobloxAssetId;
 
 #[derive(Parser)]
 #[command(author, version, about, long_about = None)]
@@ -15,7 +14,6 @@ struct Cli {
 #[derive(Subcommand)]
 enum Commands {
 	RobloxToSNF(RobloxToSNFSubcommand),
-	SourceToSNF(SourceToSNFSubcommand),
 	DownloadTextures(DownloadTexturesSubcommand),
 	ExtractTextures(ExtractTexturesSubcommand),
 	ConvertTextures(ConvertTexturesSubcommand),
@@ -33,13 +31,6 @@ struct RobloxToSNFSubcommand {
 	input_files:Vec<PathBuf>,
 }
 #[derive(Args)]
-struct SourceToSNFSubcommand {
-	#[arg(long)]
-	output_folder:PathBuf,
-	#[arg(required=true)]
-	input_files:Vec<PathBuf>,
-}
-#[derive(Args)]
 struct DownloadTexturesSubcommand {
 	#[arg(long,required=true)]
 	roblox_files:Vec<PathBuf>
@@ -76,8 +67,7 @@ struct WriteAttributesSubcommand {
 fn main() -> AResult<()> {
 	let cli = Cli::parse();
 	match cli.command {
-		Commands::RobloxToSNF(subcommand)=>roblox_to_snf(subcommand.input_files,subcommand.output_folder),
-		Commands::SourceToSNF(subcommand)=>source_to_snf(subcommand.input_files,subcommand.output_folder),
+		Commands::RobloxToSNF(subcommand)=>convert_to_snf(subcommand.input_files,subcommand.output_folder),
 		Commands::DownloadTextures(subcommand)=>download_textures(subcommand.roblox_files),
 		Commands::ExtractTextures(subcommand)=>extract_textures(vec![subcommand.bsp_file],subcommand.vpk_dir_files),
 		Commands::VPKContents(subcommand)=>vpk_contents(subcommand.input_file),
@@ -153,6 +143,22 @@ fn get_dom<R:Read+Seek>(input:&mut R)->AResult<rbx_dom_weak::WeakDom>{
 	}
 }
 
+struct RobloxAssetId(u64);
+struct RobloxAssetIdParseErr;
+impl std::str::FromStr for RobloxAssetId {
+	type Err=RobloxAssetIdParseErr;
+	fn from_str(s: &str) -> Result<Self, Self::Err>{
+		let regman=lazy_regex::regex!(r"(\d+)$");
+		if let Some(captures) = regman.captures(s) {
+			if captures.len()==2{//captures[0] is all captures concatenated, and then each individual capture
+				if let Ok(id) = captures[0].parse::<u64>() {
+					return Ok(Self(id));
+				}
+			}
+		}
+		Err(RobloxAssetIdParseErr)
+	}
+}
 /* The ones I'm interested in:
 Beam.Texture
 Decal.Texture
@@ -231,9 +237,8 @@ fn download_textures(paths:Vec<PathBuf>)->AResult<()>{
 	}
 	let texture_list_string=texture_list.into_iter().map(|id|id.to_string()).collect::<Vec<String>>();
 	println!("Texture list:{:?}",texture_list_string.join(" "));
-	std::fs::create_dir_all("textures/unprocessed")?;
 	let output=std::process::Command::new("asset-tool")
-	.args(["download","--cookie-literal","","--output-folder","textures/unprocessed/"])
+	.args(["download","environment","RBXCOOKIE","textures/unprocessed/"])
 	.args(texture_list_string)
 	.spawn()?
 	.wait_with_output()?;
@@ -267,9 +272,8 @@ fn download_meshes(paths:Vec<PathBuf>)->AResult<()>{
 	}
 	let mesh_list_string=mesh_list.into_iter().map(|id|id.to_string()).collect::<Vec<String>>();
 	println!("Mesh list:{:?}",mesh_list_string.join(" "));
-	std::fs::create_dir_all("meshes/")?;
 	let output=std::process::Command::new("asset-tool")
-	.args(["download","--cookie-literal","","--output-folder","meshes/"])
+	.args(["download","environment","RBXCOOKIE","meshes/"])
 	.args(mesh_list_string)
 	.spawn()?
 	.wait_with_output()?;
@@ -307,9 +311,9 @@ fn convert(file_thing:std::fs::DirEntry) -> AResult<()>{
 	}?.to_rgba8();//this sets a=255, arcane is actually supposed to look like that
 
 	let format=if image.width()%4!=0||image.height()%4!=0{
-		image_dds::ImageFormat::Rgba8UnormSrgb
+		image_dds::ImageFormat::R8G8B8A8Srgb
 	}else{
-		image_dds::ImageFormat::BC7RgbaUnormSrgb
+		image_dds::ImageFormat::BC7Srgb
 	};
 	//this fails if the image dimensions are not a multiple of 4
 	let dds = image_dds::dds_from_image(
@@ -320,7 +324,7 @@ fn convert(file_thing:std::fs::DirEntry) -> AResult<()>{
 	)?;
 
 	//write dds
-	let mut dest=PathBuf::from("textures");
+	let mut dest=PathBuf::from("textures/dds");
 	dest.push(file_thing.file_name());
 	dest.set_extension("dds");
 	let mut writer = std::io::BufWriter::new(std::fs::File::create(dest)?);
@@ -342,8 +346,6 @@ fn convert(file_thing:std::fs::DirEntry) -> AResult<()>{
 	Ok(())
 }
 fn convert_textures() -> AResult<()>{
-	std::fs::create_dir_all("textures/unprocessed")?;
-	std::fs::create_dir_all("textures/processed")?;
 	let start = std::time::Instant::now();
 	let mut threads=Vec::new();
 	for entry in std::fs::read_dir("textures/unprocessed")? {
@@ -490,7 +492,6 @@ fn recursive_vmt_loader<F:Fn(String)->AResult<Option<Vec<u8>>>>(find_stuff:&F,ma
 }
 
 fn extract_textures(paths:Vec<PathBuf>,vpk_paths:Vec<PathBuf>)->AResult<()>{
-	std::fs::create_dir_all("textures")?;
 	let vpk_list:Vec<vpk::VPK>=vpk_paths.into_iter().map(|vpk_path|vpk::VPK::read(&vpk_path).expect("vpk file does not exist")).collect();
 	for path in paths{
 		let mut deduplicate=std::collections::HashSet::new();
@@ -546,9 +547,9 @@ fn extract_textures(paths:Vec<PathBuf>,vpk_paths:Vec<PathBuf>)->AResult<()>{
 					let image=vtf::from_bytes(&mut stuff)?.highres_image.decode(0)?.to_rgba8();
 
 					let format=if image.width()%4!=0||image.height()%4!=0{
-						image_dds::ImageFormat::Rgba8UnormSrgb
+						image_dds::ImageFormat::R8G8B8A8Srgb
 					}else{
-						image_dds::ImageFormat::BC7RgbaUnormSrgb
+						image_dds::ImageFormat::BC7Srgb
 					};
 					//this fails if the image dimensions are not a multiple of 4
 					let dds = image_dds::dds_from_image(
@@ -559,7 +560,7 @@ fn extract_textures(paths:Vec<PathBuf>,vpk_paths:Vec<PathBuf>)->AResult<()>{
 					)?;
 
 					//write dds
-					let mut dest=PathBuf::from("textures");
+					let mut dest=PathBuf::from("textures/dds");
 					dest.push(write_file_name);
 					dest.set_extension("dds");
 					std::fs::create_dir_all(dest.parent().unwrap())?;
@@ -641,12 +642,10 @@ fn bsp_contents(path:PathBuf)->AResult<()>{
 }
 
 #[derive(Debug)]
-#[allow(dead_code)]
 enum ConvertError{
 	IO(std::io::Error),
 	SNFMap(strafesnet_snf::map::Error),
 	RbxLoader(strafesnet_rbx_loader::ReadError),
-	BspLoader(strafesnet_bsp_loader::ReadError),
 }
 impl std::fmt::Display for ConvertError{
 	fn fmt(&self,f:&mut std::fmt::Formatter<'_>)->std::fmt::Result{
@@ -655,41 +654,22 @@ impl std::fmt::Display for ConvertError{
 }
 impl std::error::Error for ConvertError{}
 
-type MapThread=std::thread::JoinHandle<Result<(),ConvertError>>;
-
-fn roblox_to_snf(pathlist:Vec<std::path::PathBuf>,output_folder:PathBuf)->AResult<()>{
-	let n_paths=pathlist.len();
+fn convert_to_snf(pathlist:Vec<std::path::PathBuf>,output_folder:PathBuf)->AResult<()>{
 	let start = std::time::Instant::now();
-	let mut threads:std::collections::VecDeque<MapThread>=std::collections::VecDeque::new();
-	let mut i=0;
-	let mut join_thread=|thread:MapThread|{
-		i+=1;
-		if let Err(e)=thread.join(){
-			println!("thread error: {:?}",e);
-		}else{
-			println!("{}/{}",i,n_paths);
-		}
-	};
+	let mut threads:Vec<std::thread::JoinHandle<Result<(),ConvertError>>>=Vec::new();
 	for path in pathlist{
-		if 32<=threads.len(){
-			join_thread(threads.pop_front().unwrap());
-		}
 		let output_folder=output_folder.clone();
-		threads.push_back(std::thread::spawn(move ||{
-			let model=strafesnet_rbx_loader::read(
+		threads.push(std::thread::spawn(move ||{
+			let dom=strafesnet_rbx_loader::read(
 				std::fs::File::open(path.as_path())
 				.map_err(ConvertError::IO)?
 			).map_err(ConvertError::RbxLoader)?;
-
-			let mut place=model.into_place();
-			place.run_scripts();
-
 			let mut loader=strafesnet_deferred_loader::roblox_legacy();
 
 			let (texture_loader,mesh_loader)=loader.get_inner_mut();
 
 			let map_step1=strafesnet_rbx_loader::convert(
-				&place,
+				&dom,
 				|name|texture_loader.acquire_render_config_id(name),
 				|name|mesh_loader.acquire_mesh_id(name),
 			);
@@ -723,83 +703,15 @@ fn roblox_to_snf(pathlist:Vec<std::path::PathBuf>,output_folder:PathBuf)->AResul
 		}));
 	}
 
-	for thread in threads{
-		join_thread(thread);
-	}
-	println!("{:?}", start.elapsed());
-	Ok(())
-}
-
-fn source_to_snf(pathlist:Vec<std::path::PathBuf>,output_folder:PathBuf)->AResult<()>{
-	let n_paths=pathlist.len();
-	let start = std::time::Instant::now();
-	let mut threads:std::collections::VecDeque<MapThread>=std::collections::VecDeque::new();
 	let mut i=0;
-	let mut join_thread=|thread:MapThread|{
+	let n_threads=threads.len();
+	for thread in threads{
 		i+=1;
 		if let Err(e)=thread.join(){
 			println!("thread error: {:?}",e);
 		}else{
-			println!("{}/{}",i,n_paths);
+			println!("{}/{}",i,n_threads);
 		}
-	};
-	for path in pathlist{
-		if 32<=threads.len(){
-			join_thread(threads.pop_front().unwrap());
-		}
-		let output_folder=output_folder.clone();
-		threads.push_back(std::thread::spawn(move ||{
-			let bsp=strafesnet_bsp_loader::read(
-				std::fs::File::open(path.as_path())
-				.map_err(ConvertError::IO)?
-			).map_err(ConvertError::BspLoader)?;
-			let mut loader=strafesnet_deferred_loader::source_legacy();
-
-				let (texture_loader,mesh_loader)=loader.get_inner_mut();
-
-				let map_step1=strafesnet_bsp_loader::convert(
-					&bsp,
-					|name|texture_loader.acquire_render_config_id(name),
-					|name|mesh_loader.acquire_mesh_id(name),
-				);
-
-				let prop_meshes=mesh_loader.load_meshes(&bsp.as_ref());
-
-				let map_step2=map_step1.add_prop_meshes(
-					//the type conflagulator 9000
-					prop_meshes.into_iter().map(|(mesh_id,loader_model)|
-						(mesh_id,strafesnet_bsp_loader::data::ModelData{
-							mdl:strafesnet_bsp_loader::data::MdlData::new(loader_model.mdl.get()),
-							vtx:strafesnet_bsp_loader::data::VtxData::new(loader_model.vtx.get()),
-							vvd:strafesnet_bsp_loader::data::VvdData::new(loader_model.vvd.get()),
-						})
-					),
-					|name|texture_loader.acquire_render_config_id(name),
-				);
-
-				let (textures,render_configs)=loader.into_render_configs().map_err(ConvertError::IO)?.consume();
-
-				let map=map_step2.add_render_configs_and_textures(
-					render_configs.into_iter(),
-					textures.into_iter().map(|(texture_id,texture)|
-						(texture_id,match texture{
-							strafesnet_deferred_loader::texture::Texture::ImageDDS(data)=>data,
-						})
-					),
-				);
-
-			let mut dest=output_folder.clone();
-			dest.push(path.file_stem().unwrap());
-			dest.set_extension("snfm");
-			let file=std::fs::File::create(dest).map_err(ConvertError::IO)?;
-
-			strafesnet_snf::map::write_map(file,map).map_err(ConvertError::SNFMap)?;
-			Ok(())
-		}));
-	}
-
-	for thread in threads{
-		join_thread(thread);
 	}
 	println!("{:?}", start.elapsed());
 	Ok(())