naive multithread

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "map-tool"
-version = "1.5.3"
+version = "1.5.0"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@@ -12,14 +12,13 @@ flate2 = "1.0.27"
 image = "0.24.7"
 image_dds = "0.1.1"
 lazy-regex = "3.1.0"
-rbx_binary = { version = "0.7.4", registry = "strafesnet" }
+rbx_binary = "0.7.1"
-rbx_dom_weak = { version = "2.7.0", registry = "strafesnet" }
+rbx_dom_weak = "2.5.0"
-rbx_reflection_database = { version = "0.2.10", registry = "strafesnet" }
+rbx_reflection_database = "0.2.7"
-rbx_xml = { version = "0.13.3", registry = "strafesnet" }
+rbx_xml = "0.13.1"
-strafesnet_bsp_loader = { version = "0.1.3", registry = "strafesnet" }
+strafesnet_bsp_loader = { git = "https://git.itzana.me/StrafesNET/bsp_loader", rev = "79b8120106e63de237b8ed59c3d3e30db51e7146" }
-strafesnet_deferred_loader = { version = "0.3.1", features = ["legacy"], registry = "strafesnet" }
+strafesnet_rbx_loader = { git = "https://git.itzana.me/StrafesNET/rbx_loader", rev = "0d2c27f60f4a448063edc57d9a23766775fb2cc5" }
-strafesnet_rbx_loader = { version = "0.3.6", registry = "strafesnet" }
+strafesnet_snf = { git = "https://git.itzana.me/StrafesNET/snf", rev = "dea408daeef576cff8ffa61356c89a9d03d95f6b" }
-strafesnet_snf = { version = "0.1.0", registry = "strafesnet" }
 vbsp = "0.5.0"
 vmdl = "0.1.1"
 vmt-parser = "0.1.1"

src/main.rs

@@ -14,7 +14,6 @@ struct Cli {
 #[derive(Subcommand)]
 enum Commands {
 	RobloxToSNF(RobloxToSNFSubcommand),
-	SourceToSNF(SourceToSNFSubcommand),
 	DownloadTextures(DownloadTexturesSubcommand),
 	ExtractTextures(ExtractTexturesSubcommand),
 	ConvertTextures(ConvertTexturesSubcommand),
@@ -32,13 +31,6 @@ struct RobloxToSNFSubcommand {
 	input_files:Vec<PathBuf>,
 }
 #[derive(Args)]
-struct SourceToSNFSubcommand {
-	#[arg(long)]
-	output_folder:PathBuf,
-	#[arg(required=true)]
-	input_files:Vec<PathBuf>,
-}
-#[derive(Args)]
 struct DownloadTexturesSubcommand {
 	#[arg(long,required=true)]
 	roblox_files:Vec<PathBuf>
@@ -75,8 +67,7 @@ struct WriteAttributesSubcommand {
 fn main() -> AResult<()> {
 	let cli = Cli::parse();
 	match cli.command {
-		Commands::RobloxToSNF(subcommand)=>roblox_to_snf(subcommand.input_files,subcommand.output_folder),
+		Commands::RobloxToSNF(subcommand)=>convert_to_snf(subcommand.input_files,subcommand.output_folder),
-		Commands::SourceToSNF(subcommand)=>source_to_snf(subcommand.input_files,subcommand.output_folder),
 		Commands::DownloadTextures(subcommand)=>download_textures(subcommand.roblox_files),
 		Commands::ExtractTextures(subcommand)=>extract_textures(vec![subcommand.bsp_file],subcommand.vpk_dir_files),
 		Commands::VPKContents(subcommand)=>vpk_contents(subcommand.input_file),
@@ -246,9 +237,8 @@ fn download_textures(paths:Vec<PathBuf>)->AResult<()>{
 	}
 	let texture_list_string=texture_list.into_iter().map(|id|id.to_string()).collect::<Vec<String>>();
 	println!("Texture list:{:?}",texture_list_string.join(" "));
-	std::fs::create_dir_all("textures/unprocessed")?;
 	let output=std::process::Command::new("asset-tool")
-	.args(["download","--cookie-literal","","--output-folder","textures/unprocessed/"])
+	.args(["download","environment","RBXCOOKIE","textures/unprocessed/"])
 	.args(texture_list_string)
 	.spawn()?
 	.wait_with_output()?;
@@ -282,9 +272,8 @@ fn download_meshes(paths:Vec<PathBuf>)->AResult<()>{
 	}
 	let mesh_list_string=mesh_list.into_iter().map(|id|id.to_string()).collect::<Vec<String>>();
 	println!("Mesh list:{:?}",mesh_list_string.join(" "));
-	std::fs::create_dir_all("meshes/")?;
 	let output=std::process::Command::new("asset-tool")
-	.args(["download","--cookie-literal","","--output-folder","meshes/"])
+	.args(["download","environment","RBXCOOKIE","meshes/"])
 	.args(mesh_list_string)
 	.spawn()?
 	.wait_with_output()?;
@@ -335,7 +324,7 @@ fn convert(file_thing:std::fs::DirEntry) -> AResult<()>{
 	)?;
 
 	//write dds
-	let mut dest=PathBuf::from("textures");
+	let mut dest=PathBuf::from("textures/dds");
 	dest.push(file_thing.file_name());
 	dest.set_extension("dds");
 	let mut writer = std::io::BufWriter::new(std::fs::File::create(dest)?);
@@ -357,8 +346,6 @@ fn convert(file_thing:std::fs::DirEntry) -> AResult<()>{
 	Ok(())
 }
 fn convert_textures() -> AResult<()>{
-	std::fs::create_dir_all("textures/unprocessed")?;
-	std::fs::create_dir_all("textures/processed")?;
 	let start = std::time::Instant::now();
 	let mut threads=Vec::new();
 	for entry in std::fs::read_dir("textures/unprocessed")? {
@@ -505,7 +492,6 @@ fn recursive_vmt_loader<F:Fn(String)->AResult<Option<Vec<u8>>>>(find_stuff:&F,ma
 }
 
 fn extract_textures(paths:Vec<PathBuf>,vpk_paths:Vec<PathBuf>)->AResult<()>{
-	std::fs::create_dir_all("textures")?;
 	let vpk_list:Vec<vpk::VPK>=vpk_paths.into_iter().map(|vpk_path|vpk::VPK::read(&vpk_path).expect("vpk file does not exist")).collect();
 	for path in paths{
 		let mut deduplicate=std::collections::HashSet::new();
@@ -574,7 +560,7 @@ fn extract_textures(paths:Vec<PathBuf>,vpk_paths:Vec<PathBuf>)->AResult<()>{
 					)?;
 
 					//write dds
-					let mut dest=PathBuf::from("textures");
+					let mut dest=PathBuf::from("textures/dds");
 					dest.push(write_file_name);
 					dest.set_extension("dds");
 					std::fs::create_dir_all(dest.parent().unwrap())?;
@@ -656,12 +642,10 @@ fn bsp_contents(path:PathBuf)->AResult<()>{
 }
 
 #[derive(Debug)]
-#[allow(dead_code)]
 enum ConvertError{
 	IO(std::io::Error),
 	SNFMap(strafesnet_snf::map::Error),
 	RbxLoader(strafesnet_rbx_loader::ReadError),
-	BspLoader(strafesnet_bsp_loader::ReadError),
 }
 impl std::fmt::Display for ConvertError{
 	fn fmt(&self,f:&mut std::fmt::Formatter<'_>)->std::fmt::Result{
@@ -670,34 +654,16 @@ impl std::fmt::Display for ConvertError{
 }
 impl std::error::Error for ConvertError{}
 
-type MapThread=std::thread::JoinHandle<Result<(),ConvertError>>;
+fn convert_to_snf(pathlist:Vec<std::path::PathBuf>,output_folder:PathBuf)->AResult<()>{
-
-fn roblox_to_snf(pathlist:Vec<std::path::PathBuf>,output_folder:PathBuf)->AResult<()>{
-	let n_paths=pathlist.len();
 	let start = std::time::Instant::now();
-	let mut threads:std::collections::VecDeque<MapThread>=std::collections::VecDeque::new();
+	let mut threads:Vec<std::thread::JoinHandle<Result<(),ConvertError>>>=Vec::new();
-	let mut i=0;
-	let mut join_thread=|thread:MapThread|{
-		i+=1;
-		if let Err(e)=thread.join(){
-			println!("thread error: {:?}",e);
-		}else{
-			println!("{}/{}",i,n_paths);
-		}
-	};
 	for path in pathlist{
-		if 32<=threads.len(){
-			join_thread(threads.pop_front().unwrap());
-		}
 		let output_folder=output_folder.clone();
-		threads.push_back(std::thread::spawn(move ||{
+		threads.push(std::thread::spawn(move ||{
-			let mut dom=strafesnet_rbx_loader::read(
+			let dom=strafesnet_rbx_loader::read(
 				std::fs::File::open(path.as_path())
 				.map_err(ConvertError::IO)?
 			).map_err(ConvertError::RbxLoader)?;
-
-			dom.run_scripts();
-
 			let mut loader=strafesnet_deferred_loader::roblox_legacy();
 
 			let (texture_loader,mesh_loader)=loader.get_inner_mut();
@@ -737,83 +703,15 @@ fn roblox_to_snf(pathlist:Vec<std::path::PathBuf>,output_folder:PathBuf)->AResul
 		}));
 	}
 
-	for thread in threads{
-		join_thread(thread);
-	}
-	println!("{:?}", start.elapsed());
-	Ok(())
-}
-
-fn source_to_snf(pathlist:Vec<std::path::PathBuf>,output_folder:PathBuf)->AResult<()>{
-	let n_paths=pathlist.len();
-	let start = std::time::Instant::now();
-	let mut threads:std::collections::VecDeque<MapThread>=std::collections::VecDeque::new();
 	let mut i=0;
-	let mut join_thread=|thread:MapThread|{
+	let n_threads=threads.len();
+	for thread in threads{
 		i+=1;
 		if let Err(e)=thread.join(){
 			println!("thread error: {:?}",e);
 		}else{
-			println!("{}/{}",i,n_paths);
+			println!("{}/{}",i,n_threads);
 		}
-	};
-	for path in pathlist{
-		if 32<=threads.len(){
-			join_thread(threads.pop_front().unwrap());
-		}
-		let output_folder=output_folder.clone();
-		threads.push_back(std::thread::spawn(move ||{
-			let bsp=strafesnet_bsp_loader::read(
-				std::fs::File::open(path.as_path())
-				.map_err(ConvertError::IO)?
-			).map_err(ConvertError::BspLoader)?;
-			let mut loader=strafesnet_deferred_loader::source_legacy();
-
-				let (texture_loader,mesh_loader)=loader.get_inner_mut();
-
-				let map_step1=strafesnet_bsp_loader::convert(
-					&bsp,
-					|name|texture_loader.acquire_render_config_id(name),
-					|name|mesh_loader.acquire_mesh_id(name),
-				);
-
-				let prop_meshes=mesh_loader.load_meshes(&bsp.as_ref());
-
-				let map_step2=map_step1.add_prop_meshes(
-					//the type conflagulator 9000
-					prop_meshes.into_iter().map(|(mesh_id,loader_model)|
-						(mesh_id,strafesnet_bsp_loader::data::ModelData{
-							mdl:strafesnet_bsp_loader::data::MdlData::new(loader_model.mdl.get()),
-							vtx:strafesnet_bsp_loader::data::VtxData::new(loader_model.vtx.get()),
-							vvd:strafesnet_bsp_loader::data::VvdData::new(loader_model.vvd.get()),
-						})
-					),
-					|name|texture_loader.acquire_render_config_id(name),
-				);
-
-				let (textures,render_configs)=loader.into_render_configs().map_err(ConvertError::IO)?.consume();
-
-				let map=map_step2.add_render_configs_and_textures(
-					render_configs.into_iter(),
-					textures.into_iter().map(|(texture_id,texture)|
-						(texture_id,match texture{
-							strafesnet_deferred_loader::texture::Texture::ImageDDS(data)=>data,
-						})
-					),
-				);
-
-			let mut dest=output_folder.clone();
-			dest.push(path.file_stem().unwrap());
-			dest.set_extension("snfm");
-			let file=std::fs::File::create(dest).map_err(ConvertError::IO)?;
-
-			strafesnet_snf::map::write_map(file,map).map_err(ConvertError::SNFMap)?;
-			Ok(())
-		}));
-	}
-
-	for thread in threads{
-		join_thread(thread);
 	}
 	println!("{:?}", start.elapsed());
 	Ok(())