Automatically create sub-directories when downloading assets; move dds files into the textures folder

the latter change is because roblox-to-snf expects the files in textures, not textures/dds

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "map-tool"
-version = "1.5.0"
+version = "1.5.3"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@@ -12,10 +12,14 @@ flate2 = "1.0.27"
 image = "0.24.7"
 image_dds = "0.1.1"
 lazy-regex = "3.1.0"
-rbx_binary = "0.7.1"
+rbx_binary = { version = "0.7.4", registry = "strafesnet" }
-rbx_dom_weak = "2.5.0"
+rbx_dom_weak = { version = "2.7.0", registry = "strafesnet" }
-rbx_reflection_database = "0.2.7"
+rbx_reflection_database = { version = "0.2.10", registry = "strafesnet" }
-rbx_xml = "0.13.1"
+rbx_xml = { version = "0.13.3", registry = "strafesnet" }
+strafesnet_bsp_loader = { version = "0.1.3", registry = "strafesnet" }
+strafesnet_deferred_loader = { version = "0.3.1", features = ["legacy"], registry = "strafesnet" }
+strafesnet_rbx_loader = { version = "0.3.6", registry = "strafesnet" }
+strafesnet_snf = { version = "0.1.0", registry = "strafesnet" }
 vbsp = "0.5.0"
 vmdl = "0.1.1"
 vmt-parser = "0.1.1"

src/main.rs

@@ -13,6 +13,8 @@ struct Cli {
 
 #[derive(Subcommand)]
 enum Commands {
+	RobloxToSNF(RobloxToSNFSubcommand),
+	SourceToSNF(SourceToSNFSubcommand),
 	DownloadTextures(DownloadTexturesSubcommand),
 	ExtractTextures(ExtractTexturesSubcommand),
 	ConvertTextures(ConvertTexturesSubcommand),
@@ -22,6 +24,20 @@ enum Commands {
 	WriteAttributes(WriteAttributesSubcommand),
 }
 
+#[derive(Args)]
+struct RobloxToSNFSubcommand {
+	#[arg(long)]
+	output_folder:PathBuf,
+	#[arg(required=true)]
+	input_files:Vec<PathBuf>,
+}
+#[derive(Args)]
+struct SourceToSNFSubcommand {
+	#[arg(long)]
+	output_folder:PathBuf,
+	#[arg(required=true)]
+	input_files:Vec<PathBuf>,
+}
 #[derive(Args)]
 struct DownloadTexturesSubcommand {
 	#[arg(long,required=true)]
@@ -59,6 +75,8 @@ struct WriteAttributesSubcommand {
 fn main() -> AResult<()> {
 	let cli = Cli::parse();
 	match cli.command {
+		Commands::RobloxToSNF(subcommand)=>roblox_to_snf(subcommand.input_files,subcommand.output_folder),
+		Commands::SourceToSNF(subcommand)=>source_to_snf(subcommand.input_files,subcommand.output_folder),
 		Commands::DownloadTextures(subcommand)=>download_textures(subcommand.roblox_files),
 		Commands::ExtractTextures(subcommand)=>extract_textures(vec![subcommand.bsp_file],subcommand.vpk_dir_files),
 		Commands::VPKContents(subcommand)=>vpk_contents(subcommand.input_file),
@@ -228,8 +246,9 @@ fn download_textures(paths:Vec<PathBuf>)->AResult<()>{
 	}
 	let texture_list_string=texture_list.into_iter().map(|id|id.to_string()).collect::<Vec<String>>();
 	println!("Texture list:{:?}",texture_list_string.join(" "));
+	std::fs::create_dir_all("textures/unprocessed")?;
 	let output=std::process::Command::new("asset-tool")
-	.args(["download","environment","RBXCOOKIE","textures/unprocessed/"])
+	.args(["download","--cookie-literal","","--output-folder","textures/unprocessed/"])
 	.args(texture_list_string)
 	.spawn()?
 	.wait_with_output()?;
@@ -263,8 +282,9 @@ fn download_meshes(paths:Vec<PathBuf>)->AResult<()>{
 	}
 	let mesh_list_string=mesh_list.into_iter().map(|id|id.to_string()).collect::<Vec<String>>();
 	println!("Mesh list:{:?}",mesh_list_string.join(" "));
+	std::fs::create_dir_all("meshes/")?;
 	let output=std::process::Command::new("asset-tool")
-	.args(["download","environment","RBXCOOKIE","meshes/"])
+	.args(["download","--cookie-literal","","--output-folder","meshes/"])
 	.args(mesh_list_string)
 	.spawn()?
 	.wait_with_output()?;
@@ -315,7 +335,7 @@ fn convert(file_thing:std::fs::DirEntry) -> AResult<()>{
 	)?;
 
 	//write dds
-	let mut dest=PathBuf::from("textures/dds");
+	let mut dest=PathBuf::from("textures");
 	dest.push(file_thing.file_name());
 	dest.set_extension("dds");
 	let mut writer = std::io::BufWriter::new(std::fs::File::create(dest)?);
@@ -337,6 +357,8 @@ fn convert(file_thing:std::fs::DirEntry) -> AResult<()>{
 	Ok(())
 }
 fn convert_textures() -> AResult<()>{
+	std::fs::create_dir_all("textures/unprocessed")?;
+	std::fs::create_dir_all("textures/processed")?;
 	let start = std::time::Instant::now();
 	let mut threads=Vec::new();
 	for entry in std::fs::read_dir("textures/unprocessed")? {
@@ -483,6 +505,7 @@ fn recursive_vmt_loader<F:Fn(String)->AResult<Option<Vec<u8>>>>(find_stuff:&F,ma
 }
 
 fn extract_textures(paths:Vec<PathBuf>,vpk_paths:Vec<PathBuf>)->AResult<()>{
+	std::fs::create_dir_all("textures")?;
 	let vpk_list:Vec<vpk::VPK>=vpk_paths.into_iter().map(|vpk_path|vpk::VPK::read(&vpk_path).expect("vpk file does not exist")).collect();
 	for path in paths{
 		let mut deduplicate=std::collections::HashSet::new();
@@ -551,7 +574,7 @@ fn extract_textures(paths:Vec<PathBuf>,vpk_paths:Vec<PathBuf>)->AResult<()>{
 					)?;
 
 					//write dds
-					let mut dest=PathBuf::from("textures/dds");
+					let mut dest=PathBuf::from("textures");
 					dest.push(write_file_name);
 					dest.set_extension("dds");
 					std::fs::create_dir_all(dest.parent().unwrap())?;
@@ -631,3 +654,167 @@ fn bsp_contents(path:PathBuf)->AResult<()>{
 	}
 	Ok(())
 }
+
+#[derive(Debug)]
+#[allow(dead_code)]
+enum ConvertError{
+	IO(std::io::Error),
+	SNFMap(strafesnet_snf::map::Error),
+	RbxLoader(strafesnet_rbx_loader::ReadError),
+	BspLoader(strafesnet_bsp_loader::ReadError),
+}
+impl std::fmt::Display for ConvertError{
+	fn fmt(&self,f:&mut std::fmt::Formatter<'_>)->std::fmt::Result{
+		write!(f,"{self:?}")
+	}
+}
+impl std::error::Error for ConvertError{}
+
+type MapThread=std::thread::JoinHandle<Result<(),ConvertError>>;
+
+fn roblox_to_snf(pathlist:Vec<std::path::PathBuf>,output_folder:PathBuf)->AResult<()>{
+	let n_paths=pathlist.len();
+	let start = std::time::Instant::now();
+	let mut threads:std::collections::VecDeque<MapThread>=std::collections::VecDeque::new();
+	let mut i=0;
+	let mut join_thread=|thread:MapThread|{
+		i+=1;
+		if let Err(e)=thread.join(){
+			println!("thread error: {:?}",e);
+		}else{
+			println!("{}/{}",i,n_paths);
+		}
+	};
+	for path in pathlist{
+		if 32<=threads.len(){
+			join_thread(threads.pop_front().unwrap());
+		}
+		let output_folder=output_folder.clone();
+		threads.push_back(std::thread::spawn(move ||{
+			let mut dom=strafesnet_rbx_loader::read(
+				std::fs::File::open(path.as_path())
+				.map_err(ConvertError::IO)?
+			).map_err(ConvertError::RbxLoader)?;
+
+			dom.run_scripts();
+
+			let mut loader=strafesnet_deferred_loader::roblox_legacy();
+
+			let (texture_loader,mesh_loader)=loader.get_inner_mut();
+
+			let map_step1=strafesnet_rbx_loader::convert(
+				&dom,
+				|name|texture_loader.acquire_render_config_id(name),
+				|name|mesh_loader.acquire_mesh_id(name),
+			);
+
+			let meshpart_meshes=mesh_loader.load_meshes().map_err(ConvertError::IO)?;
+
+			let map_step2=map_step1.add_meshpart_meshes_and_calculate_attributes(
+				meshpart_meshes.into_iter().map(|(mesh_id,loader_model)|
+					(mesh_id,strafesnet_rbx_loader::data::RobloxMeshBytes::new(loader_model.get()))
+				)
+			);
+
+			let (textures,render_configs)=loader.into_render_configs().map_err(ConvertError::IO)?.consume();
+
+			let map=map_step2.add_render_configs_and_textures(
+				render_configs.into_iter(),
+				textures.into_iter().map(|(texture_id,texture)|
+					(texture_id,match texture{
+						strafesnet_deferred_loader::texture::Texture::ImageDDS(data)=>data,
+					})
+				)
+			);
+
+			let mut dest=output_folder.clone();
+			dest.push(path.file_stem().unwrap());
+			dest.set_extension("snfm");
+			let file=std::fs::File::create(dest).map_err(ConvertError::IO)?;
+
+			strafesnet_snf::map::write_map(file,map).map_err(ConvertError::SNFMap)?;
+			Ok(())
+		}));
+	}
+
+	for thread in threads{
+		join_thread(thread);
+	}
+	println!("{:?}", start.elapsed());
+	Ok(())
+}
+
+fn source_to_snf(pathlist:Vec<std::path::PathBuf>,output_folder:PathBuf)->AResult<()>{
+	let n_paths=pathlist.len();
+	let start = std::time::Instant::now();
+	let mut threads:std::collections::VecDeque<MapThread>=std::collections::VecDeque::new();
+	let mut i=0;
+	let mut join_thread=|thread:MapThread|{
+		i+=1;
+		if let Err(e)=thread.join(){
+			println!("thread error: {:?}",e);
+		}else{
+			println!("{}/{}",i,n_paths);
+		}
+	};
+	for path in pathlist{
+		if 32<=threads.len(){
+			join_thread(threads.pop_front().unwrap());
+		}
+		let output_folder=output_folder.clone();
+		threads.push_back(std::thread::spawn(move ||{
+			let bsp=strafesnet_bsp_loader::read(
+				std::fs::File::open(path.as_path())
+				.map_err(ConvertError::IO)?
+			).map_err(ConvertError::BspLoader)?;
+			let mut loader=strafesnet_deferred_loader::source_legacy();
+
+				let (texture_loader,mesh_loader)=loader.get_inner_mut();
+
+				let map_step1=strafesnet_bsp_loader::convert(
+					&bsp,
+					|name|texture_loader.acquire_render_config_id(name),
+					|name|mesh_loader.acquire_mesh_id(name),
+				);
+
+				let prop_meshes=mesh_loader.load_meshes(&bsp.as_ref());
+
+				let map_step2=map_step1.add_prop_meshes(
+					//the type conflagulator 9000
+					prop_meshes.into_iter().map(|(mesh_id,loader_model)|
+						(mesh_id,strafesnet_bsp_loader::data::ModelData{
+							mdl:strafesnet_bsp_loader::data::MdlData::new(loader_model.mdl.get()),
+							vtx:strafesnet_bsp_loader::data::VtxData::new(loader_model.vtx.get()),
+							vvd:strafesnet_bsp_loader::data::VvdData::new(loader_model.vvd.get()),
+						})
+					),
+					|name|texture_loader.acquire_render_config_id(name),
+				);
+
+				let (textures,render_configs)=loader.into_render_configs().map_err(ConvertError::IO)?.consume();
+
+				let map=map_step2.add_render_configs_and_textures(
+					render_configs.into_iter(),
+					textures.into_iter().map(|(texture_id,texture)|
+						(texture_id,match texture{
+							strafesnet_deferred_loader::texture::Texture::ImageDDS(data)=>data,
+						})
+					),
+				);
+
+			let mut dest=output_folder.clone();
+			dest.push(path.file_stem().unwrap());
+			dest.set_extension("snfm");
+			let file=std::fs::File::create(dest).map_err(ConvertError::IO)?;
+
+			strafesnet_snf::map::write_map(file,map).map_err(ConvertError::SNFMap)?;
+			Ok(())
+		}));
+	}
+
+	for thread in threads{
+		join_thread(thread);
+	}
+	println!("{:?}", start.elapsed());
+	Ok(())
+}