Automatically create sub-directories when downloading assets; move dds files into the textures folder

the latter change is because roblox-to-snf expects the files in textures, not textures/dds

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "map-tool"
-version = "1.5.0"
+version = "1.5.3"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@@ -12,13 +12,14 @@ flate2 = "1.0.27"
 image = "0.24.7"
 image_dds = "0.1.1"
 lazy-regex = "3.1.0"
-rbx_binary = "0.7.1"
+rbx_binary = { version = "0.7.4", registry = "strafesnet" }
-rbx_dom_weak = "2.5.0"
+rbx_dom_weak = { version = "2.7.0", registry = "strafesnet" }
-rbx_reflection_database = "0.2.7"
+rbx_reflection_database = { version = "0.2.10", registry = "strafesnet" }
-rbx_xml = "0.13.1"
+rbx_xml = { version = "0.13.3", registry = "strafesnet" }
-strafesnet_bsp_loader = { git = "https://git.itzana.me/StrafesNET/bsp_loader", rev = "79b8120106e63de237b8ed59c3d3e30db51e7146" }
+strafesnet_bsp_loader = { version = "0.1.3", registry = "strafesnet" }
-strafesnet_rbx_loader = { git = "https://git.itzana.me/StrafesNET/rbx_loader", rev = "0d2c27f60f4a448063edc57d9a23766775fb2cc5" }
+strafesnet_deferred_loader = { version = "0.3.1", features = ["legacy"], registry = "strafesnet" }
-strafesnet_snf = { git = "https://git.itzana.me/StrafesNET/snf", rev = "dea408daeef576cff8ffa61356c89a9d03d95f6b" }
+strafesnet_rbx_loader = { version = "0.3.6", registry = "strafesnet" }
+strafesnet_snf = { version = "0.1.0", registry = "strafesnet" }
 vbsp = "0.5.0"
 vmdl = "0.1.1"
 vmt-parser = "0.1.1"

src/main.rs

@@ -14,6 +14,7 @@ struct Cli {
 #[derive(Subcommand)]
 enum Commands {
 	RobloxToSNF(RobloxToSNFSubcommand),
+	SourceToSNF(SourceToSNFSubcommand),
 	DownloadTextures(DownloadTexturesSubcommand),
 	ExtractTextures(ExtractTexturesSubcommand),
 	ConvertTextures(ConvertTexturesSubcommand),
@@ -31,6 +32,13 @@ struct RobloxToSNFSubcommand {
 	input_files:Vec<PathBuf>,
 }
 #[derive(Args)]
+struct SourceToSNFSubcommand {
+	#[arg(long)]
+	output_folder:PathBuf,
+	#[arg(required=true)]
+	input_files:Vec<PathBuf>,
+}
+#[derive(Args)]
 struct DownloadTexturesSubcommand {
 	#[arg(long,required=true)]
 	roblox_files:Vec<PathBuf>
@@ -67,7 +75,8 @@ struct WriteAttributesSubcommand {
 fn main() -> AResult<()> {
 	let cli = Cli::parse();
 	match cli.command {
-		Commands::RobloxToSNF(subcommand)=>convert_to_snf(subcommand.input_files,subcommand.output_folder),
+		Commands::RobloxToSNF(subcommand)=>roblox_to_snf(subcommand.input_files,subcommand.output_folder),
+		Commands::SourceToSNF(subcommand)=>source_to_snf(subcommand.input_files,subcommand.output_folder),
 		Commands::DownloadTextures(subcommand)=>download_textures(subcommand.roblox_files),
 		Commands::ExtractTextures(subcommand)=>extract_textures(vec![subcommand.bsp_file],subcommand.vpk_dir_files),
 		Commands::VPKContents(subcommand)=>vpk_contents(subcommand.input_file),
@@ -237,8 +246,9 @@ fn download_textures(paths:Vec<PathBuf>)->AResult<()>{
 	}
 	let texture_list_string=texture_list.into_iter().map(|id|id.to_string()).collect::<Vec<String>>();
 	println!("Texture list:{:?}",texture_list_string.join(" "));
+	std::fs::create_dir_all("textures/unprocessed")?;
 	let output=std::process::Command::new("asset-tool")
-	.args(["download","environment","RBXCOOKIE","textures/unprocessed/"])
+	.args(["download","--cookie-literal","","--output-folder","textures/unprocessed/"])
 	.args(texture_list_string)
 	.spawn()?
 	.wait_with_output()?;
@@ -272,8 +282,9 @@ fn download_meshes(paths:Vec<PathBuf>)->AResult<()>{
 	}
 	let mesh_list_string=mesh_list.into_iter().map(|id|id.to_string()).collect::<Vec<String>>();
 	println!("Mesh list:{:?}",mesh_list_string.join(" "));
+	std::fs::create_dir_all("meshes/")?;
 	let output=std::process::Command::new("asset-tool")
-	.args(["download","environment","RBXCOOKIE","meshes/"])
+	.args(["download","--cookie-literal","","--output-folder","meshes/"])
 	.args(mesh_list_string)
 	.spawn()?
 	.wait_with_output()?;
@@ -324,7 +335,7 @@ fn convert(file_thing:std::fs::DirEntry) -> AResult<()>{
 	)?;
 
 	//write dds
-	let mut dest=PathBuf::from("textures/dds");
+	let mut dest=PathBuf::from("textures");
 	dest.push(file_thing.file_name());
 	dest.set_extension("dds");
 	let mut writer = std::io::BufWriter::new(std::fs::File::create(dest)?);
@@ -346,6 +357,8 @@ fn convert(file_thing:std::fs::DirEntry) -> AResult<()>{
 	Ok(())
 }
 fn convert_textures() -> AResult<()>{
+	std::fs::create_dir_all("textures/unprocessed")?;
+	std::fs::create_dir_all("textures/processed")?;
 	let start = std::time::Instant::now();
 	let mut threads=Vec::new();
 	for entry in std::fs::read_dir("textures/unprocessed")? {
@@ -492,6 +505,7 @@ fn recursive_vmt_loader<F:Fn(String)->AResult<Option<Vec<u8>>>>(find_stuff:&F,ma
 }
 
 fn extract_textures(paths:Vec<PathBuf>,vpk_paths:Vec<PathBuf>)->AResult<()>{
+	std::fs::create_dir_all("textures")?;
 	let vpk_list:Vec<vpk::VPK>=vpk_paths.into_iter().map(|vpk_path|vpk::VPK::read(&vpk_path).expect("vpk file does not exist")).collect();
 	for path in paths{
 		let mut deduplicate=std::collections::HashSet::new();
@@ -560,7 +574,7 @@ fn extract_textures(paths:Vec<PathBuf>,vpk_paths:Vec<PathBuf>)->AResult<()>{
 					)?;
 
 					//write dds
-					let mut dest=PathBuf::from("textures/dds");
+					let mut dest=PathBuf::from("textures");
 					dest.push(write_file_name);
 					dest.set_extension("dds");
 					std::fs::create_dir_all(dest.parent().unwrap())?;
@@ -642,10 +656,12 @@ fn bsp_contents(path:PathBuf)->AResult<()>{
 }
 
 #[derive(Debug)]
+#[allow(dead_code)]
 enum ConvertError{
 	IO(std::io::Error),
 	SNFMap(strafesnet_snf::map::Error),
 	RbxLoader(strafesnet_rbx_loader::ReadError),
+	BspLoader(strafesnet_bsp_loader::ReadError),
 }
 impl std::fmt::Display for ConvertError{
 	fn fmt(&self,f:&mut std::fmt::Formatter<'_>)->std::fmt::Result{
@@ -654,16 +670,34 @@ impl std::fmt::Display for ConvertError{
 }
 impl std::error::Error for ConvertError{}
 
-fn convert_to_snf(pathlist:Vec<std::path::PathBuf>,output_folder:PathBuf)->AResult<()>{
+type MapThread=std::thread::JoinHandle<Result<(),ConvertError>>;
+
+fn roblox_to_snf(pathlist:Vec<std::path::PathBuf>,output_folder:PathBuf)->AResult<()>{
+	let n_paths=pathlist.len();
 	let start = std::time::Instant::now();
-	let mut threads:Vec<std::thread::JoinHandle<Result<(),ConvertError>>>=Vec::new();
+	let mut threads:std::collections::VecDeque<MapThread>=std::collections::VecDeque::new();
+	let mut i=0;
+	let mut join_thread=|thread:MapThread|{
+		i+=1;
+		if let Err(e)=thread.join(){
+			println!("thread error: {:?}",e);
+		}else{
+			println!("{}/{}",i,n_paths);
+		}
+	};
 	for path in pathlist{
+		if 32<=threads.len(){
+			join_thread(threads.pop_front().unwrap());
+		}
 		let output_folder=output_folder.clone();
-		threads.push(std::thread::spawn(move ||{
+		threads.push_back(std::thread::spawn(move ||{
-			let dom=strafesnet_rbx_loader::read(
+			let mut dom=strafesnet_rbx_loader::read(
 				std::fs::File::open(path.as_path())
 				.map_err(ConvertError::IO)?
 			).map_err(ConvertError::RbxLoader)?;
+
+			dom.run_scripts();
+
 			let mut loader=strafesnet_deferred_loader::roblox_legacy();
 
 			let (texture_loader,mesh_loader)=loader.get_inner_mut();
@@ -703,15 +737,83 @@ fn convert_to_snf(pathlist:Vec<std::path::PathBuf>,output_folder:PathBuf)->AResu
 		}));
 	}
 
-	let mut i=0;
-	let n_threads=threads.len();
 	for thread in threads{
-		i+=1;
+		join_thread(thread);
-		if let Err(e)=thread.join(){
+	}
-			println!("thread error: {:?}",e);
+	println!("{:?}", start.elapsed());
-		}else{
+	Ok(())
-			println!("{}/{}",i,n_threads);
+}
-		}
+
+fn source_to_snf(pathlist:Vec<std::path::PathBuf>,output_folder:PathBuf)->AResult<()>{
+	let n_paths=pathlist.len();
+	let start = std::time::Instant::now();
+	let mut threads:std::collections::VecDeque<MapThread>=std::collections::VecDeque::new();
+	let mut i=0;
+	let mut join_thread=|thread:MapThread|{
+		i+=1;
+		if let Err(e)=thread.join(){
+			println!("thread error: {:?}",e);
+		}else{
+			println!("{}/{}",i,n_paths);
+		}
+	};
+	for path in pathlist{
+		if 32<=threads.len(){
+			join_thread(threads.pop_front().unwrap());
+		}
+		let output_folder=output_folder.clone();
+		threads.push_back(std::thread::spawn(move ||{
+			let bsp=strafesnet_bsp_loader::read(
+				std::fs::File::open(path.as_path())
+				.map_err(ConvertError::IO)?
+			).map_err(ConvertError::BspLoader)?;
+			let mut loader=strafesnet_deferred_loader::source_legacy();
+
+				let (texture_loader,mesh_loader)=loader.get_inner_mut();
+
+				let map_step1=strafesnet_bsp_loader::convert(
+					&bsp,
+					|name|texture_loader.acquire_render_config_id(name),
+					|name|mesh_loader.acquire_mesh_id(name),
+				);
+
+				let prop_meshes=mesh_loader.load_meshes(&bsp.as_ref());
+
+				let map_step2=map_step1.add_prop_meshes(
+					//the type conflagulator 9000
+					prop_meshes.into_iter().map(|(mesh_id,loader_model)|
+						(mesh_id,strafesnet_bsp_loader::data::ModelData{
+							mdl:strafesnet_bsp_loader::data::MdlData::new(loader_model.mdl.get()),
+							vtx:strafesnet_bsp_loader::data::VtxData::new(loader_model.vtx.get()),
+							vvd:strafesnet_bsp_loader::data::VvdData::new(loader_model.vvd.get()),
+						})
+					),
+					|name|texture_loader.acquire_render_config_id(name),
+				);
+
+				let (textures,render_configs)=loader.into_render_configs().map_err(ConvertError::IO)?.consume();
+
+				let map=map_step2.add_render_configs_and_textures(
+					render_configs.into_iter(),
+					textures.into_iter().map(|(texture_id,texture)|
+						(texture_id,match texture{
+							strafesnet_deferred_loader::texture::Texture::ImageDDS(data)=>data,
+						})
+					),
+				);
+
+			let mut dest=output_folder.clone();
+			dest.push(path.file_stem().unwrap());
+			dest.set_extension("snfm");
+			let file=std::fs::File::create(dest).map_err(ConvertError::IO)?;
+
+			strafesnet_snf::map::write_map(file,map).map_err(ConvertError::SNFMap)?;
+			Ok(())
+		}));
+	}
+
+	for thread in threads{
+		join_thread(thread);
 	}
 	println!("{:?}", start.elapsed());
 	Ok(())