Add strafesnet registry

the latter change is because roblox-to-snf expects the files in textures, not textures/dds

.cargo/config.toml

@@ -0,0 +1,2 @@
+[registries.strafesnet]
+index = "sparse+https://git.itzana.me/api/packages/strafesnet/cargo/"

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "map-tool"
-version = "1.5.0"
+version = "1.5.7"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@@ -9,21 +9,22 @@ edition = "2021"
 anyhow = "1.0.75"
 clap = { version = "4.4.2", features = ["derive"] }
 flate2 = "1.0.27"
-image = "0.24.7"
+image = "0.25.2"
-image_dds = "0.1.1"
+image_dds = "0.6.0"
 lazy-regex = "3.1.0"
-rbx_binary = "0.7.1"
+rbx_binary = { version = "0.7.4", registry = "strafesnet" }
-rbx_dom_weak = "2.5.0"
+rbx_dom_weak = { version = "2.7.0", registry = "strafesnet" }
-rbx_reflection_database = "0.2.7"
+rbx_reflection_database = { version = "0.2.10", registry = "strafesnet" }
-rbx_xml = "0.13.1"
+rbx_xml = { version = "0.13.3", registry = "strafesnet" }
-strafesnet_bsp_loader = { git = "https://git.itzana.me/StrafesNET/bsp_loader", rev = "79b8120106e63de237b8ed59c3d3e30db51e7146" }
+strafesnet_bsp_loader = { version = "0.2.1", registry = "strafesnet" }
-strafesnet_rbx_loader = { git = "https://git.itzana.me/StrafesNET/rbx_loader", rev = "0d2c27f60f4a448063edc57d9a23766775fb2cc5" }
+strafesnet_deferred_loader = { version = "0.4.0", features = ["legacy"], registry = "strafesnet" }
-strafesnet_snf = { git = "https://git.itzana.me/StrafesNET/snf", rev = "dea408daeef576cff8ffa61356c89a9d03d95f6b" }
+strafesnet_rbx_loader = { version = "0.5.1", registry = "strafesnet" }
-vbsp = "0.5.0"
+strafesnet_snf = { version = "0.2.0", registry = "strafesnet" }
-vmdl = "0.1.1"
+vbsp = "0.6.0"
-vmt-parser = "0.1.1"
+vmdl = "0.2.0"
+vmt-parser = "0.2.0"
 vpk = "0.2.0"
-vtf = "0.2.1"
+vtf = "0.3.0"
 
 #[profile.release]
 #lto = true

src/main.rs

@@ -2,6 +2,7 @@ use std::{collections::HashSet,io::{Read,Seek},path::PathBuf};
 use clap::{Args,Parser,Subcommand};
 use anyhow::Result as AResult;
 use rbx_dom_weak::Instance;
+use strafesnet_deferred_loader::rbxassetid::RobloxAssetId;
 
 #[derive(Parser)]
 #[command(author, version, about, long_about = None)]
@@ -14,6 +15,7 @@ struct Cli {
 #[derive(Subcommand)]
 enum Commands {
 	RobloxToSNF(RobloxToSNFSubcommand),
+	SourceToSNF(SourceToSNFSubcommand),
 	DownloadTextures(DownloadTexturesSubcommand),
 	ExtractTextures(ExtractTexturesSubcommand),
 	ConvertTextures(ConvertTexturesSubcommand),
@@ -31,6 +33,13 @@ struct RobloxToSNFSubcommand {
 	input_files:Vec<PathBuf>,
 }
 #[derive(Args)]
+struct SourceToSNFSubcommand {
+	#[arg(long)]
+	output_folder:PathBuf,
+	#[arg(required=true)]
+	input_files:Vec<PathBuf>,
+}
+#[derive(Args)]
 struct DownloadTexturesSubcommand {
 	#[arg(long,required=true)]
 	roblox_files:Vec<PathBuf>
@@ -67,7 +76,8 @@ struct WriteAttributesSubcommand {
 fn main() -> AResult<()> {
 	let cli = Cli::parse();
 	match cli.command {
-		Commands::RobloxToSNF(subcommand)=>convert_to_snf(subcommand.input_files,subcommand.output_folder),
+		Commands::RobloxToSNF(subcommand)=>roblox_to_snf(subcommand.input_files,subcommand.output_folder),
+		Commands::SourceToSNF(subcommand)=>source_to_snf(subcommand.input_files,subcommand.output_folder),
 		Commands::DownloadTextures(subcommand)=>download_textures(subcommand.roblox_files),
 		Commands::ExtractTextures(subcommand)=>extract_textures(vec![subcommand.bsp_file],subcommand.vpk_dir_files),
 		Commands::VPKContents(subcommand)=>vpk_contents(subcommand.input_file),
@@ -143,22 +153,6 @@ fn get_dom<R:Read+Seek>(input:&mut R)->AResult<rbx_dom_weak::WeakDom>{
 	}
 }
 
-struct RobloxAssetId(u64);
-struct RobloxAssetIdParseErr;
-impl std::str::FromStr for RobloxAssetId {
-	type Err=RobloxAssetIdParseErr;
-	fn from_str(s: &str) -> Result<Self, Self::Err>{
-		let regman=lazy_regex::regex!(r"(\d+)$");
-		if let Some(captures) = regman.captures(s) {
-			if captures.len()==2{//captures[0] is all captures concatenated, and then each individual capture
-				if let Ok(id) = captures[0].parse::<u64>() {
-					return Ok(Self(id));
-				}
-			}
-		}
-		Err(RobloxAssetIdParseErr)
-	}
-}
 /* The ones I'm interested in:
 Beam.Texture
 Decal.Texture
@@ -237,8 +231,9 @@ fn download_textures(paths:Vec<PathBuf>)->AResult<()>{
 	}
 	let texture_list_string=texture_list.into_iter().map(|id|id.to_string()).collect::<Vec<String>>();
 	println!("Texture list:{:?}",texture_list_string.join(" "));
+	std::fs::create_dir_all("textures/unprocessed")?;
 	let output=std::process::Command::new("asset-tool")
-	.args(["download","environment","RBXCOOKIE","textures/unprocessed/"])
+	.args(["download","--cookie-literal","","--output-folder","textures/unprocessed/"])
 	.args(texture_list_string)
 	.spawn()?
 	.wait_with_output()?;
@@ -272,8 +267,9 @@ fn download_meshes(paths:Vec<PathBuf>)->AResult<()>{
 	}
 	let mesh_list_string=mesh_list.into_iter().map(|id|id.to_string()).collect::<Vec<String>>();
 	println!("Mesh list:{:?}",mesh_list_string.join(" "));
+	std::fs::create_dir_all("meshes/")?;
 	let output=std::process::Command::new("asset-tool")
-	.args(["download","environment","RBXCOOKIE","meshes/"])
+	.args(["download","--cookie-literal","","--output-folder","meshes/"])
 	.args(mesh_list_string)
 	.spawn()?
 	.wait_with_output()?;
@@ -311,9 +307,9 @@ fn convert(file_thing:std::fs::DirEntry) -> AResult<()>{
 	}?.to_rgba8();//this sets a=255, arcane is actually supposed to look like that
 
 	let format=if image.width()%4!=0||image.height()%4!=0{
-		image_dds::ImageFormat::R8G8B8A8Srgb
+		image_dds::ImageFormat::Rgba8UnormSrgb
 	}else{
-		image_dds::ImageFormat::BC7Srgb
+		image_dds::ImageFormat::BC7RgbaUnormSrgb
 	};
 	//this fails if the image dimensions are not a multiple of 4
 	let dds = image_dds::dds_from_image(
@@ -324,7 +320,7 @@ fn convert(file_thing:std::fs::DirEntry) -> AResult<()>{
 	)?;
 
 	//write dds
-	let mut dest=PathBuf::from("textures/dds");
+	let mut dest=PathBuf::from("textures");
 	dest.push(file_thing.file_name());
 	dest.set_extension("dds");
 	let mut writer = std::io::BufWriter::new(std::fs::File::create(dest)?);
@@ -346,6 +342,8 @@ fn convert(file_thing:std::fs::DirEntry) -> AResult<()>{
 	Ok(())
 }
 fn convert_textures() -> AResult<()>{
+	std::fs::create_dir_all("textures/unprocessed")?;
+	std::fs::create_dir_all("textures/processed")?;
 	let start = std::time::Instant::now();
 	let mut threads=Vec::new();
 	for entry in std::fs::read_dir("textures/unprocessed")? {
@@ -492,6 +490,7 @@ fn recursive_vmt_loader<F:Fn(String)->AResult<Option<Vec<u8>>>>(find_stuff:&F,ma
 }
 
 fn extract_textures(paths:Vec<PathBuf>,vpk_paths:Vec<PathBuf>)->AResult<()>{
+	std::fs::create_dir_all("textures")?;
 	let vpk_list:Vec<vpk::VPK>=vpk_paths.into_iter().map(|vpk_path|vpk::VPK::read(&vpk_path).expect("vpk file does not exist")).collect();
 	for path in paths{
 		let mut deduplicate=std::collections::HashSet::new();
@@ -547,9 +546,9 @@ fn extract_textures(paths:Vec<PathBuf>,vpk_paths:Vec<PathBuf>)->AResult<()>{
 					let image=vtf::from_bytes(&mut stuff)?.highres_image.decode(0)?.to_rgba8();
 
 					let format=if image.width()%4!=0||image.height()%4!=0{
-						image_dds::ImageFormat::R8G8B8A8Srgb
+						image_dds::ImageFormat::Rgba8UnormSrgb
 					}else{
-						image_dds::ImageFormat::BC7Srgb
+						image_dds::ImageFormat::BC7RgbaUnormSrgb
 					};
 					//this fails if the image dimensions are not a multiple of 4
 					let dds = image_dds::dds_from_image(
@@ -560,7 +559,7 @@ fn extract_textures(paths:Vec<PathBuf>,vpk_paths:Vec<PathBuf>)->AResult<()>{
 					)?;
 
 					//write dds
-					let mut dest=PathBuf::from("textures/dds");
+					let mut dest=PathBuf::from("textures");
 					dest.push(write_file_name);
 					dest.set_extension("dds");
 					std::fs::create_dir_all(dest.parent().unwrap())?;
@@ -642,10 +641,12 @@ fn bsp_contents(path:PathBuf)->AResult<()>{
 }
 
 #[derive(Debug)]
+#[allow(dead_code)]
 enum ConvertError{
 	IO(std::io::Error),
 	SNFMap(strafesnet_snf::map::Error),
 	RbxLoader(strafesnet_rbx_loader::ReadError),
+	BspLoader(strafesnet_bsp_loader::ReadError),
 }
 impl std::fmt::Display for ConvertError{
 	fn fmt(&self,f:&mut std::fmt::Formatter<'_>)->std::fmt::Result{
@@ -654,22 +655,41 @@ impl std::fmt::Display for ConvertError{
 }
 impl std::error::Error for ConvertError{}
 
-fn convert_to_snf(pathlist:Vec<std::path::PathBuf>,output_folder:PathBuf)->AResult<()>{
+type MapThread=std::thread::JoinHandle<Result<(),ConvertError>>;
+
+fn roblox_to_snf(pathlist:Vec<std::path::PathBuf>,output_folder:PathBuf)->AResult<()>{
+	let n_paths=pathlist.len();
 	let start = std::time::Instant::now();
-	let mut threads:Vec<std::thread::JoinHandle<Result<(),ConvertError>>>=Vec::new();
+	let mut threads:std::collections::VecDeque<MapThread>=std::collections::VecDeque::new();
+	let mut i=0;
+	let mut join_thread=|thread:MapThread|{
+		i+=1;
+		if let Err(e)=thread.join(){
+			println!("thread error: {:?}",e);
+		}else{
+			println!("{}/{}",i,n_paths);
+		}
+	};
 	for path in pathlist{
+		if 32<=threads.len(){
+			join_thread(threads.pop_front().unwrap());
+		}
 		let output_folder=output_folder.clone();
-		threads.push(std::thread::spawn(move ||{
+		threads.push_back(std::thread::spawn(move ||{
-			let dom=strafesnet_rbx_loader::read(
+			let model=strafesnet_rbx_loader::read(
 				std::fs::File::open(path.as_path())
 				.map_err(ConvertError::IO)?
 			).map_err(ConvertError::RbxLoader)?;
+
+			let mut place=model.into_place();
+			place.run_scripts();
+
 			let mut loader=strafesnet_deferred_loader::roblox_legacy();
 
 			let (texture_loader,mesh_loader)=loader.get_inner_mut();
 
 			let map_step1=strafesnet_rbx_loader::convert(
-				&dom,
+				&place,
 				|name|texture_loader.acquire_render_config_id(name),
 				|name|mesh_loader.acquire_mesh_id(name),
 			);
@@ -703,15 +723,83 @@ fn convert_to_snf(pathlist:Vec<std::path::PathBuf>,output_folder:PathBuf)->AResu
 		}));
 	}
 
-	let mut i=0;
-	let n_threads=threads.len();
 	for thread in threads{
-		i+=1;
+		join_thread(thread);
-		if let Err(e)=thread.join(){
+	}
-			println!("thread error: {:?}",e);
+	println!("{:?}", start.elapsed());
-		}else{
+	Ok(())
-			println!("{}/{}",i,n_threads);
+}
-		}
+
+fn source_to_snf(pathlist:Vec<std::path::PathBuf>,output_folder:PathBuf)->AResult<()>{
+	let n_paths=pathlist.len();
+	let start = std::time::Instant::now();
+	let mut threads:std::collections::VecDeque<MapThread>=std::collections::VecDeque::new();
+	let mut i=0;
+	let mut join_thread=|thread:MapThread|{
+		i+=1;
+		if let Err(e)=thread.join(){
+			println!("thread error: {:?}",e);
+		}else{
+			println!("{}/{}",i,n_paths);
+		}
+	};
+	for path in pathlist{
+		if 32<=threads.len(){
+			join_thread(threads.pop_front().unwrap());
+		}
+		let output_folder=output_folder.clone();
+		threads.push_back(std::thread::spawn(move ||{
+			let bsp=strafesnet_bsp_loader::read(
+				std::fs::File::open(path.as_path())
+				.map_err(ConvertError::IO)?
+			).map_err(ConvertError::BspLoader)?;
+			let mut loader=strafesnet_deferred_loader::source_legacy();
+
+				let (texture_loader,mesh_loader)=loader.get_inner_mut();
+
+				let map_step1=strafesnet_bsp_loader::convert(
+					&bsp,
+					|name|texture_loader.acquire_render_config_id(name),
+					|name|mesh_loader.acquire_mesh_id(name),
+				);
+
+				let prop_meshes=mesh_loader.load_meshes(&bsp.as_ref());
+
+				let map_step2=map_step1.add_prop_meshes(
+					//the type conflagulator 9000
+					prop_meshes.into_iter().map(|(mesh_id,loader_model)|
+						(mesh_id,strafesnet_bsp_loader::data::ModelData{
+							mdl:strafesnet_bsp_loader::data::MdlData::new(loader_model.mdl.get()),
+							vtx:strafesnet_bsp_loader::data::VtxData::new(loader_model.vtx.get()),
+							vvd:strafesnet_bsp_loader::data::VvdData::new(loader_model.vvd.get()),
+						})
+					),
+					|name|texture_loader.acquire_render_config_id(name),
+				);
+
+				let (textures,render_configs)=loader.into_render_configs().map_err(ConvertError::IO)?.consume();
+
+				let map=map_step2.add_render_configs_and_textures(
+					render_configs.into_iter(),
+					textures.into_iter().map(|(texture_id,texture)|
+						(texture_id,match texture{
+							strafesnet_deferred_loader::texture::Texture::ImageDDS(data)=>data,
+						})
+					),
+				);
+
+			let mut dest=output_folder.clone();
+			dest.push(path.file_stem().unwrap());
+			dest.set_extension("snfm");
+			let file=std::fs::File::create(dest).map_err(ConvertError::IO)?;
+
+			strafesnet_snf::map::write_map(file,map).map_err(ConvertError::SNFMap)?;
+			Ok(())
+		}));
+	}
+
+	for thread in threads{
+		join_thread(thread);
 	}
 	println!("{:?}", start.elapsed());
 	Ok(())