tokio-ify source-to-snf

this will probably still hit the roblox rate limit

Cargo.toml

@@ -9,9 +9,11 @@ edition = "2021"
 anyhow = "1.0.75"
 clap = { version = "4.4.2", features = ["derive"] }
 flate2 = "1.0.27"
+futures = "0.3.31"
 image = "0.25.2"
-image_dds = "0.6.0"
+image_dds = "0.7.1"
 lazy-regex = "3.1.0"
+rbx_asset = { version = "0.2.5", registry = "strafesnet" }
 rbx_binary = { version = "0.7.4", registry = "strafesnet" }
 rbx_dom_weak = { version = "2.7.0", registry = "strafesnet" }
 rbx_reflection_database = { version = "0.2.10", registry = "strafesnet" }
@@ -20,6 +22,7 @@ strafesnet_bsp_loader = { version = "0.2.1", registry = "strafesnet" }
 strafesnet_deferred_loader = { version = "0.4.0", features = ["legacy"], registry = "strafesnet" }
 strafesnet_rbx_loader = { version = "0.5.1", registry = "strafesnet" }
 strafesnet_snf = { version = "0.2.0", registry = "strafesnet" }
+tokio = { version = "1.43.0", features = ["macros", "rt-multi-thread", "fs"] }
 vbsp = "0.6.0"
 vmdl = "0.2.0"
 vmt-parser = "0.2.0"

src/main.rs

@@ -22,11 +22,12 @@ enum Commands{
 	ConvertTextures,
 }
 
-fn main() -> AResult<()> {
-	let cli = Cli::parse();
+#[tokio::main]
+async fn main()->AResult<()>{
+	let cli=Cli::parse();
 	match cli.command{
-		Commands::Roblox(commands)=>commands.run(),
-		Commands::Source(commands)=>commands.run(),
+		Commands::Roblox(commands)=>commands.run().await,
+		Commands::Source(commands)=>commands.run().await,
 		Commands::ConvertTextures=>common::convert_textures(),
 	}
 }

src/roblox.rs

@@ -4,7 +4,11 @@ use std::collections::HashSet;
 use clap::{Args,Subcommand};
 use anyhow::Result as AResult;
 use rbx_dom_weak::Instance;
-use strafesnet_deferred_loader::rbxassetid::RobloxAssetId;
+use strafesnet_deferred_loader::deferred_loader::LoadFailureMode;
+use rbxassetid::RobloxAssetId;
+use tokio::io::AsyncReadExt;
+
+const DOWNLOAD_LIMIT:usize=16;
 
 #[derive(Subcommand)]
 pub enum Commands{
@@ -21,27 +25,40 @@ pub struct RobloxToSNFSubcommand {
 }
 #[derive(Args)]
 pub struct DownloadAssetsSubcommand{
-	#[arg(long,required=true)]
-	roblox_files:Vec<PathBuf>
+	#[arg(required=true)]
+	roblox_files:Vec<PathBuf>,
+	// #[arg(long)]
+	// cookie_file:Option<String>,
 }
 
 impl Commands{
-	pub fn run(self)->AResult<()>{
+	pub async fn run(self)->AResult<()>{
 		match self{
-			Commands::RobloxToSNF(subcommand)=>roblox_to_snf(subcommand.input_files,subcommand.output_folder),
-			Commands::DownloadAssets(subcommand)=>download_assets(subcommand.roblox_files),
+			Commands::RobloxToSNF(subcommand)=>roblox_to_snf(subcommand.input_files,subcommand.output_folder).await,
+			Commands::DownloadAssets(subcommand)=>download_assets(
+				subcommand.roblox_files,
+				rbx_asset::cookie::Cookie::new("".to_string()),
+			).await,
 		}
 	}
 }
 
-fn load_dom<R:Read+Seek>(mut input:R)->AResult<rbx_dom_weak::WeakDom>{
+#[allow(unused)]
+#[derive(Debug)]
+enum LoadDomError{
+	IO(std::io::Error),
+	Binary(rbx_binary::DecodeError),
+	Xml(rbx_xml::DecodeError),
+	UnknownFormat,
+}
+fn load_dom<R:Read+Seek>(mut input:R)->Result<rbx_dom_weak::WeakDom,LoadDomError>{
 	let mut first_8=[0u8;8];
-	input.read_exact(&mut first_8)?;
-	input.rewind()?;
+	input.read_exact(&mut first_8).map_err(LoadDomError::IO)?;
+	input.rewind().map_err(LoadDomError::IO)?;
 	match &first_8{
-		b"<roblox!"=>rbx_binary::from_reader(input).map_err(anyhow::Error::msg),
-		b"<roblox "=>rbx_xml::from_reader(input,rbx_xml::DecodeOptions::default()).map_err(anyhow::Error::msg),
-		_=>Err(anyhow::Error::msg("unsupported file type")),
+		b"<roblox!"=>rbx_binary::from_reader(input).map_err(LoadDomError::Binary),
+		b"<roblox "=>rbx_xml::from_reader(input,rbx_xml::DecodeOptions::default()).map_err(LoadDomError::Xml),
+		_=>Err(LoadDomError::UnknownFormat),
 	}
 }
 
@@ -80,13 +97,13 @@ fn accumulate_content_id(content_list:&mut HashSet<RobloxAssetId>,object:&Instan
 			println!("Content failed to parse into AssetID: {:?}",content);
 		}
 	}else{
-		println!("property={} does not exist for class={}",object.class.as_str(),property);
+		println!("property={} does not exist for class={}",property,object.class.as_str());
 	}
 }
-fn read_entire_file(path:impl AsRef<Path>)->Result<Cursor<Vec<u8>>,std::io::Error>{
-	let mut file=std::fs::File::open(path)?;
+async fn read_entire_file(path:impl AsRef<Path>)->Result<Cursor<Vec<u8>>,std::io::Error>{
+	let mut file=tokio::fs::File::open(path).await?;
 	let mut data=Vec::new();
-	file.read_to_end(&mut data)?;
+	file.read_to_end(&mut data).await?;
 	Ok(Cursor::new(data))
 }
 #[derive(Default)]
@@ -123,30 +140,249 @@ impl UniqueAssets{
 		}
 	}
 }
-fn unique_assets(path:&Path)->AResult<UniqueAssets>{
+
+#[allow(unused)]
+#[derive(Debug)]
+enum UniqueAssetError{
+	IO(std::io::Error),
+	LoadDom(LoadDomError),
+}
+async fn unique_assets(path:&Path)->Result<UniqueAssets,UniqueAssetError>{
 	// read entire file
 	let mut assets=UniqueAssets::default();
-	let data=read_entire_file(path)?;
-	let dom=load_dom(data)?;
+	let data=read_entire_file(path).await.map_err(UniqueAssetError::IO)?;
+	let dom=load_dom(data).map_err(UniqueAssetError::LoadDom)?;
 	for object in dom.into_raw().1.into_values(){
 		assets.collect(&object);
 	}
 	Ok(assets)
 }
-struct UniqueAssetsResult{
-	path:std::path::PathBuf,
-	result:AResult<UniqueAssets>,
+enum DownloadType{
+	Texture(RobloxAssetId),
+	Mesh(RobloxAssetId),
+	Union(RobloxAssetId),
 }
-fn do_thread(path:std::path::PathBuf,send:std::sync::mpsc::Sender<UniqueAssetsResult>){
-	std::thread::spawn(move ||{
-		let result=unique_assets(path.as_path());
-		send.send(UniqueAssetsResult{
-			path,
-			result,
-		}).unwrap();
+impl DownloadType{
+	fn path(&self)->PathBuf{
+		match self{
+			DownloadType::Texture(asset_id)=>format!("downloaded_textures/{}",asset_id.0.to_string()).into(),
+			DownloadType::Mesh(asset_id)=>format!("meshes/{}",asset_id.0.to_string()).into(),
+			DownloadType::Union(asset_id)=>format!("unions/{}",asset_id.0.to_string()).into(),
+		}
+	}
+	fn asset_id(&self)->u64{
+		match self{
+			DownloadType::Texture(asset_id)=>asset_id.0,
+			DownloadType::Mesh(asset_id)=>asset_id.0,
+			DownloadType::Union(asset_id)=>asset_id.0,
+		}
+	}
+}
+enum DownloadResult{
+	Cached(PathBuf),
+	Data(Vec<u8>),
+	Failed,
+}
+#[derive(Default,Debug)]
+struct Stats{
+	total_assets:u32,
+	cached_assets:u32,
+	downloaded_assets:u32,
+	failed_downloads:u32,
+	timed_out_downloads:u32,
+}
+async fn download_retry(stats:&mut Stats,context:&rbx_asset::cookie::CookieContext,download_instruction:DownloadType)->Result<DownloadResult,std::io::Error>{
+	stats.total_assets+=1;
+	let download_instruction=download_instruction;
+	// check if file exists on disk
+	let path=download_instruction.path();
+	if tokio::fs::try_exists(path.as_path()).await?{
+		stats.cached_assets+=1;
+		return Ok(DownloadResult::Cached(path));
+	}
+	let asset_id=download_instruction.asset_id();
+	// if not, download file
+	let mut retry=0;
+	const BACKOFF_MUL:f32=1.3956124250860895286;//exp(1/3)
+	let mut backoff=1000f32;
+	loop{
+		let asset_result=context.get_asset(rbx_asset::cookie::GetAssetRequest{
+			asset_id,
+			version:None,
+		}).await;
+		match asset_result{
+			Ok(asset_result)=>{
+				stats.downloaded_assets+=1;
+				tokio::fs::write(path,&asset_result).await?;
+				break Ok(DownloadResult::Data(asset_result));
+			},
+			Err(rbx_asset::cookie::GetError::Response(rbx_asset::ResponseError::StatusCodeWithUrlAndBody(scwuab)))=>{
+				if scwuab.status_code.as_u16()==429{
+					if retry==12{
+						println!("Giving up asset download {asset_id}");
+						stats.timed_out_downloads+=1;
+						break Ok(DownloadResult::Failed);
+					}
+					println!("Hit roblox rate limit, waiting {:.0}ms...",backoff);
+					tokio::time::sleep(std::time::Duration::from_millis(backoff as u64)).await;
+					backoff*=BACKOFF_MUL;
+					retry+=1;
+				}else{
+					stats.failed_downloads+=1;
+					println!("weird scuwab error: {scwuab:?}");
+					break Ok(DownloadResult::Failed);
+				}
+			},
+			Err(e)=>{
+				stats.failed_downloads+=1;
+				println!("sadly error: {e}");
+				break Ok(DownloadResult::Failed);
+			},
+		}
+	}
+}
+#[allow(unused)]
+#[derive(Debug)]
+enum ConvertTextureError{
+	Io(std::io::Error),
+	Image(image::ImageError),
+	DDS(image_dds::CreateDdsError),
+	DDSWrite(image_dds::ddsfile::Error),
+}
+impl From<std::io::Error> for ConvertTextureError{
+	fn from(value:std::io::Error)->Self{
+		Self::Io(value)
+	}
+}
+impl From<image::ImageError> for ConvertTextureError{
+	fn from(value:image::ImageError)->Self{
+		Self::Image(value)
+	}
+}
+impl From<image_dds::CreateDdsError> for ConvertTextureError{
+	fn from(value:image_dds::CreateDdsError)->Self{
+		Self::DDS(value)
+	}
+}
+impl From<image_dds::ddsfile::Error> for ConvertTextureError{
+	fn from(value:image_dds::ddsfile::Error)->Self{
+		Self::DDSWrite(value)
+	}
+}
+async fn convert_texture(asset_id:RobloxAssetId,download_result:DownloadResult)->Result<(),ConvertTextureError>{
+	let data=match download_result{
+		DownloadResult::Cached(path)=>tokio::fs::read(path).await?,
+		DownloadResult::Data(data)=>data,
+		DownloadResult::Failed=>return Ok(()),
+	};
+	// image::ImageFormat::Png
+	// image::ImageFormat::Jpeg
+	let image=image::load_from_memory(&data)?.to_rgba8();
+
+	// pick format
+	let format=if image.width()%4!=0||image.height()%4!=0{
+		image_dds::ImageFormat::Rgba8UnormSrgb
+	}else{
+		image_dds::ImageFormat::BC7RgbaUnormSrgb
+	};
+
+	//this fails if the image dimensions are not a multiple of 4
+	let dds=image_dds::dds_from_image(
+		&image,
+		format,
+		image_dds::Quality::Slow,
+		image_dds::Mipmaps::GeneratedAutomatic,
+	)?;
+
+	let file_name=format!("textures/{}.dds",asset_id.0);
+	let mut file=std::fs::File::create(file_name)?;
+	dds.write(&mut file)?;
+	Ok(())
+}
+async fn download_assets(paths:Vec<PathBuf>,cookie:rbx_asset::cookie::Cookie)->AResult<()>{
+	tokio::try_join!(
+		tokio::fs::create_dir_all("downloaded_textures"),
+		tokio::fs::create_dir_all("textures"),
+		tokio::fs::create_dir_all("meshes"),
+		tokio::fs::create_dir_all("unions"),
+	)?;
+	// use mpsc
+	let thread_limit=std::thread::available_parallelism()?.get();
+	let (send_assets,mut recv_assets)=tokio::sync::mpsc::channel(DOWNLOAD_LIMIT);
+	let (send_texture,mut recv_texture)=tokio::sync::mpsc::channel(thread_limit);
+	// map decode dispatcher
+	// read files multithreaded
+	// produce UniqueAssetsResult per file
+	tokio::spawn(async move{
+		// move send so it gets dropped when all maps have been decoded
+		// closing the channel
+		let mut it=paths.into_iter();
+		static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
+		SEM.add_permits(thread_limit);
+		while let (Ok(permit),Some(path))=(SEM.acquire().await,it.next()){
+			let send=send_assets.clone();
+			tokio::spawn(async move{
+				let result=unique_assets(path.as_path()).await;
+				_=send.send(result).await;
+				drop(permit);
+			});
+		}
 	});
-}
-fn download_assets(paths:Vec<PathBuf>)->AResult<()>{
+	// download manager
+	// insert into global unique assets guy
+	// add to download queue if the asset is globally unique and does not already exist on disk
+	let mut stats=Stats::default();
+	let context=rbx_asset::cookie::CookieContext::new(cookie);
+	let mut globally_unique_assets=UniqueAssets::default();
+	// pop a job = retry_queue.pop_front() or ingest(recv.recv().await)
+	// SLOW MODE:
+	// acquire all permits
+	// drop all permits
+	// pop one job
+	// if it succeeds go into fast mode
+	// FAST MODE:
+	// acquire one permit
+	// pop a job
+	let download_thread=tokio::spawn(async move{
+		while let Some(result)=recv_assets.recv().await{
+			let unique_assets=match result{
+				Ok(unique_assets)=>unique_assets,
+				Err(e)=>{
+					println!("error: {e:?}");
+					continue;
+				},
+			};
+			for texture_id in unique_assets.textures{
+				if globally_unique_assets.textures.insert(texture_id){
+					let data=download_retry(&mut stats,&context,DownloadType::Texture(texture_id)).await?;
+					send_texture.send((texture_id,data)).await?;
+				}
+			}
+			for mesh_id in unique_assets.meshes{
+				if globally_unique_assets.meshes.insert(mesh_id){
+					download_retry(&mut stats,&context,DownloadType::Mesh(mesh_id)).await?;
+				}
+			}
+			for union_id in unique_assets.unions{
+				if globally_unique_assets.unions.insert(union_id){
+					download_retry(&mut stats,&context,DownloadType::Union(union_id)).await?;
+				}
+			}
+		}
+		dbg!(stats);
+		Ok::<(),anyhow::Error>(())
+	});
+	static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
+	SEM.add_permits(thread_limit);
+	while let (Ok(permit),Some((asset_id,download_result)))=(SEM.acquire().await,recv_texture.recv().await){
+		tokio::spawn(async move{
+			let result=convert_texture(asset_id,download_result).await;
+			drop(permit);
+			result.unwrap();
+		});
+	}
+	download_thread.await??;
+	_=SEM.acquire_many(thread_limit as u32).await.unwrap();
 	Ok(())
 }
 
@@ -155,7 +391,8 @@ fn download_assets(paths:Vec<PathBuf>)->AResult<()>{
 enum ConvertError{
 	IO(std::io::Error),
 	SNFMap(strafesnet_snf::map::Error),
-	RbxLoader(strafesnet_rbx_loader::ReadError),
+	RobloxRead(strafesnet_rbx_loader::ReadError),
+	RobloxLoad(strafesnet_rbx_loader::LoadError),
 }
 impl std::fmt::Display for ConvertError{
 	fn fmt(&self,f:&mut std::fmt::Formatter<'_>)->std::fmt::Result{
@@ -163,78 +400,49 @@ impl std::fmt::Display for ConvertError{
 	}
 }
 impl std::error::Error for ConvertError{}
+async fn convert_to_snf(path:&Path,output_folder:PathBuf)->AResult<()>{
+	let entire_file=tokio::fs::read(path).await?;
 
-type MapThread=std::thread::JoinHandle<Result<(),ConvertError>>;
+	let model=strafesnet_rbx_loader::read(
+		std::io::Cursor::new(entire_file)
+	).map_err(ConvertError::RobloxRead)?;
 
-fn roblox_to_snf(pathlist:Vec<std::path::PathBuf>,output_folder:PathBuf)->AResult<()>{
-	let n_paths=pathlist.len();
-	let start = std::time::Instant::now();
-	let mut threads:std::collections::VecDeque<MapThread>=std::collections::VecDeque::new();
-	let mut i=0;
-	let mut join_thread=|thread:MapThread|{
-		i+=1;
-		if let Err(e)=thread.join(){
-			println!("thread error: {:?}",e);
-		}else{
-			println!("{}/{}",i,n_paths);
-		}
-	};
-	for path in pathlist{
-		if 32<=threads.len(){
-			join_thread(threads.pop_front().unwrap());
-		}
-		let output_folder=output_folder.clone();
-		threads.push_back(std::thread::spawn(move ||{
-			let model=strafesnet_rbx_loader::read(
-				std::fs::File::open(path.as_path())
-				.map_err(ConvertError::IO)?
-			).map_err(ConvertError::RbxLoader)?;
+	let mut place=model.into_place();
+	place.run_scripts();
 
-			let mut place=model.into_place();
-			place.run_scripts();
+	let map=place.to_snf(LoadFailureMode::DefaultToNone).map_err(ConvertError::RobloxLoad)?;
 
-			let mut loader=strafesnet_deferred_loader::roblox_legacy();
+	let mut dest=output_folder;
+	dest.push(path.file_stem().unwrap());
+	dest.set_extension("snfm");
+	let file=std::fs::File::create(dest).map_err(ConvertError::IO)?;
 
-			let (texture_loader,mesh_loader)=loader.get_inner_mut();
+	strafesnet_snf::map::write_map(file,map).map_err(ConvertError::SNFMap)?;
 
-			let map_step1=strafesnet_rbx_loader::convert(
-				&place,
-				|name|texture_loader.acquire_render_config_id(name),
-				|name|mesh_loader.acquire_mesh_id(name),
-			);
-
-			let meshpart_meshes=mesh_loader.load_meshes().map_err(ConvertError::IO)?;
-
-			let map_step2=map_step1.add_meshpart_meshes_and_calculate_attributes(
-				meshpart_meshes.into_iter().map(|(mesh_id,loader_model)|
-					(mesh_id,strafesnet_rbx_loader::data::RobloxMeshBytes::new(loader_model.get()))
-				)
-			);
-
-			let (textures,render_configs)=loader.into_render_configs().map_err(ConvertError::IO)?.consume();
-
-			let map=map_step2.add_render_configs_and_textures(
-				render_configs.into_iter(),
-				textures.into_iter().map(|(texture_id,texture)|
-					(texture_id,match texture{
-						strafesnet_deferred_loader::texture::Texture::ImageDDS(data)=>data,
-					})
-				)
-			);
-
-			let mut dest=output_folder.clone();
-			dest.push(path.file_stem().unwrap());
-			dest.set_extension("snfm");
-			let file=std::fs::File::create(dest).map_err(ConvertError::IO)?;
-
-			strafesnet_snf::map::write_map(file,map).map_err(ConvertError::SNFMap)?;
-			Ok(())
-		}));
-	}
-
-	for thread in threads{
-		join_thread(thread);
-	}
-	println!("{:?}", start.elapsed());
+	Ok(())
+}
+
+async fn roblox_to_snf(paths:Vec<std::path::PathBuf>,output_folder:PathBuf)->AResult<()>{
+	let start=std::time::Instant::now();
+
+	let thread_limit=std::thread::available_parallelism()?.get();
+	let mut it=paths.into_iter();
+	static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
+	SEM.add_permits(thread_limit);
+
+	while let (Ok(permit),Some(path))=(SEM.acquire().await,it.next()){
+		let output_folder=output_folder.clone();
+		tokio::spawn(async move{
+			let result=convert_to_snf(path.as_path(),output_folder).await;
+			drop(permit);
+			match result{
+				Ok(())=>(),
+				Err(e)=>println!("Convert error: {e:?}"),
+			}
+		});
+	}
+	_=SEM.acquire_many(thread_limit as u32).await.unwrap();
+
+	println!("elapsed={:?}", start.elapsed());
 	Ok(())
 }

src/source.rs

@@ -1,6 +1,7 @@
-use std::path::PathBuf;
+use std::path::{Path,PathBuf};
 use clap::{Args,Subcommand};
 use anyhow::Result as AResult;
+use strafesnet_deferred_loader::deferred_loader::LoadFailureMode;
 
 #[derive(Subcommand)]
 pub enum Commands{
@@ -36,10 +37,10 @@ pub struct BSPContentsSubcommand {
 }
 
 impl Commands{
-	pub fn run(self)->AResult<()>{
+	pub async fn run(self)->AResult<()>{
 		match self{
-			Commands::SourceToSNF(subcommand)=>source_to_snf(subcommand.input_files,subcommand.output_folder),
 			Commands::ExtractTextures(subcommand)=>extract_textures(vec![subcommand.bsp_file],subcommand.vpk_dir_files),
+			Commands::SourceToSNF(subcommand)=>source_to_snf(subcommand.input_files,subcommand.output_folder).await,
 			Commands::VPKContents(subcommand)=>vpk_contents(subcommand.input_file),
 			Commands::BSPContents(subcommand)=>bsp_contents(subcommand.input_file),
 		}
@@ -277,7 +278,8 @@ fn bsp_contents(path:PathBuf)->AResult<()>{
 enum ConvertError{
 	IO(std::io::Error),
 	SNFMap(strafesnet_snf::map::Error),
-	BspLoader(strafesnet_bsp_loader::ReadError),
+	BspRead(strafesnet_bsp_loader::ReadError),
+	BspLoad(strafesnet_bsp_loader::LoadError),
 }
 impl std::fmt::Display for ConvertError{
 	fn fmt(&self,f:&mut std::fmt::Formatter<'_>)->std::fmt::Result{
@@ -286,79 +288,45 @@ impl std::fmt::Display for ConvertError{
 }
 impl std::error::Error for ConvertError{}
 
-type MapThread=std::thread::JoinHandle<Result<(),ConvertError>>;
+async fn convert_to_snf(path:&Path,output_folder:PathBuf)->AResult<()>{
+	let entire_file=tokio::fs::read(path).await?;
 
-fn source_to_snf(pathlist:Vec<std::path::PathBuf>,output_folder:PathBuf)->AResult<()>{
-	let n_paths=pathlist.len();
-	let start = std::time::Instant::now();
-	let mut threads:std::collections::VecDeque<MapThread>=std::collections::VecDeque::new();
-	let mut i=0;
-	let mut join_thread=|thread:MapThread|{
-		i+=1;
-		if let Err(e)=thread.join(){
-			println!("thread error: {:?}",e);
-		}else{
-			println!("{}/{}",i,n_paths);
-		}
-	};
-	for path in pathlist{
-		if 32<=threads.len(){
-			join_thread(threads.pop_front().unwrap());
-		}
-		let output_folder=output_folder.clone();
-		threads.push_back(std::thread::spawn(move ||{
-			let bsp=strafesnet_bsp_loader::read(
-				std::fs::File::open(path.as_path())
-				.map_err(ConvertError::IO)?
-			).map_err(ConvertError::BspLoader)?;
-			let mut loader=strafesnet_deferred_loader::source_legacy();
+	let bsp=strafesnet_bsp_loader::read(
+		std::io::Cursor::new(entire_file)
+	).map_err(ConvertError::BspRead)?;
 
-				let (texture_loader,mesh_loader)=loader.get_inner_mut();
+	let map=bsp.to_snf(LoadFailureMode::DefaultToNone).map_err(ConvertError::BspLoad)?;
 
-				let map_step1=strafesnet_bsp_loader::convert(
-					&bsp,
-					|name|texture_loader.acquire_render_config_id(name),
-					|name|mesh_loader.acquire_mesh_id(name),
-				);
+	let mut dest=output_folder;
+	dest.push(path.file_stem().unwrap());
+	dest.set_extension("snfm");
+	let file=std::fs::File::create(dest).map_err(ConvertError::IO)?;
 
-				let prop_meshes=mesh_loader.load_meshes(&bsp.as_ref());
+	strafesnet_snf::map::write_map(file,map).map_err(ConvertError::SNFMap)?;
 
-				let map_step2=map_step1.add_prop_meshes(
-					//the type conflagulator 9000
-					prop_meshes.into_iter().map(|(mesh_id,loader_model)|
-						(mesh_id,strafesnet_bsp_loader::data::ModelData{
-							mdl:strafesnet_bsp_loader::data::MdlData::new(loader_model.mdl.get()),
-							vtx:strafesnet_bsp_loader::data::VtxData::new(loader_model.vtx.get()),
-							vvd:strafesnet_bsp_loader::data::VvdData::new(loader_model.vvd.get()),
-						})
-					),
-					|name|texture_loader.acquire_render_config_id(name),
-				);
-
-				let (textures,render_configs)=loader.into_render_configs().map_err(ConvertError::IO)?.consume();
-
-				let map=map_step2.add_render_configs_and_textures(
-					render_configs.into_iter(),
-					textures.into_iter().map(|(texture_id,texture)|
-						(texture_id,match texture{
-							strafesnet_deferred_loader::texture::Texture::ImageDDS(data)=>data,
-						})
-					),
-				);
-
-			let mut dest=output_folder.clone();
-			dest.push(path.file_stem().unwrap());
-			dest.set_extension("snfm");
-			let file=std::fs::File::create(dest).map_err(ConvertError::IO)?;
-
-			strafesnet_snf::map::write_map(file,map).map_err(ConvertError::SNFMap)?;
-			Ok(())
-		}));
-	}
-
-	for thread in threads{
-		join_thread(thread);
-	}
-	println!("{:?}", start.elapsed());
+	Ok(())
+}
+async fn source_to_snf(paths:Vec<std::path::PathBuf>,output_folder:PathBuf)->AResult<()>{
+	let start=std::time::Instant::now();
+
+	let thread_limit=std::thread::available_parallelism()?.get();
+	let mut it=paths.into_iter();
+	static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
+	SEM.add_permits(thread_limit);
+
+	while let (Ok(permit),Some(path))=(SEM.acquire().await,it.next()){
+		let output_folder=output_folder.clone();
+		tokio::spawn(async move{
+			let result=convert_to_snf(path.as_path(),output_folder).await;
+			drop(permit);
+			match result{
+				Ok(())=>(),
+				Err(e)=>println!("Convert error: {e:?}"),
+			}
+		});
+	}
+	_=SEM.acquire_many(thread_limit as u32).await.unwrap();
+
+	println!("elapsed={:?}", start.elapsed());
 	Ok(())
 }