roblox: convert textures during download

src/roblox.rs

@@ -178,6 +178,11 @@ impl DownloadType{
 		}
 	}
 }
+enum DownloadResult{
+	Cached(PathBuf),
+	Data(Vec<u8>),
+	Failed,
+}
 #[derive(Default,Debug)]
 struct Stats{
 	total_assets:u32,
@@ -186,14 +191,14 @@ struct Stats{
 	failed_downloads:u32,
 	timed_out_downloads:u32,
 }
-async fn download_retry(stats:&mut Stats,context:&rbx_asset::cookie::CookieContext,download_instruction:DownloadType)->Result<(),std::io::Error>{
+async fn download_retry(stats:&mut Stats,context:&rbx_asset::cookie::CookieContext,download_instruction:DownloadType)->Result<DownloadResult,std::io::Error>{
 	stats.total_assets+=1;
 	let download_instruction=download_instruction;
 	// check if file exists on disk
 	let path=download_instruction.path();
 	if tokio::fs::try_exists(path.as_path()).await?{
 		stats.cached_assets+=1;
-		return Ok(());
+		return Ok(DownloadResult::Cached(path));
 	}
 	let asset_id=download_instruction.asset_id();
 	// if not, download file
@@ -208,15 +213,15 @@ async fn download_retry(stats:&mut Stats,context:&rbx_asset::cookie::CookieConte
 		match asset_result{
 			Ok(asset_result)=>{
 				stats.downloaded_assets+=1;
-				tokio::fs::write(path,asset_result).await?;
+				tokio::fs::write(path,&asset_result).await?;
-				break;
+				break Ok(DownloadResult::Data(asset_result));
 			},
 			Err(rbx_asset::cookie::GetError::Response(rbx_asset::ResponseError::StatusCodeWithUrlAndBody(scwuab)))=>{
 				if scwuab.status_code.as_u16()==429{
 					if retry==12{
 						println!("Giving up asset download {asset_id}");
 						stats.timed_out_downloads+=1;
-						break;
+						break Ok(DownloadResult::Failed);
 					}
 					println!("Hit roblox rate limit, waiting {:.0}ms...",backoff);
 					tokio::time::sleep(std::time::Duration::from_millis(backoff as u64)).await;
@@ -225,27 +230,85 @@ async fn download_retry(stats:&mut Stats,context:&rbx_asset::cookie::CookieConte
 				}else{
 					stats.failed_downloads+=1;
 					println!("weird scuwab error: {scwuab:?}");
-					break;
+					break Ok(DownloadResult::Failed);
 				}
 			},
 			Err(e)=>{
 				stats.failed_downloads+=1;
 				println!("sadly error: {e}");
-				break;
+				break Ok(DownloadResult::Failed);
 			},
 		}
 	}
+}
+#[allow(unused)]
+#[derive(Debug)]
+enum ConvertTextureError{
+	Io(std::io::Error),
+	Image(image::ImageError),
+	DDS(image_dds::CreateDdsError),
+	DDSWrite(image_dds::ddsfile::Error),
+}
+impl From<std::io::Error> for ConvertTextureError{
+	fn from(value:std::io::Error)->Self{
+		Self::Io(value)
+	}
+}
+impl From<image::ImageError> for ConvertTextureError{
+	fn from(value:image::ImageError)->Self{
+		Self::Image(value)
+	}
+}
+impl From<image_dds::CreateDdsError> for ConvertTextureError{
+	fn from(value:image_dds::CreateDdsError)->Self{
+		Self::DDS(value)
+	}
+}
+impl From<image_dds::ddsfile::Error> for ConvertTextureError{
+	fn from(value:image_dds::ddsfile::Error)->Self{
+		Self::DDSWrite(value)
+	}
+}
+async fn convert_texture(asset_id:RobloxAssetId,download_result:DownloadResult)->Result<(),ConvertTextureError>{
+	let data=match download_result{
+		DownloadResult::Cached(path)=>tokio::fs::read(path).await?,
+		DownloadResult::Data(data)=>data,
+		DownloadResult::Failed=>return Ok(()),
+	};
+
+	let image=image::ImageReader::new(std::io::Cursor::new(data)).decode()?.to_rgba8();
+
+	// pick format
+	let format=if image.width()%4!=0||image.height()%4!=0{
+		image_dds::ImageFormat::Rgba8UnormSrgb
+	}else{
+		image_dds::ImageFormat::BC7RgbaUnormSrgb
+	};
+
+	//this fails if the image dimensions are not a multiple of 4
+	let dds=image_dds::dds_from_image(
+		&image,
+		format,
+		image_dds::Quality::Slow,
+		image_dds::Mipmaps::GeneratedAutomatic,
+	)?;
+
+	let file_name=format!("textures/{}.dds",asset_id.0);
+	let mut file=std::fs::File::create(file_name)?;
+	dds.write(&mut file)?;
 	Ok(())
 }
 async fn download_assets(paths:Vec<PathBuf>,cookie:rbx_asset::cookie::Cookie)->AResult<()>{
 	tokio::try_join!(
 		tokio::fs::create_dir_all("downloaded_textures"),
+		tokio::fs::create_dir_all("textures"),
 		tokio::fs::create_dir_all("meshes"),
 		tokio::fs::create_dir_all("unions"),
 	)?;
 	// use mpsc
 	let thread_limit=std::thread::available_parallelism()?.get();
-	let (send,mut recv)=tokio::sync::mpsc::channel(DOWNLOAD_LIMIT);
+	let (send_assets,mut recv_assets)=tokio::sync::mpsc::channel(DOWNLOAD_LIMIT);
+	let (send_texture,mut recv_texture)=tokio::sync::mpsc::channel(thread_limit);
 	// map decode dispatcher
 	// read files multithreaded
 	// produce UniqueAssetsResult per file
@@ -256,7 +319,7 @@ async fn download_assets(paths:Vec<PathBuf>,cookie:rbx_asset::cookie::Cookie)->A
 		static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
 		SEM.add_permits(thread_limit);
 		while let (Ok(permit),Some(path))=(SEM.acquire().await,it.next()){
-			let send=send.clone();
+			let send=send_assets.clone();
 			tokio::spawn(async move{
 				let result=unique_assets(path.as_path()).await;
 				_=send.send(result).await;
@@ -279,32 +342,46 @@ async fn download_assets(paths:Vec<PathBuf>,cookie:rbx_asset::cookie::Cookie)->A
 	// FAST MODE:
 	// acquire one permit
 	// pop a job
-	while let Some(result)=recv.recv().await{
+	let download_thread=tokio::spawn(async move{
-		let unique_assets=match result{
+		while let Some(result)=recv_assets.recv().await{
-			Ok(unique_assets)=>unique_assets,
+			let unique_assets=match result{
-			Err(e)=>{
+				Ok(unique_assets)=>unique_assets,
-				println!("error: {e:?}");
+				Err(e)=>{
-				continue;
+					println!("error: {e:?}");
-			},
+					continue;
-		};
+				},
-		for texture_id in unique_assets.textures{
+			};
-			if globally_unique_assets.textures.insert(RobloxAssetId(texture_id.0)){
+			for texture_id in unique_assets.textures{
-				download_retry(&mut stats,&context,DownloadType::Texture(texture_id)).await?;
+				if globally_unique_assets.textures.insert(texture_id){
-			}
+					let data=download_retry(&mut stats,&context,DownloadType::Texture(texture_id)).await?;
-		}
+					send_texture.send((texture_id,data)).await?;
-		for mesh_id in unique_assets.meshes{
+				}
-			if globally_unique_assets.meshes.insert(RobloxAssetId(mesh_id.0)){
+			}
-				download_retry(&mut stats,&context,DownloadType::Mesh(mesh_id)).await?;
+			for mesh_id in unique_assets.meshes{
-			}
+				if globally_unique_assets.meshes.insert(mesh_id){
-		}
+					download_retry(&mut stats,&context,DownloadType::Mesh(mesh_id)).await?;
-		for union_id in unique_assets.unions{
+				}
-			if globally_unique_assets.unions.insert(RobloxAssetId(union_id.0)){
+			}
-				download_retry(&mut stats,&context,DownloadType::Union(union_id)).await?;
+			for union_id in unique_assets.unions{
+				if globally_unique_assets.unions.insert(union_id){
+					download_retry(&mut stats,&context,DownloadType::Union(union_id)).await?;
+				}
 			}
 		}
+		dbg!(stats);
+		Ok::<(),anyhow::Error>(())
+	});
+	static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
+	SEM.add_permits(thread_limit);
+	while let (Ok(permit),Some((asset_id,download_result)))=(SEM.acquire().await,recv_texture.recv().await){
+		tokio::spawn(async move{
+			let result=convert_texture(asset_id,download_result).await;
+			drop(permit);
+			result.unwrap();
+		});
 	}
-
+	download_thread.await??;
-	dbg!(stats);
+	_=SEM.acquire_many(thread_limit as u32).await.unwrap();
 	Ok(())
 }