From 2917fded43e9b45b5f7398367a8a115b1e0e0332 Mon Sep 17 00:00:00 2001
From: Quaternions <krakow20@gmail.com>
Date: Fri, 31 Jan 2025 12:03:57 -0800
Subject: [PATCH] roblox: convert textures during download
---
src/roblox.rs | 141 ++++++++++++++++++++++++++++++++++++++------------
1 file changed, 109 insertions(+), 32 deletions(-)
diff --git a/src/roblox.rs b/src/roblox.rs
index 0753cd5..d297a1a 100644
--- a/src/roblox.rs
+++ b/src/roblox.rs
@@ -178,6 +178,11 @@ impl DownloadType{
}
}
}
+enum DownloadResult{
+ Cached(PathBuf),
+ Data(Vec<u8>),
+ Failed,
+}
#[derive(Default,Debug)]
struct Stats{
total_assets:u32,
@@ -186,14 +191,14 @@ struct Stats{
failed_downloads:u32,
timed_out_downloads:u32,
}
-async fn download_retry(stats:&mut Stats,context:&rbx_asset::cookie::CookieContext,download_instruction:DownloadType)->Result<(),std::io::Error>{
+async fn download_retry(stats:&mut Stats,context:&rbx_asset::cookie::CookieContext,download_instruction:DownloadType)->Result<DownloadResult,std::io::Error>{
stats.total_assets+=1;
let download_instruction=download_instruction;
// check if file exists on disk
let path=download_instruction.path();
if tokio::fs::try_exists(path.as_path()).await?{
stats.cached_assets+=1;
- return Ok(());
+ return Ok(DownloadResult::Cached(path));
}
let asset_id=download_instruction.asset_id();
// if not, download file
@@ -208,15 +213,15 @@ async fn download_retry(stats:&mut Stats,context:&rbx_asset::cookie::CookieConte
match asset_result{
Ok(asset_result)=>{
stats.downloaded_assets+=1;
- tokio::fs::write(path,asset_result).await?;
- break;
+ tokio::fs::write(path,&asset_result).await?;
+ break Ok(DownloadResult::Data(asset_result));
},
Err(rbx_asset::cookie::GetError::Response(rbx_asset::ResponseError::StatusCodeWithUrlAndBody(scwuab)))=>{
if scwuab.status_code.as_u16()==429{
if retry==12{
println!("Giving up asset download {asset_id}");
stats.timed_out_downloads+=1;
- break;
+ break Ok(DownloadResult::Failed);
}
println!("Hit roblox rate limit, waiting {:.0}ms...",backoff);
tokio::time::sleep(std::time::Duration::from_millis(backoff as u64)).await;
@@ -225,27 +230,85 @@ async fn download_retry(stats:&mut Stats,context:&rbx_asset::cookie::CookieConte
}else{
stats.failed_downloads+=1;
println!("weird scuwab error: {scwuab:?}");
- break;
+ break Ok(DownloadResult::Failed);
}
},
Err(e)=>{
stats.failed_downloads+=1;
println!("sadly error: {e}");
- break;
+ break Ok(DownloadResult::Failed);
},
}
}
+}
+#[allow(unused)]
+#[derive(Debug)]
+enum ConvertTextureError{
+ Io(std::io::Error),
+ Image(image::ImageError),
+ DDS(image_dds::CreateDdsError),
+ DDSWrite(image_dds::ddsfile::Error),
+}
+impl From<std::io::Error> for ConvertTextureError{
+ fn from(value:std::io::Error)->Self{
+ Self::Io(value)
+ }
+}
+impl From<image::ImageError> for ConvertTextureError{
+ fn from(value:image::ImageError)->Self{
+ Self::Image(value)
+ }
+}
+impl From<image_dds::CreateDdsError> for ConvertTextureError{
+ fn from(value:image_dds::CreateDdsError)->Self{
+ Self::DDS(value)
+ }
+}
+impl From<image_dds::ddsfile::Error> for ConvertTextureError{
+ fn from(value:image_dds::ddsfile::Error)->Self{
+ Self::DDSWrite(value)
+ }
+}
+async fn convert_texture(asset_id:RobloxAssetId,download_result:DownloadResult)->Result<(),ConvertTextureError>{
+ let data=match download_result{
+ DownloadResult::Cached(path)=>tokio::fs::read(path).await?,
+ DownloadResult::Data(data)=>data,
+ DownloadResult::Failed=>return Ok(()),
+ };
+
+ let image=image::ImageReader::new(std::io::Cursor::new(data)).decode()?.to_rgba8();
+
+ // pick format
+ let format=if image.width()%4!=0||image.height()%4!=0{
+ image_dds::ImageFormat::Rgba8UnormSrgb
+ }else{
+ image_dds::ImageFormat::BC7RgbaUnormSrgb
+ };
+
+ //this fails if the image dimensions are not a multiple of 4
+ let dds=image_dds::dds_from_image(
+ &image,
+ format,
+ image_dds::Quality::Slow,
+ image_dds::Mipmaps::GeneratedAutomatic,
+ )?;
+
+ let file_name=format!("textures/{}.dds",asset_id.0);
+ let mut file=std::fs::File::create(file_name)?;
+ dds.write(&mut file)?;
Ok(())
}
async fn download_assets(paths:Vec<PathBuf>,cookie:rbx_asset::cookie::Cookie)->AResult<()>{
tokio::try_join!(
tokio::fs::create_dir_all("downloaded_textures"),
+ tokio::fs::create_dir_all("textures"),
tokio::fs::create_dir_all("meshes"),
tokio::fs::create_dir_all("unions"),
)?;
// use mpsc
let thread_limit=std::thread::available_parallelism()?.get();
- let (send,mut recv)=tokio::sync::mpsc::channel(DOWNLOAD_LIMIT);
+ let (send_assets,mut recv_assets)=tokio::sync::mpsc::channel(DOWNLOAD_LIMIT);
+ let (send_texture,mut recv_texture)=tokio::sync::mpsc::channel(thread_limit);
// map decode dispatcher
// read files multithreaded
// produce UniqueAssetsResult per file
@@ -256,7 +319,7 @@ async fn download_assets(paths:Vec<PathBuf>,cookie:rbx_asset::cookie::Cookie)->A
static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
SEM.add_permits(thread_limit);
while let (Ok(permit),Some(path))=(SEM.acquire().await,it.next()){
- let send=send.clone();
+ let send=send_assets.clone();
tokio::spawn(async move{
let result=unique_assets(path.as_path()).await;
_=send.send(result).await;
@@ -279,32 +342,46 @@ async fn download_assets(paths:Vec<PathBuf>,cookie:rbx_asset::cookie::Cookie)->A
// FAST MODE:
// acquire one permit
// pop a job
- while let Some(result)=recv.recv().await{
- let unique_assets=match result{
- Ok(unique_assets)=>unique_assets,
- Err(e)=>{
- println!("error: {e:?}");
- continue;
- },
- };
- for texture_id in unique_assets.textures{
- if globally_unique_assets.textures.insert(RobloxAssetId(texture_id.0)){
- download_retry(&mut stats,&context,DownloadType::Texture(texture_id)).await?;
- }
- }
- for mesh_id in unique_assets.meshes{
- if globally_unique_assets.meshes.insert(RobloxAssetId(mesh_id.0)){
- download_retry(&mut stats,&context,DownloadType::Mesh(mesh_id)).await?;
- }
- }
- for union_id in unique_assets.unions{
- if globally_unique_assets.unions.insert(RobloxAssetId(union_id.0)){
- download_retry(&mut stats,&context,DownloadType::Union(union_id)).await?;
+ let download_thread=tokio::spawn(async move{
+ while let Some(result)=recv_assets.recv().await{
+ let unique_assets=match result{
+ Ok(unique_assets)=>unique_assets,
+ Err(e)=>{
+ println!("error: {e:?}");
+ continue;
+ },
+ };
+ for texture_id in unique_assets.textures{
+ if globally_unique_assets.textures.insert(texture_id){
+ let data=download_retry(&mut stats,&context,DownloadType::Texture(texture_id)).await?;
+ send_texture.send((texture_id,data)).await?;
+ }
+ }
+ for mesh_id in unique_assets.meshes{
+ if globally_unique_assets.meshes.insert(mesh_id){
+ download_retry(&mut stats,&context,DownloadType::Mesh(mesh_id)).await?;
+ }
+ }
+ for union_id in unique_assets.unions{
+ if globally_unique_assets.unions.insert(union_id){
+ download_retry(&mut stats,&context,DownloadType::Union(union_id)).await?;
+ }
}
}
+ dbg!(stats);
+ Ok::<(),anyhow::Error>(())
+ });
+ static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
+ SEM.add_permits(thread_limit);
+ while let (Ok(permit),Some((asset_id,download_result)))=(SEM.acquire().await,recv_texture.recv().await){
+ tokio::spawn(async move{
+ let result=convert_texture(asset_id,download_result).await;
+ drop(permit);
+ result.unwrap();
+ });
}
-
- dbg!(stats);
+ download_thread.await??;
+ _=SEM.acquire_many(thread_limit as u32).await.unwrap();
Ok(())
}