loader + tokio rewrite

This commit is contained in:
Quaternions 2025-02-04 09:09:01 -08:00
parent 6522c255cd
commit ca50bf35c2
5 changed files with 1708 additions and 407 deletions

1166
Cargo.lock generated

File diff suppressed because it is too large Load Diff

@ -9,17 +9,22 @@ edition = "2021"
anyhow = "1.0.75"
clap = { version = "4.4.2", features = ["derive"] }
flate2 = "1.0.27"
futures = "0.3.31"
image = "0.25.2"
image_dds = "0.6.0"
image_dds = "0.7.1"
lazy-regex = "3.1.0"
rbx_asset = { version = "0.2.5", registry = "strafesnet" }
rbx_binary = { version = "0.7.4", registry = "strafesnet" }
rbx_dom_weak = { version = "2.7.0", registry = "strafesnet" }
rbx_reflection_database = { version = "0.2.10", registry = "strafesnet" }
rbx_xml = { version = "0.13.3", registry = "strafesnet" }
strafesnet_bsp_loader = { version = "0.2.1", registry = "strafesnet" }
strafesnet_deferred_loader = { version = "0.4.0", features = ["legacy"], registry = "strafesnet" }
strafesnet_rbx_loader = { version = "0.5.1", registry = "strafesnet" }
strafesnet_snf = { version = "0.2.0", registry = "strafesnet" }
rbxassetid = { version = "0.1.0", registry = "strafesnet" }
strafesnet_bsp_loader = { version = "0.3.0", registry = "strafesnet" }
strafesnet_deferred_loader = { version = "0.5.0", registry = "strafesnet" }
strafesnet_rbx_loader = { version = "0.6.0", registry = "strafesnet" }
strafesnet_snf = { version = "0.3.0", registry = "strafesnet" }
thiserror = "2.0.11"
tokio = { version = "1.43.0", features = ["macros", "rt-multi-thread", "fs"] }
vbsp = "0.6.0"
vmdl = "0.2.0"
vmt-parser = "0.2.0"

@ -20,10 +20,11 @@ enum Commands{
Source(source::Commands),
}
fn main() -> AResult<()> {
let cli = Cli::parse();
#[tokio::main]
async fn main()->AResult<()>{
let cli=Cli::parse();
match cli.command{
Commands::Roblox(commands)=>commands.run(),
Commands::Source(commands)=>commands.run(),
Commands::Roblox(commands)=>commands.run().await,
Commands::Source(commands)=>commands.run().await,
}
}

@ -4,7 +4,11 @@ use std::collections::HashSet;
use clap::{Args,Subcommand};
use anyhow::Result as AResult;
use rbx_dom_weak::Instance;
use strafesnet_deferred_loader::rbxassetid::RobloxAssetId;
use strafesnet_deferred_loader::deferred_loader::LoadFailureMode;
use rbxassetid::RobloxAssetId;
use tokio::io::AsyncReadExt;
const DOWNLOAD_LIMIT:usize=16;
#[derive(Subcommand)]
pub enum Commands{
@ -21,27 +25,40 @@ pub struct RobloxToSNFSubcommand {
}
#[derive(Args)]
pub struct DownloadAssetsSubcommand{
#[arg(long,required=true)]
roblox_files:Vec<PathBuf>
#[arg(required=true)]
roblox_files:Vec<PathBuf>,
// #[arg(long)]
// cookie_file:Option<String>,
}
impl Commands{
pub fn run(self)->AResult<()>{
pub async fn run(self)->AResult<()>{
match self{
Commands::RobloxToSNF(subcommand)=>roblox_to_snf(subcommand.input_files,subcommand.output_folder),
Commands::DownloadAssets(subcommand)=>download_assets(subcommand.roblox_files),
Commands::RobloxToSNF(subcommand)=>roblox_to_snf(subcommand.input_files,subcommand.output_folder).await,
Commands::DownloadAssets(subcommand)=>download_assets(
subcommand.roblox_files,
rbx_asset::cookie::Cookie::new("".to_string()),
).await,
}
}
}
fn load_dom<R:Read+Seek>(mut input:R)->AResult<rbx_dom_weak::WeakDom>{
#[allow(unused)]
#[derive(Debug)]
enum LoadDomError{
IO(std::io::Error),
Binary(rbx_binary::DecodeError),
Xml(rbx_xml::DecodeError),
UnknownFormat,
}
fn load_dom<R:Read+Seek>(mut input:R)->Result<rbx_dom_weak::WeakDom,LoadDomError>{
let mut first_8=[0u8;8];
input.read_exact(&mut first_8)?;
input.rewind()?;
input.read_exact(&mut first_8).map_err(LoadDomError::IO)?;
input.rewind().map_err(LoadDomError::IO)?;
match &first_8{
b"<roblox!"=>rbx_binary::from_reader(input).map_err(anyhow::Error::msg),
b"<roblox "=>rbx_xml::from_reader(input,rbx_xml::DecodeOptions::default()).map_err(anyhow::Error::msg),
_=>Err(anyhow::Error::msg("unsupported file type")),
b"<roblox!"=>rbx_binary::from_reader(input).map_err(LoadDomError::Binary),
b"<roblox "=>rbx_xml::from_reader(input,rbx_xml::DecodeOptions::default()).map_err(LoadDomError::Xml),
_=>Err(LoadDomError::UnknownFormat),
}
}
@ -80,13 +97,13 @@ fn accumulate_content_id(content_list:&mut HashSet<RobloxAssetId>,object:&Instan
println!("Content failed to parse into AssetID: {:?}",content);
}
}else{
println!("property={} does not exist for class={}",object.class.as_str(),property);
println!("property={} does not exist for class={}",property,object.class.as_str());
}
}
fn read_entire_file(path:impl AsRef<Path>)->Result<Cursor<Vec<u8>>,std::io::Error>{
let mut file=std::fs::File::open(path)?;
async fn read_entire_file(path:impl AsRef<Path>)->Result<Cursor<Vec<u8>>,std::io::Error>{
let mut file=tokio::fs::File::open(path).await?;
let mut data=Vec::new();
file.read_to_end(&mut data)?;
file.read_to_end(&mut data).await?;
Ok(Cursor::new(data))
}
#[derive(Default)]
@ -123,30 +140,232 @@ impl UniqueAssets{
}
}
}
fn unique_assets(path:&Path)->AResult<UniqueAssets>{
#[allow(unused)]
#[derive(Debug)]
enum UniqueAssetError{
IO(std::io::Error),
LoadDom(LoadDomError),
}
async fn unique_assets(path:&Path)->Result<UniqueAssets,UniqueAssetError>{
// read entire file
let mut assets=UniqueAssets::default();
let data=read_entire_file(path)?;
let dom=load_dom(data)?;
let data=read_entire_file(path).await.map_err(UniqueAssetError::IO)?;
let dom=load_dom(data).map_err(UniqueAssetError::LoadDom)?;
for object in dom.into_raw().1.into_values(){
assets.collect(&object);
}
Ok(assets)
}
struct UniqueAssetsResult{
path:std::path::PathBuf,
result:AResult<UniqueAssets>,
enum DownloadType{
Texture(RobloxAssetId),
Mesh(RobloxAssetId),
Union(RobloxAssetId),
}
fn do_thread(path:std::path::PathBuf,send:std::sync::mpsc::Sender<UniqueAssetsResult>){
std::thread::spawn(move ||{
let result=unique_assets(path.as_path());
send.send(UniqueAssetsResult{
path,
result,
}).unwrap();
impl DownloadType{
fn path(&self)->PathBuf{
match self{
DownloadType::Texture(asset_id)=>format!("downloaded_textures/{}",asset_id.0.to_string()).into(),
DownloadType::Mesh(asset_id)=>format!("meshes/{}",asset_id.0.to_string()).into(),
DownloadType::Union(asset_id)=>format!("unions/{}",asset_id.0.to_string()).into(),
}
}
fn asset_id(&self)->u64{
match self{
DownloadType::Texture(asset_id)=>asset_id.0,
DownloadType::Mesh(asset_id)=>asset_id.0,
DownloadType::Union(asset_id)=>asset_id.0,
}
}
}
enum DownloadResult{
Cached(PathBuf),
Data(Vec<u8>),
Failed,
}
#[derive(Default,Debug)]
struct Stats{
total_assets:u32,
cached_assets:u32,
downloaded_assets:u32,
failed_downloads:u32,
timed_out_downloads:u32,
}
async fn download_retry(stats:&mut Stats,context:&rbx_asset::cookie::CookieContext,download_instruction:DownloadType)->Result<DownloadResult,std::io::Error>{
stats.total_assets+=1;
let download_instruction=download_instruction;
// check if file exists on disk
let path=download_instruction.path();
if tokio::fs::try_exists(path.as_path()).await?{
stats.cached_assets+=1;
return Ok(DownloadResult::Cached(path));
}
let asset_id=download_instruction.asset_id();
// if not, download file
let mut retry=0;
const BACKOFF_MUL:f32=1.3956124250860895286;//exp(1/3)
let mut backoff=1000f32;
loop{
let asset_result=context.get_asset(rbx_asset::cookie::GetAssetRequest{
asset_id,
version:None,
}).await;
match asset_result{
Ok(asset_result)=>{
stats.downloaded_assets+=1;
tokio::fs::write(path,&asset_result).await?;
break Ok(DownloadResult::Data(asset_result));
},
Err(rbx_asset::cookie::GetError::Response(rbx_asset::ResponseError::StatusCodeWithUrlAndBody(scwuab)))=>{
if scwuab.status_code.as_u16()==429{
if retry==12{
println!("Giving up asset download {asset_id}");
stats.timed_out_downloads+=1;
break Ok(DownloadResult::Failed);
}
println!("Hit roblox rate limit, waiting {:.0}ms...",backoff);
tokio::time::sleep(std::time::Duration::from_millis(backoff as u64)).await;
backoff*=BACKOFF_MUL;
retry+=1;
}else{
stats.failed_downloads+=1;
println!("weird scuwab error: {scwuab:?}");
break Ok(DownloadResult::Failed);
}
},
Err(e)=>{
stats.failed_downloads+=1;
println!("sadly error: {e}");
break Ok(DownloadResult::Failed);
},
}
}
}
#[derive(Debug,thiserror::Error)]
enum ConvertTextureError{
#[error("Io error {0:?}")]
Io(#[from]std::io::Error),
#[error("Image error {0:?}")]
Image(#[from]image::ImageError),
#[error("DDS create error {0:?}")]
DDS(#[from]image_dds::CreateDdsError),
#[error("DDS write error {0:?}")]
DDSWrite(#[from]image_dds::ddsfile::Error),
}
async fn convert_texture(asset_id:RobloxAssetId,download_result:DownloadResult)->Result<(),ConvertTextureError>{
let data=match download_result{
DownloadResult::Cached(path)=>tokio::fs::read(path).await?,
DownloadResult::Data(data)=>data,
DownloadResult::Failed=>return Ok(()),
};
// image::ImageFormat::Png
// image::ImageFormat::Jpeg
let image=image::load_from_memory(&data)?.to_rgba8();
// pick format
let format=if image.width()%4!=0||image.height()%4!=0{
image_dds::ImageFormat::Rgba8UnormSrgb
}else{
image_dds::ImageFormat::BC7RgbaUnormSrgb
};
//this fails if the image dimensions are not a multiple of 4
let dds=image_dds::dds_from_image(
&image,
format,
image_dds::Quality::Slow,
image_dds::Mipmaps::GeneratedAutomatic,
)?;
let file_name=format!("textures/{}.dds",asset_id.0);
let mut file=std::fs::File::create(file_name)?;
dds.write(&mut file)?;
Ok(())
}
async fn download_assets(paths:Vec<PathBuf>,cookie:rbx_asset::cookie::Cookie)->AResult<()>{
tokio::try_join!(
tokio::fs::create_dir_all("downloaded_textures"),
tokio::fs::create_dir_all("textures"),
tokio::fs::create_dir_all("meshes"),
tokio::fs::create_dir_all("unions"),
)?;
// use mpsc
let thread_limit=std::thread::available_parallelism()?.get();
let (send_assets,mut recv_assets)=tokio::sync::mpsc::channel(DOWNLOAD_LIMIT);
let (send_texture,mut recv_texture)=tokio::sync::mpsc::channel(thread_limit);
// map decode dispatcher
// read files multithreaded
// produce UniqueAssetsResult per file
tokio::spawn(async move{
// move send so it gets dropped when all maps have been decoded
// closing the channel
let mut it=paths.into_iter();
static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
SEM.add_permits(thread_limit);
while let (Ok(permit),Some(path))=(SEM.acquire().await,it.next()){
let send=send_assets.clone();
tokio::spawn(async move{
let result=unique_assets(path.as_path()).await;
_=send.send(result).await;
drop(permit);
});
}
});
}
fn download_assets(paths:Vec<PathBuf>)->AResult<()>{
// download manager
// insert into global unique assets guy
// add to download queue if the asset is globally unique and does not already exist on disk
let mut stats=Stats::default();
let context=rbx_asset::cookie::CookieContext::new(cookie);
let mut globally_unique_assets=UniqueAssets::default();
// pop a job = retry_queue.pop_front() or ingest(recv.recv().await)
// SLOW MODE:
// acquire all permits
// drop all permits
// pop one job
// if it succeeds go into fast mode
// FAST MODE:
// acquire one permit
// pop a job
let download_thread=tokio::spawn(async move{
while let Some(result)=recv_assets.recv().await{
let unique_assets=match result{
Ok(unique_assets)=>unique_assets,
Err(e)=>{
println!("error: {e:?}");
continue;
},
};
for texture_id in unique_assets.textures{
if globally_unique_assets.textures.insert(texture_id){
let data=download_retry(&mut stats,&context,DownloadType::Texture(texture_id)).await?;
send_texture.send((texture_id,data)).await?;
}
}
for mesh_id in unique_assets.meshes{
if globally_unique_assets.meshes.insert(mesh_id){
download_retry(&mut stats,&context,DownloadType::Mesh(mesh_id)).await?;
}
}
for union_id in unique_assets.unions{
if globally_unique_assets.unions.insert(union_id){
download_retry(&mut stats,&context,DownloadType::Union(union_id)).await?;
}
}
}
dbg!(stats);
Ok::<(),anyhow::Error>(())
});
static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
SEM.add_permits(thread_limit);
while let (Ok(permit),Some((asset_id,download_result)))=(SEM.acquire().await,recv_texture.recv().await){
tokio::spawn(async move{
let result=convert_texture(asset_id,download_result).await;
drop(permit);
result.unwrap();
});
}
download_thread.await??;
_=SEM.acquire_many(thread_limit as u32).await.unwrap();
Ok(())
}
@ -155,7 +374,8 @@ fn download_assets(paths:Vec<PathBuf>)->AResult<()>{
enum ConvertError{
IO(std::io::Error),
SNFMap(strafesnet_snf::map::Error),
RbxLoader(strafesnet_rbx_loader::ReadError),
RobloxRead(strafesnet_rbx_loader::ReadError),
RobloxLoad(strafesnet_rbx_loader::LoadError),
}
impl std::fmt::Display for ConvertError{
fn fmt(&self,f:&mut std::fmt::Formatter<'_>)->std::fmt::Result{
@ -163,78 +383,49 @@ impl std::fmt::Display for ConvertError{
}
}
impl std::error::Error for ConvertError{}
async fn convert_to_snf(path:&Path,output_folder:PathBuf)->AResult<()>{
let entire_file=tokio::fs::read(path).await?;
type MapThread=std::thread::JoinHandle<Result<(),ConvertError>>;
let model=strafesnet_rbx_loader::read(
std::io::Cursor::new(entire_file)
).map_err(ConvertError::RobloxRead)?;
fn roblox_to_snf(pathlist:Vec<std::path::PathBuf>,output_folder:PathBuf)->AResult<()>{
let n_paths=pathlist.len();
let start = std::time::Instant::now();
let mut threads:std::collections::VecDeque<MapThread>=std::collections::VecDeque::new();
let mut i=0;
let mut join_thread=|thread:MapThread|{
i+=1;
if let Err(e)=thread.join(){
println!("thread error: {:?}",e);
}else{
println!("{}/{}",i,n_paths);
}
};
for path in pathlist{
if 32<=threads.len(){
join_thread(threads.pop_front().unwrap());
}
let output_folder=output_folder.clone();
threads.push_back(std::thread::spawn(move ||{
let model=strafesnet_rbx_loader::read(
std::fs::File::open(path.as_path())
.map_err(ConvertError::IO)?
).map_err(ConvertError::RbxLoader)?;
let mut place=model.into_place();
place.run_scripts();
let mut place=model.into_place();
place.run_scripts();
let map=place.to_snf(LoadFailureMode::DefaultToNone).map_err(ConvertError::RobloxLoad)?;
let mut loader=strafesnet_deferred_loader::roblox_legacy();
let mut dest=output_folder;
dest.push(path.file_stem().unwrap());
dest.set_extension("snfm");
let file=std::fs::File::create(dest).map_err(ConvertError::IO)?;
let (texture_loader,mesh_loader)=loader.get_inner_mut();
strafesnet_snf::map::write_map(file,map).map_err(ConvertError::SNFMap)?;
let map_step1=strafesnet_rbx_loader::convert(
&place,
|name|texture_loader.acquire_render_config_id(name),
|name|mesh_loader.acquire_mesh_id(name),
);
let meshpart_meshes=mesh_loader.load_meshes().map_err(ConvertError::IO)?;
let map_step2=map_step1.add_meshpart_meshes_and_calculate_attributes(
meshpart_meshes.into_iter().map(|(mesh_id,loader_model)|
(mesh_id,strafesnet_rbx_loader::data::RobloxMeshBytes::new(loader_model.get()))
)
);
let (textures,render_configs)=loader.into_render_configs().map_err(ConvertError::IO)?.consume();
let map=map_step2.add_render_configs_and_textures(
render_configs.into_iter(),
textures.into_iter().map(|(texture_id,texture)|
(texture_id,match texture{
strafesnet_deferred_loader::texture::Texture::ImageDDS(data)=>data,
})
)
);
let mut dest=output_folder.clone();
dest.push(path.file_stem().unwrap());
dest.set_extension("snfm");
let file=std::fs::File::create(dest).map_err(ConvertError::IO)?;
strafesnet_snf::map::write_map(file,map).map_err(ConvertError::SNFMap)?;
Ok(())
}));
}
for thread in threads{
join_thread(thread);
}
println!("{:?}", start.elapsed());
Ok(())
}
async fn roblox_to_snf(paths:Vec<std::path::PathBuf>,output_folder:PathBuf)->AResult<()>{
let start=std::time::Instant::now();
let thread_limit=std::thread::available_parallelism()?.get();
let mut it=paths.into_iter();
static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
SEM.add_permits(thread_limit);
while let (Ok(permit),Some(path))=(SEM.acquire().await,it.next()){
let output_folder=output_folder.clone();
tokio::spawn(async move{
let result=convert_to_snf(path.as_path(),output_folder).await;
drop(permit);
match result{
Ok(())=>(),
Err(e)=>println!("Convert error: {e:?}"),
}
});
}
_=SEM.acquire_many(thread_limit as u32).await.unwrap();
println!("elapsed={:?}", start.elapsed());
Ok(())
}

@ -1,6 +1,11 @@
use std::path::PathBuf;
use std::path::{Path,PathBuf};
use std::borrow::Cow;
use clap::{Args,Subcommand};
use anyhow::Result as AResult;
use futures::StreamExt;
use strafesnet_bsp_loader::loader::BspFinder;
use strafesnet_deferred_loader::loader::Loader;
use strafesnet_deferred_loader::deferred_loader::{LoadFailureMode,MeshDeferredLoader,RenderConfigDeferredLoader};
#[derive(Subcommand)]
pub enum Commands{
@ -16,13 +21,15 @@ pub struct SourceToSNFSubcommand {
output_folder:PathBuf,
#[arg(required=true)]
input_files:Vec<PathBuf>,
#[arg(long)]
vpks:Vec<PathBuf>,
}
#[derive(Args)]
pub struct ExtractTexturesSubcommand {
pub struct ExtractTexturesSubcommand{
#[arg(required=true)]
bsp_files:Vec<PathBuf>,
#[arg(long)]
bsp_file:PathBuf,
#[arg(long)]
vpk_dir_files:Vec<PathBuf>
vpks:Vec<PathBuf>,
}
#[derive(Args)]
pub struct VPKContentsSubcommand {
@ -36,10 +43,10 @@ pub struct BSPContentsSubcommand {
}
impl Commands{
pub fn run(self)->AResult<()>{
pub async fn run(self)->AResult<()>{
match self{
Commands::SourceToSNF(subcommand)=>source_to_snf(subcommand.input_files,subcommand.output_folder),
Commands::ExtractTextures(subcommand)=>extract_textures(vec![subcommand.bsp_file],subcommand.vpk_dir_files),
Commands::SourceToSNF(subcommand)=>source_to_snf(subcommand.input_files,subcommand.output_folder,subcommand.vpks).await,
Commands::ExtractTextures(subcommand)=>extract_textures(subcommand.bsp_files,subcommand.vpks).await,
Commands::VPKContents(subcommand)=>vpk_contents(subcommand.input_file),
Commands::BSPContents(subcommand)=>bsp_contents(subcommand.input_file),
}
@ -63,9 +70,9 @@ impl VMTContent{
}
}
fn get_some_texture(material:vmt_parser::material::Material)->AResult<VMTContent>{
fn get_some_texture(material:vmt_parser::material::Material)->VMTContent{
//just grab some texture from somewhere for now
Ok(match material{
match material{
vmt_parser::material::Material::LightMappedGeneric(mat)=>VMTContent::vtf(Some(mat.base_texture)),
vmt_parser::material::Material::VertexLitGeneric(mat)=>VMTContent::vtf(mat.base_texture.or(mat.decal_texture)),//this just dies if there is none
vmt_parser::material::Material::VertexLitGenericDx6(mat)=>VMTContent::vtf(mat.base_texture.or(mat.decal_texture)),
@ -84,175 +91,258 @@ fn get_some_texture(material:vmt_parser::material::Material)->AResult<VMTContent
vmt_parser::material::Material::Sky(mat)=>VMTContent::vtf(Some(mat.base_texture)),
vmt_parser::material::Material::Replacements(_mat)=>VMTContent::Unsupported,
vmt_parser::material::Material::Patch(mat)=>VMTContent::Patch(mat),
_=>return Err(anyhow::Error::msg("vmt failed to parse")),
})
}
fn get_vmt<F:Fn(String)->AResult<Option<Vec<u8>>>>(find_stuff:&F,search_name:String)->AResult<vmt_parser::material::Material>{
if let Some(stuff)=find_stuff(search_name)?{
//decode vmt and then write
let stuff=String::from_utf8(stuff)?;
let material=vmt_parser::from_str(stuff.as_str())?;
println!("vmt material={:?}",material);
return Ok(material);
_=>unreachable!(),
}
Err(anyhow::Error::msg("vmt not found"))
}
fn recursive_vmt_loader<F:Fn(String)->AResult<Option<Vec<u8>>>>(find_stuff:&F,material:vmt_parser::material::Material)->AResult<Option<Vec<u8>>>{
match get_some_texture(material)?{
VMTContent::VMT(s)=>recursive_vmt_loader(find_stuff,get_vmt(find_stuff,s)?),
#[derive(Debug,thiserror::Error)]
enum GetVMTError{
#[error("Bsp error {0:?}")]
Bsp(#[from]vbsp::BspError),
#[error("Utf8 error {0:?}")]
Utf8(#[from]std::str::Utf8Error),
#[error("Vdf error {0:?}")]
Vdf(#[from]vmt_parser::VdfError),
#[error("Vmt not found")]
NotFound,
}
fn get_vmt(finder:BspFinder,search_name:&str)->Result<vmt_parser::material::Material,GetVMTError>{
let vmt_data=finder.find(search_name)?.ok_or(GetVMTError::NotFound)?;
//decode vmt and then write
let vmt_str=core::str::from_utf8(&vmt_data)?;
let material=vmt_parser::from_str(vmt_str)?;
//println!("vmt material={:?}",material);
Ok(material)
}
#[derive(Debug,thiserror::Error)]
enum LoadVMTError{
#[error("Bsp error {0:?}")]
Bsp(#[from]vbsp::BspError),
#[error("GetVMT error {0:?}")]
GetVMT(#[from]GetVMTError),
#[error("FromUtf8 error {0:?}")]
FromUtf8(#[from]std::string::FromUtf8Error),
#[error("Vdf error {0:?}")]
Vdf(#[from]vmt_parser::VdfError),
#[error("Vmt unsupported")]
Unsupported,
#[error("Vmt unresolved")]
Unresolved,
#[error("Vmt not found")]
NotFound,
}
fn recursive_vmt_loader<'bsp,'vpk,'a>(finder:BspFinder<'bsp,'vpk>,material:vmt_parser::material::Material)->Result<Option<Cow<'a,[u8]>>,LoadVMTError>
where
'bsp:'a,
'vpk:'a,
{
match get_some_texture(material){
VMTContent::VMT(s)=>recursive_vmt_loader(finder,get_vmt(finder,s.as_str())?),
VMTContent::VTF(s)=>{
let mut texture_file_name=PathBuf::from("materials");
texture_file_name.push(s);
texture_file_name.set_extension("vtf");
find_stuff(texture_file_name.into_os_string().into_string().unwrap())
Ok(finder.find(texture_file_name.to_str().unwrap())?)
},
VMTContent::Patch(mat)=>recursive_vmt_loader(find_stuff,
mat.resolve(|search_name|{
match find_stuff(search_name.to_string())?{
Some(bytes)=>Ok(String::from_utf8(bytes)?),
None=>Err(anyhow::Error::msg("could not find vmt")),
VMTContent::Patch(mat)=>recursive_vmt_loader(finder,
mat.resolve(|search_name|
match finder.find(search_name)?{
Some(bytes)=>Ok(String::from_utf8(bytes.into_owned())?),
None=>Err(LoadVMTError::NotFound),
}
})?
)?
),
VMTContent::Unsupported=>{println!("Unsupported vmt");Ok(None)},//print and move on
VMTContent::Unresolved=>{println!("Unresolved vmt");Ok(None)},
VMTContent::Unsupported=>Err(LoadVMTError::Unsupported),
VMTContent::Unresolved=>Err(LoadVMTError::Unresolved),
}
}
fn extract_textures(paths:Vec<PathBuf>,vpk_paths:Vec<PathBuf>)->AResult<()>{
std::fs::create_dir_all("textures")?;
let vpk_list:Vec<vpk::VPK>=vpk_paths.into_iter().map(|vpk_path|vpk::VPK::read(&vpk_path).expect("vpk file does not exist")).collect();
for path in paths{
let mut deduplicate=std::collections::HashSet::new();
let bsp=vbsp::Bsp::read(std::fs::read(path)?.as_ref())?;
for texture in bsp.textures(){
deduplicate.insert(PathBuf::from(texture.name()));
}
//dedupe prop models
let mut model_dedupe=std::collections::HashSet::new();
for prop in bsp.static_props(){
model_dedupe.insert(prop.model());
}
//grab texture names from props
for model_name in model_dedupe{
//.mdl, .vvd, .dx90.vtx
let mut path=PathBuf::from(model_name);
let file_name=PathBuf::from(path.file_stem().unwrap());
path.pop();
path.push(file_name);
let mut vvd_path=path.clone();
let mut vtx_path=path.clone();
vvd_path.set_extension("vvd");
vtx_path.set_extension("dx90.vtx");
match (bsp.pack.get(model_name),bsp.pack.get(vvd_path.as_os_str().to_str().unwrap()),bsp.pack.get(vtx_path.as_os_str().to_str().unwrap())){
(Ok(Some(mdl_file)),Ok(Some(vvd_file)),Ok(Some(vtx_file)))=>{
match (vmdl::mdl::Mdl::read(mdl_file.as_ref()),vmdl::vvd::Vvd::read(vvd_file.as_ref()),vmdl::vtx::Vtx::read(vtx_file.as_ref())){
(Ok(mdl),Ok(vvd),Ok(vtx))=>{
let model=vmdl::Model::from_parts(mdl,vtx,vvd);
for texture in model.textures(){
for search_path in &texture.search_paths{
let mut path=PathBuf::from(search_path.as_str());
path.push(texture.name.as_str());
deduplicate.insert(path);
}
}
},
_=>println!("model_name={} error",model_name),
}
},
_=>println!("no model name={}",model_name),
}
}
let pack=&bsp.pack;
let vpk_list=&vpk_list;
std::thread::scope(move|s|{
let mut thread_handles=Vec::new();
for texture_name in deduplicate{
let mut found_texture=false;
//LMAO imagine having to write type names
let write_image=|mut stuff,write_file_name|{
let image=vtf::from_bytes(&mut stuff)?.highres_image.decode(0)?.to_rgba8();
let format=if image.width()%4!=0||image.height()%4!=0{
image_dds::ImageFormat::Rgba8UnormSrgb
}else{
image_dds::ImageFormat::BC7RgbaUnormSrgb
};
//this fails if the image dimensions are not a multiple of 4
let dds = image_dds::dds_from_image(
&image,
format,
image_dds::Quality::Slow,
image_dds::Mipmaps::GeneratedAutomatic,
)?;
//write dds
let mut dest=PathBuf::from("textures");
dest.push(write_file_name);
dest.set_extension("dds");
std::fs::create_dir_all(dest.parent().unwrap())?;
let mut writer = std::io::BufWriter::new(std::fs::File::create(dest)?);
dds.write(&mut writer)?;
Ok::<(),anyhow::Error>(())
};
let find_stuff=|search_file_name:String|{
println!("search_file_name={}",search_file_name);
match pack.get(search_file_name.as_str())?{
Some(file)=>return Ok(Some(file)),
_=>(),
}
//search pak list
for vpk_index in vpk_list{
if let Some(vpk_entry)=vpk_index.tree.get(search_file_name.as_str()){
return Ok(Some(match vpk_entry.get()?{
std::borrow::Cow::Borrowed(bytes)=>bytes.to_vec(),
std::borrow::Cow::Owned(bytes)=>bytes,
}));
}
}
Ok::<Option<Vec<u8>>,anyhow::Error>(None)
};
let loader=|texture_name:String|{
let mut texture_file_name=PathBuf::from("materials");
//lower case
let texture_file_name_lowercase=texture_name.to_lowercase();
texture_file_name.push(texture_file_name_lowercase.clone());
//remove stem and search for both vtf and vmt files
let stem=PathBuf::from(texture_file_name.file_stem().unwrap());
texture_file_name.pop();
texture_file_name.push(stem);
//somehow search for both files
let mut texture_file_name_vmt=texture_file_name.clone();
texture_file_name.set_extension("vtf");
texture_file_name_vmt.set_extension("vmt");
if let Some(stuff)=find_stuff(texture_file_name.to_string_lossy().to_string())?{
return Ok(Some(stuff))
}
recursive_vmt_loader(&find_stuff,get_vmt(&find_stuff,texture_file_name_vmt.to_string_lossy().to_string())?)
};
if let Some(stuff)=loader(texture_name.to_string_lossy().to_string())?{
found_texture=true;
let texture_name=texture_name.clone();
thread_handles.push(s.spawn(move||write_image(stuff,texture_name)));
}
if !found_texture{
println!("no data");
}
}
for thread in thread_handles{
match thread.join(){
Ok(Err(e))=>println!("write error: {:?}",e),
Err(e)=>println!("thread error: {:?}",e),
Ok(_)=>(),
}
}
Ok::<(),anyhow::Error>(())
})?
fn load_texture<'bsp,'vpk,'a>(finder:BspFinder<'bsp,'vpk>,texture_name:&str)->Result<Option<Cow<'a,[u8]>>,LoadVMTError>
where
'bsp:'a,
'vpk:'a,
{
let mut texture_file_name=PathBuf::from("materials");
//lower case
let texture_file_name_lowercase=texture_name.to_lowercase();
texture_file_name.push(texture_file_name_lowercase.clone());
//remove stem and search for both vtf and vmt files
let stem=PathBuf::from(texture_file_name.file_stem().unwrap());
texture_file_name.pop();
texture_file_name.push(stem);
if let Some(stuff)=finder.find(texture_file_name.to_str().unwrap())?{
return Ok(Some(stuff))
}
//somehow search for both files
let mut texture_file_name_vmt=texture_file_name.clone();
texture_file_name.set_extension("vtf");
texture_file_name_vmt.set_extension("vmt");
recursive_vmt_loader(finder,get_vmt(finder,texture_file_name_vmt.to_str().unwrap())?)
}
#[derive(Debug,thiserror::Error)]
enum ExtractTextureError{
#[error("Io error {0:?}")]
Io(#[from]std::io::Error),
#[error("Bsp error {0:?}")]
Bsp(#[from]vbsp::BspError),
#[error("MeshLoad error {0:?}")]
MeshLoad(#[from]strafesnet_bsp_loader::loader::MeshError),
#[error("Load VMT error {0:?}")]
LoadVMT(#[from]LoadVMTError),
}
async fn gimme_them_textures(path:&Path,vpk_list:&[vpk::VPK],send_texture:tokio::sync::mpsc::Sender<(Vec<u8>,String)>)->Result<(),ExtractTextureError>{
let bsp=vbsp::Bsp::read(tokio::fs::read(path).await?.as_ref())?;
let loader_bsp=strafesnet_bsp_loader::Bsp::new(bsp);
let bsp=loader_bsp.as_ref();
let mut texture_deferred_loader=RenderConfigDeferredLoader::new();
for texture in bsp.textures(){
texture_deferred_loader.acquire_render_config_id(Some(Cow::Borrowed(texture.name())));
}
let mut mesh_deferred_loader=MeshDeferredLoader::new();
for prop in bsp.static_props(){
mesh_deferred_loader.acquire_mesh_id(prop.model());
}
let finder=BspFinder{
bsp:&loader_bsp,
vpks:vpk_list
};
let mut mesh_loader=strafesnet_bsp_loader::loader::ModelLoader::new(finder);
// load models and collect requested textures
for model_path in mesh_deferred_loader.into_indices(){
let model:vmdl::Model=match mesh_loader.load(model_path){
Ok(model)=>model,
Err(e)=>{
println!("Model={model_path} Load model error: {e}");
continue;
},
};
for texture in model.textures(){
for search_path in &texture.search_paths{
let mut path=PathBuf::from(search_path.as_str());
path.push(texture.name.as_str());
let path=path.to_str().unwrap().to_owned();
texture_deferred_loader.acquire_render_config_id(Some(Cow::Owned(path)));
}
}
}
for texture_path in texture_deferred_loader.into_indices(){
match load_texture(finder,&texture_path){
Ok(Some(texture))=>send_texture.send(
(texture.into_owned(),texture_path.into_owned())
).await.unwrap(),
Ok(None)=>(),
Err(e)=>println!("Texture={texture_path} Load error: {e}"),
}
}
Ok(())
}
#[derive(Debug,thiserror::Error)]
enum ConvertTextureError{
#[error("Bsp error {0:?}")]
Bsp(#[from]vbsp::BspError),
#[error("Vtf error {0:?}")]
Vtf(#[from]vtf::Error),
#[error("DDS create error {0:?}")]
DDS(#[from]image_dds::CreateDdsError),
#[error("DDS write error {0:?}")]
DDSWrite(#[from]image_dds::ddsfile::Error),
#[error("Io error {0:?}")]
Io(#[from]std::io::Error),
}
async fn convert_texture(texture:Vec<u8>,write_file_name:impl AsRef<Path>)->Result<(),ConvertTextureError>{
let image=vtf::from_bytes(&texture)?.highres_image.decode(0)?.to_rgba8();
let format=if image.width()%4!=0||image.height()%4!=0{
image_dds::ImageFormat::Rgba8UnormSrgb
}else{
image_dds::ImageFormat::BC7RgbaUnormSrgb
};
//this fails if the image dimensions are not a multiple of 4
let dds = image_dds::dds_from_image(
&image,
format,
image_dds::Quality::Slow,
image_dds::Mipmaps::GeneratedAutomatic,
)?;
//write dds
let mut dest=PathBuf::from("textures");
dest.push(write_file_name);
dest.set_extension("dds");
std::fs::create_dir_all(dest.parent().unwrap())?;
let mut writer=std::io::BufWriter::new(std::fs::File::create(dest)?);
dds.write(&mut writer)?;
Ok(())
}
async fn read_vpks(vpk_paths:Vec<PathBuf>,thread_limit:usize)->Vec<vpk::VPK>{
futures::stream::iter(vpk_paths).map(|vpk_path|async{
// idk why it doesn't want to pass out the errors but this is fatal anyways
tokio::task::spawn_blocking(move||vpk::VPK::read(&vpk_path)).await.unwrap().unwrap()
})
.buffer_unordered(thread_limit)
.collect().await
}
async fn extract_textures(paths:Vec<PathBuf>,vpk_paths:Vec<PathBuf>)->AResult<()>{
tokio::try_join!(
tokio::fs::create_dir_all("extracted_textures"),
tokio::fs::create_dir_all("textures"),
tokio::fs::create_dir_all("meshes"),
)?;
let thread_limit=std::thread::available_parallelism()?.get();
// load vpk list
let vpk_list=read_vpks(vpk_paths,thread_limit).await;
// leak vpk_list for static lifetime?
let vpk_list:&[vpk::VPK]=vpk_list.leak();
let (send_texture,mut recv_texture)=tokio::sync::mpsc::channel(thread_limit);
let mut it=paths.into_iter();
let extract_thread=tokio::spawn(async move{
static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
SEM.add_permits(thread_limit);
while let (Ok(permit),Some(path))=(SEM.acquire().await,it.next()){
let send=send_texture.clone();
tokio::spawn(async move{
let result=gimme_them_textures(&path,vpk_list,send).await;
drop(permit);
match result{
Ok(())=>(),
Err(e)=>println!("Map={path:?} Decode error: {e:?}"),
}
});
}
});
// convert images
static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
SEM.add_permits(thread_limit);
while let (Ok(permit),Some((data,dest)))=(SEM.acquire().await,recv_texture.recv().await){
// TODO: dedup dest?
tokio::spawn(async move{
let result=convert_texture(data,dest).await;
drop(permit);
match result{
Ok(())=>(),
Err(e)=>println!("Convert error: {e:?}"),
}
});
}
extract_thread.await?;
_=SEM.acquire_many(thread_limit as u32).await?;
Ok(())
}
@ -277,7 +367,8 @@ fn bsp_contents(path:PathBuf)->AResult<()>{
enum ConvertError{
IO(std::io::Error),
SNFMap(strafesnet_snf::map::Error),
BspLoader(strafesnet_bsp_loader::ReadError),
BspRead(strafesnet_bsp_loader::ReadError),
BspLoad(strafesnet_bsp_loader::LoadError),
}
impl std::fmt::Display for ConvertError{
fn fmt(&self,f:&mut std::fmt::Formatter<'_>)->std::fmt::Result{
@ -286,79 +377,52 @@ impl std::fmt::Display for ConvertError{
}
impl std::error::Error for ConvertError{}
type MapThread=std::thread::JoinHandle<Result<(),ConvertError>>;
async fn convert_to_snf(path:&Path,vpk_list:&[vpk::VPK],output_folder:PathBuf)->AResult<()>{
let entire_file=tokio::fs::read(path).await?;
fn source_to_snf(pathlist:Vec<std::path::PathBuf>,output_folder:PathBuf)->AResult<()>{
let n_paths=pathlist.len();
let start = std::time::Instant::now();
let mut threads:std::collections::VecDeque<MapThread>=std::collections::VecDeque::new();
let mut i=0;
let mut join_thread=|thread:MapThread|{
i+=1;
if let Err(e)=thread.join(){
println!("thread error: {:?}",e);
}else{
println!("{}/{}",i,n_paths);
}
};
for path in pathlist{
if 32<=threads.len(){
join_thread(threads.pop_front().unwrap());
}
let output_folder=output_folder.clone();
threads.push_back(std::thread::spawn(move ||{
let bsp=strafesnet_bsp_loader::read(
std::fs::File::open(path.as_path())
.map_err(ConvertError::IO)?
).map_err(ConvertError::BspLoader)?;
let mut loader=strafesnet_deferred_loader::source_legacy();
let bsp=strafesnet_bsp_loader::read(
std::io::Cursor::new(entire_file)
).map_err(ConvertError::BspRead)?;
let (texture_loader,mesh_loader)=loader.get_inner_mut();
let map=bsp.to_snf(LoadFailureMode::DefaultToNone,vpk_list).map_err(ConvertError::BspLoad)?;
let map_step1=strafesnet_bsp_loader::convert(
&bsp,
|name|texture_loader.acquire_render_config_id(name),
|name|mesh_loader.acquire_mesh_id(name),
);
let mut dest=output_folder;
dest.push(path.file_stem().unwrap());
dest.set_extension("snfm");
let file=std::fs::File::create(dest).map_err(ConvertError::IO)?;
let prop_meshes=mesh_loader.load_meshes(&bsp.as_ref());
strafesnet_snf::map::write_map(file,map).map_err(ConvertError::SNFMap)?;
let map_step2=map_step1.add_prop_meshes(
//the type conflagulator 9000
prop_meshes.into_iter().map(|(mesh_id,loader_model)|
(mesh_id,strafesnet_bsp_loader::data::ModelData{
mdl:strafesnet_bsp_loader::data::MdlData::new(loader_model.mdl.get()),
vtx:strafesnet_bsp_loader::data::VtxData::new(loader_model.vtx.get()),
vvd:strafesnet_bsp_loader::data::VvdData::new(loader_model.vvd.get()),
})
),
|name|texture_loader.acquire_render_config_id(name),
);
let (textures,render_configs)=loader.into_render_configs().map_err(ConvertError::IO)?.consume();
let map=map_step2.add_render_configs_and_textures(
render_configs.into_iter(),
textures.into_iter().map(|(texture_id,texture)|
(texture_id,match texture{
strafesnet_deferred_loader::texture::Texture::ImageDDS(data)=>data,
})
),
);
let mut dest=output_folder.clone();
dest.push(path.file_stem().unwrap());
dest.set_extension("snfm");
let file=std::fs::File::create(dest).map_err(ConvertError::IO)?;
strafesnet_snf::map::write_map(file,map).map_err(ConvertError::SNFMap)?;
Ok(())
}));
}
for thread in threads{
join_thread(thread);
}
println!("{:?}", start.elapsed());
Ok(())
}
async fn source_to_snf(paths:Vec<std::path::PathBuf>,output_folder:PathBuf,vpk_paths:Vec<PathBuf>)->AResult<()>{
let start=std::time::Instant::now();
let thread_limit=std::thread::available_parallelism()?.get();
// load vpk list
let vpk_list=read_vpks(vpk_paths,thread_limit).await;
// leak vpk_list for static lifetime?
let vpk_list:&[vpk::VPK]=vpk_list.leak();
let mut it=paths.into_iter();
static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
SEM.add_permits(thread_limit);
while let (Ok(permit),Some(path))=(SEM.acquire().await,it.next()){
let output_folder=output_folder.clone();
tokio::spawn(async move{
let result=convert_to_snf(path.as_path(),vpk_list,output_folder).await;
drop(permit);
match result{
Ok(())=>(),
Err(e)=>println!("Convert error: {e:?}"),
}
});
}
_=SEM.acquire_many(thread_limit as u32).await.unwrap();
println!("elapsed={:?}", start.elapsed());
Ok(())
}