loader + tokio rewrite
This commit is contained in:
parent
6522c255cd
commit
ca50bf35c2
1166
Cargo.lock
generated
1166
Cargo.lock
generated
File diff suppressed because it is too large
Load Diff
15
Cargo.toml
15
Cargo.toml
@ -9,17 +9,22 @@ edition = "2021"
|
|||||||
anyhow = "1.0.75"
|
anyhow = "1.0.75"
|
||||||
clap = { version = "4.4.2", features = ["derive"] }
|
clap = { version = "4.4.2", features = ["derive"] }
|
||||||
flate2 = "1.0.27"
|
flate2 = "1.0.27"
|
||||||
|
futures = "0.3.31"
|
||||||
image = "0.25.2"
|
image = "0.25.2"
|
||||||
image_dds = "0.6.0"
|
image_dds = "0.7.1"
|
||||||
lazy-regex = "3.1.0"
|
lazy-regex = "3.1.0"
|
||||||
|
rbx_asset = { version = "0.2.5", registry = "strafesnet" }
|
||||||
rbx_binary = { version = "0.7.4", registry = "strafesnet" }
|
rbx_binary = { version = "0.7.4", registry = "strafesnet" }
|
||||||
rbx_dom_weak = { version = "2.7.0", registry = "strafesnet" }
|
rbx_dom_weak = { version = "2.7.0", registry = "strafesnet" }
|
||||||
rbx_reflection_database = { version = "0.2.10", registry = "strafesnet" }
|
rbx_reflection_database = { version = "0.2.10", registry = "strafesnet" }
|
||||||
rbx_xml = { version = "0.13.3", registry = "strafesnet" }
|
rbx_xml = { version = "0.13.3", registry = "strafesnet" }
|
||||||
strafesnet_bsp_loader = { version = "0.2.1", registry = "strafesnet" }
|
rbxassetid = { version = "0.1.0", registry = "strafesnet" }
|
||||||
strafesnet_deferred_loader = { version = "0.4.0", features = ["legacy"], registry = "strafesnet" }
|
strafesnet_bsp_loader = { version = "0.3.0", registry = "strafesnet" }
|
||||||
strafesnet_rbx_loader = { version = "0.5.1", registry = "strafesnet" }
|
strafesnet_deferred_loader = { version = "0.5.0", registry = "strafesnet" }
|
||||||
strafesnet_snf = { version = "0.2.0", registry = "strafesnet" }
|
strafesnet_rbx_loader = { version = "0.6.0", registry = "strafesnet" }
|
||||||
|
strafesnet_snf = { version = "0.3.0", registry = "strafesnet" }
|
||||||
|
thiserror = "2.0.11"
|
||||||
|
tokio = { version = "1.43.0", features = ["macros", "rt-multi-thread", "fs"] }
|
||||||
vbsp = "0.6.0"
|
vbsp = "0.6.0"
|
||||||
vmdl = "0.2.0"
|
vmdl = "0.2.0"
|
||||||
vmt-parser = "0.2.0"
|
vmt-parser = "0.2.0"
|
||||||
|
@ -20,10 +20,11 @@ enum Commands{
|
|||||||
Source(source::Commands),
|
Source(source::Commands),
|
||||||
}
|
}
|
||||||
|
|
||||||
fn main() -> AResult<()> {
|
#[tokio::main]
|
||||||
let cli = Cli::parse();
|
async fn main()->AResult<()>{
|
||||||
|
let cli=Cli::parse();
|
||||||
match cli.command{
|
match cli.command{
|
||||||
Commands::Roblox(commands)=>commands.run(),
|
Commands::Roblox(commands)=>commands.run().await,
|
||||||
Commands::Source(commands)=>commands.run(),
|
Commands::Source(commands)=>commands.run().await,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
389
src/roblox.rs
389
src/roblox.rs
@ -4,7 +4,11 @@ use std::collections::HashSet;
|
|||||||
use clap::{Args,Subcommand};
|
use clap::{Args,Subcommand};
|
||||||
use anyhow::Result as AResult;
|
use anyhow::Result as AResult;
|
||||||
use rbx_dom_weak::Instance;
|
use rbx_dom_weak::Instance;
|
||||||
use strafesnet_deferred_loader::rbxassetid::RobloxAssetId;
|
use strafesnet_deferred_loader::deferred_loader::LoadFailureMode;
|
||||||
|
use rbxassetid::RobloxAssetId;
|
||||||
|
use tokio::io::AsyncReadExt;
|
||||||
|
|
||||||
|
const DOWNLOAD_LIMIT:usize=16;
|
||||||
|
|
||||||
#[derive(Subcommand)]
|
#[derive(Subcommand)]
|
||||||
pub enum Commands{
|
pub enum Commands{
|
||||||
@ -21,27 +25,40 @@ pub struct RobloxToSNFSubcommand {
|
|||||||
}
|
}
|
||||||
#[derive(Args)]
|
#[derive(Args)]
|
||||||
pub struct DownloadAssetsSubcommand{
|
pub struct DownloadAssetsSubcommand{
|
||||||
#[arg(long,required=true)]
|
#[arg(required=true)]
|
||||||
roblox_files:Vec<PathBuf>
|
roblox_files:Vec<PathBuf>,
|
||||||
|
// #[arg(long)]
|
||||||
|
// cookie_file:Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Commands{
|
impl Commands{
|
||||||
pub fn run(self)->AResult<()>{
|
pub async fn run(self)->AResult<()>{
|
||||||
match self{
|
match self{
|
||||||
Commands::RobloxToSNF(subcommand)=>roblox_to_snf(subcommand.input_files,subcommand.output_folder),
|
Commands::RobloxToSNF(subcommand)=>roblox_to_snf(subcommand.input_files,subcommand.output_folder).await,
|
||||||
Commands::DownloadAssets(subcommand)=>download_assets(subcommand.roblox_files),
|
Commands::DownloadAssets(subcommand)=>download_assets(
|
||||||
|
subcommand.roblox_files,
|
||||||
|
rbx_asset::cookie::Cookie::new("".to_string()),
|
||||||
|
).await,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn load_dom<R:Read+Seek>(mut input:R)->AResult<rbx_dom_weak::WeakDom>{
|
#[allow(unused)]
|
||||||
|
#[derive(Debug)]
|
||||||
|
enum LoadDomError{
|
||||||
|
IO(std::io::Error),
|
||||||
|
Binary(rbx_binary::DecodeError),
|
||||||
|
Xml(rbx_xml::DecodeError),
|
||||||
|
UnknownFormat,
|
||||||
|
}
|
||||||
|
fn load_dom<R:Read+Seek>(mut input:R)->Result<rbx_dom_weak::WeakDom,LoadDomError>{
|
||||||
let mut first_8=[0u8;8];
|
let mut first_8=[0u8;8];
|
||||||
input.read_exact(&mut first_8)?;
|
input.read_exact(&mut first_8).map_err(LoadDomError::IO)?;
|
||||||
input.rewind()?;
|
input.rewind().map_err(LoadDomError::IO)?;
|
||||||
match &first_8{
|
match &first_8{
|
||||||
b"<roblox!"=>rbx_binary::from_reader(input).map_err(anyhow::Error::msg),
|
b"<roblox!"=>rbx_binary::from_reader(input).map_err(LoadDomError::Binary),
|
||||||
b"<roblox "=>rbx_xml::from_reader(input,rbx_xml::DecodeOptions::default()).map_err(anyhow::Error::msg),
|
b"<roblox "=>rbx_xml::from_reader(input,rbx_xml::DecodeOptions::default()).map_err(LoadDomError::Xml),
|
||||||
_=>Err(anyhow::Error::msg("unsupported file type")),
|
_=>Err(LoadDomError::UnknownFormat),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -80,13 +97,13 @@ fn accumulate_content_id(content_list:&mut HashSet<RobloxAssetId>,object:&Instan
|
|||||||
println!("Content failed to parse into AssetID: {:?}",content);
|
println!("Content failed to parse into AssetID: {:?}",content);
|
||||||
}
|
}
|
||||||
}else{
|
}else{
|
||||||
println!("property={} does not exist for class={}",object.class.as_str(),property);
|
println!("property={} does not exist for class={}",property,object.class.as_str());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
fn read_entire_file(path:impl AsRef<Path>)->Result<Cursor<Vec<u8>>,std::io::Error>{
|
async fn read_entire_file(path:impl AsRef<Path>)->Result<Cursor<Vec<u8>>,std::io::Error>{
|
||||||
let mut file=std::fs::File::open(path)?;
|
let mut file=tokio::fs::File::open(path).await?;
|
||||||
let mut data=Vec::new();
|
let mut data=Vec::new();
|
||||||
file.read_to_end(&mut data)?;
|
file.read_to_end(&mut data).await?;
|
||||||
Ok(Cursor::new(data))
|
Ok(Cursor::new(data))
|
||||||
}
|
}
|
||||||
#[derive(Default)]
|
#[derive(Default)]
|
||||||
@ -123,30 +140,232 @@ impl UniqueAssets{
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
fn unique_assets(path:&Path)->AResult<UniqueAssets>{
|
|
||||||
|
#[allow(unused)]
|
||||||
|
#[derive(Debug)]
|
||||||
|
enum UniqueAssetError{
|
||||||
|
IO(std::io::Error),
|
||||||
|
LoadDom(LoadDomError),
|
||||||
|
}
|
||||||
|
async fn unique_assets(path:&Path)->Result<UniqueAssets,UniqueAssetError>{
|
||||||
// read entire file
|
// read entire file
|
||||||
let mut assets=UniqueAssets::default();
|
let mut assets=UniqueAssets::default();
|
||||||
let data=read_entire_file(path)?;
|
let data=read_entire_file(path).await.map_err(UniqueAssetError::IO)?;
|
||||||
let dom=load_dom(data)?;
|
let dom=load_dom(data).map_err(UniqueAssetError::LoadDom)?;
|
||||||
for object in dom.into_raw().1.into_values(){
|
for object in dom.into_raw().1.into_values(){
|
||||||
assets.collect(&object);
|
assets.collect(&object);
|
||||||
}
|
}
|
||||||
Ok(assets)
|
Ok(assets)
|
||||||
}
|
}
|
||||||
struct UniqueAssetsResult{
|
enum DownloadType{
|
||||||
path:std::path::PathBuf,
|
Texture(RobloxAssetId),
|
||||||
result:AResult<UniqueAssets>,
|
Mesh(RobloxAssetId),
|
||||||
|
Union(RobloxAssetId),
|
||||||
}
|
}
|
||||||
fn do_thread(path:std::path::PathBuf,send:std::sync::mpsc::Sender<UniqueAssetsResult>){
|
impl DownloadType{
|
||||||
std::thread::spawn(move ||{
|
fn path(&self)->PathBuf{
|
||||||
let result=unique_assets(path.as_path());
|
match self{
|
||||||
send.send(UniqueAssetsResult{
|
DownloadType::Texture(asset_id)=>format!("downloaded_textures/{}",asset_id.0.to_string()).into(),
|
||||||
path,
|
DownloadType::Mesh(asset_id)=>format!("meshes/{}",asset_id.0.to_string()).into(),
|
||||||
result,
|
DownloadType::Union(asset_id)=>format!("unions/{}",asset_id.0.to_string()).into(),
|
||||||
}).unwrap();
|
}
|
||||||
|
}
|
||||||
|
fn asset_id(&self)->u64{
|
||||||
|
match self{
|
||||||
|
DownloadType::Texture(asset_id)=>asset_id.0,
|
||||||
|
DownloadType::Mesh(asset_id)=>asset_id.0,
|
||||||
|
DownloadType::Union(asset_id)=>asset_id.0,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
enum DownloadResult{
|
||||||
|
Cached(PathBuf),
|
||||||
|
Data(Vec<u8>),
|
||||||
|
Failed,
|
||||||
|
}
|
||||||
|
#[derive(Default,Debug)]
|
||||||
|
struct Stats{
|
||||||
|
total_assets:u32,
|
||||||
|
cached_assets:u32,
|
||||||
|
downloaded_assets:u32,
|
||||||
|
failed_downloads:u32,
|
||||||
|
timed_out_downloads:u32,
|
||||||
|
}
|
||||||
|
async fn download_retry(stats:&mut Stats,context:&rbx_asset::cookie::CookieContext,download_instruction:DownloadType)->Result<DownloadResult,std::io::Error>{
|
||||||
|
stats.total_assets+=1;
|
||||||
|
let download_instruction=download_instruction;
|
||||||
|
// check if file exists on disk
|
||||||
|
let path=download_instruction.path();
|
||||||
|
if tokio::fs::try_exists(path.as_path()).await?{
|
||||||
|
stats.cached_assets+=1;
|
||||||
|
return Ok(DownloadResult::Cached(path));
|
||||||
|
}
|
||||||
|
let asset_id=download_instruction.asset_id();
|
||||||
|
// if not, download file
|
||||||
|
let mut retry=0;
|
||||||
|
const BACKOFF_MUL:f32=1.3956124250860895286;//exp(1/3)
|
||||||
|
let mut backoff=1000f32;
|
||||||
|
loop{
|
||||||
|
let asset_result=context.get_asset(rbx_asset::cookie::GetAssetRequest{
|
||||||
|
asset_id,
|
||||||
|
version:None,
|
||||||
|
}).await;
|
||||||
|
match asset_result{
|
||||||
|
Ok(asset_result)=>{
|
||||||
|
stats.downloaded_assets+=1;
|
||||||
|
tokio::fs::write(path,&asset_result).await?;
|
||||||
|
break Ok(DownloadResult::Data(asset_result));
|
||||||
|
},
|
||||||
|
Err(rbx_asset::cookie::GetError::Response(rbx_asset::ResponseError::StatusCodeWithUrlAndBody(scwuab)))=>{
|
||||||
|
if scwuab.status_code.as_u16()==429{
|
||||||
|
if retry==12{
|
||||||
|
println!("Giving up asset download {asset_id}");
|
||||||
|
stats.timed_out_downloads+=1;
|
||||||
|
break Ok(DownloadResult::Failed);
|
||||||
|
}
|
||||||
|
println!("Hit roblox rate limit, waiting {:.0}ms...",backoff);
|
||||||
|
tokio::time::sleep(std::time::Duration::from_millis(backoff as u64)).await;
|
||||||
|
backoff*=BACKOFF_MUL;
|
||||||
|
retry+=1;
|
||||||
|
}else{
|
||||||
|
stats.failed_downloads+=1;
|
||||||
|
println!("weird scuwab error: {scwuab:?}");
|
||||||
|
break Ok(DownloadResult::Failed);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
Err(e)=>{
|
||||||
|
stats.failed_downloads+=1;
|
||||||
|
println!("sadly error: {e}");
|
||||||
|
break Ok(DownloadResult::Failed);
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
#[derive(Debug,thiserror::Error)]
|
||||||
|
enum ConvertTextureError{
|
||||||
|
#[error("Io error {0:?}")]
|
||||||
|
Io(#[from]std::io::Error),
|
||||||
|
#[error("Image error {0:?}")]
|
||||||
|
Image(#[from]image::ImageError),
|
||||||
|
#[error("DDS create error {0:?}")]
|
||||||
|
DDS(#[from]image_dds::CreateDdsError),
|
||||||
|
#[error("DDS write error {0:?}")]
|
||||||
|
DDSWrite(#[from]image_dds::ddsfile::Error),
|
||||||
|
}
|
||||||
|
async fn convert_texture(asset_id:RobloxAssetId,download_result:DownloadResult)->Result<(),ConvertTextureError>{
|
||||||
|
let data=match download_result{
|
||||||
|
DownloadResult::Cached(path)=>tokio::fs::read(path).await?,
|
||||||
|
DownloadResult::Data(data)=>data,
|
||||||
|
DownloadResult::Failed=>return Ok(()),
|
||||||
|
};
|
||||||
|
// image::ImageFormat::Png
|
||||||
|
// image::ImageFormat::Jpeg
|
||||||
|
let image=image::load_from_memory(&data)?.to_rgba8();
|
||||||
|
|
||||||
|
// pick format
|
||||||
|
let format=if image.width()%4!=0||image.height()%4!=0{
|
||||||
|
image_dds::ImageFormat::Rgba8UnormSrgb
|
||||||
|
}else{
|
||||||
|
image_dds::ImageFormat::BC7RgbaUnormSrgb
|
||||||
|
};
|
||||||
|
|
||||||
|
//this fails if the image dimensions are not a multiple of 4
|
||||||
|
let dds=image_dds::dds_from_image(
|
||||||
|
&image,
|
||||||
|
format,
|
||||||
|
image_dds::Quality::Slow,
|
||||||
|
image_dds::Mipmaps::GeneratedAutomatic,
|
||||||
|
)?;
|
||||||
|
|
||||||
|
let file_name=format!("textures/{}.dds",asset_id.0);
|
||||||
|
let mut file=std::fs::File::create(file_name)?;
|
||||||
|
dds.write(&mut file)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
async fn download_assets(paths:Vec<PathBuf>,cookie:rbx_asset::cookie::Cookie)->AResult<()>{
|
||||||
|
tokio::try_join!(
|
||||||
|
tokio::fs::create_dir_all("downloaded_textures"),
|
||||||
|
tokio::fs::create_dir_all("textures"),
|
||||||
|
tokio::fs::create_dir_all("meshes"),
|
||||||
|
tokio::fs::create_dir_all("unions"),
|
||||||
|
)?;
|
||||||
|
// use mpsc
|
||||||
|
let thread_limit=std::thread::available_parallelism()?.get();
|
||||||
|
let (send_assets,mut recv_assets)=tokio::sync::mpsc::channel(DOWNLOAD_LIMIT);
|
||||||
|
let (send_texture,mut recv_texture)=tokio::sync::mpsc::channel(thread_limit);
|
||||||
|
// map decode dispatcher
|
||||||
|
// read files multithreaded
|
||||||
|
// produce UniqueAssetsResult per file
|
||||||
|
tokio::spawn(async move{
|
||||||
|
// move send so it gets dropped when all maps have been decoded
|
||||||
|
// closing the channel
|
||||||
|
let mut it=paths.into_iter();
|
||||||
|
static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
|
||||||
|
SEM.add_permits(thread_limit);
|
||||||
|
while let (Ok(permit),Some(path))=(SEM.acquire().await,it.next()){
|
||||||
|
let send=send_assets.clone();
|
||||||
|
tokio::spawn(async move{
|
||||||
|
let result=unique_assets(path.as_path()).await;
|
||||||
|
_=send.send(result).await;
|
||||||
|
drop(permit);
|
||||||
|
});
|
||||||
|
}
|
||||||
});
|
});
|
||||||
}
|
// download manager
|
||||||
fn download_assets(paths:Vec<PathBuf>)->AResult<()>{
|
// insert into global unique assets guy
|
||||||
|
// add to download queue if the asset is globally unique and does not already exist on disk
|
||||||
|
let mut stats=Stats::default();
|
||||||
|
let context=rbx_asset::cookie::CookieContext::new(cookie);
|
||||||
|
let mut globally_unique_assets=UniqueAssets::default();
|
||||||
|
// pop a job = retry_queue.pop_front() or ingest(recv.recv().await)
|
||||||
|
// SLOW MODE:
|
||||||
|
// acquire all permits
|
||||||
|
// drop all permits
|
||||||
|
// pop one job
|
||||||
|
// if it succeeds go into fast mode
|
||||||
|
// FAST MODE:
|
||||||
|
// acquire one permit
|
||||||
|
// pop a job
|
||||||
|
let download_thread=tokio::spawn(async move{
|
||||||
|
while let Some(result)=recv_assets.recv().await{
|
||||||
|
let unique_assets=match result{
|
||||||
|
Ok(unique_assets)=>unique_assets,
|
||||||
|
Err(e)=>{
|
||||||
|
println!("error: {e:?}");
|
||||||
|
continue;
|
||||||
|
},
|
||||||
|
};
|
||||||
|
for texture_id in unique_assets.textures{
|
||||||
|
if globally_unique_assets.textures.insert(texture_id){
|
||||||
|
let data=download_retry(&mut stats,&context,DownloadType::Texture(texture_id)).await?;
|
||||||
|
send_texture.send((texture_id,data)).await?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for mesh_id in unique_assets.meshes{
|
||||||
|
if globally_unique_assets.meshes.insert(mesh_id){
|
||||||
|
download_retry(&mut stats,&context,DownloadType::Mesh(mesh_id)).await?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for union_id in unique_assets.unions{
|
||||||
|
if globally_unique_assets.unions.insert(union_id){
|
||||||
|
download_retry(&mut stats,&context,DownloadType::Union(union_id)).await?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
dbg!(stats);
|
||||||
|
Ok::<(),anyhow::Error>(())
|
||||||
|
});
|
||||||
|
static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
|
||||||
|
SEM.add_permits(thread_limit);
|
||||||
|
while let (Ok(permit),Some((asset_id,download_result)))=(SEM.acquire().await,recv_texture.recv().await){
|
||||||
|
tokio::spawn(async move{
|
||||||
|
let result=convert_texture(asset_id,download_result).await;
|
||||||
|
drop(permit);
|
||||||
|
result.unwrap();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
download_thread.await??;
|
||||||
|
_=SEM.acquire_many(thread_limit as u32).await.unwrap();
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -155,7 +374,8 @@ fn download_assets(paths:Vec<PathBuf>)->AResult<()>{
|
|||||||
enum ConvertError{
|
enum ConvertError{
|
||||||
IO(std::io::Error),
|
IO(std::io::Error),
|
||||||
SNFMap(strafesnet_snf::map::Error),
|
SNFMap(strafesnet_snf::map::Error),
|
||||||
RbxLoader(strafesnet_rbx_loader::ReadError),
|
RobloxRead(strafesnet_rbx_loader::ReadError),
|
||||||
|
RobloxLoad(strafesnet_rbx_loader::LoadError),
|
||||||
}
|
}
|
||||||
impl std::fmt::Display for ConvertError{
|
impl std::fmt::Display for ConvertError{
|
||||||
fn fmt(&self,f:&mut std::fmt::Formatter<'_>)->std::fmt::Result{
|
fn fmt(&self,f:&mut std::fmt::Formatter<'_>)->std::fmt::Result{
|
||||||
@ -163,78 +383,49 @@ impl std::fmt::Display for ConvertError{
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
impl std::error::Error for ConvertError{}
|
impl std::error::Error for ConvertError{}
|
||||||
|
async fn convert_to_snf(path:&Path,output_folder:PathBuf)->AResult<()>{
|
||||||
|
let entire_file=tokio::fs::read(path).await?;
|
||||||
|
|
||||||
type MapThread=std::thread::JoinHandle<Result<(),ConvertError>>;
|
let model=strafesnet_rbx_loader::read(
|
||||||
|
std::io::Cursor::new(entire_file)
|
||||||
|
).map_err(ConvertError::RobloxRead)?;
|
||||||
|
|
||||||
fn roblox_to_snf(pathlist:Vec<std::path::PathBuf>,output_folder:PathBuf)->AResult<()>{
|
let mut place=model.into_place();
|
||||||
let n_paths=pathlist.len();
|
place.run_scripts();
|
||||||
let start = std::time::Instant::now();
|
|
||||||
let mut threads:std::collections::VecDeque<MapThread>=std::collections::VecDeque::new();
|
|
||||||
let mut i=0;
|
|
||||||
let mut join_thread=|thread:MapThread|{
|
|
||||||
i+=1;
|
|
||||||
if let Err(e)=thread.join(){
|
|
||||||
println!("thread error: {:?}",e);
|
|
||||||
}else{
|
|
||||||
println!("{}/{}",i,n_paths);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
for path in pathlist{
|
|
||||||
if 32<=threads.len(){
|
|
||||||
join_thread(threads.pop_front().unwrap());
|
|
||||||
}
|
|
||||||
let output_folder=output_folder.clone();
|
|
||||||
threads.push_back(std::thread::spawn(move ||{
|
|
||||||
let model=strafesnet_rbx_loader::read(
|
|
||||||
std::fs::File::open(path.as_path())
|
|
||||||
.map_err(ConvertError::IO)?
|
|
||||||
).map_err(ConvertError::RbxLoader)?;
|
|
||||||
|
|
||||||
let mut place=model.into_place();
|
let map=place.to_snf(LoadFailureMode::DefaultToNone).map_err(ConvertError::RobloxLoad)?;
|
||||||
place.run_scripts();
|
|
||||||
|
|
||||||
let mut loader=strafesnet_deferred_loader::roblox_legacy();
|
let mut dest=output_folder;
|
||||||
|
dest.push(path.file_stem().unwrap());
|
||||||
|
dest.set_extension("snfm");
|
||||||
|
let file=std::fs::File::create(dest).map_err(ConvertError::IO)?;
|
||||||
|
|
||||||
let (texture_loader,mesh_loader)=loader.get_inner_mut();
|
strafesnet_snf::map::write_map(file,map).map_err(ConvertError::SNFMap)?;
|
||||||
|
|
||||||
let map_step1=strafesnet_rbx_loader::convert(
|
Ok(())
|
||||||
&place,
|
}
|
||||||
|name|texture_loader.acquire_render_config_id(name),
|
|
||||||
|name|mesh_loader.acquire_mesh_id(name),
|
async fn roblox_to_snf(paths:Vec<std::path::PathBuf>,output_folder:PathBuf)->AResult<()>{
|
||||||
);
|
let start=std::time::Instant::now();
|
||||||
|
|
||||||
let meshpart_meshes=mesh_loader.load_meshes().map_err(ConvertError::IO)?;
|
let thread_limit=std::thread::available_parallelism()?.get();
|
||||||
|
let mut it=paths.into_iter();
|
||||||
let map_step2=map_step1.add_meshpart_meshes_and_calculate_attributes(
|
static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
|
||||||
meshpart_meshes.into_iter().map(|(mesh_id,loader_model)|
|
SEM.add_permits(thread_limit);
|
||||||
(mesh_id,strafesnet_rbx_loader::data::RobloxMeshBytes::new(loader_model.get()))
|
|
||||||
)
|
while let (Ok(permit),Some(path))=(SEM.acquire().await,it.next()){
|
||||||
);
|
let output_folder=output_folder.clone();
|
||||||
|
tokio::spawn(async move{
|
||||||
let (textures,render_configs)=loader.into_render_configs().map_err(ConvertError::IO)?.consume();
|
let result=convert_to_snf(path.as_path(),output_folder).await;
|
||||||
|
drop(permit);
|
||||||
let map=map_step2.add_render_configs_and_textures(
|
match result{
|
||||||
render_configs.into_iter(),
|
Ok(())=>(),
|
||||||
textures.into_iter().map(|(texture_id,texture)|
|
Err(e)=>println!("Convert error: {e:?}"),
|
||||||
(texture_id,match texture{
|
}
|
||||||
strafesnet_deferred_loader::texture::Texture::ImageDDS(data)=>data,
|
});
|
||||||
})
|
}
|
||||||
)
|
_=SEM.acquire_many(thread_limit as u32).await.unwrap();
|
||||||
);
|
|
||||||
|
println!("elapsed={:?}", start.elapsed());
|
||||||
let mut dest=output_folder.clone();
|
|
||||||
dest.push(path.file_stem().unwrap());
|
|
||||||
dest.set_extension("snfm");
|
|
||||||
let file=std::fs::File::create(dest).map_err(ConvertError::IO)?;
|
|
||||||
|
|
||||||
strafesnet_snf::map::write_map(file,map).map_err(ConvertError::SNFMap)?;
|
|
||||||
Ok(())
|
|
||||||
}));
|
|
||||||
}
|
|
||||||
|
|
||||||
for thread in threads{
|
|
||||||
join_thread(thread);
|
|
||||||
}
|
|
||||||
println!("{:?}", start.elapsed());
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
536
src/source.rs
536
src/source.rs
@ -1,6 +1,11 @@
|
|||||||
use std::path::PathBuf;
|
use std::path::{Path,PathBuf};
|
||||||
|
use std::borrow::Cow;
|
||||||
use clap::{Args,Subcommand};
|
use clap::{Args,Subcommand};
|
||||||
use anyhow::Result as AResult;
|
use anyhow::Result as AResult;
|
||||||
|
use futures::StreamExt;
|
||||||
|
use strafesnet_bsp_loader::loader::BspFinder;
|
||||||
|
use strafesnet_deferred_loader::loader::Loader;
|
||||||
|
use strafesnet_deferred_loader::deferred_loader::{LoadFailureMode,MeshDeferredLoader,RenderConfigDeferredLoader};
|
||||||
|
|
||||||
#[derive(Subcommand)]
|
#[derive(Subcommand)]
|
||||||
pub enum Commands{
|
pub enum Commands{
|
||||||
@ -16,13 +21,15 @@ pub struct SourceToSNFSubcommand {
|
|||||||
output_folder:PathBuf,
|
output_folder:PathBuf,
|
||||||
#[arg(required=true)]
|
#[arg(required=true)]
|
||||||
input_files:Vec<PathBuf>,
|
input_files:Vec<PathBuf>,
|
||||||
|
#[arg(long)]
|
||||||
|
vpks:Vec<PathBuf>,
|
||||||
}
|
}
|
||||||
#[derive(Args)]
|
#[derive(Args)]
|
||||||
pub struct ExtractTexturesSubcommand {
|
pub struct ExtractTexturesSubcommand{
|
||||||
|
#[arg(required=true)]
|
||||||
|
bsp_files:Vec<PathBuf>,
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
bsp_file:PathBuf,
|
vpks:Vec<PathBuf>,
|
||||||
#[arg(long)]
|
|
||||||
vpk_dir_files:Vec<PathBuf>
|
|
||||||
}
|
}
|
||||||
#[derive(Args)]
|
#[derive(Args)]
|
||||||
pub struct VPKContentsSubcommand {
|
pub struct VPKContentsSubcommand {
|
||||||
@ -36,10 +43,10 @@ pub struct BSPContentsSubcommand {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl Commands{
|
impl Commands{
|
||||||
pub fn run(self)->AResult<()>{
|
pub async fn run(self)->AResult<()>{
|
||||||
match self{
|
match self{
|
||||||
Commands::SourceToSNF(subcommand)=>source_to_snf(subcommand.input_files,subcommand.output_folder),
|
Commands::SourceToSNF(subcommand)=>source_to_snf(subcommand.input_files,subcommand.output_folder,subcommand.vpks).await,
|
||||||
Commands::ExtractTextures(subcommand)=>extract_textures(vec![subcommand.bsp_file],subcommand.vpk_dir_files),
|
Commands::ExtractTextures(subcommand)=>extract_textures(subcommand.bsp_files,subcommand.vpks).await,
|
||||||
Commands::VPKContents(subcommand)=>vpk_contents(subcommand.input_file),
|
Commands::VPKContents(subcommand)=>vpk_contents(subcommand.input_file),
|
||||||
Commands::BSPContents(subcommand)=>bsp_contents(subcommand.input_file),
|
Commands::BSPContents(subcommand)=>bsp_contents(subcommand.input_file),
|
||||||
}
|
}
|
||||||
@ -63,9 +70,9 @@ impl VMTContent{
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn get_some_texture(material:vmt_parser::material::Material)->AResult<VMTContent>{
|
fn get_some_texture(material:vmt_parser::material::Material)->VMTContent{
|
||||||
//just grab some texture from somewhere for now
|
//just grab some texture from somewhere for now
|
||||||
Ok(match material{
|
match material{
|
||||||
vmt_parser::material::Material::LightMappedGeneric(mat)=>VMTContent::vtf(Some(mat.base_texture)),
|
vmt_parser::material::Material::LightMappedGeneric(mat)=>VMTContent::vtf(Some(mat.base_texture)),
|
||||||
vmt_parser::material::Material::VertexLitGeneric(mat)=>VMTContent::vtf(mat.base_texture.or(mat.decal_texture)),//this just dies if there is none
|
vmt_parser::material::Material::VertexLitGeneric(mat)=>VMTContent::vtf(mat.base_texture.or(mat.decal_texture)),//this just dies if there is none
|
||||||
vmt_parser::material::Material::VertexLitGenericDx6(mat)=>VMTContent::vtf(mat.base_texture.or(mat.decal_texture)),
|
vmt_parser::material::Material::VertexLitGenericDx6(mat)=>VMTContent::vtf(mat.base_texture.or(mat.decal_texture)),
|
||||||
@ -84,175 +91,258 @@ fn get_some_texture(material:vmt_parser::material::Material)->AResult<VMTContent
|
|||||||
vmt_parser::material::Material::Sky(mat)=>VMTContent::vtf(Some(mat.base_texture)),
|
vmt_parser::material::Material::Sky(mat)=>VMTContent::vtf(Some(mat.base_texture)),
|
||||||
vmt_parser::material::Material::Replacements(_mat)=>VMTContent::Unsupported,
|
vmt_parser::material::Material::Replacements(_mat)=>VMTContent::Unsupported,
|
||||||
vmt_parser::material::Material::Patch(mat)=>VMTContent::Patch(mat),
|
vmt_parser::material::Material::Patch(mat)=>VMTContent::Patch(mat),
|
||||||
_=>return Err(anyhow::Error::msg("vmt failed to parse")),
|
_=>unreachable!(),
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
fn get_vmt<F:Fn(String)->AResult<Option<Vec<u8>>>>(find_stuff:&F,search_name:String)->AResult<vmt_parser::material::Material>{
|
|
||||||
if let Some(stuff)=find_stuff(search_name)?{
|
|
||||||
//decode vmt and then write
|
|
||||||
let stuff=String::from_utf8(stuff)?;
|
|
||||||
let material=vmt_parser::from_str(stuff.as_str())?;
|
|
||||||
println!("vmt material={:?}",material);
|
|
||||||
return Ok(material);
|
|
||||||
}
|
}
|
||||||
Err(anyhow::Error::msg("vmt not found"))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn recursive_vmt_loader<F:Fn(String)->AResult<Option<Vec<u8>>>>(find_stuff:&F,material:vmt_parser::material::Material)->AResult<Option<Vec<u8>>>{
|
#[derive(Debug,thiserror::Error)]
|
||||||
match get_some_texture(material)?{
|
enum GetVMTError{
|
||||||
VMTContent::VMT(s)=>recursive_vmt_loader(find_stuff,get_vmt(find_stuff,s)?),
|
#[error("Bsp error {0:?}")]
|
||||||
|
Bsp(#[from]vbsp::BspError),
|
||||||
|
#[error("Utf8 error {0:?}")]
|
||||||
|
Utf8(#[from]std::str::Utf8Error),
|
||||||
|
#[error("Vdf error {0:?}")]
|
||||||
|
Vdf(#[from]vmt_parser::VdfError),
|
||||||
|
#[error("Vmt not found")]
|
||||||
|
NotFound,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get_vmt(finder:BspFinder,search_name:&str)->Result<vmt_parser::material::Material,GetVMTError>{
|
||||||
|
let vmt_data=finder.find(search_name)?.ok_or(GetVMTError::NotFound)?;
|
||||||
|
//decode vmt and then write
|
||||||
|
let vmt_str=core::str::from_utf8(&vmt_data)?;
|
||||||
|
let material=vmt_parser::from_str(vmt_str)?;
|
||||||
|
//println!("vmt material={:?}",material);
|
||||||
|
Ok(material)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug,thiserror::Error)]
|
||||||
|
enum LoadVMTError{
|
||||||
|
#[error("Bsp error {0:?}")]
|
||||||
|
Bsp(#[from]vbsp::BspError),
|
||||||
|
#[error("GetVMT error {0:?}")]
|
||||||
|
GetVMT(#[from]GetVMTError),
|
||||||
|
#[error("FromUtf8 error {0:?}")]
|
||||||
|
FromUtf8(#[from]std::string::FromUtf8Error),
|
||||||
|
#[error("Vdf error {0:?}")]
|
||||||
|
Vdf(#[from]vmt_parser::VdfError),
|
||||||
|
#[error("Vmt unsupported")]
|
||||||
|
Unsupported,
|
||||||
|
#[error("Vmt unresolved")]
|
||||||
|
Unresolved,
|
||||||
|
#[error("Vmt not found")]
|
||||||
|
NotFound,
|
||||||
|
}
|
||||||
|
fn recursive_vmt_loader<'bsp,'vpk,'a>(finder:BspFinder<'bsp,'vpk>,material:vmt_parser::material::Material)->Result<Option<Cow<'a,[u8]>>,LoadVMTError>
|
||||||
|
where
|
||||||
|
'bsp:'a,
|
||||||
|
'vpk:'a,
|
||||||
|
{
|
||||||
|
match get_some_texture(material){
|
||||||
|
VMTContent::VMT(s)=>recursive_vmt_loader(finder,get_vmt(finder,s.as_str())?),
|
||||||
VMTContent::VTF(s)=>{
|
VMTContent::VTF(s)=>{
|
||||||
let mut texture_file_name=PathBuf::from("materials");
|
let mut texture_file_name=PathBuf::from("materials");
|
||||||
texture_file_name.push(s);
|
texture_file_name.push(s);
|
||||||
texture_file_name.set_extension("vtf");
|
texture_file_name.set_extension("vtf");
|
||||||
find_stuff(texture_file_name.into_os_string().into_string().unwrap())
|
Ok(finder.find(texture_file_name.to_str().unwrap())?)
|
||||||
},
|
},
|
||||||
VMTContent::Patch(mat)=>recursive_vmt_loader(find_stuff,
|
VMTContent::Patch(mat)=>recursive_vmt_loader(finder,
|
||||||
mat.resolve(|search_name|{
|
mat.resolve(|search_name|
|
||||||
match find_stuff(search_name.to_string())?{
|
match finder.find(search_name)?{
|
||||||
Some(bytes)=>Ok(String::from_utf8(bytes)?),
|
Some(bytes)=>Ok(String::from_utf8(bytes.into_owned())?),
|
||||||
None=>Err(anyhow::Error::msg("could not find vmt")),
|
None=>Err(LoadVMTError::NotFound),
|
||||||
}
|
}
|
||||||
})?
|
)?
|
||||||
),
|
),
|
||||||
VMTContent::Unsupported=>{println!("Unsupported vmt");Ok(None)},//print and move on
|
VMTContent::Unsupported=>Err(LoadVMTError::Unsupported),
|
||||||
VMTContent::Unresolved=>{println!("Unresolved vmt");Ok(None)},
|
VMTContent::Unresolved=>Err(LoadVMTError::Unresolved),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
fn load_texture<'bsp,'vpk,'a>(finder:BspFinder<'bsp,'vpk>,texture_name:&str)->Result<Option<Cow<'a,[u8]>>,LoadVMTError>
|
||||||
fn extract_textures(paths:Vec<PathBuf>,vpk_paths:Vec<PathBuf>)->AResult<()>{
|
where
|
||||||
std::fs::create_dir_all("textures")?;
|
'bsp:'a,
|
||||||
let vpk_list:Vec<vpk::VPK>=vpk_paths.into_iter().map(|vpk_path|vpk::VPK::read(&vpk_path).expect("vpk file does not exist")).collect();
|
'vpk:'a,
|
||||||
for path in paths{
|
{
|
||||||
let mut deduplicate=std::collections::HashSet::new();
|
let mut texture_file_name=PathBuf::from("materials");
|
||||||
let bsp=vbsp::Bsp::read(std::fs::read(path)?.as_ref())?;
|
//lower case
|
||||||
for texture in bsp.textures(){
|
let texture_file_name_lowercase=texture_name.to_lowercase();
|
||||||
deduplicate.insert(PathBuf::from(texture.name()));
|
texture_file_name.push(texture_file_name_lowercase.clone());
|
||||||
}
|
//remove stem and search for both vtf and vmt files
|
||||||
//dedupe prop models
|
let stem=PathBuf::from(texture_file_name.file_stem().unwrap());
|
||||||
let mut model_dedupe=std::collections::HashSet::new();
|
texture_file_name.pop();
|
||||||
for prop in bsp.static_props(){
|
texture_file_name.push(stem);
|
||||||
model_dedupe.insert(prop.model());
|
if let Some(stuff)=finder.find(texture_file_name.to_str().unwrap())?{
|
||||||
}
|
return Ok(Some(stuff))
|
||||||
|
|
||||||
//grab texture names from props
|
|
||||||
for model_name in model_dedupe{
|
|
||||||
//.mdl, .vvd, .dx90.vtx
|
|
||||||
let mut path=PathBuf::from(model_name);
|
|
||||||
let file_name=PathBuf::from(path.file_stem().unwrap());
|
|
||||||
path.pop();
|
|
||||||
path.push(file_name);
|
|
||||||
let mut vvd_path=path.clone();
|
|
||||||
let mut vtx_path=path.clone();
|
|
||||||
vvd_path.set_extension("vvd");
|
|
||||||
vtx_path.set_extension("dx90.vtx");
|
|
||||||
match (bsp.pack.get(model_name),bsp.pack.get(vvd_path.as_os_str().to_str().unwrap()),bsp.pack.get(vtx_path.as_os_str().to_str().unwrap())){
|
|
||||||
(Ok(Some(mdl_file)),Ok(Some(vvd_file)),Ok(Some(vtx_file)))=>{
|
|
||||||
match (vmdl::mdl::Mdl::read(mdl_file.as_ref()),vmdl::vvd::Vvd::read(vvd_file.as_ref()),vmdl::vtx::Vtx::read(vtx_file.as_ref())){
|
|
||||||
(Ok(mdl),Ok(vvd),Ok(vtx))=>{
|
|
||||||
let model=vmdl::Model::from_parts(mdl,vtx,vvd);
|
|
||||||
for texture in model.textures(){
|
|
||||||
for search_path in &texture.search_paths{
|
|
||||||
let mut path=PathBuf::from(search_path.as_str());
|
|
||||||
path.push(texture.name.as_str());
|
|
||||||
deduplicate.insert(path);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
_=>println!("model_name={} error",model_name),
|
|
||||||
}
|
|
||||||
},
|
|
||||||
_=>println!("no model name={}",model_name),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let pack=&bsp.pack;
|
|
||||||
let vpk_list=&vpk_list;
|
|
||||||
std::thread::scope(move|s|{
|
|
||||||
let mut thread_handles=Vec::new();
|
|
||||||
for texture_name in deduplicate{
|
|
||||||
let mut found_texture=false;
|
|
||||||
//LMAO imagine having to write type names
|
|
||||||
let write_image=|mut stuff,write_file_name|{
|
|
||||||
let image=vtf::from_bytes(&mut stuff)?.highres_image.decode(0)?.to_rgba8();
|
|
||||||
|
|
||||||
let format=if image.width()%4!=0||image.height()%4!=0{
|
|
||||||
image_dds::ImageFormat::Rgba8UnormSrgb
|
|
||||||
}else{
|
|
||||||
image_dds::ImageFormat::BC7RgbaUnormSrgb
|
|
||||||
};
|
|
||||||
//this fails if the image dimensions are not a multiple of 4
|
|
||||||
let dds = image_dds::dds_from_image(
|
|
||||||
&image,
|
|
||||||
format,
|
|
||||||
image_dds::Quality::Slow,
|
|
||||||
image_dds::Mipmaps::GeneratedAutomatic,
|
|
||||||
)?;
|
|
||||||
|
|
||||||
//write dds
|
|
||||||
let mut dest=PathBuf::from("textures");
|
|
||||||
dest.push(write_file_name);
|
|
||||||
dest.set_extension("dds");
|
|
||||||
std::fs::create_dir_all(dest.parent().unwrap())?;
|
|
||||||
let mut writer = std::io::BufWriter::new(std::fs::File::create(dest)?);
|
|
||||||
dds.write(&mut writer)?;
|
|
||||||
Ok::<(),anyhow::Error>(())
|
|
||||||
};
|
|
||||||
let find_stuff=|search_file_name:String|{
|
|
||||||
println!("search_file_name={}",search_file_name);
|
|
||||||
match pack.get(search_file_name.as_str())?{
|
|
||||||
Some(file)=>return Ok(Some(file)),
|
|
||||||
_=>(),
|
|
||||||
}
|
|
||||||
//search pak list
|
|
||||||
for vpk_index in vpk_list{
|
|
||||||
if let Some(vpk_entry)=vpk_index.tree.get(search_file_name.as_str()){
|
|
||||||
return Ok(Some(match vpk_entry.get()?{
|
|
||||||
std::borrow::Cow::Borrowed(bytes)=>bytes.to_vec(),
|
|
||||||
std::borrow::Cow::Owned(bytes)=>bytes,
|
|
||||||
}));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Ok::<Option<Vec<u8>>,anyhow::Error>(None)
|
|
||||||
};
|
|
||||||
let loader=|texture_name:String|{
|
|
||||||
let mut texture_file_name=PathBuf::from("materials");
|
|
||||||
//lower case
|
|
||||||
let texture_file_name_lowercase=texture_name.to_lowercase();
|
|
||||||
texture_file_name.push(texture_file_name_lowercase.clone());
|
|
||||||
//remove stem and search for both vtf and vmt files
|
|
||||||
let stem=PathBuf::from(texture_file_name.file_stem().unwrap());
|
|
||||||
texture_file_name.pop();
|
|
||||||
texture_file_name.push(stem);
|
|
||||||
//somehow search for both files
|
|
||||||
let mut texture_file_name_vmt=texture_file_name.clone();
|
|
||||||
texture_file_name.set_extension("vtf");
|
|
||||||
texture_file_name_vmt.set_extension("vmt");
|
|
||||||
if let Some(stuff)=find_stuff(texture_file_name.to_string_lossy().to_string())?{
|
|
||||||
return Ok(Some(stuff))
|
|
||||||
}
|
|
||||||
recursive_vmt_loader(&find_stuff,get_vmt(&find_stuff,texture_file_name_vmt.to_string_lossy().to_string())?)
|
|
||||||
};
|
|
||||||
if let Some(stuff)=loader(texture_name.to_string_lossy().to_string())?{
|
|
||||||
found_texture=true;
|
|
||||||
let texture_name=texture_name.clone();
|
|
||||||
thread_handles.push(s.spawn(move||write_image(stuff,texture_name)));
|
|
||||||
}
|
|
||||||
if !found_texture{
|
|
||||||
println!("no data");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for thread in thread_handles{
|
|
||||||
match thread.join(){
|
|
||||||
Ok(Err(e))=>println!("write error: {:?}",e),
|
|
||||||
Err(e)=>println!("thread error: {:?}",e),
|
|
||||||
Ok(_)=>(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Ok::<(),anyhow::Error>(())
|
|
||||||
})?
|
|
||||||
}
|
}
|
||||||
|
//somehow search for both files
|
||||||
|
let mut texture_file_name_vmt=texture_file_name.clone();
|
||||||
|
texture_file_name.set_extension("vtf");
|
||||||
|
texture_file_name_vmt.set_extension("vmt");
|
||||||
|
recursive_vmt_loader(finder,get_vmt(finder,texture_file_name_vmt.to_str().unwrap())?)
|
||||||
|
}
|
||||||
|
#[derive(Debug,thiserror::Error)]
|
||||||
|
enum ExtractTextureError{
|
||||||
|
#[error("Io error {0:?}")]
|
||||||
|
Io(#[from]std::io::Error),
|
||||||
|
#[error("Bsp error {0:?}")]
|
||||||
|
Bsp(#[from]vbsp::BspError),
|
||||||
|
#[error("MeshLoad error {0:?}")]
|
||||||
|
MeshLoad(#[from]strafesnet_bsp_loader::loader::MeshError),
|
||||||
|
#[error("Load VMT error {0:?}")]
|
||||||
|
LoadVMT(#[from]LoadVMTError),
|
||||||
|
}
|
||||||
|
async fn gimme_them_textures(path:&Path,vpk_list:&[vpk::VPK],send_texture:tokio::sync::mpsc::Sender<(Vec<u8>,String)>)->Result<(),ExtractTextureError>{
|
||||||
|
let bsp=vbsp::Bsp::read(tokio::fs::read(path).await?.as_ref())?;
|
||||||
|
let loader_bsp=strafesnet_bsp_loader::Bsp::new(bsp);
|
||||||
|
let bsp=loader_bsp.as_ref();
|
||||||
|
|
||||||
|
let mut texture_deferred_loader=RenderConfigDeferredLoader::new();
|
||||||
|
for texture in bsp.textures(){
|
||||||
|
texture_deferred_loader.acquire_render_config_id(Some(Cow::Borrowed(texture.name())));
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut mesh_deferred_loader=MeshDeferredLoader::new();
|
||||||
|
for prop in bsp.static_props(){
|
||||||
|
mesh_deferred_loader.acquire_mesh_id(prop.model());
|
||||||
|
}
|
||||||
|
|
||||||
|
let finder=BspFinder{
|
||||||
|
bsp:&loader_bsp,
|
||||||
|
vpks:vpk_list
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut mesh_loader=strafesnet_bsp_loader::loader::ModelLoader::new(finder);
|
||||||
|
// load models and collect requested textures
|
||||||
|
for model_path in mesh_deferred_loader.into_indices(){
|
||||||
|
let model:vmdl::Model=match mesh_loader.load(model_path){
|
||||||
|
Ok(model)=>model,
|
||||||
|
Err(e)=>{
|
||||||
|
println!("Model={model_path} Load model error: {e}");
|
||||||
|
continue;
|
||||||
|
},
|
||||||
|
};
|
||||||
|
for texture in model.textures(){
|
||||||
|
for search_path in &texture.search_paths{
|
||||||
|
let mut path=PathBuf::from(search_path.as_str());
|
||||||
|
path.push(texture.name.as_str());
|
||||||
|
let path=path.to_str().unwrap().to_owned();
|
||||||
|
texture_deferred_loader.acquire_render_config_id(Some(Cow::Owned(path)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for texture_path in texture_deferred_loader.into_indices(){
|
||||||
|
match load_texture(finder,&texture_path){
|
||||||
|
Ok(Some(texture))=>send_texture.send(
|
||||||
|
(texture.into_owned(),texture_path.into_owned())
|
||||||
|
).await.unwrap(),
|
||||||
|
Ok(None)=>(),
|
||||||
|
Err(e)=>println!("Texture={texture_path} Load error: {e}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
#[derive(Debug,thiserror::Error)]
|
||||||
|
enum ConvertTextureError{
|
||||||
|
#[error("Bsp error {0:?}")]
|
||||||
|
Bsp(#[from]vbsp::BspError),
|
||||||
|
#[error("Vtf error {0:?}")]
|
||||||
|
Vtf(#[from]vtf::Error),
|
||||||
|
#[error("DDS create error {0:?}")]
|
||||||
|
DDS(#[from]image_dds::CreateDdsError),
|
||||||
|
#[error("DDS write error {0:?}")]
|
||||||
|
DDSWrite(#[from]image_dds::ddsfile::Error),
|
||||||
|
#[error("Io error {0:?}")]
|
||||||
|
Io(#[from]std::io::Error),
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn convert_texture(texture:Vec<u8>,write_file_name:impl AsRef<Path>)->Result<(),ConvertTextureError>{
|
||||||
|
let image=vtf::from_bytes(&texture)?.highres_image.decode(0)?.to_rgba8();
|
||||||
|
|
||||||
|
let format=if image.width()%4!=0||image.height()%4!=0{
|
||||||
|
image_dds::ImageFormat::Rgba8UnormSrgb
|
||||||
|
}else{
|
||||||
|
image_dds::ImageFormat::BC7RgbaUnormSrgb
|
||||||
|
};
|
||||||
|
//this fails if the image dimensions are not a multiple of 4
|
||||||
|
let dds = image_dds::dds_from_image(
|
||||||
|
&image,
|
||||||
|
format,
|
||||||
|
image_dds::Quality::Slow,
|
||||||
|
image_dds::Mipmaps::GeneratedAutomatic,
|
||||||
|
)?;
|
||||||
|
|
||||||
|
//write dds
|
||||||
|
let mut dest=PathBuf::from("textures");
|
||||||
|
dest.push(write_file_name);
|
||||||
|
dest.set_extension("dds");
|
||||||
|
std::fs::create_dir_all(dest.parent().unwrap())?;
|
||||||
|
let mut writer=std::io::BufWriter::new(std::fs::File::create(dest)?);
|
||||||
|
dds.write(&mut writer)?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn read_vpks(vpk_paths:Vec<PathBuf>,thread_limit:usize)->Vec<vpk::VPK>{
|
||||||
|
futures::stream::iter(vpk_paths).map(|vpk_path|async{
|
||||||
|
// idk why it doesn't want to pass out the errors but this is fatal anyways
|
||||||
|
tokio::task::spawn_blocking(move||vpk::VPK::read(&vpk_path)).await.unwrap().unwrap()
|
||||||
|
})
|
||||||
|
.buffer_unordered(thread_limit)
|
||||||
|
.collect().await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn extract_textures(paths:Vec<PathBuf>,vpk_paths:Vec<PathBuf>)->AResult<()>{
|
||||||
|
tokio::try_join!(
|
||||||
|
tokio::fs::create_dir_all("extracted_textures"),
|
||||||
|
tokio::fs::create_dir_all("textures"),
|
||||||
|
tokio::fs::create_dir_all("meshes"),
|
||||||
|
)?;
|
||||||
|
let thread_limit=std::thread::available_parallelism()?.get();
|
||||||
|
|
||||||
|
// load vpk list
|
||||||
|
let vpk_list=read_vpks(vpk_paths,thread_limit).await;
|
||||||
|
|
||||||
|
// leak vpk_list for static lifetime?
|
||||||
|
let vpk_list:&[vpk::VPK]=vpk_list.leak();
|
||||||
|
|
||||||
|
let (send_texture,mut recv_texture)=tokio::sync::mpsc::channel(thread_limit);
|
||||||
|
let mut it=paths.into_iter();
|
||||||
|
let extract_thread=tokio::spawn(async move{
|
||||||
|
static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
|
||||||
|
SEM.add_permits(thread_limit);
|
||||||
|
while let (Ok(permit),Some(path))=(SEM.acquire().await,it.next()){
|
||||||
|
let send=send_texture.clone();
|
||||||
|
tokio::spawn(async move{
|
||||||
|
let result=gimme_them_textures(&path,vpk_list,send).await;
|
||||||
|
drop(permit);
|
||||||
|
match result{
|
||||||
|
Ok(())=>(),
|
||||||
|
Err(e)=>println!("Map={path:?} Decode error: {e:?}"),
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// convert images
|
||||||
|
static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
|
||||||
|
SEM.add_permits(thread_limit);
|
||||||
|
while let (Ok(permit),Some((data,dest)))=(SEM.acquire().await,recv_texture.recv().await){
|
||||||
|
// TODO: dedup dest?
|
||||||
|
tokio::spawn(async move{
|
||||||
|
let result=convert_texture(data,dest).await;
|
||||||
|
drop(permit);
|
||||||
|
match result{
|
||||||
|
Ok(())=>(),
|
||||||
|
Err(e)=>println!("Convert error: {e:?}"),
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
extract_thread.await?;
|
||||||
|
_=SEM.acquire_many(thread_limit as u32).await?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -277,7 +367,8 @@ fn bsp_contents(path:PathBuf)->AResult<()>{
|
|||||||
enum ConvertError{
|
enum ConvertError{
|
||||||
IO(std::io::Error),
|
IO(std::io::Error),
|
||||||
SNFMap(strafesnet_snf::map::Error),
|
SNFMap(strafesnet_snf::map::Error),
|
||||||
BspLoader(strafesnet_bsp_loader::ReadError),
|
BspRead(strafesnet_bsp_loader::ReadError),
|
||||||
|
BspLoad(strafesnet_bsp_loader::LoadError),
|
||||||
}
|
}
|
||||||
impl std::fmt::Display for ConvertError{
|
impl std::fmt::Display for ConvertError{
|
||||||
fn fmt(&self,f:&mut std::fmt::Formatter<'_>)->std::fmt::Result{
|
fn fmt(&self,f:&mut std::fmt::Formatter<'_>)->std::fmt::Result{
|
||||||
@ -286,79 +377,52 @@ impl std::fmt::Display for ConvertError{
|
|||||||
}
|
}
|
||||||
impl std::error::Error for ConvertError{}
|
impl std::error::Error for ConvertError{}
|
||||||
|
|
||||||
type MapThread=std::thread::JoinHandle<Result<(),ConvertError>>;
|
async fn convert_to_snf(path:&Path,vpk_list:&[vpk::VPK],output_folder:PathBuf)->AResult<()>{
|
||||||
|
let entire_file=tokio::fs::read(path).await?;
|
||||||
|
|
||||||
fn source_to_snf(pathlist:Vec<std::path::PathBuf>,output_folder:PathBuf)->AResult<()>{
|
let bsp=strafesnet_bsp_loader::read(
|
||||||
let n_paths=pathlist.len();
|
std::io::Cursor::new(entire_file)
|
||||||
let start = std::time::Instant::now();
|
).map_err(ConvertError::BspRead)?;
|
||||||
let mut threads:std::collections::VecDeque<MapThread>=std::collections::VecDeque::new();
|
|
||||||
let mut i=0;
|
|
||||||
let mut join_thread=|thread:MapThread|{
|
|
||||||
i+=1;
|
|
||||||
if let Err(e)=thread.join(){
|
|
||||||
println!("thread error: {:?}",e);
|
|
||||||
}else{
|
|
||||||
println!("{}/{}",i,n_paths);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
for path in pathlist{
|
|
||||||
if 32<=threads.len(){
|
|
||||||
join_thread(threads.pop_front().unwrap());
|
|
||||||
}
|
|
||||||
let output_folder=output_folder.clone();
|
|
||||||
threads.push_back(std::thread::spawn(move ||{
|
|
||||||
let bsp=strafesnet_bsp_loader::read(
|
|
||||||
std::fs::File::open(path.as_path())
|
|
||||||
.map_err(ConvertError::IO)?
|
|
||||||
).map_err(ConvertError::BspLoader)?;
|
|
||||||
let mut loader=strafesnet_deferred_loader::source_legacy();
|
|
||||||
|
|
||||||
let (texture_loader,mesh_loader)=loader.get_inner_mut();
|
let map=bsp.to_snf(LoadFailureMode::DefaultToNone,vpk_list).map_err(ConvertError::BspLoad)?;
|
||||||
|
|
||||||
let map_step1=strafesnet_bsp_loader::convert(
|
let mut dest=output_folder;
|
||||||
&bsp,
|
dest.push(path.file_stem().unwrap());
|
||||||
|name|texture_loader.acquire_render_config_id(name),
|
dest.set_extension("snfm");
|
||||||
|name|mesh_loader.acquire_mesh_id(name),
|
let file=std::fs::File::create(dest).map_err(ConvertError::IO)?;
|
||||||
);
|
|
||||||
|
|
||||||
let prop_meshes=mesh_loader.load_meshes(&bsp.as_ref());
|
strafesnet_snf::map::write_map(file,map).map_err(ConvertError::SNFMap)?;
|
||||||
|
|
||||||
let map_step2=map_step1.add_prop_meshes(
|
Ok(())
|
||||||
//the type conflagulator 9000
|
}
|
||||||
prop_meshes.into_iter().map(|(mesh_id,loader_model)|
|
async fn source_to_snf(paths:Vec<std::path::PathBuf>,output_folder:PathBuf,vpk_paths:Vec<PathBuf>)->AResult<()>{
|
||||||
(mesh_id,strafesnet_bsp_loader::data::ModelData{
|
let start=std::time::Instant::now();
|
||||||
mdl:strafesnet_bsp_loader::data::MdlData::new(loader_model.mdl.get()),
|
|
||||||
vtx:strafesnet_bsp_loader::data::VtxData::new(loader_model.vtx.get()),
|
let thread_limit=std::thread::available_parallelism()?.get();
|
||||||
vvd:strafesnet_bsp_loader::data::VvdData::new(loader_model.vvd.get()),
|
|
||||||
})
|
// load vpk list
|
||||||
),
|
let vpk_list=read_vpks(vpk_paths,thread_limit).await;
|
||||||
|name|texture_loader.acquire_render_config_id(name),
|
|
||||||
);
|
// leak vpk_list for static lifetime?
|
||||||
|
let vpk_list:&[vpk::VPK]=vpk_list.leak();
|
||||||
let (textures,render_configs)=loader.into_render_configs().map_err(ConvertError::IO)?.consume();
|
|
||||||
|
let mut it=paths.into_iter();
|
||||||
let map=map_step2.add_render_configs_and_textures(
|
static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
|
||||||
render_configs.into_iter(),
|
SEM.add_permits(thread_limit);
|
||||||
textures.into_iter().map(|(texture_id,texture)|
|
|
||||||
(texture_id,match texture{
|
while let (Ok(permit),Some(path))=(SEM.acquire().await,it.next()){
|
||||||
strafesnet_deferred_loader::texture::Texture::ImageDDS(data)=>data,
|
let output_folder=output_folder.clone();
|
||||||
})
|
tokio::spawn(async move{
|
||||||
),
|
let result=convert_to_snf(path.as_path(),vpk_list,output_folder).await;
|
||||||
);
|
drop(permit);
|
||||||
|
match result{
|
||||||
let mut dest=output_folder.clone();
|
Ok(())=>(),
|
||||||
dest.push(path.file_stem().unwrap());
|
Err(e)=>println!("Convert error: {e:?}"),
|
||||||
dest.set_extension("snfm");
|
}
|
||||||
let file=std::fs::File::create(dest).map_err(ConvertError::IO)?;
|
});
|
||||||
|
}
|
||||||
strafesnet_snf::map::write_map(file,map).map_err(ConvertError::SNFMap)?;
|
_=SEM.acquire_many(thread_limit as u32).await.unwrap();
|
||||||
Ok(())
|
|
||||||
}));
|
println!("elapsed={:?}", start.elapsed());
|
||||||
}
|
|
||||||
|
|
||||||
for thread in threads{
|
|
||||||
join_thread(thread);
|
|
||||||
}
|
|
||||||
println!("{:?}", start.elapsed());
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
Loading…
x
Reference in New Issue
Block a user