wip
This commit is contained in:
parent
612174df5a
commit
a7486782b6
@ -241,33 +241,16 @@ async fn download_retry(stats:&mut Stats,context:&rbx_asset::cookie::CookieConte
|
||||
}
|
||||
}
|
||||
}
|
||||
#[allow(unused)]
|
||||
#[derive(Debug)]
|
||||
#[derive(Debug,thiserror::Error)]
|
||||
enum ConvertTextureError{
|
||||
Io(std::io::Error),
|
||||
Image(image::ImageError),
|
||||
DDS(image_dds::CreateDdsError),
|
||||
DDSWrite(image_dds::ddsfile::Error),
|
||||
}
|
||||
impl From<std::io::Error> for ConvertTextureError{
|
||||
fn from(value:std::io::Error)->Self{
|
||||
Self::Io(value)
|
||||
}
|
||||
}
|
||||
impl From<image::ImageError> for ConvertTextureError{
|
||||
fn from(value:image::ImageError)->Self{
|
||||
Self::Image(value)
|
||||
}
|
||||
}
|
||||
impl From<image_dds::CreateDdsError> for ConvertTextureError{
|
||||
fn from(value:image_dds::CreateDdsError)->Self{
|
||||
Self::DDS(value)
|
||||
}
|
||||
}
|
||||
impl From<image_dds::ddsfile::Error> for ConvertTextureError{
|
||||
fn from(value:image_dds::ddsfile::Error)->Self{
|
||||
Self::DDSWrite(value)
|
||||
}
|
||||
#[error("Io error {0:?}")]
|
||||
Io(#[from]std::io::Error),
|
||||
#[error("Image error {0:?}")]
|
||||
Image(#[from]image::ImageError),
|
||||
#[error("DDS create error {0:?}")]
|
||||
DDS(#[from]image_dds::CreateDdsError),
|
||||
#[error("DDS write error {0:?}")]
|
||||
DDSWrite(#[from]image_dds::ddsfile::Error),
|
||||
}
|
||||
async fn convert_texture(asset_id:RobloxAssetId,download_result:DownloadResult)->Result<(),ConvertTextureError>{
|
||||
let data=match download_result{
|
||||
|
||||
360
src/source.rs
360
src/source.rs
@ -1,7 +1,10 @@
|
||||
use std::path::{Path,PathBuf};
|
||||
use std::borrow::Cow;
|
||||
use clap::{Args,Subcommand};
|
||||
use anyhow::Result as AResult;
|
||||
use strafesnet_deferred_loader::deferred_loader::LoadFailureMode;
|
||||
use futures::StreamExt;
|
||||
use strafesnet_deferred_loader::loader::Loader;
|
||||
use strafesnet_deferred_loader::deferred_loader::{LoadFailureMode,MeshDeferredLoader,RenderConfigDeferredLoader};
|
||||
|
||||
#[derive(Subcommand)]
|
||||
pub enum Commands{
|
||||
@ -19,11 +22,11 @@ pub struct SourceToSNFSubcommand {
|
||||
input_files:Vec<PathBuf>,
|
||||
}
|
||||
#[derive(Args)]
|
||||
pub struct ExtractTexturesSubcommand {
|
||||
pub struct ExtractTexturesSubcommand{
|
||||
#[arg(required=true)]
|
||||
bsp_files:Vec<PathBuf>,
|
||||
#[arg(long)]
|
||||
bsp_file:PathBuf,
|
||||
#[arg(long)]
|
||||
vpk_dir_files:Vec<PathBuf>
|
||||
vpk_dir_file:Vec<PathBuf>,
|
||||
}
|
||||
#[derive(Args)]
|
||||
pub struct VPKContentsSubcommand {
|
||||
@ -39,8 +42,8 @@ pub struct BSPContentsSubcommand {
|
||||
impl Commands{
|
||||
pub async fn run(self)->AResult<()>{
|
||||
match self{
|
||||
Commands::ExtractTextures(subcommand)=>extract_textures(vec![subcommand.bsp_file],subcommand.vpk_dir_files),
|
||||
Commands::SourceToSNF(subcommand)=>source_to_snf(subcommand.input_files,subcommand.output_folder).await,
|
||||
Commands::ExtractTextures(subcommand)=>extract_textures(subcommand.bsp_files,subcommand.vpk_dir_file).await,
|
||||
Commands::VPKContents(subcommand)=>vpk_contents(subcommand.input_file),
|
||||
Commands::BSPContents(subcommand)=>bsp_contents(subcommand.input_file),
|
||||
}
|
||||
@ -121,139 +124,220 @@ fn recursive_vmt_loader<F:Fn(String)->AResult<Option<Vec<u8>>>>(find_stuff:&F,ma
|
||||
VMTContent::Unresolved=>{println!("Unresolved vmt");Ok(None)},
|
||||
}
|
||||
}
|
||||
|
||||
fn extract_textures(paths:Vec<PathBuf>,vpk_paths:Vec<PathBuf>)->AResult<()>{
|
||||
std::fs::create_dir_all("textures")?;
|
||||
let vpk_list:Vec<vpk::VPK>=vpk_paths.into_iter().map(|vpk_path|vpk::VPK::read(&vpk_path).expect("vpk file does not exist")).collect();
|
||||
for path in paths{
|
||||
let mut deduplicate=std::collections::HashSet::new();
|
||||
let bsp=vbsp::Bsp::read(std::fs::read(path)?.as_ref())?;
|
||||
for texture in bsp.textures(){
|
||||
deduplicate.insert(PathBuf::from(texture.name()));
|
||||
}
|
||||
//dedupe prop models
|
||||
let mut model_dedupe=std::collections::HashSet::new();
|
||||
for prop in bsp.static_props(){
|
||||
model_dedupe.insert(prop.model());
|
||||
}
|
||||
|
||||
//grab texture names from props
|
||||
for model_name in model_dedupe{
|
||||
//.mdl, .vvd, .dx90.vtx
|
||||
let mut path=PathBuf::from(model_name);
|
||||
let file_name=PathBuf::from(path.file_stem().unwrap());
|
||||
path.pop();
|
||||
path.push(file_name);
|
||||
let mut vvd_path=path.clone();
|
||||
let mut vtx_path=path.clone();
|
||||
vvd_path.set_extension("vvd");
|
||||
vtx_path.set_extension("dx90.vtx");
|
||||
match (bsp.pack.get(model_name),bsp.pack.get(vvd_path.as_os_str().to_str().unwrap()),bsp.pack.get(vtx_path.as_os_str().to_str().unwrap())){
|
||||
(Ok(Some(mdl_file)),Ok(Some(vvd_file)),Ok(Some(vtx_file)))=>{
|
||||
match (vmdl::mdl::Mdl::read(mdl_file.as_ref()),vmdl::vvd::Vvd::read(vvd_file.as_ref()),vmdl::vtx::Vtx::read(vtx_file.as_ref())){
|
||||
(Ok(mdl),Ok(vvd),Ok(vtx))=>{
|
||||
let model=vmdl::Model::from_parts(mdl,vtx,vvd);
|
||||
for texture in model.textures(){
|
||||
for search_path in &texture.search_paths{
|
||||
let mut path=PathBuf::from(search_path.as_str());
|
||||
path.push(texture.name.as_str());
|
||||
deduplicate.insert(path);
|
||||
}
|
||||
}
|
||||
},
|
||||
_=>println!("model_name={} error",model_name),
|
||||
}
|
||||
},
|
||||
_=>println!("no model name={}",model_name),
|
||||
}
|
||||
}
|
||||
|
||||
let pack=&bsp.pack;
|
||||
let vpk_list=&vpk_list;
|
||||
std::thread::scope(move|s|{
|
||||
let mut thread_handles=Vec::new();
|
||||
for texture_name in deduplicate{
|
||||
let mut found_texture=false;
|
||||
//LMAO imagine having to write type names
|
||||
let write_image=|mut stuff,write_file_name|{
|
||||
let image=vtf::from_bytes(&mut stuff)?.highres_image.decode(0)?.to_rgba8();
|
||||
|
||||
let format=if image.width()%4!=0||image.height()%4!=0{
|
||||
image_dds::ImageFormat::Rgba8UnormSrgb
|
||||
}else{
|
||||
image_dds::ImageFormat::BC7RgbaUnormSrgb
|
||||
};
|
||||
//this fails if the image dimensions are not a multiple of 4
|
||||
let dds = image_dds::dds_from_image(
|
||||
&image,
|
||||
format,
|
||||
image_dds::Quality::Slow,
|
||||
image_dds::Mipmaps::GeneratedAutomatic,
|
||||
)?;
|
||||
|
||||
//write dds
|
||||
let mut dest=PathBuf::from("textures");
|
||||
dest.push(write_file_name);
|
||||
dest.set_extension("dds");
|
||||
std::fs::create_dir_all(dest.parent().unwrap())?;
|
||||
let mut writer = std::io::BufWriter::new(std::fs::File::create(dest)?);
|
||||
dds.write(&mut writer)?;
|
||||
Ok::<(),anyhow::Error>(())
|
||||
};
|
||||
let find_stuff=|search_file_name:String|{
|
||||
println!("search_file_name={}",search_file_name);
|
||||
match pack.get(search_file_name.as_str())?{
|
||||
Some(file)=>return Ok(Some(file)),
|
||||
_=>(),
|
||||
}
|
||||
//search pak list
|
||||
for vpk_index in vpk_list{
|
||||
if let Some(vpk_entry)=vpk_index.tree.get(search_file_name.as_str()){
|
||||
return Ok(Some(match vpk_entry.get()?{
|
||||
std::borrow::Cow::Borrowed(bytes)=>bytes.to_vec(),
|
||||
std::borrow::Cow::Owned(bytes)=>bytes,
|
||||
}));
|
||||
}
|
||||
}
|
||||
Ok::<Option<Vec<u8>>,anyhow::Error>(None)
|
||||
};
|
||||
let loader=|texture_name:String|{
|
||||
let mut texture_file_name=PathBuf::from("materials");
|
||||
//lower case
|
||||
let texture_file_name_lowercase=texture_name.to_lowercase();
|
||||
texture_file_name.push(texture_file_name_lowercase.clone());
|
||||
//remove stem and search for both vtf and vmt files
|
||||
let stem=PathBuf::from(texture_file_name.file_stem().unwrap());
|
||||
texture_file_name.pop();
|
||||
texture_file_name.push(stem);
|
||||
//somehow search for both files
|
||||
let mut texture_file_name_vmt=texture_file_name.clone();
|
||||
texture_file_name.set_extension("vtf");
|
||||
texture_file_name_vmt.set_extension("vmt");
|
||||
if let Some(stuff)=find_stuff(texture_file_name.to_string_lossy().to_string())?{
|
||||
return Ok(Some(stuff))
|
||||
}
|
||||
recursive_vmt_loader(&find_stuff,get_vmt(&find_stuff,texture_file_name_vmt.to_string_lossy().to_string())?)
|
||||
};
|
||||
if let Some(stuff)=loader(texture_name.to_string_lossy().to_string())?{
|
||||
found_texture=true;
|
||||
let texture_name=texture_name.clone();
|
||||
thread_handles.push(s.spawn(move||write_image(stuff,texture_name)));
|
||||
}
|
||||
if !found_texture{
|
||||
println!("no data");
|
||||
}
|
||||
}
|
||||
for thread in thread_handles{
|
||||
match thread.join(){
|
||||
Ok(Err(e))=>println!("write error: {:?}",e),
|
||||
Err(e)=>println!("thread error: {:?}",e),
|
||||
Ok(_)=>(),
|
||||
}
|
||||
}
|
||||
Ok::<(),anyhow::Error>(())
|
||||
})?
|
||||
#[derive(Debug,thiserror::Error)]
|
||||
enum ExtractTextureError{
|
||||
#[error("Io error {0:?}")]
|
||||
Io(#[from]std::io::Error),
|
||||
#[error("Bsp error {0:?}")]
|
||||
Bsp(#[from]vbsp::BspError),
|
||||
#[error("MeshLoad error {0:?}")]
|
||||
MeshLoad(#[from]strafesnet_bsp_loader::loader::MeshError),
|
||||
}
|
||||
fn find_stuff(){
|
||||
println!("search_file_name={}",search_file_name);
|
||||
match pack.get(search_file_name.as_str())?{
|
||||
Some(file)=>return Ok(Some(file)),
|
||||
_=>(),
|
||||
}
|
||||
//search pak list
|
||||
for vpk_index in vpk_list{
|
||||
if let Some(vpk_entry)=vpk_index.tree.get(search_file_name.as_str()){
|
||||
return Ok(Some(match vpk_entry.get()?{
|
||||
std::borrow::Cow::Borrowed(bytes)=>bytes.to_vec(),
|
||||
std::borrow::Cow::Owned(bytes)=>bytes,
|
||||
}));
|
||||
}
|
||||
}
|
||||
Ok::<Option<Vec<u8>>,anyhow::Error>(None)
|
||||
}
|
||||
fn load_texture(){
|
||||
let mut texture_file_name=PathBuf::from("materials");
|
||||
//lower case
|
||||
let texture_file_name_lowercase=texture_name.to_lowercase();
|
||||
texture_file_name.push(texture_file_name_lowercase.clone());
|
||||
//remove stem and search for both vtf and vmt files
|
||||
let stem=PathBuf::from(texture_file_name.file_stem().unwrap());
|
||||
texture_file_name.pop();
|
||||
texture_file_name.push(stem);
|
||||
//somehow search for both files
|
||||
let mut texture_file_name_vmt=texture_file_name.clone();
|
||||
texture_file_name.set_extension("vtf");
|
||||
texture_file_name_vmt.set_extension("vmt");
|
||||
if let Some(stuff)=find_stuff(texture_file_name.to_string_lossy().to_string())?{
|
||||
return Ok(Some(stuff))
|
||||
}
|
||||
recursive_vmt_loader(&find_stuff,get_vmt(&find_stuff,texture_file_name_vmt.to_string_lossy().to_string())?)
|
||||
}
|
||||
async fn gimme_them_textures(path:PathBuf)->Result<(),ExtractTextureError>{
|
||||
let mut deduplicate=std::collections::HashSet::new();
|
||||
let bsp=vbsp::Bsp::read(tokio::fs::read(path).await?.as_ref())?;
|
||||
|
||||
let mut texture_deferred_loader=RenderConfigDeferredLoader::new();
|
||||
for texture in bsp.textures(){
|
||||
texture_deferred_loader.acquire_render_config_id(Some(Cow::Borrowed(texture.name())));
|
||||
}
|
||||
|
||||
let mut mesh_deferred_loader=MeshDeferredLoader::new();
|
||||
for prop in bsp.static_props(){
|
||||
mesh_deferred_loader.acquire_mesh_id(prop.model());
|
||||
}
|
||||
|
||||
// TODO: include more packs in mesh loader
|
||||
let mut mesh_loader=strafesnet_bsp_loader::loader::MeshLoader::new(&strafesnet_bsp_loader::Bsp::new(bsp),&mut texture_deferred_loader);
|
||||
|
||||
// load models and collect requested textures
|
||||
for &model_path in mesh_deferred_loader.indices(){
|
||||
// TODO: fix loader to return pre-converted mesh
|
||||
let model:vmdl::Model=mesh_loader.load(model_path)?;
|
||||
for texture in model.textures(){
|
||||
for search_path in &texture.search_paths{
|
||||
let mut path=PathBuf::from(search_path.as_str());
|
||||
path.push(texture.name.as_str());
|
||||
let path=path.to_str().unwrap().to_owned();
|
||||
texture_deferred_loader.acquire_render_config_id(Some(Cow::Owned(path)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for &texture_path in texture_deferred_loader.indices(){
|
||||
if let Some(texture)=load_texture(texture_path)?{
|
||||
send_texture.send(texture).unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
let pack=&bsp.pack;
|
||||
let vpk_list=&vpk_list;
|
||||
std::thread::scope(move|s|{
|
||||
let mut thread_handles=Vec::new();
|
||||
for texture_name in deduplicate{
|
||||
let mut found_texture=false;
|
||||
//LMAO imagine having to write type names
|
||||
let write_image=|mut stuff,write_file_name|{
|
||||
let image=vtf::from_bytes(&mut stuff)?.highres_image.decode(0)?.to_rgba8();
|
||||
|
||||
let format=if image.width()%4!=0||image.height()%4!=0{
|
||||
image_dds::ImageFormat::Rgba8UnormSrgb
|
||||
}else{
|
||||
image_dds::ImageFormat::BC7RgbaUnormSrgb
|
||||
};
|
||||
//this fails if the image dimensions are not a multiple of 4
|
||||
let dds = image_dds::dds_from_image(
|
||||
&image,
|
||||
format,
|
||||
image_dds::Quality::Slow,
|
||||
image_dds::Mipmaps::GeneratedAutomatic,
|
||||
)?;
|
||||
|
||||
//write dds
|
||||
let mut dest=PathBuf::from("textures");
|
||||
dest.push(write_file_name);
|
||||
dest.set_extension("dds");
|
||||
std::fs::create_dir_all(dest.parent().unwrap())?;
|
||||
let mut writer = std::io::BufWriter::new(std::fs::File::create(dest)?);
|
||||
dds.write(&mut writer)?;
|
||||
Ok::<(),anyhow::Error>(())
|
||||
};
|
||||
let find_stuff=|search_file_name:String|{
|
||||
println!("search_file_name={}",search_file_name);
|
||||
match pack.get(search_file_name.as_str())?{
|
||||
Some(file)=>return Ok(Some(file)),
|
||||
_=>(),
|
||||
}
|
||||
//search pak list
|
||||
for vpk_index in vpk_list{
|
||||
if let Some(vpk_entry)=vpk_index.tree.get(search_file_name.as_str()){
|
||||
return Ok(Some(match vpk_entry.get()?{
|
||||
std::borrow::Cow::Borrowed(bytes)=>bytes.to_vec(),
|
||||
std::borrow::Cow::Owned(bytes)=>bytes,
|
||||
}));
|
||||
}
|
||||
}
|
||||
Ok::<Option<Vec<u8>>,anyhow::Error>(None)
|
||||
};
|
||||
let loader=|texture_name:String|{
|
||||
let mut texture_file_name=PathBuf::from("materials");
|
||||
//lower case
|
||||
let texture_file_name_lowercase=texture_name.to_lowercase();
|
||||
texture_file_name.push(texture_file_name_lowercase.clone());
|
||||
//remove stem and search for both vtf and vmt files
|
||||
let stem=PathBuf::from(texture_file_name.file_stem().unwrap());
|
||||
texture_file_name.pop();
|
||||
texture_file_name.push(stem);
|
||||
//somehow search for both files
|
||||
let mut texture_file_name_vmt=texture_file_name.clone();
|
||||
texture_file_name.set_extension("vtf");
|
||||
texture_file_name_vmt.set_extension("vmt");
|
||||
if let Some(stuff)=find_stuff(texture_file_name.to_string_lossy().to_string())?{
|
||||
return Ok(Some(stuff))
|
||||
}
|
||||
recursive_vmt_loader(&find_stuff,get_vmt(&find_stuff,texture_file_name_vmt.to_string_lossy().to_string())?)
|
||||
};
|
||||
if let Some(stuff)=loader(texture_name.to_string_lossy().to_string())?{
|
||||
found_texture=true;
|
||||
let texture_name=texture_name.clone();
|
||||
thread_handles.push(s.spawn(move||write_image(stuff,texture_name)));
|
||||
}
|
||||
if !found_texture{
|
||||
println!("no data");
|
||||
}
|
||||
}
|
||||
for thread in thread_handles{
|
||||
match thread.join(){
|
||||
Ok(Err(e))=>println!("write error: {:?}",e),
|
||||
Err(e)=>println!("thread error: {:?}",e),
|
||||
Ok(_)=>(),
|
||||
}
|
||||
}
|
||||
Ok::<(),anyhow::Error>(())
|
||||
})?
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn extract_textures(paths:Vec<PathBuf>,vpk_paths:Vec<PathBuf>)->AResult<()>{
|
||||
tokio::try_join!(
|
||||
tokio::fs::create_dir_all("extracted_textures"),
|
||||
tokio::fs::create_dir_all("textures"),
|
||||
tokio::fs::create_dir_all("meshes"),
|
||||
)?;
|
||||
let thread_limit=std::thread::available_parallelism()?.get();
|
||||
|
||||
// load vpk list
|
||||
let vpk_list=futures::stream::iter(vpk_paths).map(|vpk_path|async{
|
||||
// idk why it doesn't want to pass out the errors but this is fatal anyways
|
||||
tokio::task::spawn_blocking(move||vpk::VPK::read(&vpk_path)).await.unwrap().unwrap()
|
||||
})
|
||||
.buffer_unordered(thread_limit)
|
||||
.collect::<Vec<vpk::VPK>>().await;
|
||||
|
||||
// leak vpk_list for static lifetime?
|
||||
|
||||
let (send_texture,mut recv_texture)=tokio::sync::mpsc::channel(thread_limit);
|
||||
let mut it=paths.into_iter();
|
||||
let extract_thread=tokio::spawn(async move{
|
||||
static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
|
||||
SEM.add_permits(thread_limit);
|
||||
while let (Ok(permit),Some(path))=(SEM.acquire().await,it.next()){
|
||||
let send=send_texture.clone();
|
||||
tokio::spawn(async move{
|
||||
let result=gimme_them_textures(path,send).await;
|
||||
drop(permit);
|
||||
result.unwrap();
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// convert images
|
||||
static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
|
||||
SEM.add_permits(thread_limit);
|
||||
while let (Ok(permit),Some(data))=(SEM.acquire().await,recv_texture.recv().await){
|
||||
tokio::spawn(async move{
|
||||
let result=convert_texture(data).await;
|
||||
drop(permit);
|
||||
result.unwrap();
|
||||
});
|
||||
}
|
||||
extract_thread.await??;
|
||||
SEM.acquire_many(thread_limit as u32).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user