From a5079f21d72a7fa199787918637fa5780cea56bb Mon Sep 17 00:00:00 2001
From: Quaternions <krakow20@gmail.com>
Date: Mon, 27 Jan 2025 08:47:51 -0800
Subject: [PATCH] wip download_assets (pre-tokio)
---
src/roblox.rs | 185 +++++++++++++++++++++-----------------------------
1 file changed, 76 insertions(+), 109 deletions(-)
diff --git a/src/roblox.rs b/src/roblox.rs
index 53a7a5b..b22b62f 100644
--- a/src/roblox.rs
+++ b/src/roblox.rs
@@ -1,5 +1,5 @@
-use std::path::PathBuf;
-use std::io::{Read,Seek};
+use std::path::{Path,PathBuf};
+use std::io::{Cursor,Read,Seek};
use std::collections::HashSet;
use clap::{Args,Subcommand};
use anyhow::Result as AResult;
@@ -9,8 +9,7 @@ use strafesnet_deferred_loader::rbxassetid::RobloxAssetId;
#[derive(Subcommand)]
pub enum Commands{
RobloxToSNF(RobloxToSNFSubcommand),
- DownloadTextures(DownloadTexturesSubcommand),
- DownloadMeshes(DownloadMeshesSubcommand),
+ DownloadAssets(DownloadAssetsSubcommand),
}
#[derive(Args)]
@@ -21,12 +20,7 @@ pub struct RobloxToSNFSubcommand {
input_files:Vec<PathBuf>,
}
#[derive(Args)]
-pub struct DownloadTexturesSubcommand {
- #[arg(long,required=true)]
- roblox_files:Vec<PathBuf>
-}
-#[derive(Args)]
-pub struct DownloadMeshesSubcommand {
+pub struct DownloadAssetsSubcommand{
#[arg(long,required=true)]
roblox_files:Vec<PathBuf>
}
@@ -35,27 +29,19 @@ impl Commands{
pub fn run(self)->AResult<()>{
match self{
Commands::RobloxToSNF(subcommand)=>roblox_to_snf(subcommand.input_files,subcommand.output_folder),
- Commands::DownloadTextures(subcommand)=>download_textures(subcommand.roblox_files),
- Commands::DownloadMeshes(subcommand)=>download_meshes(subcommand.roblox_files),
+ Commands::DownloadAssets(subcommand)=>download_assets(subcommand.roblox_files),
}
}
}
-fn load_dom<R:Read+Seek>(input:&mut R)->AResult<rbx_dom_weak::WeakDom>{
+fn load_dom<R:Read+Seek>(mut input:R)->AResult<rbx_dom_weak::WeakDom>{
let mut first_8=[0u8;8];
- if let (Ok(()),Ok(()))=(std::io::Read::read_exact(input, &mut first_8),std::io::Seek::rewind(input)){
- match &first_8[0..4]{
- b"<rob"=>{
- match &first_8[4..8]{
- b"lox!"=>rbx_binary::from_reader(input).map_err(anyhow::Error::msg),
- b"lox "=>rbx_xml::from_reader(input,rbx_xml::DecodeOptions::default()).map_err(anyhow::Error::msg),
- other=>Err(anyhow::Error::msg(format!("Unknown Roblox file type {:?}",other))),
- }
- },
- _=>Err(anyhow::Error::msg("unsupported file type")),
- }
- }else{
- Err(anyhow::Error::msg("peek failed"))
+ input.read_exact(&mut first_8)?;
+ input.rewind()?;
+ match &first_8{
+ b"<roblox!"=>rbx_binary::from_reader(input).map_err(anyhow::Error::msg),
+ b"<roblox "=>rbx_xml::from_reader(input,rbx_xml::DecodeOptions::default()).map_err(anyhow::Error::msg),
+ _=>Err(anyhow::Error::msg("unsupported file type")),
}
}
@@ -85,10 +71,11 @@ SurfaceAppearance.NormalMap
SurfaceAppearance.RoughnessMap
SurfaceAppearance.TexturePack
*/
-fn accumulate_content_id(content_list:&mut HashSet<u64>,object:&Instance,property:&str){
+fn accumulate_content_id(content_list:&mut HashSet<RobloxAssetId>,object:&Instance,property:&str){
if let Some(rbx_dom_weak::types::Variant::Content(content))=object.properties.get(property){
- if let Ok(asset_id)=AsRef::<str>::as_ref(content).parse::<RobloxAssetId>(){
- content_list.insert(asset_id.0);
+ let url:&str=content.as_ref();
+ if let Ok(asset_id)=url.parse(){
+ content_list.insert(asset_id);
}else{
println!("Content failed to parse into AssetID: {:?}",content);
}
@@ -96,90 +83,70 @@ fn accumulate_content_id(content_list:&mut HashSet<u64>,object:&Instance,propert
println!("property={} does not exist for class={}",object.class.as_str(),property);
}
}
-fn download_textures(paths:Vec<PathBuf>)->AResult<()>{
- println!("Reading files, this could take a hot minute...");
- let mut texture_list=HashSet::new();
- for path in paths{
- let file=match std::fs::File::open(path.as_path()){
- Ok(file)=>file,
- Err(e)=>{
- println!("file error {e}");
- continue;
- }
- };
- let mut input=std::io::BufReader::new(file);
- match load_dom(&mut input){
- Ok(dom)=>{
- for object in dom.into_raw().1.into_values(){
- match object.class.as_str(){
- "Beam"=>accumulate_content_id(&mut texture_list,&object,"Texture"),
- "Decal"=>accumulate_content_id(&mut texture_list,&object,"Texture"),
- "Texture"=>accumulate_content_id(&mut texture_list,&object,"Texture"),
- "FileMesh"=>accumulate_content_id(&mut texture_list,&object,"TextureId"),
- "MeshPart"=>accumulate_content_id(&mut texture_list,&object,"TextureID"),
- "ParticleEmitter"=>accumulate_content_id(&mut texture_list,&object,"Texture"),
- "Sky"=>{
- accumulate_content_id(&mut texture_list,&object,"MoonTextureId");
- accumulate_content_id(&mut texture_list,&object,"SkyboxBk");
- accumulate_content_id(&mut texture_list,&object,"SkyboxDn");
- accumulate_content_id(&mut texture_list,&object,"SkyboxFt");
- accumulate_content_id(&mut texture_list,&object,"SkyboxLf");
- accumulate_content_id(&mut texture_list,&object,"SkyboxRt");
- accumulate_content_id(&mut texture_list,&object,"SkyboxUp");
- accumulate_content_id(&mut texture_list,&object,"SunTextureId");
- },
- _=>(),
- }
- }
- },
- Err(e)=>println!("error loading map {:?}: {:?}",path.file_name(),e),
- }
- }
- let texture_list_string=texture_list.into_iter().map(|id|id.to_string()).collect::<Vec<String>>();
- println!("Texture list:{:?}",texture_list_string.join(" "));
- std::fs::create_dir_all("textures/unprocessed")?;
- let output=std::process::Command::new("asset-tool")
- .args(["download","--cookie-literal","","--output-folder","textures/unprocessed/"])
- .args(texture_list_string)
- .spawn()?
- .wait_with_output()?;
- println!("Asset tool exit_success:{}",output.status.success());
- Ok(())
+fn read_entire_file(path:impl AsRef<Path>)->Result<Cursor<Vec<u8>>,std::io::Error>{
+ let mut file=std::fs::File::open(path)?;
+ let mut data=Vec::new();
+ file.read_to_end(&mut data)?;
+ Ok(Cursor::new(data))
}
-fn download_meshes(paths:Vec<PathBuf>)->AResult<()>{
- println!("Reading files, this could take a hot minute...");
- let mut mesh_list=HashSet::new();
- for path in paths{
- let file=match std::fs::File::open(path.as_path()){
- Ok(file)=>file,
- Err(e)=>{
- println!("file error {e}");
- continue;
- }
- };
- let mut input=std::io::BufReader::new(file);
- match load_dom(&mut input){
- Ok(dom)=>{
- for object in dom.into_raw().1.into_values(){
- match object.class.as_str(){
- "MeshPart"=>accumulate_content_id(&mut mesh_list,&object,"MeshId"),
- "SpecialMesh"=>accumulate_content_id(&mut mesh_list,&object,"MeshId"),
- _=>(),
- }
- }
+#[derive(Default)]
+struct UniqueAssets{
+ meshes:HashSet<RobloxAssetId>,
+ unions:HashSet<RobloxAssetId>,
+ textures:HashSet<RobloxAssetId>,
+}
+impl UniqueAssets{
+ fn collect(&mut self,object:&Instance){
+ match object.class.as_str(){
+ "Beam"=>accumulate_content_id(&mut self.textures,object,"Texture"),
+ "Decal"=>accumulate_content_id(&mut self.textures,object,"Texture"),
+ "Texture"=>accumulate_content_id(&mut self.textures,object,"Texture"),
+ "FileMesh"=>accumulate_content_id(&mut self.textures,object,"TextureId"),
+ "MeshPart"=>{
+ accumulate_content_id(&mut self.textures,object,"TextureID");
+ accumulate_content_id(&mut self.meshes,object,"MeshId");
},
- Err(e)=>println!("error loading map {:?}: {:?}",path.file_name(),e),
+ "SpecialMesh"=>accumulate_content_id(&mut self.meshes,object,"MeshId"),
+ "ParticleEmitter"=>accumulate_content_id(&mut self.textures,object,"Texture"),
+ "Sky"=>{
+ accumulate_content_id(&mut self.textures,object,"MoonTextureId");
+ accumulate_content_id(&mut self.textures,object,"SkyboxBk");
+ accumulate_content_id(&mut self.textures,object,"SkyboxDn");
+ accumulate_content_id(&mut self.textures,object,"SkyboxFt");
+ accumulate_content_id(&mut self.textures,object,"SkyboxLf");
+ accumulate_content_id(&mut self.textures,object,"SkyboxRt");
+ accumulate_content_id(&mut self.textures,object,"SkyboxUp");
+ accumulate_content_id(&mut self.textures,object,"SunTextureId");
+ },
+ "UnionOperation"=>accumulate_content_id(&mut self.unions,object,"AssetId"),
+ _=>(),
}
}
- let mesh_list_string=mesh_list.into_iter().map(|id|id.to_string()).collect::<Vec<String>>();
- println!("Mesh list:{:?}",mesh_list_string.join(" "));
- std::fs::create_dir_all("meshes/")?;
- let output=std::process::Command::new("asset-tool")
- .args(["download","--cookie-literal","","--output-folder","meshes/"])
- .args(mesh_list_string)
- .spawn()?
- .wait_with_output()?;
- println!("Asset tool exit_success:{}",output.status.success());
+}
+fn unique_assets(path:&Path)->AResult<UniqueAssets>{
+ // read entire file
+ let mut assets=UniqueAssets::default();
+ let data=read_entire_file(path)?;
+ let dom=load_dom(data)?;
+ for object in dom.into_raw().1.into_values(){
+ assets.collect(&object);
+ }
+ Ok(assets)
+}
+struct UniqueAssetsResult{
+ path:std::path::PathBuf,
+ result:AResult<UniqueAssets>,
+}
+fn do_thread(path:std::path::PathBuf,send:std::sync::mpsc::Sender<UniqueAssetsResult>){
+ std::thread::spawn(move ||{
+ let result=unique_assets(path.as_path());
+ send.send(UniqueAssetsResult{
+ path,
+ result,
+ }).unwrap();
+ });
+}
+fn download_assets(paths:Vec<PathBuf>)->AResult<()>{
Ok(())
}