wip download_assets (pre-tokio)

src/roblox.rs

@@ -1,5 +1,5 @@
-use std::path::PathBuf;
-use std::io::{Read,Seek};
+use std::path::{Path,PathBuf};
+use std::io::{Cursor,Read,Seek};
 use std::collections::HashSet;
 use clap::{Args,Subcommand};
 use anyhow::Result as AResult;
@@ -9,8 +9,7 @@ use strafesnet_deferred_loader::rbxassetid::RobloxAssetId;
 #[derive(Subcommand)]
 pub enum Commands{
 	RobloxToSNF(RobloxToSNFSubcommand),
-	DownloadTextures(DownloadTexturesSubcommand),
-	DownloadMeshes(DownloadMeshesSubcommand),
+	DownloadAssets(DownloadAssetsSubcommand),
 }
 
 #[derive(Args)]
@@ -21,12 +20,7 @@ pub struct RobloxToSNFSubcommand {
 	input_files:Vec<PathBuf>,
 }
 #[derive(Args)]
-pub struct DownloadTexturesSubcommand {
-	#[arg(long,required=true)]
-	roblox_files:Vec<PathBuf>
-}
-#[derive(Args)]
-pub struct DownloadMeshesSubcommand {
+pub struct DownloadAssetsSubcommand{
 	#[arg(long,required=true)]
 	roblox_files:Vec<PathBuf>
 }
@@ -35,28 +29,20 @@ impl Commands{
 	pub fn run(self)->AResult<()>{
 		match self{
 			Commands::RobloxToSNF(subcommand)=>roblox_to_snf(subcommand.input_files,subcommand.output_folder),
-			Commands::DownloadTextures(subcommand)=>download_textures(subcommand.roblox_files),
-			Commands::DownloadMeshes(subcommand)=>download_meshes(subcommand.roblox_files),
+			Commands::DownloadAssets(subcommand)=>download_assets(subcommand.roblox_files),
 		}
 	}
 }
 
-fn load_dom<R:Read+Seek>(input:&mut R)->AResult<rbx_dom_weak::WeakDom>{
+fn load_dom<R:Read+Seek>(mut input:R)->AResult<rbx_dom_weak::WeakDom>{
 	let mut first_8=[0u8;8];
-	if let (Ok(()),Ok(()))=(std::io::Read::read_exact(input, &mut first_8),std::io::Seek::rewind(input)){
-		match &first_8[0..4]{
-			b"<rob"=>{
-				match &first_8[4..8]{
-					b"lox!"=>rbx_binary::from_reader(input).map_err(anyhow::Error::msg),
-					b"lox "=>rbx_xml::from_reader(input,rbx_xml::DecodeOptions::default()).map_err(anyhow::Error::msg),
-					other=>Err(anyhow::Error::msg(format!("Unknown Roblox file type {:?}",other))),
-				}
-			},
+	input.read_exact(&mut first_8)?;
+	input.rewind()?;
+	match &first_8{
+		b"<roblox!"=>rbx_binary::from_reader(input).map_err(anyhow::Error::msg),
+		b"<roblox "=>rbx_xml::from_reader(input,rbx_xml::DecodeOptions::default()).map_err(anyhow::Error::msg),
 		_=>Err(anyhow::Error::msg("unsupported file type")),
 	}
-	}else{
-		Err(anyhow::Error::msg("peek failed"))
-	}
 }
 
 /* The ones I'm interested in:
@@ -85,10 +71,11 @@ SurfaceAppearance.NormalMap
 SurfaceAppearance.RoughnessMap
 SurfaceAppearance.TexturePack
 */
-fn accumulate_content_id(content_list:&mut HashSet<u64>,object:&Instance,property:&str){
+fn accumulate_content_id(content_list:&mut HashSet<RobloxAssetId>,object:&Instance,property:&str){
 	if let Some(rbx_dom_weak::types::Variant::Content(content))=object.properties.get(property){
-		if let Ok(asset_id)=AsRef::<str>::as_ref(content).parse::<RobloxAssetId>(){
-			content_list.insert(asset_id.0);
+		let url:&str=content.as_ref();
+		if let Ok(asset_id)=url.parse(){
+			content_list.insert(asset_id);
 		}else{
 			println!("Content failed to parse into AssetID: {:?}",content);
 		}
@@ -96,90 +83,70 @@ fn accumulate_content_id(content_list:&mut HashSet<u64>,object:&Instance,propert
 		println!("property={} does not exist for class={}",object.class.as_str(),property);
 	}
 }
-fn download_textures(paths:Vec<PathBuf>)->AResult<()>{
-	println!("Reading files, this could take a hot minute...");
-	let mut texture_list=HashSet::new();
-	for path in paths{
-		let file=match std::fs::File::open(path.as_path()){
-			Ok(file)=>file,
-			Err(e)=>{
-				println!("file error {e}");
-				continue;
+fn read_entire_file(path:impl AsRef<Path>)->Result<Cursor<Vec<u8>>,std::io::Error>{
+	let mut file=std::fs::File::open(path)?;
+	let mut data=Vec::new();
+	file.read_to_end(&mut data)?;
+	Ok(Cursor::new(data))
 }
-		};
-		let mut input=std::io::BufReader::new(file);
-		match load_dom(&mut input){
-			Ok(dom)=>{
-				for object in dom.into_raw().1.into_values(){
+#[derive(Default)]
+struct UniqueAssets{
+	meshes:HashSet<RobloxAssetId>,
+	unions:HashSet<RobloxAssetId>,
+	textures:HashSet<RobloxAssetId>,
+}
+impl UniqueAssets{
+	fn collect(&mut self,object:&Instance){
 		match object.class.as_str(){
-						"Beam"=>accumulate_content_id(&mut texture_list,&object,"Texture"),
-						"Decal"=>accumulate_content_id(&mut texture_list,&object,"Texture"),
-						"Texture"=>accumulate_content_id(&mut texture_list,&object,"Texture"),
-						"FileMesh"=>accumulate_content_id(&mut texture_list,&object,"TextureId"),
-						"MeshPart"=>accumulate_content_id(&mut texture_list,&object,"TextureID"),
-						"ParticleEmitter"=>accumulate_content_id(&mut texture_list,&object,"Texture"),
+			"Beam"=>accumulate_content_id(&mut self.textures,object,"Texture"),
+			"Decal"=>accumulate_content_id(&mut self.textures,object,"Texture"),
+			"Texture"=>accumulate_content_id(&mut self.textures,object,"Texture"),
+			"FileMesh"=>accumulate_content_id(&mut self.textures,object,"TextureId"),
+			"MeshPart"=>{
+				accumulate_content_id(&mut self.textures,object,"TextureID");
+				accumulate_content_id(&mut self.meshes,object,"MeshId");
+			},
+			"SpecialMesh"=>accumulate_content_id(&mut self.meshes,object,"MeshId"),
+			"ParticleEmitter"=>accumulate_content_id(&mut self.textures,object,"Texture"),
 			"Sky"=>{
-							accumulate_content_id(&mut texture_list,&object,"MoonTextureId");
-							accumulate_content_id(&mut texture_list,&object,"SkyboxBk");
-							accumulate_content_id(&mut texture_list,&object,"SkyboxDn");
-							accumulate_content_id(&mut texture_list,&object,"SkyboxFt");
-							accumulate_content_id(&mut texture_list,&object,"SkyboxLf");
-							accumulate_content_id(&mut texture_list,&object,"SkyboxRt");
-							accumulate_content_id(&mut texture_list,&object,"SkyboxUp");
-							accumulate_content_id(&mut texture_list,&object,"SunTextureId");
+				accumulate_content_id(&mut self.textures,object,"MoonTextureId");
+				accumulate_content_id(&mut self.textures,object,"SkyboxBk");
+				accumulate_content_id(&mut self.textures,object,"SkyboxDn");
+				accumulate_content_id(&mut self.textures,object,"SkyboxFt");
+				accumulate_content_id(&mut self.textures,object,"SkyboxLf");
+				accumulate_content_id(&mut self.textures,object,"SkyboxRt");
+				accumulate_content_id(&mut self.textures,object,"SkyboxUp");
+				accumulate_content_id(&mut self.textures,object,"SunTextureId");
 			},
+			"UnionOperation"=>accumulate_content_id(&mut self.unions,object,"AssetId"),
 			_=>(),
 		}
 	}
-			},
-			Err(e)=>println!("error loading map {:?}: {:?}",path.file_name(),e),
 }
-	}
-	let texture_list_string=texture_list.into_iter().map(|id|id.to_string()).collect::<Vec<String>>();
-	println!("Texture list:{:?}",texture_list_string.join(" "));
-	std::fs::create_dir_all("textures/unprocessed")?;
-	let output=std::process::Command::new("asset-tool")
-	.args(["download","--cookie-literal","","--output-folder","textures/unprocessed/"])
-	.args(texture_list_string)
-	.spawn()?
-	.wait_with_output()?;
-	println!("Asset tool exit_success:{}",output.status.success());
-	Ok(())
-}
-fn download_meshes(paths:Vec<PathBuf>)->AResult<()>{
-	println!("Reading files, this could take a hot minute...");
-	let mut mesh_list=HashSet::new();
-	for path in paths{
-		let file=match std::fs::File::open(path.as_path()){
-			Ok(file)=>file,
-			Err(e)=>{
-				println!("file error {e}");
-				continue;
-			}
-		};
-		let mut input=std::io::BufReader::new(file);
-		match load_dom(&mut input){
-			Ok(dom)=>{
+fn unique_assets(path:&Path)->AResult<UniqueAssets>{
+	// read entire file
+	let mut assets=UniqueAssets::default();
+	let data=read_entire_file(path)?;
+	let dom=load_dom(data)?;
 	for object in dom.into_raw().1.into_values(){
-					match object.class.as_str(){
-						"MeshPart"=>accumulate_content_id(&mut mesh_list,&object,"MeshId"),
-						"SpecialMesh"=>accumulate_content_id(&mut mesh_list,&object,"MeshId"),
-						_=>(),
+		assets.collect(&object);
 	}
+	Ok(assets)
 }
-			},
-			Err(e)=>println!("error loading map {:?}: {:?}",path.file_name(),e),
+struct UniqueAssetsResult{
+	path:std::path::PathBuf,
+	result:AResult<UniqueAssets>,
 }
+fn do_thread(path:std::path::PathBuf,send:std::sync::mpsc::Sender<UniqueAssetsResult>){
+	std::thread::spawn(move ||{
+		let result=unique_assets(path.as_path());
+		send.send(UniqueAssetsResult{
+			path,
+			result,
+		}).unwrap();
+	});
 }
-	let mesh_list_string=mesh_list.into_iter().map(|id|id.to_string()).collect::<Vec<String>>();
-	println!("Mesh list:{:?}",mesh_list_string.join(" "));
-	std::fs::create_dir_all("meshes/")?;
-	let output=std::process::Command::new("asset-tool")
-	.args(["download","--cookie-literal","","--output-folder","meshes/"])
-	.args(mesh_list_string)
-	.spawn()?
-	.wait_with_output()?;
-	println!("Asset tool exit_success:{}",output.status.success());
+fn download_assets(paths:Vec<PathBuf>)->AResult<()>{
 	Ok(())
 }