From 0d1692f25300a37f5780a865d71641a7de64b7ed Mon Sep 17 00:00:00 2001 From: Quaternions Date: Fri, 8 Mar 2024 09:39:27 -0800 Subject: [PATCH] compile feature --- src/main.rs | 525 +++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 523 insertions(+), 2 deletions(-) diff --git a/src/main.rs b/src/main.rs index c8eb690..c7e062f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,6 +3,7 @@ use clap::{Args,Parser,Subcommand}; use anyhow::Result as AResult; use futures::StreamExt; use rbx_dom_weak::types::Ref; +use tokio::io::AsyncReadExt; type AssetID=u64; type AssetIDFileMap=Vec<(AssetID,std::path::PathBuf)>; @@ -163,7 +164,12 @@ async fn main()->AResult<()>{ }).collect() ).await, Commands::Upload=>upload_list(cookie.unwrap(),cli.group,vec![(cli.asset_id.unwrap(),cli.output.unwrap())]).await, - Commands::Compile=>compile(cli.input.unwrap(),cli.output.unwrap()), + Commands::Compile=>compile(CompileConfig{ + input_folder:cli.input.unwrap(), + output_file:cli.output.unwrap(), + template:None, + style:None, + }).await, Commands::Decompile=>decompile(DecompileConfig{ style:decompile_style.unwrap(), input_file:cli.input.unwrap(), @@ -1080,6 +1086,521 @@ async fn download_and_decompile_history_into_git(config:DownloadAndDecompileHist Ok(()) } -fn compile(_folder:std::path::PathBuf,_file:std::path::PathBuf)->AResult<()>{ +//holy smokes what am I doing lmao +//This giant machine is supposed to search for files according to style rules +//e.g. ScriptName.server.lua or init.lua +//Obviously I got carried away +//I could use an enum! +//I could use a struct! +//I could use a trait! +//I could use an error! +//I could use a match! +//I could use a function! +//eventually: +#[derive(Debug)] +enum QueryResolveError{ + NotFound,//0 results + Ambiguous,//>1 results + JoinError(tokio::task::JoinError), + IO(std::io::Error), +} +impl std::fmt::Display for QueryResolveError{ + fn fmt(&self,f:&mut std::fmt::Formatter<'_>)->std::fmt::Result{ + write!(f,"{self:?}") + } +} +impl std::error::Error for QueryResolveError{} + +struct FileWithName{ + file:tokio::fs::File, + name:String, +} + +async fn get_file_async(mut path:std::path::PathBuf,file_name:impl AsRef)->Result{ + let name=file_name.as_ref().to_str().unwrap().to_owned(); + path.push(file_name); + match tokio::fs::File::open(path).await{ + Ok(file)=>Ok(FileWithName{file,name}), + Err(e)=>match e.kind(){ + std::io::ErrorKind::NotFound=>Err(QueryResolveError::NotFound), + _=>Err(QueryResolveError::IO(e)), + }, + } +} +type QueryHintResult=Result; +trait Query{ + async fn resolve(self)->QueryHintResult; +} +type QueryHandle=tokio::task::JoinHandle>; +struct QuerySingle{ + script:QueryHandle, +} +impl QuerySingle{ + fn rox(search_path:&std::path::PathBuf,search_name:&str)->Self{ + Self{ + script:tokio::spawn(get_file_async(search_path.clone(),format!("{}.lua",search_name))) + } + } +} +impl Query for QuerySingle{ + async fn resolve(self)->QueryHintResult{ + match self.script.await{ + Ok(Ok(file))=>Ok(FileHint{file,hint:ScriptHint::ModuleScript}), + Ok(Err(e))=>Err(e), + Err(e)=>Err(QueryResolveError::JoinError(e)), + } + } +} +struct QueryTriple{ + module:QueryHandle, + server:QueryHandle, + client:QueryHandle, +} +impl QueryTriple{ + fn rox_rojo(search_path:&std::path::PathBuf,search_name:&str,search_module:bool)->Self{ + //this should be implemented as constructors of Triplet and Quadruplet to fully support Trey's suggestion + let module_name=if search_module{ + format!("{}.module.lua",search_name) + }else{ + format!("{}.lua",search_name) + }; + Self{ + module:tokio::spawn(get_file_async(search_path.clone(),module_name)), + server:tokio::spawn(get_file_async(search_path.clone(),format!("{}.server.lua",search_name))), + client:tokio::spawn(get_file_async(search_path.clone(),format!("{}.client.lua",search_name))), + } + } + fn rojo(search_path:&std::path::PathBuf)->Self{ + QueryTriple::rox_rojo(search_path,"init",false) + } +} +//these functions can be achieved with macros, but I have not learned that yet +fn mega_triple_join(query_triplet:(QueryHintResult,QueryHintResult,QueryHintResult))->QueryHintResult{ + match query_triplet{ + //unambiguously locate file + (Ok(f),Err(QueryResolveError::NotFound),Err(QueryResolveError::NotFound)) + |(Err(QueryResolveError::NotFound),Ok(f),Err(QueryResolveError::NotFound)) + |(Err(QueryResolveError::NotFound),Err(QueryResolveError::NotFound),Ok(f))=>Ok(f), + //multiple files located + (Ok(_),Ok(_),Err(QueryResolveError::NotFound)) + |(Ok(_),Err(QueryResolveError::NotFound),Ok(_)) + |(Err(QueryResolveError::NotFound),Ok(_),Ok(_)) + |(Ok(_),Ok(_),Ok(_))=>Err(QueryResolveError::Ambiguous), + //no files located + (Err(QueryResolveError::NotFound),Err(QueryResolveError::NotFound),Err(QueryResolveError::NotFound))=>Err(QueryResolveError::NotFound), + //other error + (Err(e),_,_) + |(_,Err(e),_) + |(_,_,Err(e))=>Err(e), + } +} +//LETS GOOOOOOOOOOOOOOOO +fn mega_quadruple_join(query_quad:(QueryHintResult,QueryHintResult,QueryHintResult,QueryHintResult))->QueryHintResult{ + match query_quad{ + //unambiguously locate file + (Ok(f),Err(QueryResolveError::NotFound),Err(QueryResolveError::NotFound),Err(QueryResolveError::NotFound)) + |(Err(QueryResolveError::NotFound),Ok(f),Err(QueryResolveError::NotFound),Err(QueryResolveError::NotFound)) + |(Err(QueryResolveError::NotFound),Err(QueryResolveError::NotFound),Ok(f),Err(QueryResolveError::NotFound)) + |(Err(QueryResolveError::NotFound),Err(QueryResolveError::NotFound),Err(QueryResolveError::NotFound),Ok(f))=>Ok(f), + //multiple files located + (Ok(_),Ok(_),Ok(_),Err(QueryResolveError::NotFound)) + |(Ok(_),Ok(_),Err(QueryResolveError::NotFound),Ok(_)) + |(Ok(_),Ok(_),Err(QueryResolveError::NotFound),Err(QueryResolveError::NotFound)) + |(Ok(_),Err(QueryResolveError::NotFound),Ok(_),Ok(_)) + |(Ok(_),Err(QueryResolveError::NotFound),Ok(_),Err(QueryResolveError::NotFound)) + |(Ok(_),Err(QueryResolveError::NotFound),Err(QueryResolveError::NotFound),Ok(_)) + |(Err(QueryResolveError::NotFound),Ok(_),Ok(_),Ok(_)) + |(Err(QueryResolveError::NotFound),Ok(_),Ok(_),Err(QueryResolveError::NotFound)) + |(Err(QueryResolveError::NotFound),Ok(_),Err(QueryResolveError::NotFound),Ok(_)) + |(Err(QueryResolveError::NotFound),Err(QueryResolveError::NotFound),Ok(_),Ok(_)) + |(Ok(_),Ok(_),Ok(_),Ok(_))=>Err(QueryResolveError::Ambiguous), + //no files located + (Err(QueryResolveError::NotFound),Err(QueryResolveError::NotFound),Err(QueryResolveError::NotFound),Err(QueryResolveError::NotFound))=>Err(QueryResolveError::NotFound), + //other error + (Err(e),_,_,_) + |(_,Err(e),_,_) + |(_,_,Err(e),_) + |(_,_,_,Err(e))=>Err(e), + } +} +impl Query for QueryTriple{ + async fn resolve(self)->QueryHintResult{ + let (module,server,client)=tokio::join!(self.module,self.server,self.client); + mega_triple_join(( + module.map_err(|e|QueryResolveError::JoinError(e))?.map(|file|FileHint{file,hint:ScriptHint::ModuleScript}), + server.map_err(|e|QueryResolveError::JoinError(e))?.map(|file|FileHint{file,hint:ScriptHint::Script}), + client.map_err(|e|QueryResolveError::JoinError(e))?.map(|file|FileHint{file,hint:ScriptHint::LocalScript}), + )) + } +} +struct QueryQuad{ + module_implicit:QueryHandle, + module_explicit:QueryHandle, + server:QueryHandle, + client:QueryHandle, +} +impl QueryQuad{ + fn rox_rojo(search_path:&std::path::PathBuf,search_name:&str)->Self{ + let fill=QueryTriple::rox_rojo(search_path,search_name,true); + Self{ + module_implicit:QuerySingle::rox(search_path,search_name).script,//Script.lua + module_explicit:fill.module,//Script.module.lua + server:fill.server, + client:fill.client, + } + } +} +impl Query for QueryQuad{ + async fn resolve(self)->QueryHintResult{ + let (module_implicit,module_explicit,server,client)=tokio::join!(self.module_implicit,self.module_explicit,self.server,self.client); + mega_quadruple_join(( + module_implicit.map_err(|e|QueryResolveError::JoinError(e))?.map(|file|FileHint{file,hint:ScriptHint::ModuleScript}), + module_explicit.map_err(|e|QueryResolveError::JoinError(e))?.map(|file|FileHint{file,hint:ScriptHint::ModuleScript}), + server.map_err(|e|QueryResolveError::JoinError(e))?.map(|file|FileHint{file,hint:ScriptHint::Script}), + client.map_err(|e|QueryResolveError::JoinError(e))?.map(|file|FileHint{file,hint:ScriptHint::LocalScript}), + )) + } +} + +struct ScriptWithOverrides{ + overrides:PropertiesOverride, + source:String, +} + +fn extract_script_overrides(mut source:String)->AResult{ + let mut overrides=PropertiesOverride::default(); + let mut count=0; + for line in source.lines(){ + //only string type properties are supported atm + if let Some(captures)=lazy_regex::regex!(r#"^\-\-\!\s*Properties\.([A-z]\w*)\s*\=\s*"(\w+)"$"#) + .captures(line){ + count+=line.len(); + match &captures[1]{ + "Name"=>overrides.name=Some(captures[2].to_owned()), + "ClassName"=>overrides.class=Some(captures[2].to_owned()), + other=>Err(anyhow::Error::msg(format!("Unimplemented property {other}")))?, + } + }else{ + break; + } + } + Ok(ScriptWithOverrides{overrides,source:source.split_off(count)}) +} + +async fn script_node(search_name:&str,mut file:FileWithName,hint:ScriptHint)->AResult{ + //read entire file + let mut buf=String::new(); + file.file.read_to_string(&mut buf).await?; + //regex script according to Properties lines at the top + let script_with_overrides=extract_script_overrides(buf)?; + //script + Ok(CompileNode{ + blacklist:Some(file.name), + name:script_with_overrides.overrides.name.unwrap_or_else(||search_name.to_owned()), + class:match (script_with_overrides.overrides.class.as_deref(),hint){ + (Some("ModuleScript"),_) + |(None,ScriptHint::ModuleScript)=>CompileClass::ModuleScript(script_with_overrides.source), + (Some("LocalScript"),_) + |(None,ScriptHint::Script)=>CompileClass::LocalScript(script_with_overrides.source), + (Some("Script"),_) + |(None,ScriptHint::LocalScript)=>CompileClass::Script(script_with_overrides.source), + other=>Err(anyhow::Error::msg(format!("Invalid hint or class {other:?}")))?, + }, + }) +} + +async fn model_node(search_name:&str,mut file:FileWithName)->AResult{ + //read entire file + let mut buf=Vec::new(); + file.file.read_to_end(&mut buf).await?; + //model + Ok(CompileNode{ + blacklist:Some(file.name), + name:search_name.to_owned(),//wrong but gets overwritten by internal model name + class:CompileClass::Model(buf), + }) +} + +async fn locate_override_file(entry:&tokio::fs::DirEntry,style:Option)->AResult{ + let contents_folder=entry.path(); + let file_name=entry.file_name(); + let search_name=file_name.to_str().unwrap(); + //scan inside the folder for an object to define the class of the folder + let script_query=async {match style{ + Some(DecompileStyle::Rox)=>QuerySingle::rox(&contents_folder,search_name).resolve().await, + Some(DecompileStyle::RoxRojo)=>QueryQuad::rox_rojo(&contents_folder,search_name).resolve().await, + Some(DecompileStyle::Rojo)=>QueryTriple::rojo(&contents_folder).resolve().await, + //try all three and complain if there is ambiguity + None=>mega_triple_join(tokio::join!( + QuerySingle::rox(&contents_folder,search_name).resolve(), + //true=search for module here to avoid ambiguity with QuerySingle::rox results + QueryTriple::rox_rojo(&contents_folder,search_name,true).resolve(), + QueryTriple::rojo(&contents_folder).resolve(), + )) + }}; + //model files are rox & rox-rojo only, so it's a lot less work... + let model_query=get_file_async(contents_folder.clone(),format!("{}.rbxmx",search_name)); + //model? script? both? + Ok(match tokio::join!(script_query,model_query){ + (Ok(FileHint{file,hint}),Err(QueryResolveError::NotFound))=>script_node(search_name,file,hint).await?, + (Err(QueryResolveError::NotFound),Ok(file))=>model_node(search_name,file).await?, + (Ok(_),Ok(_))=>Err(QueryResolveError::Ambiguous)?, + //neither + (Err(QueryResolveError::NotFound),Err(QueryResolveError::NotFound))=>CompileNode{ + name:search_name.to_owned(), + blacklist:None, + class:CompileClass::Folder, + }, + //other error + (Err(e),_) + |(_,Err(e))=>Err(e)? + }) +} + + +enum FileDiscernment{ + Model, + Script(ScriptHint), +} + +async fn discern_file(entry:&tokio::fs::DirEntry,style:Option)->AResult{ + let mut file_name=entry + .file_name() + .into_string() + .map_err(|e|anyhow::Error::msg(format!("insane file name {e:?}")))?; + //reject goobers + let is_goober=match style{ + Some(DecompileStyle::Rojo)=>true, + _=>false, + }; + let (ext_len,file_discernment)={ + if let Some(captures)=lazy_regex::regex!(r"^.*(.module.lua|.client.lua|.server.lua)$") + .captures(file_name.as_str()){ + let ext=&captures[1]; + (ext.len(),match ext{ + ".module.lua"=>{ + if is_goober{ + Err(anyhow::Error::msg(format!("File extension {ext} not supported in style {style:?}")))?; + } + FileDiscernment::Script(ScriptHint::ModuleScript) + }, + ".client.lua"=>FileDiscernment::Script(ScriptHint::LocalScript), + ".server.lua"=>FileDiscernment::Script(ScriptHint::Script), + _=>panic!("Regex failed"), + }) + }else if let Some(captures)=lazy_regex::regex!(r"^.*(.rbxmx|.lua)$") + .captures(file_name.as_str()){ + let ext=&captures[1]; + (ext.len(),match ext{ + ".rbxmx"=>{ + if is_goober{ + Err(anyhow::Error::msg(format!("File extension {ext} not supported in style {style:?}")))?; + } + FileDiscernment::Model + }, + ".lua"=>FileDiscernment::Script(ScriptHint::ModuleScript), + _=>panic!("Regex failed"), + }) + }else{ + return Err(anyhow::Error::msg("No file extension")); + } + }; + file_name.truncate(file_name.len()-ext_len); + let file=tokio::fs::File::open(entry.path()).await?; + Ok(match file_discernment{ + FileDiscernment::Model=>model_node(file_name.as_str(),FileWithName{file,name:file_name.clone()}).await?, + FileDiscernment::Script(hint)=>script_node(file_name.as_str(),FileWithName{file,name:file_name.clone()},hint).await?, + }) +} + +#[derive(Debug)] +enum ScriptHint{ + Script, + LocalScript, + ModuleScript, +} +struct FileHint{ + file:FileWithName, + hint:ScriptHint, +} + +enum PreparedData{ + Model(rbx_dom_weak::WeakDom), + Builder(rbx_dom_weak::InstanceBuilder), +} + +enum CompileClass{ + Folder, + Script(String), + LocalScript(String), + ModuleScript(String), + Model(Vec), +} + +struct CompileNode{ + name:String, + blacklist:Option, + class:CompileClass, +} + +enum CompileStackInstruction{ + TraverseReferent(rbx_dom_weak::types::Ref,Option), + PopFolder, +} + +struct CompileConfig{ + input_folder:std::path::PathBuf, + output_file:std::path::PathBuf, + template:Option, + style:Option, +} + +fn script_builder(class:&str,name:&str,source:String)->rbx_dom_weak::InstanceBuilder{ + let mut builder=rbx_dom_weak::InstanceBuilder::new(class); + builder.set_name(name); + builder.add_property("Source",rbx_dom_weak::types::Variant::String(source)); + builder +} + +enum TooComplicated{ + Stop, + Value(T), + Skip, +} + +async fn compile(config:CompileConfig)->AResult<()>{ + //basically decompile in reverse order + //load template dom + let input={ + let template_path=config.template.unwrap_or_else(||{ + let mut template_path=config.input_folder.clone(); + template_path.push("template.rbxlx"); + template_path + }); + //mr dom doesn't like tokio files + std::io::BufReader::new(std::fs::File::open(template_path)?) + }; + let mut dom=load_dom(input)?; + + //add in scripts and models + let mut folder=config.input_folder.clone(); + folder.push("src"); + let mut stack:Vec=dom.root().children().into_iter().map(|&referent|CompileStackInstruction::TraverseReferent(referent,None)).collect(); + while let Some(instruction)=stack.pop(){ + match instruction{ + CompileStackInstruction::TraverseReferent(item_ref,blacklist)=>{ + let sans={ + let item=dom.get_by_ref(item_ref).ok_or(anyhow::Error::msg("null child ref"))?; + sanitize(item.name.as_str()).to_string() + }; + folder.push(sans.as_str()); + stack.push(CompileStackInstruction::PopFolder); + //check if a folder exists with item.name + if let Ok(dir)=tokio::fs::read_dir(folder.as_path()).await{ + let mut exist_names:std::collections::HashSet={ + let item=dom.get_by_ref(item_ref).ok_or(anyhow::Error::msg("null child ref"))?; + //push existing dom children objects onto stack (unrelated to exist_names) + stack.extend(item.children().into_iter().map(|&referent|CompileStackInstruction::TraverseReferent(referent,None))); + //get names of existing objects + item.children().into_iter().map(|&child_ref|{ + let child=dom.get_by_ref(child_ref).ok_or(anyhow::Error::msg("null child ref"))?; + Ok::<_,anyhow::Error>(sanitize(child.name.as_str()).to_string()) + }).collect::>()? + }; + if let Some(dont)=blacklist{ + exist_names.insert(dont); + } + //generate children from folder contents UNLESS! item already has a child of the same name + + let style=config.style; + let exist_names=&exist_names; + futures::stream::unfold(dir,|mut dir1|async{ + //thread the needle! follow the path that dir takes! + let ret1={ + //capture a scoped mutable reference so we can forward dir to the next call even on an error + let dir2=&mut dir1; + (||async move{//error catcher so I can use ? + let ret2=if let Some(entry)=dir2.next_entry().await?{ + //cull early even if supporting things with identical names is possible + if exist_names.contains(entry.file_name().to_str().unwrap()){ + TooComplicated::Skip + }else{ + TooComplicated::Value(entry) + } + }else{ + TooComplicated::Stop + }; + Ok::<_,anyhow::Error>(ret2) + })().await + }; + match ret1{ + Ok(TooComplicated::Stop)=>None, + Ok(TooComplicated::Skip)=>Some((Ok(None),dir1)), + Ok(TooComplicated::Value(v))=>Some((Ok(Some(v)),dir1)), + Err(e)=>Some((Err(e),dir1)), + } + }) + + //gotta spawn off the worker threads (Model is slow) + .then(|bog|async{ + match bog{ + Ok(Some(entry))=>tokio::spawn(async move{ + let met=entry.metadata().await?; + //discern that bad boy + let compile_class=match met.is_dir(){ + true=>locate_override_file(&entry,style).await?, + false=>discern_file(&entry,style).await?, + }; + //prepare data structure + Ok(Some((compile_class.blacklist,match compile_class.class{ + CompileClass::Folder=>PreparedData::Builder(rbx_dom_weak::InstanceBuilder::new("Folder").with_name(compile_class.name.as_str())), + CompileClass::Script(source)=>PreparedData::Builder(script_builder("Script",compile_class.name.as_str(),source)), + CompileClass::LocalScript(source)=>PreparedData::Builder(script_builder("LocalScript",compile_class.name.as_str(),source)), + CompileClass::ModuleScript(source)=>PreparedData::Builder(script_builder("ModuleScript",compile_class.name.as_str(),source)), + CompileClass::Model(buf)=>PreparedData::Model(rbx_xml::from_reader_default(std::io::Cursor::new(buf))?), + }))) + }).await?, + Ok(None)=>Ok(None), + Err(e)=>Err(e), + } + }) + + //is this even what I want? + .map(|f|async{f}).buffer_unordered(32) + + //begin processing immediately + .fold((&mut stack,&mut dom),|(stack,dom),bog:Result<_,anyhow::Error>|async{ + //push child objects onto dom serially as they arrive + match bog{ + Ok(Some((blacklist,data)))=>{ + let referent=match data{ + PreparedData::Model(mut model_dom)=>{ + let referent=model_dom.root().children()[0]; + model_dom.transfer(referent,dom,item_ref); + referent + }, + PreparedData::Builder(script)=>dom.insert(item_ref,script), + }; + //new children need to be traversed + stack.push(CompileStackInstruction::TraverseReferent(referent,blacklist)); + }, + Ok(None)=>(), + Err(e)=>println!("error lole {e:?}"), + } + (stack,dom) + }).await; + } + }, + CompileStackInstruction::PopFolder=>assert!(folder.pop(),"pop folder bad"), + } + } + + let mut output_place=config.output_file.clone(); + if output_place.extension().is_none()&&tokio::fs::try_exists(output_place.as_path()).await?{ + output_place.push("place.rbxl"); + } + let output=std::io::BufWriter::new(std::fs::File::create(output_place)?); + //write inner objects + rbx_binary::to_writer(output,&dom,dom.root().children())?; Ok(()) }