compile feature

This commit is contained in:
Quaternions 2024-03-08 09:39:27 -08:00
parent 7ad7fa3d41
commit 0d1692f253

View File

@ -3,6 +3,7 @@ use clap::{Args,Parser,Subcommand};
use anyhow::Result as AResult;
use futures::StreamExt;
use rbx_dom_weak::types::Ref;
use tokio::io::AsyncReadExt;
type AssetID=u64;
type AssetIDFileMap=Vec<(AssetID,std::path::PathBuf)>;
@ -163,7 +164,12 @@ async fn main()->AResult<()>{
}).collect()
).await,
Commands::Upload=>upload_list(cookie.unwrap(),cli.group,vec![(cli.asset_id.unwrap(),cli.output.unwrap())]).await,
Commands::Compile=>compile(cli.input.unwrap(),cli.output.unwrap()),
Commands::Compile=>compile(CompileConfig{
input_folder:cli.input.unwrap(),
output_file:cli.output.unwrap(),
template:None,
style:None,
}).await,
Commands::Decompile=>decompile(DecompileConfig{
style:decompile_style.unwrap(),
input_file:cli.input.unwrap(),
@ -1080,6 +1086,521 @@ async fn download_and_decompile_history_into_git(config:DownloadAndDecompileHist
Ok(())
}
fn compile(_folder:std::path::PathBuf,_file:std::path::PathBuf)->AResult<()>{
//holy smokes what am I doing lmao
//This giant machine is supposed to search for files according to style rules
//e.g. ScriptName.server.lua or init.lua
//Obviously I got carried away
//I could use an enum!
//I could use a struct!
//I could use a trait!
//I could use an error!
//I could use a match!
//I could use a function!
//eventually:
#[derive(Debug)]
enum QueryResolveError{
NotFound,//0 results
Ambiguous,//>1 results
JoinError(tokio::task::JoinError),
IO(std::io::Error),
}
impl std::fmt::Display for QueryResolveError{
fn fmt(&self,f:&mut std::fmt::Formatter<'_>)->std::fmt::Result{
write!(f,"{self:?}")
}
}
impl std::error::Error for QueryResolveError{}
struct FileWithName{
file:tokio::fs::File,
name:String,
}
async fn get_file_async(mut path:std::path::PathBuf,file_name:impl AsRef<std::path::Path>)->Result<FileWithName,QueryResolveError>{
let name=file_name.as_ref().to_str().unwrap().to_owned();
path.push(file_name);
match tokio::fs::File::open(path).await{
Ok(file)=>Ok(FileWithName{file,name}),
Err(e)=>match e.kind(){
std::io::ErrorKind::NotFound=>Err(QueryResolveError::NotFound),
_=>Err(QueryResolveError::IO(e)),
},
}
}
type QueryHintResult=Result<FileHint,QueryResolveError>;
trait Query{
async fn resolve(self)->QueryHintResult;
}
type QueryHandle=tokio::task::JoinHandle<Result<FileWithName,QueryResolveError>>;
struct QuerySingle{
script:QueryHandle,
}
impl QuerySingle{
fn rox(search_path:&std::path::PathBuf,search_name:&str)->Self{
Self{
script:tokio::spawn(get_file_async(search_path.clone(),format!("{}.lua",search_name)))
}
}
}
impl Query for QuerySingle{
async fn resolve(self)->QueryHintResult{
match self.script.await{
Ok(Ok(file))=>Ok(FileHint{file,hint:ScriptHint::ModuleScript}),
Ok(Err(e))=>Err(e),
Err(e)=>Err(QueryResolveError::JoinError(e)),
}
}
}
struct QueryTriple{
module:QueryHandle,
server:QueryHandle,
client:QueryHandle,
}
impl QueryTriple{
fn rox_rojo(search_path:&std::path::PathBuf,search_name:&str,search_module:bool)->Self{
//this should be implemented as constructors of Triplet and Quadruplet to fully support Trey's suggestion
let module_name=if search_module{
format!("{}.module.lua",search_name)
}else{
format!("{}.lua",search_name)
};
Self{
module:tokio::spawn(get_file_async(search_path.clone(),module_name)),
server:tokio::spawn(get_file_async(search_path.clone(),format!("{}.server.lua",search_name))),
client:tokio::spawn(get_file_async(search_path.clone(),format!("{}.client.lua",search_name))),
}
}
fn rojo(search_path:&std::path::PathBuf)->Self{
QueryTriple::rox_rojo(search_path,"init",false)
}
}
//these functions can be achieved with macros, but I have not learned that yet
fn mega_triple_join(query_triplet:(QueryHintResult,QueryHintResult,QueryHintResult))->QueryHintResult{
match query_triplet{
//unambiguously locate file
(Ok(f),Err(QueryResolveError::NotFound),Err(QueryResolveError::NotFound))
|(Err(QueryResolveError::NotFound),Ok(f),Err(QueryResolveError::NotFound))
|(Err(QueryResolveError::NotFound),Err(QueryResolveError::NotFound),Ok(f))=>Ok(f),
//multiple files located
(Ok(_),Ok(_),Err(QueryResolveError::NotFound))
|(Ok(_),Err(QueryResolveError::NotFound),Ok(_))
|(Err(QueryResolveError::NotFound),Ok(_),Ok(_))
|(Ok(_),Ok(_),Ok(_))=>Err(QueryResolveError::Ambiguous),
//no files located
(Err(QueryResolveError::NotFound),Err(QueryResolveError::NotFound),Err(QueryResolveError::NotFound))=>Err(QueryResolveError::NotFound),
//other error
(Err(e),_,_)
|(_,Err(e),_)
|(_,_,Err(e))=>Err(e),
}
}
//LETS GOOOOOOOOOOOOOOOO
fn mega_quadruple_join(query_quad:(QueryHintResult,QueryHintResult,QueryHintResult,QueryHintResult))->QueryHintResult{
match query_quad{
//unambiguously locate file
(Ok(f),Err(QueryResolveError::NotFound),Err(QueryResolveError::NotFound),Err(QueryResolveError::NotFound))
|(Err(QueryResolveError::NotFound),Ok(f),Err(QueryResolveError::NotFound),Err(QueryResolveError::NotFound))
|(Err(QueryResolveError::NotFound),Err(QueryResolveError::NotFound),Ok(f),Err(QueryResolveError::NotFound))
|(Err(QueryResolveError::NotFound),Err(QueryResolveError::NotFound),Err(QueryResolveError::NotFound),Ok(f))=>Ok(f),
//multiple files located
(Ok(_),Ok(_),Ok(_),Err(QueryResolveError::NotFound))
|(Ok(_),Ok(_),Err(QueryResolveError::NotFound),Ok(_))
|(Ok(_),Ok(_),Err(QueryResolveError::NotFound),Err(QueryResolveError::NotFound))
|(Ok(_),Err(QueryResolveError::NotFound),Ok(_),Ok(_))
|(Ok(_),Err(QueryResolveError::NotFound),Ok(_),Err(QueryResolveError::NotFound))
|(Ok(_),Err(QueryResolveError::NotFound),Err(QueryResolveError::NotFound),Ok(_))
|(Err(QueryResolveError::NotFound),Ok(_),Ok(_),Ok(_))
|(Err(QueryResolveError::NotFound),Ok(_),Ok(_),Err(QueryResolveError::NotFound))
|(Err(QueryResolveError::NotFound),Ok(_),Err(QueryResolveError::NotFound),Ok(_))
|(Err(QueryResolveError::NotFound),Err(QueryResolveError::NotFound),Ok(_),Ok(_))
|(Ok(_),Ok(_),Ok(_),Ok(_))=>Err(QueryResolveError::Ambiguous),
//no files located
(Err(QueryResolveError::NotFound),Err(QueryResolveError::NotFound),Err(QueryResolveError::NotFound),Err(QueryResolveError::NotFound))=>Err(QueryResolveError::NotFound),
//other error
(Err(e),_,_,_)
|(_,Err(e),_,_)
|(_,_,Err(e),_)
|(_,_,_,Err(e))=>Err(e),
}
}
impl Query for QueryTriple{
async fn resolve(self)->QueryHintResult{
let (module,server,client)=tokio::join!(self.module,self.server,self.client);
mega_triple_join((
module.map_err(|e|QueryResolveError::JoinError(e))?.map(|file|FileHint{file,hint:ScriptHint::ModuleScript}),
server.map_err(|e|QueryResolveError::JoinError(e))?.map(|file|FileHint{file,hint:ScriptHint::Script}),
client.map_err(|e|QueryResolveError::JoinError(e))?.map(|file|FileHint{file,hint:ScriptHint::LocalScript}),
))
}
}
struct QueryQuad{
module_implicit:QueryHandle,
module_explicit:QueryHandle,
server:QueryHandle,
client:QueryHandle,
}
impl QueryQuad{
fn rox_rojo(search_path:&std::path::PathBuf,search_name:&str)->Self{
let fill=QueryTriple::rox_rojo(search_path,search_name,true);
Self{
module_implicit:QuerySingle::rox(search_path,search_name).script,//Script.lua
module_explicit:fill.module,//Script.module.lua
server:fill.server,
client:fill.client,
}
}
}
impl Query for QueryQuad{
async fn resolve(self)->QueryHintResult{
let (module_implicit,module_explicit,server,client)=tokio::join!(self.module_implicit,self.module_explicit,self.server,self.client);
mega_quadruple_join((
module_implicit.map_err(|e|QueryResolveError::JoinError(e))?.map(|file|FileHint{file,hint:ScriptHint::ModuleScript}),
module_explicit.map_err(|e|QueryResolveError::JoinError(e))?.map(|file|FileHint{file,hint:ScriptHint::ModuleScript}),
server.map_err(|e|QueryResolveError::JoinError(e))?.map(|file|FileHint{file,hint:ScriptHint::Script}),
client.map_err(|e|QueryResolveError::JoinError(e))?.map(|file|FileHint{file,hint:ScriptHint::LocalScript}),
))
}
}
struct ScriptWithOverrides{
overrides:PropertiesOverride,
source:String,
}
fn extract_script_overrides(mut source:String)->AResult<ScriptWithOverrides>{
let mut overrides=PropertiesOverride::default();
let mut count=0;
for line in source.lines(){
//only string type properties are supported atm
if let Some(captures)=lazy_regex::regex!(r#"^\-\-\!\s*Properties\.([A-z]\w*)\s*\=\s*"(\w+)"$"#)
.captures(line){
count+=line.len();
match &captures[1]{
"Name"=>overrides.name=Some(captures[2].to_owned()),
"ClassName"=>overrides.class=Some(captures[2].to_owned()),
other=>Err(anyhow::Error::msg(format!("Unimplemented property {other}")))?,
}
}else{
break;
}
}
Ok(ScriptWithOverrides{overrides,source:source.split_off(count)})
}
async fn script_node(search_name:&str,mut file:FileWithName,hint:ScriptHint)->AResult<CompileNode>{
//read entire file
let mut buf=String::new();
file.file.read_to_string(&mut buf).await?;
//regex script according to Properties lines at the top
let script_with_overrides=extract_script_overrides(buf)?;
//script
Ok(CompileNode{
blacklist:Some(file.name),
name:script_with_overrides.overrides.name.unwrap_or_else(||search_name.to_owned()),
class:match (script_with_overrides.overrides.class.as_deref(),hint){
(Some("ModuleScript"),_)
|(None,ScriptHint::ModuleScript)=>CompileClass::ModuleScript(script_with_overrides.source),
(Some("LocalScript"),_)
|(None,ScriptHint::Script)=>CompileClass::LocalScript(script_with_overrides.source),
(Some("Script"),_)
|(None,ScriptHint::LocalScript)=>CompileClass::Script(script_with_overrides.source),
other=>Err(anyhow::Error::msg(format!("Invalid hint or class {other:?}")))?,
},
})
}
async fn model_node(search_name:&str,mut file:FileWithName)->AResult<CompileNode>{
//read entire file
let mut buf=Vec::new();
file.file.read_to_end(&mut buf).await?;
//model
Ok(CompileNode{
blacklist:Some(file.name),
name:search_name.to_owned(),//wrong but gets overwritten by internal model name
class:CompileClass::Model(buf),
})
}
async fn locate_override_file(entry:&tokio::fs::DirEntry,style:Option<DecompileStyle>)->AResult<CompileNode>{
let contents_folder=entry.path();
let file_name=entry.file_name();
let search_name=file_name.to_str().unwrap();
//scan inside the folder for an object to define the class of the folder
let script_query=async {match style{
Some(DecompileStyle::Rox)=>QuerySingle::rox(&contents_folder,search_name).resolve().await,
Some(DecompileStyle::RoxRojo)=>QueryQuad::rox_rojo(&contents_folder,search_name).resolve().await,
Some(DecompileStyle::Rojo)=>QueryTriple::rojo(&contents_folder).resolve().await,
//try all three and complain if there is ambiguity
None=>mega_triple_join(tokio::join!(
QuerySingle::rox(&contents_folder,search_name).resolve(),
//true=search for module here to avoid ambiguity with QuerySingle::rox results
QueryTriple::rox_rojo(&contents_folder,search_name,true).resolve(),
QueryTriple::rojo(&contents_folder).resolve(),
))
}};
//model files are rox & rox-rojo only, so it's a lot less work...
let model_query=get_file_async(contents_folder.clone(),format!("{}.rbxmx",search_name));
//model? script? both?
Ok(match tokio::join!(script_query,model_query){
(Ok(FileHint{file,hint}),Err(QueryResolveError::NotFound))=>script_node(search_name,file,hint).await?,
(Err(QueryResolveError::NotFound),Ok(file))=>model_node(search_name,file).await?,
(Ok(_),Ok(_))=>Err(QueryResolveError::Ambiguous)?,
//neither
(Err(QueryResolveError::NotFound),Err(QueryResolveError::NotFound))=>CompileNode{
name:search_name.to_owned(),
blacklist:None,
class:CompileClass::Folder,
},
//other error
(Err(e),_)
|(_,Err(e))=>Err(e)?
})
}
enum FileDiscernment{
Model,
Script(ScriptHint),
}
async fn discern_file(entry:&tokio::fs::DirEntry,style:Option<DecompileStyle>)->AResult<CompileNode>{
let mut file_name=entry
.file_name()
.into_string()
.map_err(|e|anyhow::Error::msg(format!("insane file name {e:?}")))?;
//reject goobers
let is_goober=match style{
Some(DecompileStyle::Rojo)=>true,
_=>false,
};
let (ext_len,file_discernment)={
if let Some(captures)=lazy_regex::regex!(r"^.*(.module.lua|.client.lua|.server.lua)$")
.captures(file_name.as_str()){
let ext=&captures[1];
(ext.len(),match ext{
".module.lua"=>{
if is_goober{
Err(anyhow::Error::msg(format!("File extension {ext} not supported in style {style:?}")))?;
}
FileDiscernment::Script(ScriptHint::ModuleScript)
},
".client.lua"=>FileDiscernment::Script(ScriptHint::LocalScript),
".server.lua"=>FileDiscernment::Script(ScriptHint::Script),
_=>panic!("Regex failed"),
})
}else if let Some(captures)=lazy_regex::regex!(r"^.*(.rbxmx|.lua)$")
.captures(file_name.as_str()){
let ext=&captures[1];
(ext.len(),match ext{
".rbxmx"=>{
if is_goober{
Err(anyhow::Error::msg(format!("File extension {ext} not supported in style {style:?}")))?;
}
FileDiscernment::Model
},
".lua"=>FileDiscernment::Script(ScriptHint::ModuleScript),
_=>panic!("Regex failed"),
})
}else{
return Err(anyhow::Error::msg("No file extension"));
}
};
file_name.truncate(file_name.len()-ext_len);
let file=tokio::fs::File::open(entry.path()).await?;
Ok(match file_discernment{
FileDiscernment::Model=>model_node(file_name.as_str(),FileWithName{file,name:file_name.clone()}).await?,
FileDiscernment::Script(hint)=>script_node(file_name.as_str(),FileWithName{file,name:file_name.clone()},hint).await?,
})
}
#[derive(Debug)]
enum ScriptHint{
Script,
LocalScript,
ModuleScript,
}
struct FileHint{
file:FileWithName,
hint:ScriptHint,
}
enum PreparedData{
Model(rbx_dom_weak::WeakDom),
Builder(rbx_dom_weak::InstanceBuilder),
}
enum CompileClass{
Folder,
Script(String),
LocalScript(String),
ModuleScript(String),
Model(Vec<u8>),
}
struct CompileNode{
name:String,
blacklist:Option<String>,
class:CompileClass,
}
enum CompileStackInstruction{
TraverseReferent(rbx_dom_weak::types::Ref,Option<String>),
PopFolder,
}
struct CompileConfig{
input_folder:std::path::PathBuf,
output_file:std::path::PathBuf,
template:Option<std::path::PathBuf>,
style:Option<DecompileStyle>,
}
fn script_builder(class:&str,name:&str,source:String)->rbx_dom_weak::InstanceBuilder{
let mut builder=rbx_dom_weak::InstanceBuilder::new(class);
builder.set_name(name);
builder.add_property("Source",rbx_dom_weak::types::Variant::String(source));
builder
}
enum TooComplicated<T>{
Stop,
Value(T),
Skip,
}
async fn compile(config:CompileConfig)->AResult<()>{
//basically decompile in reverse order
//load template dom
let input={
let template_path=config.template.unwrap_or_else(||{
let mut template_path=config.input_folder.clone();
template_path.push("template.rbxlx");
template_path
});
//mr dom doesn't like tokio files
std::io::BufReader::new(std::fs::File::open(template_path)?)
};
let mut dom=load_dom(input)?;
//add in scripts and models
let mut folder=config.input_folder.clone();
folder.push("src");
let mut stack:Vec<CompileStackInstruction>=dom.root().children().into_iter().map(|&referent|CompileStackInstruction::TraverseReferent(referent,None)).collect();
while let Some(instruction)=stack.pop(){
match instruction{
CompileStackInstruction::TraverseReferent(item_ref,blacklist)=>{
let sans={
let item=dom.get_by_ref(item_ref).ok_or(anyhow::Error::msg("null child ref"))?;
sanitize(item.name.as_str()).to_string()
};
folder.push(sans.as_str());
stack.push(CompileStackInstruction::PopFolder);
//check if a folder exists with item.name
if let Ok(dir)=tokio::fs::read_dir(folder.as_path()).await{
let mut exist_names:std::collections::HashSet<String>={
let item=dom.get_by_ref(item_ref).ok_or(anyhow::Error::msg("null child ref"))?;
//push existing dom children objects onto stack (unrelated to exist_names)
stack.extend(item.children().into_iter().map(|&referent|CompileStackInstruction::TraverseReferent(referent,None)));
//get names of existing objects
item.children().into_iter().map(|&child_ref|{
let child=dom.get_by_ref(child_ref).ok_or(anyhow::Error::msg("null child ref"))?;
Ok::<_,anyhow::Error>(sanitize(child.name.as_str()).to_string())
}).collect::<AResult<_>>()?
};
if let Some(dont)=blacklist{
exist_names.insert(dont);
}
//generate children from folder contents UNLESS! item already has a child of the same name
let style=config.style;
let exist_names=&exist_names;
futures::stream::unfold(dir,|mut dir1|async{
//thread the needle! follow the path that dir takes!
let ret1={
//capture a scoped mutable reference so we can forward dir to the next call even on an error
let dir2=&mut dir1;
(||async move{//error catcher so I can use ?
let ret2=if let Some(entry)=dir2.next_entry().await?{
//cull early even if supporting things with identical names is possible
if exist_names.contains(entry.file_name().to_str().unwrap()){
TooComplicated::Skip
}else{
TooComplicated::Value(entry)
}
}else{
TooComplicated::Stop
};
Ok::<_,anyhow::Error>(ret2)
})().await
};
match ret1{
Ok(TooComplicated::Stop)=>None,
Ok(TooComplicated::Skip)=>Some((Ok(None),dir1)),
Ok(TooComplicated::Value(v))=>Some((Ok(Some(v)),dir1)),
Err(e)=>Some((Err(e),dir1)),
}
})
//gotta spawn off the worker threads (Model is slow)
.then(|bog|async{
match bog{
Ok(Some(entry))=>tokio::spawn(async move{
let met=entry.metadata().await?;
//discern that bad boy
let compile_class=match met.is_dir(){
true=>locate_override_file(&entry,style).await?,
false=>discern_file(&entry,style).await?,
};
//prepare data structure
Ok(Some((compile_class.blacklist,match compile_class.class{
CompileClass::Folder=>PreparedData::Builder(rbx_dom_weak::InstanceBuilder::new("Folder").with_name(compile_class.name.as_str())),
CompileClass::Script(source)=>PreparedData::Builder(script_builder("Script",compile_class.name.as_str(),source)),
CompileClass::LocalScript(source)=>PreparedData::Builder(script_builder("LocalScript",compile_class.name.as_str(),source)),
CompileClass::ModuleScript(source)=>PreparedData::Builder(script_builder("ModuleScript",compile_class.name.as_str(),source)),
CompileClass::Model(buf)=>PreparedData::Model(rbx_xml::from_reader_default(std::io::Cursor::new(buf))?),
})))
}).await?,
Ok(None)=>Ok(None),
Err(e)=>Err(e),
}
})
//is this even what I want?
.map(|f|async{f}).buffer_unordered(32)
//begin processing immediately
.fold((&mut stack,&mut dom),|(stack,dom),bog:Result<_,anyhow::Error>|async{
//push child objects onto dom serially as they arrive
match bog{
Ok(Some((blacklist,data)))=>{
let referent=match data{
PreparedData::Model(mut model_dom)=>{
let referent=model_dom.root().children()[0];
model_dom.transfer(referent,dom,item_ref);
referent
},
PreparedData::Builder(script)=>dom.insert(item_ref,script),
};
//new children need to be traversed
stack.push(CompileStackInstruction::TraverseReferent(referent,blacklist));
},
Ok(None)=>(),
Err(e)=>println!("error lole {e:?}"),
}
(stack,dom)
}).await;
}
},
CompileStackInstruction::PopFolder=>assert!(folder.pop(),"pop folder bad"),
}
}
let mut output_place=config.output_file.clone();
if output_place.extension().is_none()&&tokio::fs::try_exists(output_place.as_path()).await?{
output_place.push("place.rbxl");
}
let output=std::io::BufWriter::new(std::fs::File::create(output_place)?);
//write inner objects
rbx_binary::to_writer(output,&dom,dom.root().children())?;
Ok(())
}