StrafesNET/asset-tool
3
Fork 1
Code Issues Pull Requests Packages Projects Releases 4 Wiki Activity
d9531f1d4e
asset-tool/src/main.rs
Quaternions d9531f1d4e custom file stem
2024-01-24 16:25:00 -08:00

1532 lines
48 KiB
Rust
Raw Blame History

use std::io::Read;
use clap::{Args,Parser,Subcommand};
use anyhow::Result as AResult;
use futures::StreamExt;
use rbx_dom_weak::types::Ref;
use tokio::io::AsyncReadExt;
type AssetID=u64;
type AssetIDFileMap=Vec<(AssetID,std::path::PathBuf)>;
const CONCURRENT_DECODE:usize=8;
const CONCURRENT_REQUESTS:usize=32;
#[derive(Parser)]
#[command(author,version,about,long_about=None)]
#[command(propagate_version = true)]
struct Cli{
//asset options
#[arg(short,long)]
group:Option<u64>,
#[arg(long)]
asset_id:Option<AssetID>,
//idk how to do this better
#[arg(long)]
cookie_literal:Option<String>,
#[arg(long)]
cookie_env:Option<String>,
#[arg(long)]
cookie_file:Option<std::path::PathBuf>,
//TODO: read the versions.json file instead of doing this
//TODO: write file dates instead of versions.json
#[arg(long)]
start_version:Option<u64>,
#[arg(long)]
end_version:Option<u64>,
#[arg(long)]
r#continue:bool,
//decompile options
#[arg(long)]
no_models:Option<bool>,
#[arg(long)]
no_scripts:Option<bool>,
#[arg(long)]
no_template:Option<bool>,
#[arg(long)]
style:Option<String>,
//git options
#[arg(long)]
git_committer_name:Option<String>,
#[arg(long)]
git_committer_email:Option<String>,
#[arg(short,long)]
input:Option<std::path::PathBuf>,
#[arg(short,long)]
output:Option<std::path::PathBuf>,
#[command(subcommand)]
command:Commands,
}
#[derive(Subcommand)]
enum Commands{
DownloadHistory,
Download(AssetIDList),
Upload,
Compile,
Decompile,
DecompileHistoryIntoGit,
DownloadAndDecompileHistoryIntoGit,
}
#[derive(Clone,Copy)]
enum DecompileStyle{
Rox,
Rojo,
RoxRojo,
}
#[derive(Args)]
struct AssetIDList{
asset_ids:Vec<AssetID>
}
#[derive(Args)]
struct PathBufList{
paths:Vec<std::path::PathBuf>
}
#[derive(serde::Deserialize)]
#[allow(nonstandard_style,dead_code)]
struct VersionPage{
previousPageCursor:Option<String>,
nextPageCursor:Option<String>,
data:Vec<AssetVersion>,
}
#[derive(serde::Deserialize,serde::Serialize)]
#[allow(nonstandard_style,dead_code)]
struct AssetVersion{
Id:u64,
assetId:AssetID,
assetVersionNumber:u64,
creatorType:String,
creatorTargetId:u64,
creatingUniverseId:Option<u64>,
created:chrono::DateTime<chrono::Utc>,
isPublished:bool,
}
#[tokio::main]
async fn main()->AResult<()>{
let cli=Cli::parse();
let cookie_enum={
match (cli.cookie_literal,cli.cookie_env,cli.cookie_file){
(Some(literal),None,None)=>Some(Cookie::Literal(literal)),
(None,Some(env_var),None)=>Some(Cookie::Environment(env_var)),
(None,None,Some(path))=>Some(Cookie::File(path)),
(None,None,None)=>None,
_=>Err(anyhow::Error::msg("Cookie was specified multiple times."))?,
}
};
let cookie=match cookie_enum{
Some(c)=>Some(format!(".ROBLOSECURITY={}",match c{
Cookie::Literal(s)=>s,
Cookie::Environment(var)=>std::env::var(var)?,
Cookie::File(path)=>tokio::fs::read_to_string(path).await?,
})),
None=>None,
};
let decompile_style=match cli.style.as_deref(){
Some("rox")
|Some("Rox")=>Some(DecompileStyle::Rox),
Some("rojo")
|Some("Rojo")=>Some(DecompileStyle::Rojo),
Some("rox-rojo")
|Some("rojo-rox")
|Some("roxrojo")
|Some("rojorox")
|Some("RoxRojo")
|Some("RojoRox")=>Some(DecompileStyle::RoxRojo),
None=>None,
_=>Err(anyhow::Error::msg("Invalid style"))?,
};
match cli.command{
Commands::DownloadHistory=>download_history(DownloadHistoryConfig{
continue_from_versions:cli.r#continue,
end_version:cli.end_version,
start_version:cli.start_version.unwrap_or(0),
output_folder:cli.output.unwrap(),
cookie:cookie.unwrap(),
asset_id:cli.asset_id.unwrap(),
}).await,
Commands::Download(asset_id_list)=>download_list(
cookie.unwrap(),
asset_id_list.asset_ids.into_iter().map(|asset_id|{
let mut path=cli.output.clone().unwrap();
path.push(asset_id.to_string());
(asset_id,path)
}).collect()
).await,
Commands::Upload=>upload_list(cookie.unwrap(),cli.group,vec![(cli.asset_id.unwrap(),cli.output.unwrap())]).await,
Commands::Compile=>compile(CompileConfig{
input_folder:cli.input.unwrap(),
output_file:cli.output.unwrap(),
template:None,
style:None,
}).await,
Commands::Decompile=>decompile(DecompileConfig{
style:decompile_style.unwrap(),
input_file:cli.input.unwrap(),
output_folder:cli.output.unwrap(),
write_template:!cli.no_template.unwrap_or(false),
write_models:!cli.no_models.unwrap_or(false),
write_scripts:!cli.no_scripts.unwrap_or(false),
}).await,
Commands::DecompileHistoryIntoGit=>decompile_history_into_git(DecompileHistoryConfig{
git_committer_name:cli.git_committer_name.unwrap(),
git_committer_email:cli.git_committer_email.unwrap(),
input_folder:cli.input.unwrap(),
output_folder:cli.output.unwrap(),
style:decompile_style.unwrap(),
write_template:!cli.no_template.unwrap_or(false),
write_models:!cli.no_models.unwrap_or(false),
write_scripts:!cli.no_scripts.unwrap_or(false),
}).await,
Commands::DownloadAndDecompileHistoryIntoGit=>download_and_decompile_history_into_git(DownloadAndDecompileHistoryConfig{
git_committer_name:cli.git_committer_name.unwrap(),
git_committer_email:cli.git_committer_email.unwrap(),
cookie:cookie.unwrap(),
asset_id:cli.asset_id.unwrap(),
output_folder:cli.output.unwrap(),
style:decompile_style.unwrap(),
write_template:!cli.no_template.unwrap_or(false),
write_models:!cli.no_models.unwrap_or(false),
write_scripts:!cli.no_scripts.unwrap_or(false),
}).await,
}
}
enum Cookie{
Literal(String),
Environment(String),
File(std::path::PathBuf),
}
enum ReaderType<R:Read>{
GZip(flate2::read::GzDecoder<std::io::BufReader<R>>),
Raw(std::io::BufReader<R>),
}
fn maybe_gzip_decode<R:Read>(input:R)->AResult<ReaderType<R>>{
let mut buf=std::io::BufReader::new(input);
let peek=std::io::BufRead::fill_buf(&mut buf)?;
match &peek[0..2]{
b"\x1f\x8b"=>Ok(ReaderType::GZip(flate2::read::GzDecoder::new(buf))),
_=>Ok(ReaderType::Raw(buf)),
}
}
async fn upload_list(cookie:String,group:Option<u64>,asset_id_file_map:AssetIDFileMap)->AResult<()>{
let client=reqwest::Client::new();
futures::stream::iter(asset_id_file_map.into_iter()
.map(|(asset_id,file)|{
let client=&client;
let cookie=cookie.as_str();
let group=&group;
async move{
let mut url=reqwest::Url::parse("https://data.roblox.com/Data/Upload.ashx?json=1&type=Model&genreTypeId=1")?;
//url borrow scope
{
let mut query=url.query_pairs_mut();//borrow here
query.append_pair("assetid",asset_id.to_string().as_str());
match group{
Some(group_id)=>{query.append_pair("groupId",group_id.to_string().as_str());},
None=>(),
}
}
let body=tokio::fs::read_to_string(file).await?;
let mut resp=client.post(url.clone())
.header("Cookie",cookie)
.body(body.clone())
.send().await?;
//This is called a CSRF challenge apparently
if resp.status()==reqwest::StatusCode::FORBIDDEN{
if let Some(csrf_token)=resp.headers().get("X-CSRF-Token"){
resp=client.post(url)
.header("X-CSRF-Token",csrf_token)
.header("Cookie",cookie)
.body(body)
.send().await?;
}else{
Err(anyhow::Error::msg("Roblox returned 403 with no CSRF"))?;
}
}
Ok((asset_id,resp.bytes().await?))
}
}))
.buffer_unordered(CONCURRENT_REQUESTS)
.for_each(|b:AResult<_>|async{
match b{
Ok((asset_id,body))=>{
println!("asset_id={} response.body={:?}",asset_id,body);
},
Err(e)=>eprintln!("ul error: {}",e),
}
}).await;
Ok(())
}
fn read_readable(mut readable:impl Read)->AResult<Vec<u8>>{
let mut contents=Vec::new();
readable.read_to_end(&mut contents)?;
Ok(contents)
}
async fn download_list(cookie:String,asset_id_file_map:AssetIDFileMap)->AResult<()>{
let client=reqwest::Client::new();
futures::stream::iter(asset_id_file_map.into_iter()
.map(|(asset_id,file)|{
let client=&client;
let cookie=cookie.as_str();
async move{
let resp=client.get(format!("https://assetdelivery.roblox.com/v1/asset/?ID={}",asset_id))
.header("Cookie",cookie)
.send().await?;
Ok((file,resp.bytes().await?))
}
}))
.buffer_unordered(CONCURRENT_REQUESTS)
.for_each(|b:AResult<_>|async{
match b{
Ok((dest,body))=>{
let contents=match maybe_gzip_decode(&mut std::io::Cursor::new(body)){
Ok(ReaderType::GZip(readable))=>read_readable(readable),
Ok(ReaderType::Raw(readable))=>read_readable(readable),
Err(e)=>Err(e),
};
match contents{
Ok(data)=>match tokio::fs::write(dest,data).await{
Err(e)=>eprintln!("fs error: {}",e),
_=>(),
},
Err(e)=>eprintln!("gzip error: {}",e),
};
},
Err(e)=>eprintln!("dl error: {}",e),
}
}).await;
Ok(())
}
async fn download_page(client:&reqwest::Client,cookie:&str,asset_id:AssetID,cursor:Option<String>)->AResult<VersionPage>{
let mut url=reqwest::Url::parse(format!("https://develop.roblox.com/v1/assets/{}/saved-versions",asset_id).as_str())?;
//url borrow scope
{
let mut query=url.query_pairs_mut();//borrow here
//query.append_pair("sortOrder","Asc");
//query.append_pair("limit","100");
//query.append_pair("count","100");
match cursor.as_deref(){
Some(next_page)=>{query.append_pair("cursor",next_page);}
None=>(),
}
}
println!("page url={}",url);
let resp=client.get(url)
.header("Cookie",cookie)
.send().await?;
Ok(resp.json::<VersionPage>().await?)
}
async fn get_version_history(client:&reqwest::Client,cookie:&str,asset_id:AssetID)->AResult<Vec<AssetVersion>>{
let mut cursor:Option<String>=None;
let mut asset_list=Vec::new();
loop{
let mut page=download_page(client,cookie,asset_id,cursor).await?;
asset_list.append(&mut page.data);
if page.nextPageCursor.is_none(){
break;
}
cursor=page.nextPageCursor;
}
asset_list.sort_by(|a,b|a.assetVersionNumber.cmp(&b.assetVersionNumber));
Ok(asset_list)
}
async fn download_asset_version(client:&reqwest::Client,cookie:&str,asset_id_str:&str,asset_version_str:&str)->AResult<reqwest::Response>{
let mut url=reqwest::Url::parse("https://assetdelivery.roblox.com/v1/asset/")?;
//url borrow scope
{
let mut query=url.query_pairs_mut();//borrow here
query.append_pair("ID",asset_id_str);
query.append_pair("version",asset_version_str);
}
println!("download url={}",url);
for i in 0..8{
let resp=client.get(url.clone())
.header("Cookie",cookie)
.send().await?;
if !resp.status().is_success(){
println!("request {} failed",i);
continue;
}
return Ok(resp);
}
Err(anyhow::Error::msg("all requests failed"))
}
struct DownloadHistoryConfig{
continue_from_versions:bool,
end_version:Option<u64>,
start_version:u64,
output_folder:std::path::PathBuf,
cookie:String,
asset_id:AssetID,
}
async fn download_history(mut config:DownloadHistoryConfig)->AResult<()>{
let mut asset_list_contents=std::collections::HashSet::new();
let mut asset_list:Vec<AssetVersion>=Vec::new();
if config.end_version.is_none()&&config.continue_from_versions{
//load prexisting versions list
let mut versions_path=config.output_folder.clone();
versions_path.push("versions.json");
match std::fs::File::open(versions_path){
Ok(versions_file)=>asset_list.append(&mut serde_json::from_reader(versions_file)?),
Err(e)=>match e.kind(){
std::io::ErrorKind::NotFound=>Err(anyhow::Error::msg("Cannot continue from versions.json - file does not exist"))?,
_=>Err(e)?,
}
}
//write down which versions are contained
for asset_version in &asset_list{
asset_list_contents.insert(asset_version.assetVersionNumber);
}
//find the highest number
match asset_list.iter().map(|asset_version|asset_version.assetVersionNumber).max(){
Some(max)=>{
//count down contiguously until a number is missing
for i in (1..=max).rev(){
if !asset_list_contents.contains(&i){
//that is end_version
config.end_version=Some(i);
break;
}
}
//if all versions are contained, set start_version to the max + 1
if config.end_version.is_none(){
config.start_version=max+1;
}
},
None=>Err(anyhow::Error::msg("Cannot continue from versions.json - there are no previous versions"))?,
}
}
let client=reqwest::Client::new();
let asset_id_string=config.asset_id.to_string();
//limit concurrent downloads
let mut join_set=tokio::task::JoinSet::new();
//poll paged list of all asset versions
let mut cursor:Option<String>=None;
loop{
let mut page=download_page(&client,config.cookie.as_str(),config.asset_id,cursor).await?;
let client=&client;
let cookie=config.cookie.clone();
let asset_id_str=asset_id_string.clone();
let output_folder=config.output_folder.clone();
let data=&page.data;
let asset_list_contents=&asset_list_contents;
let join_set=&mut join_set;
let error_catcher=||async move{
let mut cancel_paging=false;
for asset_version in data{
let version_number=asset_version.assetVersionNumber;
//skip assets beyond specified end_version
if config.end_version.is_some_and(|v|v<version_number){
continue;
}
//skip assets lower than start_version and cancel paging asset versions
if version_number<config.start_version{
cancel_paging=true;
continue;//don't trust roblox returned order
}
//skip previously downloaded assets
if asset_list_contents.contains(&version_number){
continue;
}
while CONCURRENT_REQUESTS<=join_set.len(){
join_set.join_next().await.unwrap()??;
}
let client=client.clone();
let cookie=cookie.clone();
let asset_id_str=asset_id_str.clone();
let mut path=output_folder.clone();
path.push(format!("{}_v{}.rbxl",config.asset_id,version_number));
join_set.spawn(async move{
let resp=download_asset_version(&client,cookie.as_str(),asset_id_str.as_str(),version_number.to_string().as_str()).await?;
let contents=match maybe_gzip_decode(std::io::Cursor::new(resp.bytes().await?))?{
ReaderType::GZip(readable)=>read_readable(readable)?,
ReaderType::Raw(readable)=>read_readable(readable)?,
};
tokio::fs::write(path,contents).await?;
Ok::<_,anyhow::Error>(())
});
}
Ok::<_,anyhow::Error>(cancel_paging)
};
let cancel_paging=match error_catcher().await{
Ok(cancel)=>cancel,
Err(e)=>{
println!("download error: {}",e);
//cancel download and write versions
true
},
};
if page.nextPageCursor.is_none()||cancel_paging{
for asset_version in page.data.into_iter(){
if !(asset_list_contents.contains(&asset_version.assetVersionNumber)
||config.end_version.is_some_and(|v|v<asset_version.assetVersionNumber)
||asset_version.assetVersionNumber<config.start_version){
asset_list.push(asset_version);
}
}
break;
}else{
asset_list.append(&mut page.data);
}
cursor=page.nextPageCursor;
}
asset_list.sort_by(|a,b|a.assetVersionNumber.cmp(&b.assetVersionNumber));
let mut path=config.output_folder.clone();
path.set_file_name("versions.json");
tokio::fs::write(path,serde_json::to_string(&asset_list)?).await?;
while let Some(result)=join_set.join_next().await{
result??;
}
Ok(())
}
fn load_dom<R:Read>(input:R)->AResult<rbx_dom_weak::WeakDom>{
let mut buf=std::io::BufReader::new(input);
let peek=std::io::BufRead::fill_buf(&mut buf)?;
match &peek[0..4]{
b"<rob"=>{
match &peek[4..8]{
b"lox!"=>return rbx_binary::from_reader(buf).map_err(anyhow::Error::msg),
b"lox "=>return rbx_xml::from_reader_default(buf).map_err(anyhow::Error::msg),
other=>Err(anyhow::Error::msg(format!("Unknown Roblox file type {:?}",other))),
}
},
_=>Err(anyhow::Error::msg("unsupported file type")),
}
}
#[derive(PartialEq)]
enum Class{
Folder,
ModuleScript,
LocalScript,
Script,
Model,
}
struct TreeNode{
name:String,
referent:Ref,
parent:Ref,
class:Class,
children:Vec<Ref>,
}
impl TreeNode{
fn new(name:String,referent:Ref,parent:Ref,class:Class)->Self{
Self{
name,
referent,
parent,
class,
children:Vec::new(),
}
}
}
enum TrimStackInstruction{
Referent(Ref),
IncrementScript,
DecrementScript,
}
enum WriteStackInstruction<'a>{
Node(&'a TreeNode,u32),//(Node,NameTally)
PushFolder(String),
PopFolder,
Destroy(Ref),
}
#[derive(Default)]
struct PropertiesOverride{
name:Option<String>,
class:Option<String>,
}
impl PropertiesOverride{
fn is_some(&self)->bool{
self.name.is_some()
||self.class.is_some()
}
}
impl std::fmt::Display for PropertiesOverride{
fn fmt(&self,f:&mut std::fmt::Formatter<'_>)->std::fmt::Result{
if let Some(name)=self.name.as_deref(){
writeln!(f,"--!Properties.Name = \"{}\"",name)?;
}
if let Some(class)=self.class.as_deref(){
writeln!(f,"--!Properties.ClassName = \"{}\"",class)?;
}
Ok(())
}
}
fn sanitize<'a>(s:&'a str)->std::borrow::Cow<'a,str>{
lazy_regex::regex!(r"[^A-z0-9.-]").replace_all(s,"_")
}
fn write_item(dom:&rbx_dom_weak::WeakDom,mut file:std::path::PathBuf,node:&TreeNode,node_name_override:String,mut properties:PropertiesOverride,style:DecompileStyle,write_models:bool,write_scripts:bool)->AResult<()>{
file.push(sanitize(node_name_override.as_str()).as_ref());
match node.class{
Class::Folder=>(),
Class::ModuleScript|Class::LocalScript|Class::Script=>{
if !write_scripts{
return Ok(())
}
//set extension
match style{
DecompileStyle::Rox=>assert!(file.set_extension("lua"),"could not set extension"),
DecompileStyle::RoxRojo|DecompileStyle::Rojo=>{
match properties.class.as_deref(){
Some("LocalScript")=>{
file.set_extension("client.lua");
properties.class=None;
},
Some("Script")=>{
file.set_extension("server.lua");
properties.class=None;
},
// Some("ModuleScript")=>{
// file.set_extension("module");
// properties.class=None;
// },
None=>assert!(file.set_extension("lua"),"could not set extension"),
Some(other)=>Err(anyhow::Error::msg(format!("Attempt to write a {} as a script",other)))?,
}
}
}
if let Some(item)=dom.get_by_ref(node.referent){
//TODO: delete disabled scripts
if let Some(rbx_dom_weak::types::Variant::String(source))=item.properties.get("Source"){
if properties.is_some(){
//rox style
let source=properties.to_string()+source.as_str();
std::fs::write(file,source)?;
}else{
std::fs::write(file,source)?;
}
}
}
},
Class::Model=>{
if !write_models{
return Ok(())
}
assert!(file.set_extension("rbxmx"));
let output=std::io::BufWriter::new(std::fs::File::create(file)?);
rbx_xml::to_writer_default(output,dom,&[node.referent])?;
},
}
Ok(())
}
struct DecompiledContext{
dom:rbx_dom_weak::WeakDom,
tree_refs:std::collections::HashMap<rbx_dom_weak::types::Ref,TreeNode>,
}
fn generate_decompiled_context<R:Read>(input:R)->AResult<DecompiledContext>{
let dom=load_dom(input)?;
let mut tree_refs=std::collections::HashMap::new();
tree_refs.insert(dom.root_ref(),TreeNode::new(
"src".to_owned(),
dom.root_ref(),
Ref::none(),
Class::Folder
));
//run rules
let mut stack=vec![dom.root()];
while let Some(item)=stack.pop(){
let class=match item.class.as_str(){
"ModuleScript"=>Class::ModuleScript,
"LocalScript"=>Class::LocalScript,
"Script"=>Class::Script,
"Model"=>Class::Model,
_=>Class::Folder,
};
let skip=match class{
Class::Model=>true,
_=>false,
};
if let Some(parent_node)=tree_refs.get_mut(&item.parent()){
let referent=item.referent();
let node=TreeNode::new(item.name.clone(),referent,parent_node.referent,class);
parent_node.children.push(referent);
tree_refs.insert(referent,node);
}
//look no further, turn this node and all its children into a model
if skip{
continue;
}
for &referent in item.children(){
if let Some(c)=dom.get_by_ref(referent){
stack.push(c);
}
}
}
//trim empty folders
let mut script_count=0;
let mut stack:Vec<TrimStackInstruction>=tree_refs.get(&dom.root_ref()).unwrap().children
.iter().map(|&c|TrimStackInstruction::Referent(c)).collect();
while let Some(instruction)=stack.pop(){
match instruction{
TrimStackInstruction::IncrementScript=>script_count+=1,
TrimStackInstruction::DecrementScript=>script_count-=1,
TrimStackInstruction::Referent(referent)=>{
let mut delete=None;
if let Some(node)=tree_refs.get_mut(&referent){
if node.class==Class::Folder&&script_count!=0{
node.class=Class::Model
}
if node.class==Class::Folder&&node.children.len()==0{
delete=Some(node.parent);
}else{
//how the hell do I do this better without recursion
let is_script=match node.class{
Class::ModuleScript|Class::LocalScript|Class::Script=>true,
_=>false,
};
//stack is popped from back
if is_script{
stack.push(TrimStackInstruction::DecrementScript);
}
for &child_referent in &node.children{
stack.push(TrimStackInstruction::Referent(child_referent));
}
if is_script{
stack.push(TrimStackInstruction::IncrementScript);
}
}
}
//trim referent
if let Some(parent_ref)=delete{
let parent_node=tree_refs.get_mut(&parent_ref)
.expect("parent_ref does not exist in tree_refs");
parent_node.children.remove(
parent_node.children.iter()
.position(|&r|r==referent)
.expect("parent.children does not contain referent")
);
tree_refs.remove(&referent);
}
},
}
}
Ok(DecompiledContext{
dom,
tree_refs,
})
}
struct WriteConfig{
style:DecompileStyle,
output_folder:std::path::PathBuf,
write_template:bool,
write_models:bool,
write_scripts:bool,
}
async fn write_files(config:WriteConfig,mut context:DecompiledContext)->AResult<()>{
let mut write_queue=Vec::new();
let mut destroy_queue=Vec::new();
let mut name_tally=std::collections::HashMap::<String,u32>::new();
let mut folder=config.output_folder.clone();
let mut stack=vec![WriteStackInstruction::Node(context.tree_refs.get(&context.dom.root_ref()).unwrap(),0)];
while let Some(instruction)=stack.pop(){
match instruction{
WriteStackInstruction::PushFolder(component)=>folder.push(component),
WriteStackInstruction::PopFolder=>assert!(folder.pop(),"weirdness"),
WriteStackInstruction::Destroy(referent)=>destroy_queue.push(referent),
WriteStackInstruction::Node(node,name_count)=>{
//track properties that must be overriden to compile folder structure back into a place file
let mut properties=PropertiesOverride::default();
let has_children=node.children.len()!=0;
match node.class{
Class::Folder=>(),
Class::ModuleScript=>(),//.lua files are ModuleScript by default
Class::LocalScript=>properties.class=Some("LocalScript".to_owned()),
Class::Script=>properties.class=Some("Script".to_owned()),
Class::Model=>(),
}
let name_override=if 0<name_count{
properties.name=Some(node.name.clone());
format!("{}_{}",node.name,name_count)
}else{
node.name.clone()
};
if has_children{
//push temp subfolder
let mut subfolder=folder.clone();
subfolder.push(sanitize(name_override.as_str()).as_ref());
//make folder
tokio::fs::create_dir(subfolder.clone()).await?;
let name_final=match config.style{
DecompileStyle::Rox
|DecompileStyle::RoxRojo=>name_override.clone(),
DecompileStyle::Rojo=>"init".to_owned(),
};
//write item in subfolder
write_queue.push((subfolder,node,name_final,properties,config.style));
}else{
//write item
write_queue.push((folder.clone(),node,name_override.clone(),properties,config.style));
}
//queue item to be deleted from dom after child objects are handled (stack is popped from the back)
match node.class{
Class::Folder=>(),
_=>stack.push(WriteStackInstruction::Destroy(node.referent)),
}
if has_children{
stack.push(WriteStackInstruction::PopFolder);
name_tally.clear();
for referent in &node.children{
if let Some(c)=context.tree_refs.get(referent){
let v=name_tally.entry(c.name.clone()).and_modify(|v|*v+=1).or_default();
stack.push(WriteStackInstruction::Node(c,*v));
}
}
stack.push(WriteStackInstruction::PushFolder(sanitize(name_override.as_str()).to_string()));
}
},
}
}
//run the async
{
let dom=&context.dom;
let write_models=config.write_models;
let write_scripts=config.write_scripts;
let results:Vec<AResult<()>>=rayon::iter::ParallelIterator::collect(rayon::iter::ParallelIterator::map(rayon::iter::IntoParallelIterator::into_par_iter(write_queue),|(write_path,node,node_name_override,properties,style)|{
write_item(&dom,write_path,node,node_name_override,properties,style,write_models,write_scripts)
}));
for result in results{
result?;
}
}
//run the destroy
for destroy_ref in destroy_queue{
context.dom.destroy(destroy_ref);
}
//write what remains in template.rbxlx
if config.write_template{
let mut file=config.output_folder.clone();
file.push("template");
assert!(file.set_extension("rbxlx"));
let output=std::io::BufWriter::new(std::fs::File::create(file)?);
rbx_xml::to_writer_default(output,&context.dom,&[context.dom.root_ref()])?;
}
Ok(())
}
struct DecompileConfig{
style:DecompileStyle,
input_file:std::path::PathBuf,
output_folder:std::path::PathBuf,
write_template:bool,
write_models:bool,
write_scripts:bool,
}
async fn decompile(config:DecompileConfig)->AResult<()>{
//rules:
//Class Script|LocalScript|ModuleScript->$Name.lua
//Class Model->$Name.rbxmx
//overrides.json per-folder [Override{name,class}]
//Everything else goes into template.rbxlx
//read file
let context=generate_decompiled_context(std::io::BufReader::new(std::fs::File::open(config.input_file)?))?;
//generate folders, models, and scripts
//delete models and scripts from dom
write_files(WriteConfig{
style:config.style,
output_folder:config.output_folder,
write_template:config.write_template,
write_models:config.write_models,
write_scripts:config.write_scripts,
},context).await?;
Ok(())
}
struct WriteCommitConfig{
git_committer_name:String,
git_committer_email:String,
output_folder:std::path::PathBuf,
style:DecompileStyle,
write_template:bool,
write_models:bool,
write_scripts:bool,
}
async fn write_commit(config:WriteCommitConfig,b:Result<AResult<(AssetVersion,DecompiledContext)>,tokio::task::JoinError>,repo:&git2::Repository)->AResult<()>{
let (asset_version,context)=b??;
println!("writing files for version {}",asset_version.assetVersionNumber);
//clean output dir
if config.write_models||config.write_scripts{
let mut src=config.output_folder.clone();
src.push("src");
match std::fs::remove_dir_all(src){
Ok(())=>(),
Err(e)=>println!("remove_dir_all src failed {}",e),
}
}
if config.write_template{
let mut template=config.output_folder.clone();
template.push("template.rbxlx");
match std::fs::remove_file(template){
Ok(())=>(),
Err(e)=>println!("remove_file template.rbxlx failed {}",e),
}
}
//write files
write_files(WriteConfig{
style:config.style,
output_folder:config.output_folder.clone(),
write_template:config.write_template,
write_models:config.write_models,
write_scripts:config.write_scripts,
},context).await?;
let date=asset_version.created;
//let sig=repo.signature()?; //this pulls default name and email
let sig=git2::Signature::new(config.git_committer_name.as_str(),config.git_committer_email.as_str(),&git2::Time::new(date.timestamp(),0)).unwrap();
let tree_id={
let mut tree_index = repo.index()?;
tree_index.add_all([config.output_folder.as_path()].iter(),git2::IndexAddOption::DEFAULT,None)?;
match tree_index.update_all([config.output_folder.as_path()].iter(),None){
Ok(_)=>(),
Err(e)=>println!("tree_index.update_all error: {}",e),
}
tree_index.write()?;
tree_index.write_tree()?
};
let tree=repo.find_tree(tree_id)?;
let mut parents=Vec::new();
match repo.head(){
Ok(reference)=>{
let commit=reference.peel_to_commit()?;
//test tree against commit tree to see if there is any changes
let commit_tree=commit.tree()?;
let diff=repo.diff_tree_to_tree(Some(&commit_tree),Some(&tree),None)?;
if diff.deltas().count()==0{
println!("no changes");
return Ok(());
}
parents.push(commit);
},
Err(e)=>println!("repo head error {:?}",e),
};
repo.commit(
Some("HEAD"),//update_ref
&sig,//author
&sig,//commiter
&format!("v{}", asset_version.assetVersionNumber),//message
&tree,//tree (basically files)
parents.iter().collect::<Vec<&git2::Commit<'_>>>().as_slice(),//parents
)?;
//commit
Ok(())
}
struct DecompileHistoryConfig{
git_committer_name:String,
git_committer_email:String,
input_folder:std::path::PathBuf,
style:DecompileStyle,
output_folder:std::path::PathBuf,
write_template:bool,
write_models:bool,
write_scripts:bool,
}
async fn decompile_history_into_git(config:DecompileHistoryConfig)->AResult<()>{
//use prexisting versions list
let mut versions_path=config.input_folder.clone();
versions_path.push("versions.json");
let asset_list:Vec<AssetVersion>=serde_json::from_reader(std::fs::File::open(versions_path)?)?;
let repo=git2::Repository::init(config.output_folder.as_path())?;
//decompile all versions
futures::stream::iter(asset_list.into_iter()
.map(|asset_version|{
let mut file_path=config.input_folder.clone();
tokio::task::spawn_blocking(move||{
file_path.push(format!("{}_v{}.rbxl",asset_version.assetId,asset_version.assetVersionNumber));
let file=std::fs::File::open(file_path)?;
let contents=generate_decompiled_context(file)?;
Ok::<_,anyhow::Error>((asset_version,contents))
})
}))
.buffered(CONCURRENT_DECODE)
.for_each(|join_handle_result|async{
match write_commit(WriteCommitConfig{
git_committer_name:config.git_committer_name.clone(),
git_committer_email:config.git_committer_email.clone(),
style:config.style,
output_folder:config.output_folder.clone(),
write_template:config.write_template,
write_models:config.write_models,
write_scripts:config.write_scripts,
},join_handle_result,&repo).await{
Ok(())=>(),
Err(e)=>println!("decompile/write/commit error: {}",e),
}
}).await;
Ok(())
}
struct DownloadAndDecompileHistoryConfig{
cookie:String,
asset_id:AssetID,
git_committer_name:String,
git_committer_email:String,
style:DecompileStyle,
output_folder:std::path::PathBuf,
write_template:bool,
write_models:bool,
write_scripts:bool,
}
async fn download_and_decompile_history_into_git(config:DownloadAndDecompileHistoryConfig)->AResult<()>{
let client=reqwest::Client::new();
//poll paged list of all asset versions
let asset_list=get_version_history(&client,&config.cookie.as_str(),config.asset_id).await?;
let repo=git2::Repository::init(config.output_folder.clone())?;
//download all versions
let asset_id_string=config.asset_id.to_string();
futures::stream::iter(asset_list.into_iter()
.map(|asset_version|{
let client=client.clone();
let cookie=config.cookie.clone();
let asset_id_str=asset_id_string.clone();
tokio::task::spawn(async move{
let resp=download_asset_version(&client,cookie.as_str(),asset_id_str.as_str(),asset_version.assetVersionNumber.to_string().as_str()).await?;
let contents=match maybe_gzip_decode(std::io::Cursor::new(resp.bytes().await?))?{
ReaderType::GZip(readable)=>generate_decompiled_context(readable)?,
ReaderType::Raw(readable)=>generate_decompiled_context(readable)?,
};
Ok::<_,anyhow::Error>((asset_version,contents))
})
}))
.buffered(CONCURRENT_DECODE)
.for_each(|join_handle_result|async{
match write_commit(WriteCommitConfig{
style:config.style,
git_committer_name:config.git_committer_name.clone(),
git_committer_email:config.git_committer_email.clone(),
output_folder:config.output_folder.clone(),
write_template:config.write_template,
write_models:config.write_models,
write_scripts:config.write_scripts,
},join_handle_result,&repo).await{
Ok(())=>(),
Err(e)=>println!("download/unzip/decompile/write/commit error: {}",e),
}
}).await;
Ok(())
}
//holy smokes what am I doing lmao
//This giant machine is supposed to search for files according to style rules
//e.g. ScriptName.server.lua or init.lua
//Obviously I got carried away
//I could use an enum!
//I could use a struct!
//I could use a trait!
//I could use an error!
//I could use a match!
//I could use a function!
//eventually:
#[derive(Debug)]
enum QueryResolveError{
NotFound,//0 results
Ambiguous,//>1 results
JoinError(tokio::task::JoinError),
IO(std::io::Error),
}
impl std::fmt::Display for QueryResolveError{
fn fmt(&self,f:&mut std::fmt::Formatter<'_>)->std::fmt::Result{
write!(f,"{self:?}")
}
}
impl std::error::Error for QueryResolveError{}
async fn get_file_async(mut path:std::path::PathBuf,file_name:impl AsRef<std::path::Path>)->Result<tokio::fs::File,QueryResolveError>{
path.push(file_name);
match tokio::fs::File::open(path).await{
Ok(file)=>Ok(file),
Err(e)=>match e.kind(){
std::io::ErrorKind::NotFound=>Err(QueryResolveError::NotFound),
_=>Err(QueryResolveError::IO(e)),
},
}
}
type QueryHintResult=Result<FileHint,QueryResolveError>;
trait Query{
async fn resolve(self)->QueryHintResult;
}
type QueryHandle=tokio::task::JoinHandle<Result<tokio::fs::File,QueryResolveError>>;
struct QuerySingle{
script:QueryHandle,
}
impl QuerySingle{
fn rox(search_path:&std::path::PathBuf,search_name:&str)->Self{
Self{
script:tokio::spawn(get_file_async(search_path.clone(),format!("{}.lua",search_name)))
}
}
}
impl Query for QuerySingle{
async fn resolve(self)->QueryHintResult{
match self.script.await{
Ok(Ok(file))=>Ok(FileHint{file,hint:ScriptHint::ModuleScript}),
Ok(Err(e))=>Err(e),
Err(e)=>Err(QueryResolveError::JoinError(e)),
}
}
}
struct QueryTriple{
module:QueryHandle,
server:QueryHandle,
client:QueryHandle,
}
impl QueryTriple{
fn rox_rojo(search_path:&std::path::PathBuf,search_name:&str,search_module:bool)->Self{
//this should be implemented as constructors of Triplet and Quadruplet to fully support Trey's suggestion
let module_name=if search_module{
format!("{}.module.lua",search_name)
}else{
format!("{}.lua",search_name)
};
Self{
module:tokio::spawn(get_file_async(search_path.clone(),module_name)),
server:tokio::spawn(get_file_async(search_path.clone(),format!("{}.server.lua",search_name))),
client:tokio::spawn(get_file_async(search_path.clone(),format!("{}.client.lua",search_name))),
}
}
fn rojo(search_path:&std::path::PathBuf)->Self{
QueryTriple::rox_rojo(search_path,"init",false)
}
}
//these functions can be achieved with macros, but I have not learned that yet
fn mega_triple_join(query_triplet:(QueryHintResult,QueryHintResult,QueryHintResult))->QueryHintResult{
match query_triplet{
//unambiguously locate file
(Ok(f),Err(QueryResolveError::NotFound),Err(QueryResolveError::NotFound))
|(Err(QueryResolveError::NotFound),Ok(f),Err(QueryResolveError::NotFound))
|(Err(QueryResolveError::NotFound),Err(QueryResolveError::NotFound),Ok(f))=>Ok(f),
//multiple files located
(Ok(_),Ok(_),Err(QueryResolveError::NotFound))
|(Ok(_),Err(QueryResolveError::NotFound),Ok(_))
|(Err(QueryResolveError::NotFound),Ok(_),Ok(_))
|(Ok(_),Ok(_),Ok(_))=>Err(QueryResolveError::Ambiguous),
//no files located
(Err(QueryResolveError::NotFound),Err(QueryResolveError::NotFound),Err(QueryResolveError::NotFound))=>Err(QueryResolveError::NotFound),
//other error
(Err(e),_,_)
|(_,Err(e),_)
|(_,_,Err(e))=>Err(e),
}
}
//LETS GOOOOOOOOOOOOOOOO
fn mega_quadruple_join(query_quad:(QueryHintResult,QueryHintResult,QueryHintResult,QueryHintResult))->QueryHintResult{
match query_quad{
//unambiguously locate file
(Ok(f),Err(QueryResolveError::NotFound),Err(QueryResolveError::NotFound),Err(QueryResolveError::NotFound))
|(Err(QueryResolveError::NotFound),Ok(f),Err(QueryResolveError::NotFound),Err(QueryResolveError::NotFound))
|(Err(QueryResolveError::NotFound),Err(QueryResolveError::NotFound),Ok(f),Err(QueryResolveError::NotFound))
|(Err(QueryResolveError::NotFound),Err(QueryResolveError::NotFound),Err(QueryResolveError::NotFound),Ok(f))=>Ok(f),
//multiple files located
(Ok(_),Ok(_),Ok(_),Err(QueryResolveError::NotFound))
|(Ok(_),Ok(_),Err(QueryResolveError::NotFound),Ok(_))
|(Ok(_),Ok(_),Err(QueryResolveError::NotFound),Err(QueryResolveError::NotFound))
|(Ok(_),Err(QueryResolveError::NotFound),Ok(_),Ok(_))
|(Ok(_),Err(QueryResolveError::NotFound),Ok(_),Err(QueryResolveError::NotFound))
|(Ok(_),Err(QueryResolveError::NotFound),Err(QueryResolveError::NotFound),Ok(_))
|(Err(QueryResolveError::NotFound),Ok(_),Ok(_),Ok(_))
|(Err(QueryResolveError::NotFound),Ok(_),Ok(_),Err(QueryResolveError::NotFound))
|(Err(QueryResolveError::NotFound),Ok(_),Err(QueryResolveError::NotFound),Ok(_))
|(Err(QueryResolveError::NotFound),Err(QueryResolveError::NotFound),Ok(_),Ok(_))
|(Ok(_),Ok(_),Ok(_),Ok(_))=>Err(QueryResolveError::Ambiguous),
//no files located
(Err(QueryResolveError::NotFound),Err(QueryResolveError::NotFound),Err(QueryResolveError::NotFound),Err(QueryResolveError::NotFound))=>Err(QueryResolveError::NotFound),
//other error
(Err(e),_,_,_)
|(_,Err(e),_,_)
|(_,_,Err(e),_)
|(_,_,_,Err(e))=>Err(e),
}
}
impl Query for QueryTriple{
async fn resolve(self)->QueryHintResult{
let (module,server,client)=tokio::join!(self.module,self.server,self.client);
mega_triple_join((
module.map_err(|e|QueryResolveError::JoinError(e))?.map(|file|FileHint{file,hint:ScriptHint::ModuleScript}),
server.map_err(|e|QueryResolveError::JoinError(e))?.map(|file|FileHint{file,hint:ScriptHint::Script}),
client.map_err(|e|QueryResolveError::JoinError(e))?.map(|file|FileHint{file,hint:ScriptHint::LocalScript}),
))
}
}
struct QueryQuad{
module_implicit:QueryHandle,
module_explicit:QueryHandle,
server:QueryHandle,
client:QueryHandle,
}
impl QueryQuad{
fn rox_rojo(search_path:&std::path::PathBuf,search_name:&str)->Self{
let fill=QueryTriple::rox_rojo(search_path,search_name,true);
Self{
module_implicit:QuerySingle::rox(search_path,search_name).script,//Script.lua
module_explicit:fill.module,//Script.module.lua
server:fill.server,
client:fill.client,
}
}
}
impl Query for QueryQuad{
async fn resolve(self)->QueryHintResult{
let (module_implicit,module_explicit,server,client)=tokio::join!(self.module_implicit,self.module_explicit,self.server,self.client);
mega_quadruple_join((
module_implicit.map_err(|e|QueryResolveError::JoinError(e))?.map(|file|FileHint{file,hint:ScriptHint::ModuleScript}),
module_explicit.map_err(|e|QueryResolveError::JoinError(e))?.map(|file|FileHint{file,hint:ScriptHint::ModuleScript}),
server.map_err(|e|QueryResolveError::JoinError(e))?.map(|file|FileHint{file,hint:ScriptHint::Script}),
client.map_err(|e|QueryResolveError::JoinError(e))?.map(|file|FileHint{file,hint:ScriptHint::LocalScript}),
))
}
}
struct ScriptWithOverrides{
overrides:PropertiesOverride,
source:String,
}
fn extract_script_overrides(mut source:String)->AResult<ScriptWithOverrides>{
let mut overrides=PropertiesOverride::default();
let mut count=0;
for line in source.lines(){
//only string type properties are supported atm
if let Some(captures)=lazy_regex::regex!(r#"^\-\-\!\s*Properties\.([A-z]\w*)\s*\=\s*"(\w+)"$"#)
.captures(line){
count+=line.len();
match &captures[1]{
"Name"=>overrides.name=Some(captures[2].to_owned()),
"ClassName"=>overrides.class=Some(captures[2].to_owned()),
other=>Err(anyhow::Error::msg(format!("Unimplemented property {other}")))?,
}
}else{
break;
}
}
Ok(ScriptWithOverrides{overrides,source:source.split_off(count)})
}
async fn script_node(search_name:&str,mut file:tokio::fs::File,hint:ScriptHint)->AResult<CompileNode>{
//read entire file
let mut buf=String::new();
file.read_to_string(&mut buf).await?;
//regex script according to Properties lines at the top
let script_with_overrides=extract_script_overrides(buf)?;
//script
Ok(CompileNode{
name:script_with_overrides.overrides.name.unwrap_or_else(||search_name.to_owned()),
class:match (script_with_overrides.overrides.class.as_deref(),hint){
(Some("ModuleScript"),_)
|(None,ScriptHint::ModuleScript)=>CompileClass::ModuleScript(script_with_overrides.source),
(Some("LocalScript"),_)
|(None,ScriptHint::Script)=>CompileClass::LocalScript(script_with_overrides.source),
(Some("Script"),_)
|(None,ScriptHint::LocalScript)=>CompileClass::Script(script_with_overrides.source),
other=>Err(anyhow::Error::msg(format!("Invalid hint or class {other:?}")))?,
},
})
}
async fn model_node(search_name:&str,mut file:tokio::fs::File)->AResult<CompileNode>{
//read entire file
let mut buf=Vec::new();
file.read_to_end(&mut buf).await?;
//model
Ok(CompileNode{
name:search_name.to_owned(),
class:CompileClass::Model(buf),
})
}
async fn locate_override_file(entry:&tokio::fs::DirEntry,style:Option<DecompileStyle>)->AResult<CompileNode>{
let contents_folder=entry.path();
let file_name=entry.file_name();
let search_name=file_name.to_str().unwrap();
//scan inside the folder for an object to define the class of the folder
let script_query=async {match style{
Some(DecompileStyle::Rox)=>QuerySingle::rox(&contents_folder,search_name).resolve().await,
Some(DecompileStyle::RoxRojo)=>QueryQuad::rox_rojo(&contents_folder,search_name).resolve().await,
Some(DecompileStyle::Rojo)=>QueryTriple::rojo(&contents_folder).resolve().await,
//try all three and complain if there is ambiguity
None=>mega_triple_join(tokio::join!(
QuerySingle::rox(&contents_folder,search_name).resolve(),
//true=search for module here to avoid ambiguity with QuerySingle::rox results
QueryTriple::rox_rojo(&contents_folder,search_name,true).resolve(),
QueryTriple::rojo(&contents_folder).resolve(),
))
}};
//model files are rox & rox-rojo only, so it's a lot less work...
let model_query=get_file_async(contents_folder.clone(),format!("{}.rbxmx",search_name));
//model? script? both?
Ok(match tokio::join!(script_query,model_query){
(Ok(FileHint{file,hint}),Err(QueryResolveError::NotFound))=>script_node(search_name,file,hint).await?,
(Err(QueryResolveError::NotFound),Ok(file))=>model_node(search_name,file).await?,
(Ok(_),Ok(_))=>Err(QueryResolveError::Ambiguous)?,
//neither
(Err(QueryResolveError::NotFound),Err(QueryResolveError::NotFound))=>CompileNode{
name:search_name.to_owned(),
class:CompileClass::Folder,
},
//other error
(Err(e),_)
|(_,Err(e))=>Err(e)?
})
}
enum FileDiscernment{
Model,
Script(ScriptHint),
}
async fn discern_file(entry:&tokio::fs::DirEntry,style:Option<DecompileStyle>)->AResult<CompileNode>{
let path=entry.path();
let (ext_len,file_discernment)=match path.extension(){
Some(extension)=>{
let ext=extension.to_str().unwrap_or("");
(ext.len(),match ext{
"lua"
|"module.lua"=>FileDiscernment::Script(ScriptHint::ModuleScript),
"client.lua"=>FileDiscernment::Script(ScriptHint::LocalScript),
"server.lua"=>FileDiscernment::Script(ScriptHint::Script),
"rbxmx"=>FileDiscernment::Model,
other=>Err(anyhow::Error::msg(format!("Weird file extension: {other:?}")))?,
})
},
None=>Err(anyhow::Error::msg("No file extension"))?,
};
//reject goobers
match (style,&file_discernment){
(Some(DecompileStyle::Rojo),FileDiscernment::Script(ScriptHint::ModuleScript))
|(Some(DecompileStyle::Rojo),FileDiscernment::Model)=>Err(anyhow::Error::msg("Invalid file extension for style"))?,
_=>(),
}
let mut search_name=entry
.file_name()
.into_string()
.map_err(|e|anyhow::Error::msg(format!("insane file name {e:?}")))?;
search_name.truncate(search_name.len()-ext_len-1);//remove period
let file=tokio::fs::File::open(path.as_path()).await?;
Ok(match file_discernment{
FileDiscernment::Model=>model_node(search_name.as_str(),file).await?,
FileDiscernment::Script(hint)=>script_node(search_name.as_str(),file,hint).await?,
})
}
#[derive(Debug)]
enum ScriptHint{
Script,
LocalScript,
ModuleScript,
}
struct FileHint{
file:tokio::fs::File,
hint:ScriptHint,
}
enum PreparedData{
Model(rbx_dom_weak::WeakDom),
Builder(rbx_dom_weak::InstanceBuilder),
}
enum CompileClass{
Folder,
Script(String),
LocalScript(String),
ModuleScript(String),
Model(Vec<u8>),
}
struct CompileNode{
name:String,
class:CompileClass,
}
enum CompileStackInstruction{
TraverseReferent(rbx_dom_weak::types::Ref),
PopFolder,
}
struct CompileConfig{
input_folder:std::path::PathBuf,
output_file:std::path::PathBuf,
template:Option<std::path::PathBuf>,
style:Option<DecompileStyle>,
}
fn script_builder(class:&str,name:&str,source:String)->rbx_dom_weak::InstanceBuilder{
let mut builder=rbx_dom_weak::InstanceBuilder::new(class);
builder.set_name(name);
builder.add_property("Source",rbx_dom_weak::types::Variant::String(source));
builder
}
async fn compile(config:CompileConfig)->AResult<()>{
//basically decompile in reverse order
//load template dom
let input={
let template_path=config.template.unwrap_or_else(||{
let mut template_path=config.input_folder.clone();
template_path.push("template.rbxlx");
template_path
});
//mr dom doesn't like tokio files
std::io::BufReader::new(std::fs::File::open(template_path)?)
};
let mut dom=load_dom(input)?;
//add in scripts and models
let mut folder=config.input_folder.clone();
folder.push("src");
let mut stack:Vec<CompileStackInstruction>=dom.get_by_ref(dom.root().children()[0]).unwrap().children().into_iter().map(|&referent|CompileStackInstruction::TraverseReferent(referent)).collect();
while let Some(instruction)=stack.pop(){
match instruction{
CompileStackInstruction::TraverseReferent(item_ref)=>{
let sans={
let item=dom.get_by_ref(item_ref).ok_or(anyhow::Error::msg("null child ref"))?;
sanitize(item.name.as_str()).to_string()
};
folder.push(sans.as_str());
stack.push(CompileStackInstruction::PopFolder);
//check if a folder exists with item.name
if let Ok(mut dir)=tokio::fs::read_dir(folder.as_path()).await{
let mut exist_names=std::collections::HashSet::new();
{
let item=dom.get_by_ref(item_ref).ok_or(anyhow::Error::msg("null child ref"))?;
for &child_ref in item.children(){
let child=dom.get_by_ref(child_ref).ok_or(anyhow::Error::msg("null child ref"))?;
let child_sans=sanitize(child.name.as_str()).to_string();
exist_names.insert(child_sans);
}
}
//generate children from folder contents UNLESS! item already has a child of the same name
let mut join_set=tokio::task::JoinSet::new();
//I wish I could make the join_next() loop begin processing immediately,
//but I don't know an ergonomic way to do that.
//this will probably be fine considering there won't be millions of files in the directories
while let Some(entry)=dir.next_entry().await?{
//cull early even if supporting things with identical names is possible
if !exist_names.contains(entry.file_name().to_str().unwrap()){
let style=config.style;
join_set.spawn(async move{
let met=entry.metadata().await?;
let compile_class=if met.is_dir(){
locate_override_file(&entry,style).await?
}else{
discern_file(&entry,style).await?
};
//discern that bad boy
Ok::<_,anyhow::Error>(
//prepare data structure
match compile_class.class{
CompileClass::Folder=>PreparedData::Builder(rbx_dom_weak::InstanceBuilder::new("Folder").with_name(compile_class.name.as_str())),
CompileClass::Script(source)=>PreparedData::Builder(script_builder("Script",compile_class.name.as_str(),source)),
CompileClass::LocalScript(source)=>PreparedData::Builder(script_builder("LocalScript",compile_class.name.as_str(),source)),
CompileClass::ModuleScript(source)=>PreparedData::Builder(script_builder("ModuleScript",compile_class.name.as_str(),source)),
CompileClass::Model(buf)=>PreparedData::Model(rbx_xml::from_reader_default(std::io::Cursor::new(buf))?),
}
)
});
}
}
//this is only able to begin after dir iterator is exhausted
//push child objects onto dom
while let Some(goober)=join_set.join_next().await{
match goober??{
PreparedData::Model(mut model_dom)=>model_dom.transfer(model_dom.root().children()[0],&mut dom,item_ref),
PreparedData::Builder(script)=>{dom.insert(item_ref,script);},
}
}
//push dom children objects onto stack
let item=dom.get_by_ref(item_ref).ok_or(anyhow::Error::msg("null child ref"))?;
stack.extend(item.children().into_iter().map(|&referent|CompileStackInstruction::TraverseReferent(referent)));
}
},
CompileStackInstruction::PopFolder=>assert!(folder.pop(),"pop folder bad"),
}
}
let mut output_place=config.output_file.clone();
if output_place.extension().is_none()&&tokio::fs::try_exists(output_place.as_path()).await?{
output_place.push("place.rbxl");
}
let output=std::io::BufWriter::new(std::fs::File::create(output_place)?);
rbx_binary::to_writer(output,&dom,&[dom.root_ref()])?;
Ok(())
}
Reference in New Issue View Git Blame Copy Permalink