use std::io::Read; use clap::{Args,Parser,Subcommand}; use anyhow::Result as AResult; use futures::StreamExt; use rbx_dom_weak::types::Ref; type AssetID=u64; type AssetIDFileMap=Vec<(AssetID,std::path::PathBuf)>; const CONCURRENT_DECODE:usize=8; const CONCURRENT_REQUESTS:usize=32; #[derive(Parser)] #[command(author,version,about,long_about=None)] #[command(propagate_version = true)] struct Cli{ //asset options #[arg(short,long)] group:Option, #[arg(long)] asset_id:Option, //idk how to do this better #[arg(long)] cookie_literal:Option, #[arg(long)] cookie_env:Option, #[arg(long)] cookie_file:Option, //TODO: read the versions.json file instead of doing this //TODO: write file dates instead of versions.json #[arg(long)] start_version:Option, #[arg(long)] end_version:Option, #[arg(long)] r#continue:bool, //decompile options #[arg(long)] no_models:Option, #[arg(long)] no_scripts:Option, #[arg(long)] no_template:Option, #[arg(long)] style:Option, //git options #[arg(long)] git_committer_name:Option, #[arg(long)] git_committer_email:Option, #[arg(short,long)] input:Option, #[arg(short,long)] output:Option, #[command(subcommand)] command:Commands, } #[derive(Subcommand)] enum Commands{ DownloadHistory, Download(AssetIDList), Upload, Compile, Decompile, DecompileHistoryIntoGit, DownloadAndDecompileHistoryIntoGit, } #[derive(Clone,Copy)] enum DecompileStyle{ Rox, Rojo, RoxRojo, } #[derive(Args)] struct AssetIDList{ asset_ids:Vec } #[derive(Args)] struct PathBufList{ paths:Vec } #[derive(serde::Deserialize)] #[allow(nonstandard_style,dead_code)] struct VersionPage{ previousPageCursor:Option, nextPageCursor:Option, data:Vec, } #[derive(serde::Deserialize,serde::Serialize)] #[allow(nonstandard_style,dead_code)] struct AssetVersion{ Id:u64, assetId:AssetID, assetVersionNumber:u64, creatorType:String, creatorTargetId:u64, creatingUniverseId:Option, created:chrono::DateTime, isPublished:bool, } #[tokio::main] async fn main()->AResult<()>{ let cli=Cli::parse(); let cookie_enum={ match (cli.cookie_literal,cli.cookie_env,cli.cookie_file){ (Some(literal),None,None)=>Some(Cookie::Literal(literal)), (None,Some(env_var),None)=>Some(Cookie::Environment(env_var)), (None,None,Some(path))=>Some(Cookie::File(path)), (None,None,None)=>None, _=>return Err(anyhow::Error::msg("Cookie was specified multiple times.")), } }; let cookie=match cookie_enum{ Some(c)=>Some(format!(".ROBLOSECURITY={}",match c{ Cookie::Literal(s)=>s, Cookie::Environment(var)=>std::env::var(var)?, Cookie::File(path)=>tokio::fs::read_to_string(path).await?, })), None=>None, }; let decompile_style=match cli.style.as_deref(){ Some("rox") |Some("Rox")=>Some(DecompileStyle::Rox), Some("rojo") |Some("Rojo")=>Some(DecompileStyle::Rojo), Some("rox-rojo") |Some("rojo-rox") |Some("roxrojo") |Some("rojorox") |Some("RoxRojo") |Some("RojoRox")=>Some(DecompileStyle::RoxRojo), None=>None, _=>return Err(anyhow::Error::msg("Invalid style")), }; match cli.command{ Commands::DownloadHistory=>download_history(DownloadHistoryConfig{ continue_from_versions:cli.r#continue, end_version:cli.end_version, start_version:cli.start_version.unwrap_or(0), output_folder:cli.output.unwrap(), cookie:cookie.unwrap(), asset_id:cli.asset_id.unwrap(), }).await, Commands::Download(asset_id_list)=>download_list( cookie.unwrap(), asset_id_list.asset_ids.into_iter().map(|asset_id|{ let mut path=cli.output.clone().unwrap(); path.push(asset_id.to_string()); (asset_id,path) }).collect() ).await, Commands::Upload=>upload_list(cookie.unwrap(),cli.group,vec![(cli.asset_id.unwrap(),cli.output.unwrap())]).await, Commands::Compile=>compile(cli.input.unwrap(),cli.output.unwrap()), Commands::Decompile=>decompile(DecompileConfig{ style:decompile_style.unwrap(), input_file:cli.input.unwrap(), output_folder:cli.output.unwrap(), write_template:!cli.no_template.unwrap_or(false), write_models:!cli.no_models.unwrap_or(false), write_scripts:!cli.no_scripts.unwrap_or(false), }).await, Commands::DecompileHistoryIntoGit=>decompile_history_into_git(DecompileHistoryConfig{ git_committer_name:cli.git_committer_name.unwrap(), git_committer_email:cli.git_committer_email.unwrap(), input_folder:cli.input.unwrap(), output_folder:cli.output.unwrap(), style:decompile_style.unwrap(), write_template:!cli.no_template.unwrap_or(false), write_models:!cli.no_models.unwrap_or(false), write_scripts:!cli.no_scripts.unwrap_or(false), }).await, Commands::DownloadAndDecompileHistoryIntoGit=>download_and_decompile_history_into_git(DownloadAndDecompileHistoryConfig{ git_committer_name:cli.git_committer_name.unwrap(), git_committer_email:cli.git_committer_email.unwrap(), cookie:cookie.unwrap(), asset_id:cli.asset_id.unwrap(), output_folder:cli.output.unwrap(), style:decompile_style.unwrap(), write_template:!cli.no_template.unwrap_or(false), write_models:!cli.no_models.unwrap_or(false), write_scripts:!cli.no_scripts.unwrap_or(false), }).await, } } enum Cookie{ Literal(String), Environment(String), File(std::path::PathBuf), } enum ReaderType{ GZip(flate2::read::GzDecoder>), Raw(std::io::BufReader), } fn maybe_gzip_decode(input:R)->AResult>{ let mut buf=std::io::BufReader::new(input); let peek=std::io::BufRead::fill_buf(&mut buf)?; match &peek[0..2]{ b"\x1f\x8b"=>Ok(ReaderType::GZip(flate2::read::GzDecoder::new(buf))), _=>Ok(ReaderType::Raw(buf)), } } async fn upload_list(cookie:String,group:Option,asset_id_file_map:AssetIDFileMap)->AResult<()>{ let client=reqwest::Client::new(); futures::stream::iter(asset_id_file_map.into_iter() .map(|(asset_id,file)|{ let client=&client; let cookie=cookie.as_str(); let group=&group; async move{ let mut url=reqwest::Url::parse("https://data.roblox.com/Data/Upload.ashx?json=1&type=Model&genreTypeId=1")?; //url borrow scope { let mut query=url.query_pairs_mut();//borrow here query.append_pair("assetid",asset_id.to_string().as_str()); match group{ Some(group_id)=>{query.append_pair("groupId",group_id.to_string().as_str());}, None=>(), } } let body=tokio::fs::read_to_string(file).await?; let mut resp=client.post(url.clone()) .header("Cookie",cookie) .body(body.clone()) .send().await?; //This is called a CSRF challenge apparently if resp.status()==reqwest::StatusCode::FORBIDDEN{ if let Some(csrf_token)=resp.headers().get("X-CSRF-Token"){ resp=client.post(url) .header("X-CSRF-Token",csrf_token) .header("Cookie",cookie) .body(body) .send().await?; }else{ return Err(anyhow::Error::msg("Roblox returned 403 with no CSRF")); } } Ok((asset_id,resp.bytes().await?)) } })) .buffer_unordered(CONCURRENT_REQUESTS) .for_each(|b:AResult<_>|async{ match b{ Ok((asset_id,body))=>{ println!("asset_id={} response.body={:?}",asset_id,body); }, Err(e)=>eprintln!("ul error: {}",e), } }).await; Ok(()) } fn read_readable(mut readable:impl Read)->AResult>{ let mut contents=Vec::new(); readable.read_to_end(&mut contents)?; Ok(contents) } async fn download_list(cookie:String,asset_id_file_map:AssetIDFileMap)->AResult<()>{ let client=reqwest::Client::new(); futures::stream::iter(asset_id_file_map.into_iter() .map(|(asset_id,file)|{ let client=&client; let cookie=cookie.as_str(); async move{ let resp=client.get(format!("https://assetdelivery.roblox.com/v1/asset/?ID={}",asset_id)) .header("Cookie",cookie) .send().await?; Ok((file,resp.bytes().await?)) } })) .buffer_unordered(CONCURRENT_REQUESTS) .for_each(|b:AResult<_>|async{ match b{ Ok((dest,body))=>{ let contents=match maybe_gzip_decode(&mut std::io::Cursor::new(body)){ Ok(ReaderType::GZip(readable))=>read_readable(readable), Ok(ReaderType::Raw(readable))=>read_readable(readable), Err(e)=>Err(e), }; match contents{ Ok(data)=>match tokio::fs::write(dest,data).await{ Err(e)=>eprintln!("fs error: {}",e), _=>(), }, Err(e)=>eprintln!("gzip error: {}",e), }; }, Err(e)=>eprintln!("dl error: {}",e), } }).await; Ok(()) } async fn download_page(client:&reqwest::Client,cookie:&str,asset_id:AssetID,cursor:Option)->AResult{ let mut url=reqwest::Url::parse(format!("https://develop.roblox.com/v1/assets/{}/saved-versions",asset_id).as_str())?; //url borrow scope { let mut query=url.query_pairs_mut();//borrow here //query.append_pair("sortOrder","Asc"); //query.append_pair("limit","100"); //query.append_pair("count","100"); match cursor.as_deref(){ Some(next_page)=>{query.append_pair("cursor",next_page);} None=>(), } } println!("page url={}",url); let resp=client.get(url) .header("Cookie",cookie) .send().await?; Ok(resp.json::().await?) } async fn get_version_history(client:&reqwest::Client,cookie:&str,asset_id:AssetID)->AResult>{ let mut cursor:Option=None; let mut asset_list=Vec::new(); loop{ let mut page=download_page(client,cookie,asset_id,cursor).await?; asset_list.append(&mut page.data); if page.nextPageCursor.is_none(){ break; } cursor=page.nextPageCursor; } asset_list.sort_by(|a,b|a.assetVersionNumber.cmp(&b.assetVersionNumber)); Ok(asset_list) } async fn download_asset_version(client:&reqwest::Client,cookie:&str,asset_id_str:&str,asset_version_str:&str)->AResult{ let mut url=reqwest::Url::parse("https://assetdelivery.roblox.com/v1/asset/")?; //url borrow scope { let mut query=url.query_pairs_mut();//borrow here query.append_pair("ID",asset_id_str); query.append_pair("version",asset_version_str); } println!("download url={}",url); for i in 0..8{ let resp=client.get(url.clone()) .header("Cookie",cookie) .send().await?; if !resp.status().is_success(){ println!("request {} failed",i); continue; } return Ok(resp); } Err(anyhow::Error::msg("all requests failed")) } struct DownloadHistoryConfig{ continue_from_versions:bool, end_version:Option, start_version:u64, output_folder:std::path::PathBuf, cookie:String, asset_id:AssetID, } async fn download_history(mut config:DownloadHistoryConfig)->AResult<()>{ let mut asset_list_contents=std::collections::HashSet::new(); let mut asset_list:Vec=Vec::new(); if config.end_version.is_none()&&config.continue_from_versions{ //load prexisting versions list let mut versions_path=config.output_folder.clone(); versions_path.push("versions.json"); match std::fs::File::open(versions_path){ Ok(versions_file)=>asset_list.append(&mut serde_json::from_reader(versions_file)?), Err(e)=>match e.kind(){ std::io::ErrorKind::NotFound=>return Err(anyhow::Error::msg("Cannot continue from versions.json - file does not exist")), _=>Err(e)?, } } //write down which versions are contained for asset_version in &asset_list{ asset_list_contents.insert(asset_version.assetVersionNumber); } //find the highest number match asset_list.iter().map(|asset_version|asset_version.assetVersionNumber).max(){ Some(max)=>{ //count down contiguously until a number is missing for i in (1..=max).rev(){ if !asset_list_contents.contains(&i){ //that is end_version config.end_version=Some(i); break; } } //if all versions are contained, set start_version to the max + 1 if config.end_version.is_none(){ config.start_version=max+1; } }, None=>Err(anyhow::Error::msg("Cannot continue from versions.json - there are no previous versions"))?, } } let client=reqwest::Client::new(); let asset_id_string=config.asset_id.to_string(); //limit concurrent downloads let mut join_set=tokio::task::JoinSet::new(); //poll paged list of all asset versions let mut cursor:Option=None; loop{ let mut page=download_page(&client,config.cookie.as_str(),config.asset_id,cursor).await?; let client=&client; let cookie=config.cookie.clone(); let asset_id_str=asset_id_string.clone(); let output_folder=config.output_folder.clone(); let data=&page.data; let asset_list_contents=&asset_list_contents; let join_set=&mut join_set; let error_catcher=||async move{ let mut cancel_paging=false; for asset_version in data{ let version_number=asset_version.assetVersionNumber; //skip assets beyond specified end_version if config.end_version.is_some_and(|v|vread_readable(readable)?, ReaderType::Raw(readable)=>read_readable(readable)?, }; tokio::fs::write(path,contents).await?; Ok::<_,anyhow::Error>(()) }); } Ok::<_,anyhow::Error>(cancel_paging) }; let cancel_paging=match error_catcher().await{ Ok(cancel)=>cancel, Err(e)=>{ println!("download error: {}",e); //cancel download and write versions true }, }; if page.nextPageCursor.is_none()||cancel_paging{ for asset_version in page.data.into_iter(){ if !(asset_list_contents.contains(&asset_version.assetVersionNumber) ||config.end_version.is_some_and(|v|v(input:R)->AResult{ let mut buf=std::io::BufReader::new(input); let peek=std::io::BufRead::fill_buf(&mut buf)?; match &peek[0..4]{ b"{ match &peek[4..8]{ b"lox!"=>return rbx_binary::from_reader(buf).map_err(anyhow::Error::msg), b"lox "=>return rbx_xml::from_reader_default(buf).map_err(anyhow::Error::msg), other=>Err(anyhow::Error::msg(format!("Unknown Roblox file type {:?}",other))), } }, _=>Err(anyhow::Error::msg("unsupported file type")), } } #[derive(PartialEq)] enum Class{ Folder, ModuleScript, LocalScript, Script, Model, } struct TreeNode{ name:String, referent:Ref, parent:Ref, class:Class, children:Vec, } impl TreeNode{ fn new(name:String,referent:Ref,parent:Ref,class:Class)->Self{ Self{ name, referent, parent, class, children:Vec::new(), } } } enum TrimStackInstruction{ Referent(Ref), IncrementScript, DecrementScript, } enum WriteStackInstruction<'a>{ Node(&'a TreeNode,u32),//(Node,NameTally) PushFolder(String), PopFolder, Destroy(Ref), } #[derive(Default)] struct PropertiesOverride{ name:Option, class_name:Option, } impl PropertiesOverride{ fn is_some(&self)->bool{ self.name.is_some() ||self.class_name.is_some() } } impl std::fmt::Display for PropertiesOverride{ fn fmt(&self,f:&mut std::fmt::Formatter<'_>)->std::fmt::Result{ if let Some(name)=self.name.as_deref(){ writeln!(f,"--! Properties.Name=\"{}\"",name)?; } if let Some(class_name)=self.class_name.as_deref(){ writeln!(f,"--! Properties.ClassName=\"{}\"",class_name)?; } Ok(()) } } fn sanitize<'a>(s:&'a str)->std::borrow::Cow<'a,str>{ lazy_regex::regex!(r"[^a-zA-Z0-9._-]").replace_all(s,"_") } fn write_item(dom:&rbx_dom_weak::WeakDom,mut file:std::path::PathBuf,node:&TreeNode,node_name_override:String,mut properties:PropertiesOverride,style:DecompileStyle,write_models:bool,write_scripts:bool)->AResult<()>{ file.push(sanitize(node_name_override.as_str()).as_ref()); match node.class{ Class::Folder=>(), Class::ModuleScript|Class::LocalScript|Class::Script=>{ if !write_scripts{ return Ok(()) } //set extension match style{ DecompileStyle::Rox=>assert!(file.set_extension("lua"),"could not set extension"), DecompileStyle::RoxRojo|DecompileStyle::Rojo=>{ match properties.class_name.as_deref(){ Some("LocalScript")=>{ file.set_extension("client.lua"); properties.class_name=None; }, Some("Script")=>{ file.set_extension("server.lua"); properties.class_name=None; }, // Some("ModuleScript")=>{ // file.set_extension("module"); // properties.class_name=None; // }, None=>assert!(file.set_extension("lua"),"could not set extension"), Some(other)=>return Err(anyhow::Error::msg(format!("Attempt to write a {} as a script",other))), } } } if let Some(item)=dom.get_by_ref(node.referent){ //TODO: delete disabled scripts if let Some(rbx_dom_weak::types::Variant::String(source))=item.properties.get("Source"){ if properties.is_some(){ //rox style let source=properties.to_string()+source.as_str(); std::fs::write(file,source)?; }else{ std::fs::write(file,source)?; } } } }, Class::Model=>{ if !write_models{ return Ok(()) } assert!(file.set_extension("rbxmx")); let output=std::io::BufWriter::new(std::fs::File::create(file)?); rbx_xml::to_writer_default(output,dom,&[node.referent])?; }, } Ok(()) } struct DecompiledContext{ dom:rbx_dom_weak::WeakDom, tree_refs:std::collections::HashMap, } fn generate_decompiled_context(input:R)->AResult{ let dom=load_dom(input)?; let mut tree_refs=std::collections::HashMap::new(); tree_refs.insert(dom.root_ref(),TreeNode::new( "src".to_string(), dom.root_ref(), Ref::none(), Class::Folder )); //run rules let mut stack=vec![dom.root()]; while let Some(item)=stack.pop(){ let class=match item.class.as_str(){ "ModuleScript"=>Class::ModuleScript, "LocalScript"=>Class::LocalScript, "Script"=>Class::Script, "Model"=>Class::Model, _=>Class::Folder, }; let skip=match class{ Class::Model=>true, _=>false, }; if let Some(parent_node)=tree_refs.get_mut(&item.parent()){ let referent=item.referent(); let node=TreeNode::new(item.name.clone(),referent,parent_node.referent,class); parent_node.children.push(referent); tree_refs.insert(referent,node); } //look no further, turn this node and all its children into a model if skip{ continue; } for &referent in item.children(){ if let Some(c)=dom.get_by_ref(referent){ stack.push(c); } } } //trim empty folders let mut script_count=0; let mut stack:Vec=tree_refs.get(&dom.root_ref()).unwrap().children .iter().map(|&c|TrimStackInstruction::Referent(c)).collect(); while let Some(instruction)=stack.pop(){ match instruction{ TrimStackInstruction::IncrementScript=>script_count+=1, TrimStackInstruction::DecrementScript=>script_count-=1, TrimStackInstruction::Referent(referent)=>{ let mut delete=None; if let Some(node)=tree_refs.get_mut(&referent){ if node.class==Class::Folder&&script_count!=0{ node.class=Class::Model } if node.class==Class::Folder&&node.children.len()==0{ delete=Some(node.parent); }else{ //how the hell do I do this better without recursion let is_script=match node.class{ Class::ModuleScript|Class::LocalScript|Class::Script=>true, _=>false, }; //stack is popped from back if is_script{ stack.push(TrimStackInstruction::DecrementScript); } for &child_referent in &node.children{ stack.push(TrimStackInstruction::Referent(child_referent)); } if is_script{ stack.push(TrimStackInstruction::IncrementScript); } } } //trim referent if let Some(parent_ref)=delete{ let parent_node=tree_refs.get_mut(&parent_ref) .expect("parent_ref does not exist in tree_refs"); parent_node.children.remove( parent_node.children.iter() .position(|&r|r==referent) .expect("parent.children does not contain referent") ); tree_refs.remove(&referent); } }, } } Ok(DecompiledContext{ dom, tree_refs, }) } struct WriteConfig{ style:DecompileStyle, output_folder:std::path::PathBuf, write_template:bool, write_models:bool, write_scripts:bool, } async fn write_files(config:WriteConfig,mut context:DecompiledContext)->AResult<()>{ let mut write_queue=Vec::new(); let mut destroy_queue=Vec::new(); let mut name_tally=std::collections::HashMap::::new(); let mut folder=config.output_folder.clone(); let mut stack=vec![WriteStackInstruction::Node(context.tree_refs.get(&context.dom.root_ref()).unwrap(),0)]; while let Some(instruction)=stack.pop(){ match instruction{ WriteStackInstruction::PushFolder(component)=>folder.push(component), WriteStackInstruction::PopFolder=>assert!(folder.pop(),"weirdness"), WriteStackInstruction::Destroy(referent)=>destroy_queue.push(referent), WriteStackInstruction::Node(node,name_count)=>{ //track properties that must be overriden to compile folder structure back into a place file let mut properties=PropertiesOverride::default(); let has_children=node.children.len()!=0; match node.class{ Class::Folder=>(), Class::ModuleScript=>(),//.lua files are ModuleScript by default Class::LocalScript=>properties.class_name=Some("LocalScript".to_string()), Class::Script=>properties.class_name=Some("Script".to_string()), Class::Model=>(), } let name_override=if 0name_override.clone(), DecompileStyle::Rojo=>"init".to_owned(), }; //write item in subfolder write_queue.push((subfolder,node,name_final,properties,config.style)); }else{ //write item write_queue.push((folder.clone(),node,name_override.clone(),properties,config.style)); } //queue item to be deleted from dom after child objects are handled (stack is popped from the back) match node.class{ Class::Folder=>(), _=>stack.push(WriteStackInstruction::Destroy(node.referent)), } if has_children{ stack.push(WriteStackInstruction::PopFolder); name_tally.clear(); for referent in &node.children{ if let Some(c)=context.tree_refs.get(referent){ let v=name_tally.entry(c.name.clone()).and_modify(|v|*v+=1).or_default(); stack.push(WriteStackInstruction::Node(c,*v)); } } stack.push(WriteStackInstruction::PushFolder(sanitize(name_override.as_str()).to_string())); } }, } } //run the async { let dom=&context.dom; let write_models=config.write_models; let write_scripts=config.write_scripts; let results:Vec>=rayon::iter::ParallelIterator::collect(rayon::iter::ParallelIterator::map(rayon::iter::IntoParallelIterator::into_par_iter(write_queue),|(write_path,node,node_name_override,properties,style)|{ write_item(&dom,write_path,node,node_name_override,properties,style,write_models,write_scripts) })); for result in results{ result?; } } //run the destroy for destroy_ref in destroy_queue{ context.dom.destroy(destroy_ref); } //write what remains in template.rbxlx if config.write_template{ let mut file=config.output_folder.clone(); file.push("template"); assert!(file.set_extension("rbxlx")); let output=std::io::BufWriter::new(std::fs::File::create(file)?); rbx_xml::to_writer_default(output,&context.dom,&[context.dom.root_ref()])?; } Ok(()) } struct DecompileConfig{ style:DecompileStyle, input_file:std::path::PathBuf, output_folder:std::path::PathBuf, write_template:bool, write_models:bool, write_scripts:bool, } async fn decompile(config:DecompileConfig)->AResult<()>{ //rules: //Class Script|LocalScript|ModuleScript->$Name.lua //Class Model->$Name.rbxmx //overrides.json per-folder [Override{name,class}] //Everything else goes into template.rbxlx //read file let context=generate_decompiled_context(std::io::BufReader::new(std::fs::File::open(config.input_file)?))?; //generate folders, models, and scripts //delete models and scripts from dom write_files(WriteConfig{ style:config.style, output_folder:config.output_folder, write_template:config.write_template, write_models:config.write_models, write_scripts:config.write_scripts, },context).await?; Ok(()) } struct WriteCommitConfig{ git_committer_name:String, git_committer_email:String, output_folder:std::path::PathBuf, style:DecompileStyle, write_template:bool, write_models:bool, write_scripts:bool, } async fn write_commit(config:WriteCommitConfig,b:Result,tokio::task::JoinError>,repo:&git2::Repository)->AResult<()>{ let (asset_version,context)=b??; println!("writing files for version {}",asset_version.assetVersionNumber); //clean output dir if config.write_models||config.write_scripts{ let mut src=config.output_folder.clone(); src.push("src"); match std::fs::remove_dir_all(src){ Ok(())=>(), Err(e)=>println!("remove_dir_all src failed {}",e), } } if config.write_template{ let mut template=config.output_folder.clone(); template.push("template.rbxlx"); match std::fs::remove_file(template){ Ok(())=>(), Err(e)=>println!("remove_file template.rbxlx failed {}",e), } } //write files write_files(WriteConfig{ style:config.style, output_folder:config.output_folder.clone(), write_template:config.write_template, write_models:config.write_models, write_scripts:config.write_scripts, },context).await?; let date=asset_version.created; //let sig=repo.signature()?; //this pulls default name and email let sig=git2::Signature::new(config.git_committer_name.as_str(),config.git_committer_email.as_str(),&git2::Time::new(date.timestamp(),0)).unwrap(); let tree_id={ let mut tree_index = repo.index()?; tree_index.add_all([config.output_folder.as_path()].iter(),git2::IndexAddOption::DEFAULT,None)?; match tree_index.update_all([config.output_folder.as_path()].iter(),None){ Ok(_)=>(), Err(e)=>println!("tree_index.update_all error: {}",e), } tree_index.write()?; tree_index.write_tree()? }; let tree=repo.find_tree(tree_id)?; let mut parents=Vec::new(); match repo.head(){ Ok(reference)=>{ let commit=reference.peel_to_commit()?; //test tree against commit tree to see if there is any changes let commit_tree=commit.tree()?; let diff=repo.diff_tree_to_tree(Some(&commit_tree),Some(&tree),None)?; if diff.deltas().count()==0{ println!("no changes"); return Ok(()); } parents.push(commit); }, Err(e)=>println!("repo head error {:?}",e), }; repo.commit( Some("HEAD"),//update_ref &sig,//author &sig,//commiter &format!("v{}", asset_version.assetVersionNumber),//message &tree,//tree (basically files) parents.iter().collect::>>().as_slice(),//parents )?; //commit Ok(()) } struct DecompileHistoryConfig{ git_committer_name:String, git_committer_email:String, input_folder:std::path::PathBuf, style:DecompileStyle, output_folder:std::path::PathBuf, write_template:bool, write_models:bool, write_scripts:bool, } async fn decompile_history_into_git(config:DecompileHistoryConfig)->AResult<()>{ //use prexisting versions list let mut versions_path=config.input_folder.clone(); versions_path.push("versions.json"); let asset_list:Vec=serde_json::from_reader(std::fs::File::open(versions_path)?)?; let repo=git2::Repository::init(config.output_folder.clone())?; //decompile all versions futures::stream::iter(asset_list.into_iter() .map(|asset_version|{ let mut file_path=config.input_folder.clone(); tokio::task::spawn_blocking(move||{ file_path.push(format!("{}_v{}.rbxl",asset_version.assetId,asset_version.assetVersionNumber)); let file=std::fs::File::open(file_path)?; let contents=generate_decompiled_context(file)?; Ok::<_,anyhow::Error>((asset_version,contents)) }) })) .buffered(CONCURRENT_DECODE) .for_each(|join_handle_result|async{ match write_commit(WriteCommitConfig{ git_committer_name:config.git_committer_name.clone(), git_committer_email:config.git_committer_email.clone(), style:config.style, output_folder:config.output_folder.clone(), write_template:config.write_template, write_models:config.write_models, write_scripts:config.write_scripts, },join_handle_result,&repo).await{ Ok(())=>(), Err(e)=>println!("decompile/write/commit error: {}",e), } }).await; Ok(()) } struct DownloadAndDecompileHistoryConfig{ cookie:String, asset_id:AssetID, git_committer_name:String, git_committer_email:String, style:DecompileStyle, output_folder:std::path::PathBuf, write_template:bool, write_models:bool, write_scripts:bool, } async fn download_and_decompile_history_into_git(config:DownloadAndDecompileHistoryConfig)->AResult<()>{ let client=reqwest::Client::new(); //poll paged list of all asset versions let asset_list=get_version_history(&client,&config.cookie.as_str(),config.asset_id).await?; let repo=git2::Repository::init(config.output_folder.clone())?; //download all versions let asset_id_string=config.asset_id.to_string(); futures::stream::iter(asset_list.into_iter() .map(|asset_version|{ let client=client.clone(); let cookie=config.cookie.clone(); let asset_id_str=asset_id_string.clone(); tokio::task::spawn(async move{ let resp=download_asset_version(&client,cookie.as_str(),asset_id_str.as_str(),asset_version.assetVersionNumber.to_string().as_str()).await?; let contents=match maybe_gzip_decode(std::io::Cursor::new(resp.bytes().await?))?{ ReaderType::GZip(readable)=>generate_decompiled_context(readable)?, ReaderType::Raw(readable)=>generate_decompiled_context(readable)?, }; Ok::<_,anyhow::Error>((asset_version,contents)) }) })) .buffered(CONCURRENT_DECODE) .for_each(|join_handle_result|async{ match write_commit(WriteCommitConfig{ style:config.style, git_committer_name:config.git_committer_name.clone(), git_committer_email:config.git_committer_email.clone(), output_folder:config.output_folder.clone(), write_template:config.write_template, write_models:config.write_models, write_scripts:config.write_scripts, },join_handle_result,&repo).await{ Ok(())=>(), Err(e)=>println!("download/unzip/decompile/write/commit error: {}",e), } }).await; Ok(()) } fn compile(_folder:std::path::PathBuf,_file:std::path::PathBuf)->AResult<()>{ Ok(()) }