use std::{io::Read,path::PathBuf}; use clap::{Args,Parser,Subcommand}; use anyhow::Result as AResult; use futures::StreamExt; use rbx_dom_weak::types::Ref; use tokio::io::AsyncReadExt; use rbx_asset::context::{RobloxContext,InventoryItem,AssetVersion}; type AssetID=u64; type AssetIDFileMap=Vec<(AssetID,PathBuf)>; const CONCURRENT_DECODE:usize=8; const CONCURRENT_REQUESTS:usize=32; #[derive(Parser)] #[command(author,version,about,long_about=None)] #[command(propagate_version = true)] struct Cli{ #[command(subcommand)] command:Commands, } #[derive(Subcommand)] enum Commands{ DownloadHistory(DownloadHistorySubcommand), Download(DownloadSubcommand), DownloadGroupInventoryJson(DownloadGroupInventoryJsonSubcommand), Create(CreateSubcommand), Upload(UploadSubcommand), Compile(CompileSubcommand), Decompile(DecompileSubcommand), DecompileHistoryIntoGit(DecompileHistoryIntoGitSubcommand), DownloadAndDecompileHistoryIntoGit(DownloadAndDecompileHistoryIntoGitSubcommand), } #[derive(Args)] struct DownloadHistorySubcommand{ #[arg(long)] asset_id:AssetID, #[arg(long)] cookie_type:CookieType, #[arg(long)] cookie:String, #[arg(long)] output_folder:Option, #[arg(long)] continue_from_versions:Option, #[arg(long)] start_version:Option, #[arg(long)] end_version:Option, } #[derive(Args)] struct DownloadSubcommand{ #[arg(long)] cookie_type:CookieType, #[arg(long)] cookie:String, #[arg(long)] output_folder:Option, #[arg(required=true)] asset_ids:Vec, } #[derive(Args)] struct DownloadGroupInventoryJsonSubcommand{ #[arg(long)] cookie_type:CookieType, #[arg(long)] cookie:String, #[arg(long)] output_folder:Option, #[arg(long)] group:u64, } #[derive(Args)] struct CreateSubcommand{ #[arg(long)] cookie_type:CookieType, #[arg(long)] cookie:String, #[arg(long)] model_name:String, #[arg(long)] description:Option, #[arg(long)] input_file:PathBuf, #[arg(long)] group:Option, #[arg(long)] free_model:Option, #[arg(long)] allow_comments:Option, } #[derive(Args)] struct UploadSubcommand{ #[arg(long)] asset_id:AssetID, #[arg(long)] cookie_type:CookieType, #[arg(long)] cookie:String, #[arg(long)] input_file:PathBuf, #[arg(long)] group:Option, } #[derive(Args)] struct CompileSubcommand{ #[arg(long)] input_folder:Option, #[arg(long)] output_file:PathBuf, #[arg(long)] style:Option, #[arg(long)] template:Option, } #[derive(Args)] struct DecompileSubcommand{ #[arg(long)] input_file:PathBuf, #[arg(long)] output_folder:Option, #[arg(long)] style:DecompileStyle, #[arg(long)] write_template:Option, #[arg(long)] write_models:Option, #[arg(long)] write_scripts:Option, } #[derive(Args)] struct DecompileHistoryIntoGitSubcommand{ #[arg(long)] input_folder:PathBuf, //currently output folder must be the current folder due to git2 limitations //output_folder:cli.output.unwrap(), #[arg(long)] style:DecompileStyle, #[arg(long)] git_committer_name:String, #[arg(long)] git_committer_email:String, #[arg(long)] write_template:Option, #[arg(long)] write_models:Option, #[arg(long)] write_scripts:Option, } #[derive(Args)] struct DownloadAndDecompileHistoryIntoGitSubcommand{ #[arg(long)] asset_id:AssetID, #[arg(long)] cookie_type:CookieType, #[arg(long)] cookie:String, //currently output folder must be the current folder due to git2 limitations //output_folder:cli.output.unwrap(), #[arg(long)] style:DecompileStyle, #[arg(long)] git_committer_name:String, #[arg(long)] git_committer_email:String, #[arg(long)] write_template:Option, #[arg(long)] write_models:Option, #[arg(long)] write_scripts:Option, } #[derive(Clone,clap::ValueEnum)] enum CookieType{ Literal, Environment, File, } #[derive(Clone,Copy,Debug,clap::ValueEnum)] pub enum DecompileStyle{ Rox, Rojo, RoxRojo, } impl DecompileStyle{ fn rox(&self)->rox_compiler::types::DecompileStyle{ match self{ DecompileStyle::Rox=>rox_compiler::types::DecompileStyle::Rox, DecompileStyle::Rojo=>rox_compiler::types::DecompileStyle::Rojo, DecompileStyle::RoxRojo=>rox_compiler::types::DecompileStyle::RoxRojo, } } } #[tokio::main] async fn main()->AResult<()>{ let cli=Cli::parse(); match cli.command{ Commands::DownloadHistory(subcommand)=>download_history(DownloadHistoryConfig{ continue_from_versions:subcommand.continue_from_versions.unwrap_or(false), end_version:subcommand.end_version, start_version:subcommand.start_version.unwrap_or(0), output_folder:subcommand.output_folder.unwrap_or_else(||std::env::current_dir().unwrap()), cookie:Cookie::from_type(subcommand.cookie_type,subcommand.cookie).await?.0, asset_id:subcommand.asset_id, }).await, Commands::Download(subcommand)=>{ let output_folder=subcommand.output_folder.unwrap_or_else(||std::env::current_dir().unwrap()); download_list( Cookie::from_type(subcommand.cookie_type,subcommand.cookie).await?.0, subcommand.asset_ids.into_iter().map(|asset_id|{ let mut path=output_folder.clone(); path.push(asset_id.to_string()); (asset_id,path) }).collect() ).await }, Commands::DownloadGroupInventoryJson(subcommand)=>download_group_inventory_json( Cookie::from_type(subcommand.cookie_type,subcommand.cookie).await?.0, subcommand.group, subcommand.output_folder.unwrap_or_else(||std::env::current_dir().unwrap()), ).await, Commands::Create(subcommand)=>create(CreateConfig{ cookie:Cookie::from_type(subcommand.cookie_type,subcommand.cookie).await?.0, group:subcommand.group, input_file:subcommand.input_file, model_name:subcommand.model_name, description:subcommand.description.unwrap_or_else(||String::with_capacity(0)), free_model:subcommand.free_model.unwrap_or(false), allow_comments:subcommand.allow_comments.unwrap_or(false), }).await, Commands::Upload(subcommand)=>upload_list( Cookie::from_type(subcommand.cookie_type,subcommand.cookie).await?.0, subcommand.group, vec![(subcommand.asset_id,subcommand.input_file)] ).await, Commands::Compile(subcommand)=>compile(CompileConfig{ input_folder:subcommand.input_folder.unwrap_or_else(||std::env::current_dir().unwrap()), output_file:subcommand.output_file, template:subcommand.template, style:subcommand.style, }).await, Commands::Decompile(subcommand)=>decompile(DecompileConfig{ style:subcommand.style, input_file:subcommand.input_file, output_folder:subcommand.output_folder.unwrap_or_else(||std::env::current_dir().unwrap()), write_template:subcommand.write_template.unwrap_or(false), write_models:subcommand.write_models.unwrap_or(false), write_scripts:subcommand.write_scripts.unwrap_or(true), }).await, Commands::DecompileHistoryIntoGit(subcommand)=>decompile_history_into_git(DecompileHistoryConfig{ git_committer_name:subcommand.git_committer_name, git_committer_email:subcommand.git_committer_email, input_folder:subcommand.input_folder, output_folder:std::env::current_dir()?, style:subcommand.style, write_template:subcommand.write_template.unwrap_or(false), write_models:subcommand.write_models.unwrap_or(false), write_scripts:subcommand.write_scripts.unwrap_or(true), }).await, Commands::DownloadAndDecompileHistoryIntoGit(subcommand)=>download_and_decompile_history_into_git(DownloadAndDecompileHistoryConfig{ git_committer_name:subcommand.git_committer_name, git_committer_email:subcommand.git_committer_email, cookie:Cookie::from_type(subcommand.cookie_type,subcommand.cookie).await?.0, asset_id:subcommand.asset_id, output_folder:std::env::current_dir()?, style:subcommand.style, write_template:subcommand.write_template.unwrap_or(false), write_models:subcommand.write_models.unwrap_or(false), write_scripts:subcommand.write_scripts.unwrap_or(true), }).await, } } struct Cookie(String); impl Cookie{ async fn from_type(cookie_type:CookieType,cookie_string:String)->AResult{ Ok(Self(format!(".ROBLOSECURITY={}",match cookie_type{ CookieType::Literal=>cookie_string, CookieType::Environment=>std::env::var(cookie_string)?, CookieType::File=>tokio::fs::read_to_string(cookie_string).await?, }))) } } struct CreateConfig{ cookie:String, model_name:String, description:String, input_file:PathBuf, group:Option, free_model:bool, allow_comments:bool, } async fn create(config:CreateConfig)->AResult<()>{ let resp=RobloxContext::new(config.cookie) .create(rbx_asset::context::CreateRequest{ name:config.model_name, description:config.description, ispublic:config.free_model, allowComments:config.allow_comments, groupId:config.group, },tokio::fs::read(config.input_file).await?).await?; println!("UploadResponse={:?}",resp); Ok(()) } async fn upload_list(cookie:String,group:Option,asset_id_file_map:AssetIDFileMap)->AResult<()>{ let context=RobloxContext::new(cookie); //this is calling map on the vec because the closure produces an iterator of futures futures::stream::iter(asset_id_file_map.into_iter() .map(|(asset_id,file)|{ let context=&context; async move{ Ok((asset_id,context.upload(rbx_asset::context::UploadRequest{ assetid:asset_id, name:None, description:None, ispublic:None, allowComments:None, groupId:group, },tokio::fs::read(file).await?).await?)) } })) .buffer_unordered(CONCURRENT_REQUESTS) .for_each(|b:AResult<_>|async{ match b{ Ok((asset_id,body))=>{ println!("asset_id={} UploadResponse={:?}",asset_id,body); }, Err(e)=>eprintln!("ul error: {}",e), } }).await; Ok(()) } async fn download_list(cookie:String,asset_id_file_map:AssetIDFileMap)->AResult<()>{ let context=RobloxContext::new(cookie); futures::stream::iter(asset_id_file_map.into_iter() .map(|(asset_id,file)|{ let context=&context; async move{ Ok((file,context.download(rbx_asset::context::DownloadRequest{asset_id,version:None}).await?)) } })) .buffer_unordered(CONCURRENT_REQUESTS) .for_each(|b:AResult<_>|async{ match b{ Ok((dest,data))=>{ match tokio::fs::write(dest,data).await{ Err(e)=>eprintln!("fs error: {}",e), _=>(), } }, Err(e)=>eprintln!("dl error: {}",e), } }).await; Ok(()) } async fn get_inventory_pages(context:&RobloxContext,group:u64)->AResult>{ let mut cursor:Option=None; let mut asset_list=Vec::new(); loop{ let mut page=context.inventory_page(rbx_asset::context::InventoryPageRequest{group,cursor}).await?; asset_list.append(&mut page.data); if page.nextPageCursor.is_none(){ break; } cursor=page.nextPageCursor; } Ok(asset_list) } async fn download_group_inventory_json(cookie:String,group:u64,output_folder:PathBuf)->AResult<()>{ let context=RobloxContext::new(cookie); let item_list=get_inventory_pages(&context,group).await?; let mut path=output_folder.clone(); path.set_file_name("versions.json"); tokio::fs::write(path,serde_json::to_string(&item_list)?).await?; Ok(()) } async fn get_version_history(context:&RobloxContext,asset_id:AssetID)->AResult>{ let mut cursor:Option=None; let mut asset_list=Vec::new(); loop{ let mut page=context.history_page(rbx_asset::context::HistoryPageRequest{asset_id,cursor}).await?; asset_list.append(&mut page.data); if page.nextPageCursor.is_none(){ break; } cursor=page.nextPageCursor; } asset_list.sort_by(|a,b|a.assetVersionNumber.cmp(&b.assetVersionNumber)); Ok(asset_list) } struct DownloadHistoryConfig{ continue_from_versions:bool, end_version:Option, start_version:u64, output_folder:PathBuf, cookie:String, asset_id:AssetID, } async fn download_history(mut config:DownloadHistoryConfig)->AResult<()>{ let mut asset_list_contents=std::collections::HashSet::new(); let mut asset_list:Vec=Vec::new(); if config.end_version.is_none()&&config.continue_from_versions{ //load prexisting versions list let mut versions_path=config.output_folder.clone(); versions_path.push("versions.json"); match std::fs::File::open(versions_path){ Ok(versions_file)=>asset_list.append(&mut serde_json::from_reader(versions_file)?), Err(e)=>match e.kind(){ std::io::ErrorKind::NotFound=>Err(anyhow::Error::msg("Cannot continue from versions.json - file does not exist"))?, _=>Err(e)?, } } //write down which versions are contained for asset_version in &asset_list{ asset_list_contents.insert(asset_version.assetVersionNumber); } //find the highest number match asset_list.iter().map(|asset_version|asset_version.assetVersionNumber).max(){ Some(max)=>{ //count down contiguously until a number is missing for i in (1..=max).rev(){ if !asset_list_contents.contains(&i){ //that is end_version config.end_version=Some(i); break; } } //if all versions are contained, set start_version to the max + 1 if config.end_version.is_none(){ config.start_version=max+1; } }, None=>Err(anyhow::Error::msg("Cannot continue from versions.json - there are no previous versions"))?, } } let context=RobloxContext::new(config.cookie); //limit concurrent downloads let mut join_set=tokio::task::JoinSet::new(); //poll paged list of all asset versions let mut cursor:Option=None; loop{ let mut page=context.history_page(rbx_asset::context::HistoryPageRequest{asset_id:config.asset_id,cursor}).await?; let context=&context; let output_folder=config.output_folder.clone(); let data=&page.data; let asset_list_contents=&asset_list_contents; let join_set=&mut join_set; let error_catcher=||async move{ let mut cancel_paging=false; for asset_version in data{ let version_number=asset_version.assetVersionNumber; //skip assets beyond specified end_version if config.end_version.is_some_and(|v|v(()) }); } Ok::<_,anyhow::Error>(cancel_paging) }; let cancel_paging=match error_catcher().await{ Ok(cancel)=>cancel, Err(e)=>{ println!("download error: {}",e); //cancel download and write versions true }, }; if page.nextPageCursor.is_none()||cancel_paging{ for asset_version in page.data.into_iter(){ if !(asset_list_contents.contains(&asset_version.assetVersionNumber) ||config.end_version.is_some_and(|v|v(input:R)->AResult{ let mut buf=std::io::BufReader::new(input); let peek=std::io::BufRead::fill_buf(&mut buf)?; match &peek[0..4]{ b"{ match &peek[4..8]{ b"lox!"=>rbx_binary::from_reader(buf).map_err(anyhow::Error::msg), b"lox "=>rbx_xml::from_reader_default(buf).map_err(anyhow::Error::msg), other=>Err(anyhow::Error::msg(format!("Unknown Roblox file type {:?}",other))), } }, _=>Err(anyhow::Error::msg("unsupported file type")), } } struct DecompileConfig{ style:DecompileStyle, input_file:PathBuf, output_folder:PathBuf, write_template:bool, write_models:bool, write_scripts:bool, } async fn decompile(config:DecompileConfig)->AResult<()>{ //rules: //Class Script|LocalScript|ModuleScript->$Name.lua //Class Model->$Name.rbxmx //overrides.json per-folder [Override{name,class}] //Everything else goes into template.rbxlx //read file let context=generate_decompiled_context(std::io::BufReader::new(std::fs::File::open(config.input_file)?))?; //generate folders, models, and scripts //delete models and scripts from dom write_files(WriteConfig{ style:config.style, output_folder:config.output_folder, write_template:config.write_template, write_models:config.write_models, write_scripts:config.write_scripts, },context).await?; Ok(()) } struct WriteCommitConfig{ git_committer_name:String, git_committer_email:String, output_folder:PathBuf, style:DecompileStyle, write_template:bool, write_models:bool, write_scripts:bool, } async fn write_commit(config:WriteCommitConfig,b:Result,tokio::task::JoinError>,repo:&git2::Repository)->AResult<()>{ let (asset_version,context)=b??; println!("writing files for version {}",asset_version.assetVersionNumber); //clean output dir if config.write_models||config.write_scripts{ let mut src=config.output_folder.clone(); src.push("src"); match std::fs::remove_dir_all(src){ Ok(())=>(), Err(e)=>println!("remove_dir_all src failed {}",e), } } if config.write_template{ let mut template=config.output_folder.clone(); template.push("template.rbxlx"); match std::fs::remove_file(template){ Ok(())=>(), Err(e)=>println!("remove_file template.rbxlx failed {}",e), } } //write files write_files(WriteConfig{ style:config.style, output_folder:config.output_folder.clone(), write_template:config.write_template, write_models:config.write_models, write_scripts:config.write_scripts, },context).await?; let date=asset_version.created; //let sig=repo.signature()?; //this pulls default name and email let sig=git2::Signature::new(config.git_committer_name.as_str(),config.git_committer_email.as_str(),&git2::Time::new(date.timestamp(),0)).unwrap(); let tree_id={ let mut tree_index = repo.index()?; match tree_index.add_all(std::iter::once(config.output_folder.as_path()),git2::IndexAddOption::DEFAULT,None){ Ok(_)=>(), Err(e)=>println!("tree_index.add_all error: {}",e), } match tree_index.update_all(std::iter::once(config.output_folder.as_path()),None){ Ok(_)=>(), Err(e)=>println!("tree_index.update_all error: {}",e), } tree_index.write()?; tree_index.write_tree()? }; let tree=repo.find_tree(tree_id)?; let mut parents=Vec::new(); match repo.head(){ Ok(reference)=>{ let commit=reference.peel_to_commit()?; //test tree against commit tree to see if there is any changes let commit_tree=commit.tree()?; let diff=repo.diff_tree_to_tree(Some(&commit_tree),Some(&tree),None)?; if diff.get_delta(0).is_none(){ println!("no changes"); return Ok(()); } parents.push(commit); }, Err(e)=>println!("repo head error {:?}",e), }; repo.commit( Some("HEAD"),//update_ref &sig,//author &sig,//commiter &format!("v{}", asset_version.assetVersionNumber),//message &tree,//tree (basically files) parents.iter().collect::>>().as_slice(),//parents )?; //commit Ok(()) } struct DecompileHistoryConfig{ git_committer_name:String, git_committer_email:String, input_folder:PathBuf, style:DecompileStyle, output_folder:PathBuf, write_template:bool, write_models:bool, write_scripts:bool, } async fn decompile_history_into_git(config:DecompileHistoryConfig)->AResult<()>{ //use prexisting versions list let mut versions_path=config.input_folder.clone(); versions_path.push("versions.json"); let asset_list:Vec=serde_json::from_reader(std::fs::File::open(versions_path)?)?; let repo=git2::Repository::init(config.output_folder.as_path())?; //decompile all versions futures::stream::iter(asset_list.into_iter() .map(|asset_version|{ let mut file_path=config.input_folder.clone(); tokio::task::spawn_blocking(move||{ file_path.push(format!("{}_v{}.rbxl",asset_version.assetId,asset_version.assetVersionNumber)); let file=std::fs::File::open(file_path)?; let contents=generate_decompiled_context(file)?; Ok::<_,anyhow::Error>((asset_version,contents)) }) })) .buffered(CONCURRENT_DECODE) .for_each(|join_handle_result|async{ match write_commit(WriteCommitConfig{ git_committer_name:config.git_committer_name.clone(), git_committer_email:config.git_committer_email.clone(), style:config.style, output_folder:config.output_folder.clone(), write_template:config.write_template, write_models:config.write_models, write_scripts:config.write_scripts, },join_handle_result,&repo).await{ Ok(())=>(), Err(e)=>println!("decompile/write/commit error: {}",e), } }).await; Ok(()) } struct DownloadAndDecompileHistoryConfig{ cookie:String, asset_id:AssetID, git_committer_name:String, git_committer_email:String, style:DecompileStyle, output_folder:PathBuf, write_template:bool, write_models:bool, write_scripts:bool, } async fn download_and_decompile_history_into_git(config:DownloadAndDecompileHistoryConfig)->AResult<()>{ let context=RobloxContext::new(config.cookie); //poll paged list of all asset versions let asset_list=get_version_history(&context,config.asset_id).await?; let repo=git2::Repository::init(config.output_folder.clone())?; //download all versions let asset_id=config.asset_id; futures::stream::iter(asset_list.into_iter() .map(|asset_version|{ let context=context.clone(); tokio::task::spawn(async move{ let file=context.download(rbx_asset::context::DownloadRequest{asset_id,version:Some(asset_version.assetVersionNumber)}).await?; Ok::<_,anyhow::Error>((asset_version,generate_decompiled_context(std::io::Cursor::new(file))?)) }) })) .buffered(CONCURRENT_DECODE) .for_each(|join_handle_result|async{ match write_commit(WriteCommitConfig{ style:config.style, git_committer_name:config.git_committer_name.clone(), git_committer_email:config.git_committer_email.clone(), output_folder:config.output_folder.clone(), write_template:config.write_template, write_models:config.write_models, write_scripts:config.write_scripts, },join_handle_result,&repo).await{ Ok(())=>(), Err(e)=>println!("download/unzip/decompile/write/commit error: {}",e), } }).await; Ok(()) } struct CompileConfig{ input_folder:PathBuf, output_file:PathBuf, template:Option, style:Option, } async fn compile(config:CompileConfig)->AResult<()>{ //basically decompile in reverse order //load template dom let mut dom=match config.template{ //mr dom doesn't like tokio files Some(template_path)=>load_dom(std::io::BufReader::new(std::fs::File::open(template_path)?))?, None=>rbx_dom_weak::WeakDom::default(), }; //hack to traverse root folder as the root object dom.root_mut().name="src".to_owned(); something_something_dom_write(&mut dom).await?; let mut output_place=config.output_file.clone(); if output_place.extension().is_none()&&tokio::fs::try_exists(output_place.as_path()).await?{ output_place.push("place.rbxl"); } let output=std::io::BufWriter::new(std::fs::File::create(output_place)?); //write inner objects rbx_binary::to_writer(output,&dom,dom.root().children())?; Ok(()) }