Compare commits
2 Commits
feature/gi
...
quat-histo
| Author | SHA1 | Date | |
|---|---|---|---|
| 32a9d75a62 | |||
| 1712e0c7e3 |
78
Cargo.lock
generated
78
Cargo.lock
generated
@ -116,7 +116,6 @@ dependencies = [
|
||||
"clap",
|
||||
"flate2",
|
||||
"futures",
|
||||
"git2",
|
||||
"lazy-regex",
|
||||
"pollster",
|
||||
"rbx_binary",
|
||||
@ -211,7 +210,6 @@ version = "1.0.83"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0"
|
||||
dependencies = [
|
||||
"jobserver",
|
||||
"libc",
|
||||
]
|
||||
|
||||
@ -527,21 +525,6 @@ version = "0.28.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253"
|
||||
|
||||
[[package]]
|
||||
name = "git2"
|
||||
version = "0.18.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fbf97ba92db08df386e10c8ede66a2a0369bd277090afd8710e19e38de9ec0cd"
|
||||
dependencies = [
|
||||
"bitflags 2.4.1",
|
||||
"libc",
|
||||
"libgit2-sys",
|
||||
"log",
|
||||
"openssl-probe",
|
||||
"openssl-sys",
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "h2"
|
||||
version = "0.3.22"
|
||||
@ -726,15 +709,6 @@ version = "1.0.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c"
|
||||
|
||||
[[package]]
|
||||
name = "jobserver"
|
||||
version = "0.1.27"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8c37f63953c4c63420ed5fd3d6d398c719489b9f872b9fa683262f8edd363c7d"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "js-sys"
|
||||
version = "0.3.66"
|
||||
@ -779,46 +753,6 @@ version = "0.2.151"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4"
|
||||
|
||||
[[package]]
|
||||
name = "libgit2-sys"
|
||||
version = "0.16.1+1.7.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f2a2bb3680b094add03bb3732ec520ece34da31a8cd2d633d1389d0f0fb60d0c"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"libc",
|
||||
"libssh2-sys",
|
||||
"libz-sys",
|
||||
"openssl-sys",
|
||||
"pkg-config",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libssh2-sys"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2dc8a030b787e2119a731f1951d6a773e2280c660f8ec4b0f5e1505a386e71ee"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"libc",
|
||||
"libz-sys",
|
||||
"openssl-sys",
|
||||
"pkg-config",
|
||||
"vcpkg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libz-sys"
|
||||
version = "1.1.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "295c17e837573c8c821dbaeb3cceb3d745ad082f7572191409e69cbc1b3fd050"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"libc",
|
||||
"pkg-config",
|
||||
"vcpkg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "linux-raw-sys"
|
||||
version = "0.4.12"
|
||||
@ -1035,9 +969,9 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.76"
|
||||
version = "1.0.75"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "95fc56cda0b5c3325f5fbbd7ff9fda9e02bb00bb3dac51252d2f1bfa1cb8cc8c"
|
||||
checksum = "907a61bd0f64c2f29cd1cf1dc34d05176426a3f504a78010f08416ddb7b13708"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
@ -1352,18 +1286,18 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.195"
|
||||
version = "1.0.194"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "63261df402c67811e9ac6def069e4786148c4563f4b50fd4bf30aa370d626b02"
|
||||
checksum = "0b114498256798c94a0689e1a15fec6005dee8ac1f41de56404b67afc2a4b773"
|
||||
dependencies = [
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.195"
|
||||
version = "1.0.194"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "46fe8f8603d81ba86327b23a2e9cdf49e1255fb94a4c5f297f6ee0547178ea2c"
|
||||
checksum = "a3385e45322e8f9931410f01b3031ec534c3947d0e94c18049af4d9f9907d4e0"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
|
||||
@ -11,7 +11,6 @@ chrono = { version = "0.4.31", features = ["serde"] }
|
||||
clap = { version = "4.4.2", features = ["derive"] }
|
||||
flate2 = "1.0.28"
|
||||
futures = "0.3.30"
|
||||
git2 = "0.18.1"
|
||||
lazy-regex = "3.1.0"
|
||||
pollster = "0.3.0"
|
||||
rbx_binary = "0.7.1"
|
||||
@ -19,7 +18,7 @@ rbx_dom_weak = "2.5.0"
|
||||
rbx_reflection_database = "0.2.7"
|
||||
rbx_xml = "0.13.1"
|
||||
reqwest = { version = "0.11.23", features = ["cookies", "json"] }
|
||||
serde = { version = "1.0.195", features = ["derive"] }
|
||||
serde = { version = "1.0.194", features = ["derive"] }
|
||||
serde_json = "1.0.111"
|
||||
tokio = { version = "1.35.1", features = ["macros", "rt-multi-thread", "fs"] }
|
||||
|
||||
|
||||
742
src/main.rs
742
src/main.rs
@ -1,12 +1,26 @@
|
||||
use std::io::Read;
|
||||
use std::io::{Read,Seek};
|
||||
use clap::{Args,Parser,Subcommand};
|
||||
use anyhow::Result as AResult;
|
||||
use futures::StreamExt;
|
||||
use rbx_dom_weak::types::Ref;
|
||||
use serde::Serialize;
|
||||
|
||||
type AssetID=u64;
|
||||
type AssetIDFileMap=Vec<(AssetID,std::path::PathBuf)>;
|
||||
const CONCURRENT_REQUESTS:usize=32;
|
||||
const CONCURRENT_REQUESTS:usize=8;
|
||||
|
||||
/// Parse a single key-value pair
|
||||
fn parse_key_val<T,U>(s:&str)->AResult<(T,U)>
|
||||
where
|
||||
T:std::str::FromStr,
|
||||
T::Err:std::error::Error+Send+Sync+'static,
|
||||
U:std::str::FromStr,
|
||||
U::Err:std::error::Error+Send+Sync+'static,
|
||||
{
|
||||
let pos=s
|
||||
.find('=')
|
||||
.ok_or_else(||anyhow::Error::msg(format!("invalid KEY=value: no `=` found in `{s}`")))?;
|
||||
Ok((s[..pos].parse()?,s[pos+1..].parse()?))
|
||||
}
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(author,version,about,long_about=None)]
|
||||
@ -22,21 +36,8 @@ struct Cli{
|
||||
#[arg(long)]
|
||||
cookie_file:Option<std::path::PathBuf>,
|
||||
|
||||
#[arg(long)]
|
||||
no_models:Option<bool>,
|
||||
#[arg(long)]
|
||||
no_scripts:Option<bool>,
|
||||
#[arg(long)]
|
||||
no_template:Option<bool>,
|
||||
|
||||
#[arg(long)]
|
||||
asset_id:Option<AssetID>,
|
||||
|
||||
#[arg(short,long)]
|
||||
input:Option<std::path::PathBuf>,
|
||||
|
||||
#[arg(short,long)]
|
||||
output:Option<std::path::PathBuf>,
|
||||
#[arg(long,value_parser=parse_key_val::<AssetID,std::path::PathBuf>)]
|
||||
asset_id:(AssetID,std::path::PathBuf),
|
||||
|
||||
#[command(subcommand)]
|
||||
command:Commands,
|
||||
@ -44,12 +45,8 @@ struct Cli{
|
||||
|
||||
#[derive(Subcommand)]
|
||||
enum Commands{
|
||||
DownloadHistory,
|
||||
Download,
|
||||
Upload,
|
||||
Compile,
|
||||
Decompile,
|
||||
DecompileHistoryIntoGit,
|
||||
}
|
||||
|
||||
#[derive(Args)]
|
||||
@ -81,47 +78,71 @@ async fn main()->AResult<()>{
|
||||
|
||||
let cookie_enum={
|
||||
match (cli.cookie_literal,cli.cookie_env,cli.cookie_file){
|
||||
(Some(literal),None,None)=>Some(Cookie::Literal(literal)),
|
||||
(None,Some(env_var),None)=>Some(Cookie::Environment(env_var)),
|
||||
(None,None,Some(path))=>Some(Cookie::File(path)),
|
||||
(None,None,None)=>None,
|
||||
_=>return Err(anyhow::Error::msg("Cookie was specified multiple times.")),
|
||||
(Some(literal),None,None)=>Cookie::Literal(literal),
|
||||
(None,Some(env_var),None)=>Cookie::Environment(env_var),
|
||||
(None,None,Some(path))=>Cookie::File(path),
|
||||
_=>return Err(anyhow::Error::msg("Cookie was not specified or was specified multiple times.")),
|
||||
}
|
||||
};
|
||||
let cookie=match cookie_enum{
|
||||
Some(c)=>Some(format!(".ROBLOSECURITY={}",match c{
|
||||
Cookie::Literal(s)=>s,
|
||||
Cookie::Environment(var)=>std::env::var(var)?,
|
||||
Cookie::File(path)=>tokio::fs::read_to_string(path).await?,
|
||||
})),
|
||||
None=>None,
|
||||
};
|
||||
let cookie=format!(".ROBLOSECURITY={}",match cookie_enum{
|
||||
Cookie::Literal(s)=>s,
|
||||
Cookie::Environment(var)=>std::env::var(var)?,
|
||||
Cookie::File(path)=>tokio::fs::read_to_string(path).await?.trim().to_string(),
|
||||
});
|
||||
|
||||
match cli.command{
|
||||
Commands::DownloadHistory=>download_history(DownloadHistoryConfig{
|
||||
output_folder:cli.output.unwrap(),
|
||||
cookie:cookie.unwrap(),
|
||||
asset_id:cli.asset_id.unwrap(),
|
||||
}).await,
|
||||
Commands::Download=>download_list(cookie.unwrap(),vec![(cli.asset_id.unwrap(),cli.output.unwrap())]).await,
|
||||
Commands::Upload=>upload_list(cookie.unwrap(),cli.group,vec![(cli.asset_id.unwrap(),cli.output.unwrap())]).await,
|
||||
Commands::Compile=>compile(cli.input.unwrap(),cli.output.unwrap()),
|
||||
Commands::Decompile=>decompile(DecompileConfig{
|
||||
input_file:cli.input.unwrap(),
|
||||
output_folder:cli.output.unwrap(),
|
||||
write_template:!cli.no_template.unwrap_or(false),
|
||||
write_models:!cli.no_models.unwrap_or(false),
|
||||
write_scripts:!cli.no_scripts.unwrap_or(false),
|
||||
}),
|
||||
Commands::DecompileHistoryIntoGit=>decompile_history_into_git(DecompileHistoryConfig{
|
||||
cookie:cookie.unwrap(),
|
||||
asset_id:cli.asset_id.unwrap(),
|
||||
output_folder:cli.output.unwrap(),
|
||||
write_template:!cli.no_template.unwrap_or(false),
|
||||
write_models:!cli.no_models.unwrap_or(false),
|
||||
write_scripts:!cli.no_scripts.unwrap_or(false),
|
||||
}).await,
|
||||
}
|
||||
let group=match cli.group{
|
||||
Some(group_id)=>Owner::Group(group_id),
|
||||
None=>Owner::User,
|
||||
};
|
||||
{
|
||||
let client=reqwest::Client::new();
|
||||
let mut cursor:Option<String>=None;
|
||||
let mut asset_list=Vec::new();
|
||||
loop{
|
||||
let mut url=reqwest::Url::parse("https://develop.roblox.com/v1/assets/252877716/saved-versions")?;
|
||||
//url borrow scope
|
||||
{
|
||||
let mut query=url.query_pairs_mut();//borrow here
|
||||
//query.append_pair("sortOrder","Asc");
|
||||
//query.append_pair("limit","100");
|
||||
//query.append_pair("count","100");
|
||||
match &cursor{
|
||||
Some(next_page)=>{query.append_pair("cursor",next_page);}
|
||||
None=>(),
|
||||
}
|
||||
}
|
||||
println!("url={}",url);
|
||||
let resp=client.get(url)
|
||||
.header("Cookie",cookie.clone())
|
||||
.send().await?;
|
||||
println!("resp:{:?}",resp);
|
||||
match resp.json::<VersionPage>().await{
|
||||
Ok(mut page)=>{
|
||||
asset_list.append(&mut page.data);
|
||||
if page.nextPageCursor.is_none(){
|
||||
break;
|
||||
}
|
||||
cursor=page.nextPageCursor;
|
||||
},
|
||||
Err(e)=>panic!("error: {}",e),
|
||||
}
|
||||
}
|
||||
asset_list.sort_by(|a,b|a.assetVersionNumber.cmp(&b.assetVersionNumber));
|
||||
let mut path=std::path::PathBuf::new();
|
||||
path.set_file_name("versions.json");
|
||||
tokio::fs::write(path,serde_json::to_string(&asset_list)?).await?;
|
||||
};
|
||||
Ok(())
|
||||
|
||||
// match cli.command{
|
||||
// Commands::Download=>download_list(cookie,vec![cli.asset_id]).await,
|
||||
// Commands::Upload=>upload_list(cookie,group,vec![cli.asset_id]).await,
|
||||
// }
|
||||
}
|
||||
|
||||
enum Owner{
|
||||
Group(u64),
|
||||
User
|
||||
}
|
||||
|
||||
enum Cookie{
|
||||
@ -130,36 +151,39 @@ enum Cookie{
|
||||
File(std::path::PathBuf),
|
||||
}
|
||||
|
||||
enum ReaderType<R:Read>{
|
||||
GZip(flate2::read::GzDecoder<std::io::BufReader<R>>),
|
||||
Raw(std::io::BufReader<R>),
|
||||
enum ReaderType<'a,R:Read+Seek>{
|
||||
GZip(flate2::read::GzDecoder<&'a mut R>),
|
||||
Raw(&'a mut R),
|
||||
}
|
||||
|
||||
fn maybe_gzip_decode<R:Read>(input:R)->AResult<ReaderType<R>>{
|
||||
let mut buf=std::io::BufReader::new(input);
|
||||
let peek=std::io::BufRead::fill_buf(&mut buf)?;
|
||||
match &peek[0..2]{
|
||||
b"\x1f\x8b"=>Ok(ReaderType::GZip(flate2::read::GzDecoder::new(buf))),
|
||||
_=>Ok(ReaderType::Raw(buf)),
|
||||
fn maybe_gzip_decode<R:Read+Seek>(input:&mut R)->AResult<ReaderType<R>>{
|
||||
let mut first_2=[0u8;2];
|
||||
if let (Ok(()),Ok(()))=(std::io::Read::read_exact(input,&mut first_2),std::io::Seek::rewind(input)){
|
||||
match &first_2{
|
||||
b"\x1f\x8b"=>Ok(ReaderType::GZip(flate2::read::GzDecoder::new(input))),
|
||||
_=>Ok(ReaderType::Raw(input)),
|
||||
}
|
||||
}else{
|
||||
Err(anyhow::Error::msg("failed to peek"))
|
||||
}
|
||||
}
|
||||
|
||||
async fn upload_list(cookie:String,group:Option<u64>,asset_id_file_map:AssetIDFileMap)->AResult<()>{
|
||||
async fn upload_list(cookie:String,owner:Owner,asset_id_file_map:AssetIDFileMap)->AResult<()>{
|
||||
let client=reqwest::Client::new();
|
||||
futures::stream::iter(asset_id_file_map)
|
||||
.map(|(asset_id,file)|{
|
||||
let client=&client;
|
||||
let cookie=cookie.as_str();
|
||||
let group=&group;
|
||||
let owner=&owner;
|
||||
async move{
|
||||
let mut url=reqwest::Url::parse("https://data.roblox.com/Data/Upload.ashx?json=1&type=Model&genreTypeId=1")?;
|
||||
//url borrow scope
|
||||
{
|
||||
let mut query=url.query_pairs_mut();//borrow here
|
||||
query.append_pair("assetid",asset_id.to_string().as_str());
|
||||
match group{
|
||||
Some(group_id)=>{query.append_pair("groupId",group_id.to_string().as_str());},
|
||||
None=>(),
|
||||
match owner{
|
||||
Owner::Group(group_id)=>{query.append_pair("groupId",group_id.to_string().as_str());},
|
||||
Owner::User=>(),
|
||||
}
|
||||
}
|
||||
|
||||
@ -203,17 +227,30 @@ fn read_readable(mut readable:impl Read)->AResult<Vec<u8>>{
|
||||
Ok(contents)
|
||||
}
|
||||
|
||||
const BHOP_PLACEID:u64=252877716;
|
||||
|
||||
async fn download_list(cookie:String,asset_id_file_map:AssetIDFileMap)->AResult<()>{
|
||||
let client=reqwest::Client::new();
|
||||
futures::stream::iter(asset_id_file_map)
|
||||
.map(|(asset_id,file)|{
|
||||
futures::stream::iter(1..=1006)
|
||||
.map(|version_id|{
|
||||
let client=&client;
|
||||
let cookie=cookie.as_str();
|
||||
async move{
|
||||
let resp=client.get(format!("https://assetdelivery.roblox.com/v1/asset/?ID={}",asset_id))
|
||||
let mut url=reqwest::Url::parse("https://assetdelivery.roblox.com/v1/asset/")?;
|
||||
//url borrow scope
|
||||
{
|
||||
let mut query=url.query_pairs_mut();//borrow here
|
||||
query.append_pair("ID",BHOP_PLACEID.to_string().as_str());
|
||||
query.append_pair("version",version_id.to_string().as_str());
|
||||
}
|
||||
println!("url={}",url);
|
||||
let resp=client.get(url)
|
||||
.header("Cookie",cookie)
|
||||
.send().await?;
|
||||
Ok((file,resp.bytes().await?))
|
||||
|
||||
let mut path=std::path::PathBuf::new();
|
||||
path.set_file_name(BHOP_PLACEID.to_string()+"_v"+version_id.to_string().as_str()+".rbxl");
|
||||
Ok((path,resp.bytes().await?))
|
||||
}
|
||||
})
|
||||
.buffer_unordered(CONCURRENT_REQUESTS)
|
||||
@ -238,552 +275,3 @@ async fn download_list(cookie:String,asset_id_file_map:AssetIDFileMap)->AResult<
|
||||
}).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn get_version_history(client:&reqwest::Client,cookie:&str,asset_id:AssetID)->AResult<Vec<AssetVersion>>{
|
||||
let mut cursor:Option<String>=None;
|
||||
let mut asset_list=Vec::new();
|
||||
loop{
|
||||
let mut url=reqwest::Url::parse(format!("https://develop.roblox.com/v1/assets/{}/saved-versions",asset_id).as_str())?;
|
||||
//url borrow scope
|
||||
{
|
||||
let mut query=url.query_pairs_mut();//borrow here
|
||||
//query.append_pair("sortOrder","Asc");
|
||||
//query.append_pair("limit","100");
|
||||
//query.append_pair("count","100");
|
||||
match &cursor{
|
||||
Some(next_page)=>{query.append_pair("cursor",next_page);}
|
||||
None=>(),
|
||||
}
|
||||
}
|
||||
println!("page url={}",url);
|
||||
let resp=client.get(url)
|
||||
.header("Cookie",cookie)
|
||||
.send().await?;
|
||||
match resp.json::<VersionPage>().await{
|
||||
Ok(mut page)=>{
|
||||
asset_list.append(&mut page.data);
|
||||
if page.nextPageCursor.is_none(){
|
||||
break;
|
||||
}
|
||||
cursor=page.nextPageCursor;
|
||||
},
|
||||
Err(e)=>panic!("error: {}",e),
|
||||
}
|
||||
}
|
||||
asset_list.sort_by(|a,b|a.assetVersionNumber.cmp(&b.assetVersionNumber));
|
||||
Ok(asset_list)
|
||||
}
|
||||
|
||||
async fn download_asset_version(client:&reqwest::Client,cookie:&str,asset_id_str:&str,asset_version_str:&str)->AResult<reqwest::Response>{
|
||||
let mut url=reqwest::Url::parse("https://assetdelivery.roblox.com/v1/asset/")?;
|
||||
//url borrow scope
|
||||
{
|
||||
let mut query=url.query_pairs_mut();//borrow here
|
||||
query.append_pair("ID",asset_id_str);
|
||||
query.append_pair("version",asset_version_str);
|
||||
}
|
||||
println!("download url={}",url);
|
||||
for i in 0..8{
|
||||
let resp=client.get(url.clone())
|
||||
.header("Cookie",cookie)
|
||||
.send().await?;
|
||||
|
||||
if !resp.status().is_success(){
|
||||
println!("request {} failed",i);
|
||||
continue;
|
||||
}
|
||||
|
||||
return Ok(resp);
|
||||
}
|
||||
Err(anyhow::Error::msg("all requests failed"))
|
||||
}
|
||||
|
||||
struct DownloadHistoryConfig{
|
||||
output_folder:std::path::PathBuf,
|
||||
cookie:String,
|
||||
asset_id:AssetID,
|
||||
}
|
||||
|
||||
async fn download_history(config:DownloadHistoryConfig)->AResult<()>{
|
||||
let client=reqwest::Client::new();
|
||||
|
||||
//poll paged list of all asset versions
|
||||
let asset_list=get_version_history(&client,&config.cookie.as_str(),config.asset_id).await?;
|
||||
let mut path=config.output_folder.clone();
|
||||
path.set_file_name("versions.json");
|
||||
tokio::fs::write(path,serde_json::to_string(&asset_list)?).await?;
|
||||
|
||||
//download all versions
|
||||
let asset_id_string=config.asset_id.to_string();
|
||||
futures::stream::iter(asset_list)
|
||||
.map(|asset_version|{
|
||||
let client=&client;
|
||||
let cookie=config.cookie.as_str();
|
||||
let asset_id_str=asset_id_string.as_str();
|
||||
let output_folder=config.output_folder.clone();
|
||||
async move{
|
||||
let resp=download_asset_version(client,cookie,asset_id_str,asset_version.assetVersionNumber.to_string().as_str()).await?;
|
||||
let contents=match maybe_gzip_decode(std::io::Cursor::new(resp.bytes().await?))?{
|
||||
ReaderType::GZip(readable)=>read_readable(readable)?,
|
||||
ReaderType::Raw(readable)=>read_readable(readable)?,
|
||||
};
|
||||
|
||||
let mut path=output_folder;
|
||||
path.set_file_name(format!("{}_v{}.rbxl",config.asset_id,asset_version.assetVersionNumber));
|
||||
|
||||
Ok((path,contents))
|
||||
}
|
||||
})
|
||||
.buffer_unordered(CONCURRENT_REQUESTS)
|
||||
.for_each(|b:AResult<_>|async{
|
||||
match b{
|
||||
Ok((dest,data))=>{
|
||||
match tokio::fs::write(dest,data).await{
|
||||
Err(e)=>eprintln!("fs error: {}",e),
|
||||
_=>(),
|
||||
}
|
||||
},
|
||||
Err(e)=>eprintln!("dl error: {}",e),
|
||||
}
|
||||
}).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn load_dom<R:Read>(input:R)->AResult<rbx_dom_weak::WeakDom>{
|
||||
let mut buf=std::io::BufReader::new(input);
|
||||
let peek=std::io::BufRead::fill_buf(&mut buf)?;
|
||||
match &peek[0..4]{
|
||||
b"<rob"=>{
|
||||
match &peek[4..8]{
|
||||
b"lox!"=>return rbx_binary::from_reader(buf).map_err(anyhow::Error::msg),
|
||||
b"lox "=>return rbx_xml::from_reader_default(buf).map_err(anyhow::Error::msg),
|
||||
other=>Err(anyhow::Error::msg(format!("Unknown Roblox file type {:?}",other))),
|
||||
}
|
||||
},
|
||||
_=>Err(anyhow::Error::msg("unsupported file type")),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(PartialEq)]
|
||||
enum Class{
|
||||
Folder,
|
||||
ModuleScript,
|
||||
LocalScript,
|
||||
Script,
|
||||
Model,
|
||||
}
|
||||
|
||||
struct TreeNode{
|
||||
name:String,
|
||||
referent:Ref,
|
||||
parent:Ref,
|
||||
class:Class,
|
||||
children:Vec<Ref>,
|
||||
}
|
||||
impl TreeNode{
|
||||
fn new(name:String,referent:Ref,parent:Ref,class:Class)->Self{
|
||||
Self{
|
||||
name,
|
||||
referent,
|
||||
parent,
|
||||
class,
|
||||
children:Vec::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
enum TrimStackInstruction{
|
||||
Referent(Ref),
|
||||
IncrementScript,
|
||||
DecrementScript,
|
||||
}
|
||||
|
||||
enum WriteStackInstruction<'a>{
|
||||
Node(&'a TreeNode,u32),//(Node,NameTally)
|
||||
PushFolder(String),
|
||||
PopFolder,
|
||||
Destroy(Ref),
|
||||
}
|
||||
|
||||
#[derive(Default,serde::Deserialize,serde::Serialize)]
|
||||
struct PropertiesOverride{
|
||||
//Name:Option<String>,
|
||||
ClassName:Option<String>,
|
||||
}
|
||||
impl PropertiesOverride{
|
||||
fn is_some(&self)->bool{
|
||||
self.ClassName.is_some()
|
||||
}
|
||||
}
|
||||
|
||||
fn sanitize<'a>(s:&'a str)->std::borrow::Cow<'a,str>{
|
||||
lazy_regex::regex!(r"[^a-zA-Z0-9._-]").replace_all(s,"_")
|
||||
}
|
||||
|
||||
fn write_item(dom:&rbx_dom_weak::WeakDom,mut file:std::path::PathBuf,node:&TreeNode,node_name_override:&str,write_models:bool,write_scripts:bool)->AResult<()>{
|
||||
file.push(sanitize(node_name_override).as_ref());
|
||||
match node.class{
|
||||
Class::Folder=>(),
|
||||
Class::ModuleScript|Class::LocalScript|Class::Script=>{
|
||||
if !write_scripts{
|
||||
return Ok(())
|
||||
}
|
||||
assert!(file.set_extension("lua"),"could not set extension");
|
||||
assert!(dom.get_by_ref(node.referent).is_some_and(|item|{
|
||||
//TODO: delete disabled scripts
|
||||
if let Some(rbx_dom_weak::types::Variant::String(source))=item.properties.get("Source"){
|
||||
std::fs::write(file,source).is_ok()
|
||||
}else{false}
|
||||
}),"no string property or file failed to write");
|
||||
},
|
||||
Class::Model=>{
|
||||
if !write_models{
|
||||
return Ok(())
|
||||
}
|
||||
assert!(file.set_extension("rbxmx"));
|
||||
let output=std::io::BufWriter::new(std::fs::File::create(file)?);
|
||||
rbx_xml::to_writer_default(output,dom,&[node.referent])?;
|
||||
},
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
struct DecompiledContext{
|
||||
dom:rbx_dom_weak::WeakDom,
|
||||
tree_refs:std::collections::HashMap<rbx_dom_weak::types::Ref,TreeNode>,
|
||||
}
|
||||
|
||||
fn generate_decompiled_context<R:Read>(input:R)->AResult<DecompiledContext>{
|
||||
let dom=load_dom(input)?;
|
||||
|
||||
let mut tree_refs=std::collections::HashMap::new();
|
||||
tree_refs.insert(dom.root_ref(),TreeNode::new(
|
||||
"src".to_string(),
|
||||
dom.root_ref(),
|
||||
Ref::none(),
|
||||
Class::Folder
|
||||
));
|
||||
|
||||
//run rules
|
||||
let mut stack=vec![dom.root()];
|
||||
while let Some(item)=stack.pop(){
|
||||
let class=match item.class.as_str(){
|
||||
"ModuleScript"=>Class::ModuleScript,
|
||||
"LocalScript"=>Class::LocalScript,
|
||||
"Script"=>Class::Script,
|
||||
"Model"=>Class::Model,
|
||||
_=>Class::Folder,
|
||||
};
|
||||
let skip=match class{
|
||||
Class::Model=>true,
|
||||
_=>false,
|
||||
};
|
||||
if let Some(parent_node)=tree_refs.get_mut(&item.parent()){
|
||||
let referent=item.referent();
|
||||
let node=TreeNode::new(item.name.clone(),referent,parent_node.referent,class);
|
||||
parent_node.children.push(referent);
|
||||
tree_refs.insert(referent,node);
|
||||
}
|
||||
//look no further, turn this node and all its children into a model
|
||||
if skip{
|
||||
continue;
|
||||
}
|
||||
for &referent in item.children(){
|
||||
if let Some(c)=dom.get_by_ref(referent){
|
||||
stack.push(c);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//trim empty folders
|
||||
let mut script_count=0;
|
||||
let mut stack:Vec<TrimStackInstruction>=tree_refs.get(&dom.root_ref()).unwrap().children
|
||||
.iter().map(|&c|TrimStackInstruction::Referent(c)).collect();
|
||||
while let Some(instruction)=stack.pop(){
|
||||
match instruction{
|
||||
TrimStackInstruction::IncrementScript=>script_count+=1,
|
||||
TrimStackInstruction::DecrementScript=>script_count-=1,
|
||||
TrimStackInstruction::Referent(referent)=>{
|
||||
let mut delete=None;
|
||||
if let Some(node)=tree_refs.get_mut(&referent){
|
||||
if node.class==Class::Folder&&script_count!=0{
|
||||
node.class=Class::Model
|
||||
}
|
||||
if node.class==Class::Folder&&node.children.len()==0{
|
||||
delete=Some(node.parent);
|
||||
}else{
|
||||
//how the hell do I do this better without recursion
|
||||
let is_script=match node.class{
|
||||
Class::ModuleScript|Class::LocalScript|Class::Script=>true,
|
||||
_=>false,
|
||||
};
|
||||
//stack is popped from back
|
||||
if is_script{
|
||||
stack.push(TrimStackInstruction::DecrementScript);
|
||||
}
|
||||
for &child_referent in &node.children{
|
||||
stack.push(TrimStackInstruction::Referent(child_referent));
|
||||
}
|
||||
if is_script{
|
||||
stack.push(TrimStackInstruction::IncrementScript);
|
||||
}
|
||||
}
|
||||
}
|
||||
//trim referent
|
||||
if let Some(parent_ref)=delete{
|
||||
let parent_node=tree_refs.get_mut(&parent_ref)
|
||||
.expect("parent_ref does not exist in tree_refs");
|
||||
parent_node.children.remove(
|
||||
parent_node.children.iter()
|
||||
.position(|&r|r==referent)
|
||||
.expect("parent.children does not contain referent")
|
||||
);
|
||||
tree_refs.remove(&referent);
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
Ok(DecompiledContext{
|
||||
dom,
|
||||
tree_refs,
|
||||
})
|
||||
}
|
||||
|
||||
struct WriteConfig{
|
||||
output_folder:std::path::PathBuf,
|
||||
write_template:bool,
|
||||
write_models:bool,
|
||||
write_scripts:bool,
|
||||
}
|
||||
|
||||
fn write_files(config:WriteConfig,mut context:DecompiledContext)->AResult<()>{
|
||||
let mut name_tally=std::collections::HashMap::<String,u32>::new();
|
||||
let mut folder=config.output_folder.clone();
|
||||
let mut stack=vec![WriteStackInstruction::Node(context.tree_refs.get(&context.dom.root_ref()).unwrap(),0)];
|
||||
while let Some(instruction)=stack.pop(){
|
||||
match instruction{
|
||||
WriteStackInstruction::PushFolder(component)=>folder.push(component),
|
||||
WriteStackInstruction::PopFolder=>assert!(folder.pop(),"weirdness"),
|
||||
WriteStackInstruction::Destroy(referent)=>context.dom.destroy(referent),
|
||||
WriteStackInstruction::Node(node,name_count)=>{
|
||||
//properties.json to override class or other simple properties
|
||||
let mut properties=PropertiesOverride::default();
|
||||
let has_children=node.children.len()!=0;
|
||||
match node.class{
|
||||
Class::Folder=>(),
|
||||
Class::ModuleScript=>{
|
||||
//.lua files are ModuleScript by default
|
||||
if has_children{
|
||||
properties.ClassName=Some("ModuleScript".to_string())
|
||||
}
|
||||
},
|
||||
Class::LocalScript=>properties.ClassName=Some("LocalScript".to_string()),
|
||||
Class::Script=>properties.ClassName=Some("Script".to_string()),
|
||||
Class::Model=>(),
|
||||
}
|
||||
let name_override=if 0<name_count{
|
||||
format!("{}_{}",node.name,name_count)
|
||||
}else{
|
||||
node.name.clone()
|
||||
};
|
||||
if has_children||properties.is_some(){
|
||||
//push temp subfolder
|
||||
let mut subfolder=folder.clone();
|
||||
subfolder.push(sanitize(name_override.as_str()).as_ref());
|
||||
//make folder
|
||||
std::fs::create_dir(subfolder.clone())?;
|
||||
//write properties
|
||||
if properties.is_some(){
|
||||
let mut file=subfolder.clone();
|
||||
file.push("properties");
|
||||
assert!(file.set_extension("json"),"could not set extension");
|
||||
std::fs::write(file,serde_json::to_string(&properties)?)?
|
||||
}
|
||||
//write item in subfolder
|
||||
write_item(&context.dom,subfolder,node,name_override.as_str(),config.write_models,config.write_scripts)?;
|
||||
}else{
|
||||
//write item
|
||||
write_item(&context.dom,folder.clone(),node,name_override.as_str(),config.write_models,config.write_scripts)?;
|
||||
}
|
||||
//queue item to be deleted from dom after child objects are handled (stack is popped from the back)
|
||||
match node.class{
|
||||
Class::Folder=>(),
|
||||
_=>stack.push(WriteStackInstruction::Destroy(node.referent)),
|
||||
}
|
||||
if has_children{
|
||||
stack.push(WriteStackInstruction::PopFolder);
|
||||
name_tally.clear();
|
||||
for referent in &node.children{
|
||||
if let Some(c)=context.tree_refs.get(referent){
|
||||
let v=name_tally.entry(c.name.clone()).and_modify(|v|*v+=1).or_default();
|
||||
stack.push(WriteStackInstruction::Node(c,*v));
|
||||
}
|
||||
}
|
||||
stack.push(WriteStackInstruction::PushFolder(sanitize(name_override.as_str()).to_string()));
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
//write what remains in template.rbxlx
|
||||
if config.write_template{
|
||||
let mut file=config.output_folder.clone();
|
||||
file.push("template");
|
||||
assert!(file.set_extension("rbxlx"));
|
||||
let output=std::io::BufWriter::new(std::fs::File::create(file)?);
|
||||
rbx_xml::to_writer_default(output,&context.dom,&[context.dom.root_ref()])?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
struct DecompileConfig{
|
||||
input_file:std::path::PathBuf,
|
||||
output_folder:std::path::PathBuf,
|
||||
write_template:bool,
|
||||
write_models:bool,
|
||||
write_scripts:bool,
|
||||
}
|
||||
|
||||
fn decompile(config:DecompileConfig)->AResult<()>{
|
||||
//rules:
|
||||
//Class Script|LocalScript|ModuleScript->$Name.lua
|
||||
//Class Model->$Name.rbxmx
|
||||
//overrides.json per-folder [Override{name,class}]
|
||||
//Everything else goes into template.rbxlx
|
||||
|
||||
//read file
|
||||
let context=generate_decompiled_context(std::io::BufReader::new(std::fs::File::open(config.input_file)?))?;
|
||||
|
||||
//generate folders, models, and scripts
|
||||
//delete models and scripts from dom
|
||||
write_files(WriteConfig{
|
||||
output_folder:config.output_folder,
|
||||
write_template:config.write_template,
|
||||
write_models:config.write_models,
|
||||
write_scripts:config.write_scripts,
|
||||
},context)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
struct WriteCommitConfig{
|
||||
output_folder:std::path::PathBuf,
|
||||
write_template:bool,
|
||||
write_models:bool,
|
||||
write_scripts:bool,
|
||||
}
|
||||
|
||||
fn write_commit(config:WriteCommitConfig,b:AResult<(AssetVersion,DecompiledContext)>,repo:&git2::Repository)->AResult<()>{
|
||||
let (asset_version,context)=b?;
|
||||
|
||||
//clean output dir
|
||||
{
|
||||
let mut src=config.output_folder.clone();
|
||||
src.push("src");
|
||||
match std::fs::remove_dir_all(src){
|
||||
Ok(())=>(),
|
||||
Err(e)=>println!("remove_dir_all src failed {}",e),
|
||||
}
|
||||
let mut template=config.output_folder.clone();
|
||||
template.push("template.rbxlx");
|
||||
match std::fs::remove_file(template){
|
||||
Ok(())=>(),
|
||||
Err(e)=>println!("remove_file template.rbxlx failed {}",e),
|
||||
}
|
||||
}
|
||||
|
||||
//write files
|
||||
write_files(WriteConfig{
|
||||
output_folder:config.output_folder.clone(),
|
||||
write_template:config.write_template,
|
||||
write_models:config.write_models,
|
||||
write_scripts:config.write_scripts,
|
||||
},context)?;
|
||||
|
||||
|
||||
let date=asset_version.created;
|
||||
let sig=git2::Signature::new("roblox-git-history","git_history@strafes.net",&git2::Time::new(date.timestamp(),0)).unwrap();
|
||||
let tree_id={
|
||||
let mut tree_index = repo.index()?;
|
||||
tree_index.add_all([config.output_folder].iter(),git2::IndexAddOption::DEFAULT,None)?;
|
||||
tree_index.write_tree()?
|
||||
};
|
||||
let tree=repo.find_tree(tree_id)?;
|
||||
|
||||
match repo.head(){
|
||||
Ok(reference)=>repo.commit(
|
||||
Some("HEAD"),//update_ref
|
||||
&sig,//author
|
||||
&sig,//commiter
|
||||
&format!("v{}", asset_version.assetVersionNumber),//message
|
||||
&tree,//tree (basically files)
|
||||
&[&reference.peel_to_commit()?],//parents
|
||||
)?,
|
||||
Err(_)=>repo.commit(
|
||||
Some("HEAD"),//update_ref
|
||||
&sig,//author
|
||||
&sig,//commiter
|
||||
&format!("v{}", asset_version.assetVersionNumber),//message
|
||||
&tree,//tree (basically files)
|
||||
&[],//parents
|
||||
)?,
|
||||
};
|
||||
|
||||
//commit
|
||||
Ok(())
|
||||
}
|
||||
|
||||
struct DecompileHistoryConfig{
|
||||
cookie:String,
|
||||
asset_id:AssetID,
|
||||
output_folder:std::path::PathBuf,
|
||||
write_template:bool,
|
||||
write_models:bool,
|
||||
write_scripts:bool,
|
||||
}
|
||||
|
||||
async fn decompile_history_into_git(config:DecompileHistoryConfig)->AResult<()>{
|
||||
let client=reqwest::Client::new();
|
||||
|
||||
//poll paged list of all asset versions
|
||||
let asset_list=get_version_history(&client,&config.cookie.as_str(),config.asset_id).await?;
|
||||
|
||||
let repo=git2::Repository::init(config.output_folder.clone())?;
|
||||
|
||||
//download all versions
|
||||
let asset_id_string=config.asset_id.to_string();
|
||||
futures::stream::iter(asset_list)
|
||||
.map(|asset_version|{
|
||||
let client=&client;
|
||||
let cookie=config.cookie.as_str();
|
||||
let asset_id_str=asset_id_string.as_str();
|
||||
async move{
|
||||
let resp=download_asset_version(client,cookie,asset_id_str,asset_version.assetVersionNumber.to_string().as_str()).await?;
|
||||
let contents=match maybe_gzip_decode(std::io::Cursor::new(resp.bytes().await?))?{
|
||||
ReaderType::GZip(readable)=>generate_decompiled_context(readable)?,
|
||||
ReaderType::Raw(readable)=>generate_decompiled_context(readable)?,
|
||||
};
|
||||
|
||||
Ok((asset_version,contents))
|
||||
}
|
||||
})
|
||||
.buffered(CONCURRENT_REQUESTS)
|
||||
.for_each(|b:AResult<_>|async{
|
||||
match write_commit(WriteCommitConfig{
|
||||
output_folder:config.output_folder.clone(),
|
||||
write_template:config.write_template,
|
||||
write_models:config.write_models,
|
||||
write_scripts:config.write_scripts,
|
||||
},b,&repo){
|
||||
Ok(())=>(),
|
||||
Err(e)=>println!("download/unzip/decompile/write/commit error: {}",e),
|
||||
}
|
||||
}).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn compile(_folder:std::path::PathBuf,_file:std::path::PathBuf)->AResult<()>{
|
||||
Ok(())
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user