StrafesNET/asset-tool
3
Fork 1
Code Issues1 Pull Requests Packages Projects Releases4 Wiki Activity

Compare commits

..

1 Commits
feature/gi ... feature/ta

Author SHA1 Message Date
Quaternions
f0de2a41ad pass tally into stack instruction to generate name override 2024-01-06 15:01:56 -08:00
3 changed files with 96 additions and 357 deletions

66
Cargo.lock generated

@ -116,7 +116,6 @@ dependencies = [
"clap",
"flate2",
"futures",
"git2",
"lazy-regex",
"pollster",
"rbx_binary",
@ -211,7 +210,6 @@ version = "1.0.83"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0"
dependencies = [
"jobserver",
"libc",
]
@ -527,21 +525,6 @@ version = "0.28.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253"
[[package]]
name = "git2"
version = "0.18.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fbf97ba92db08df386e10c8ede66a2a0369bd277090afd8710e19e38de9ec0cd"
dependencies = [
"bitflags 2.4.1",
"libc",
"libgit2-sys",
"log",
"openssl-probe",
"openssl-sys",
"url",
]
[[package]]
name = "h2"
version = "0.3.22"
@ -726,15 +709,6 @@ version = "1.0.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c"
[[package]]
name = "jobserver"
version = "0.1.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8c37f63953c4c63420ed5fd3d6d398c719489b9f872b9fa683262f8edd363c7d"
dependencies = [
"libc",
]
[[package]]
name = "js-sys"
version = "0.3.66"
@ -779,46 +753,6 @@ version = "0.2.151"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4"
[[package]]
name = "libgit2-sys"
version = "0.16.1+1.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2a2bb3680b094add03bb3732ec520ece34da31a8cd2d633d1389d0f0fb60d0c"
dependencies = [
"cc",
"libc",
"libssh2-sys",
"libz-sys",
"openssl-sys",
"pkg-config",
]
[[package]]
name = "libssh2-sys"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2dc8a030b787e2119a731f1951d6a773e2280c660f8ec4b0f5e1505a386e71ee"
dependencies = [
"cc",
"libc",
"libz-sys",
"openssl-sys",
"pkg-config",
"vcpkg",
]
[[package]]
name = "libz-sys"
version = "1.1.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "295c17e837573c8c821dbaeb3cceb3d745ad082f7572191409e69cbc1b3fd050"
dependencies = [
"cc",
"libc",
"pkg-config",
"vcpkg",
]
[[package]]
name = "linux-raw-sys"
version = "0.4.12"

1
Cargo.toml

@ -11,7 +11,6 @@ chrono = { version = "0.4.31", features = ["serde"] }
clap = { version = "4.4.2", features = ["derive"] }
flate2 = "1.0.28"
futures = "0.3.30"
git2 = "0.18.1"
lazy-regex = "3.1.0"
pollster = "0.3.0"
rbx_binary = "0.7.1"

386
src/main.rs

@ -1,4 +1,4 @@
use std::io::Read;
use std::io::{Read,Seek};
use clap::{Args,Parser,Subcommand};
use anyhow::Result as AResult;
use futures::StreamExt;
@ -22,13 +22,6 @@ struct Cli{
#[arg(long)]
cookie_file:Option<std::path::PathBuf>,
#[arg(long)]
no_models:Option<bool>,
#[arg(long)]
no_scripts:Option<bool>,
#[arg(long)]
no_template:Option<bool>,
#[arg(long)]
asset_id:Option<AssetID>,
@ -49,7 +42,6 @@ enum Commands{
Upload,
Compile,
Decompile,
DecompileHistoryIntoGit,
}
#[derive(Args)]
@ -98,29 +90,11 @@ async fn main()->AResult<()>{
};
match cli.command{
Commands::DownloadHistory=>download_history(DownloadHistoryConfig{
output_folder:cli.output.unwrap(),
cookie:cookie.unwrap(),
asset_id:cli.asset_id.unwrap(),
}).await,
Commands::DownloadHistory=>download_history(cookie.unwrap(),cli.asset_id.unwrap()).await,
Commands::Download=>download_list(cookie.unwrap(),vec![(cli.asset_id.unwrap(),cli.output.unwrap())]).await,
Commands::Upload=>upload_list(cookie.unwrap(),cli.group,vec![(cli.asset_id.unwrap(),cli.output.unwrap())]).await,
Commands::Compile=>compile(cli.input.unwrap(),cli.output.unwrap()),
Commands::Decompile=>decompile(DecompileConfig{
input_file:cli.input.unwrap(),
output_folder:cli.output.unwrap(),
write_template:!cli.no_template.unwrap_or(false),
write_models:!cli.no_models.unwrap_or(false),
write_scripts:!cli.no_scripts.unwrap_or(false),
}),
Commands::DecompileHistoryIntoGit=>decompile_history_into_git(DecompileHistoryConfig{
cookie:cookie.unwrap(),
asset_id:cli.asset_id.unwrap(),
output_folder:cli.output.unwrap(),
write_template:!cli.no_template.unwrap_or(false),
write_models:!cli.no_models.unwrap_or(false),
write_scripts:!cli.no_scripts.unwrap_or(false),
}).await,
Commands::Decompile=>decompile(cli.input.unwrap(),cli.output.unwrap()),
}
}
@ -130,17 +104,20 @@ enum Cookie{
File(std::path::PathBuf),
}
enum ReaderType<R:Read>{
GZip(flate2::read::GzDecoder<std::io::BufReader<R>>),
Raw(std::io::BufReader<R>),
enum ReaderType<'a,R:Read+Seek>{
GZip(flate2::read::GzDecoder<&'a mut R>),
Raw(&'a mut R),
}
fn maybe_gzip_decode<R:Read>(input:R)->AResult<ReaderType<R>>{
let mut buf=std::io::BufReader::new(input);
let peek=std::io::BufRead::fill_buf(&mut buf)?;
match &peek[0..2]{
b"\x1f\x8b"=>Ok(ReaderType::GZip(flate2::read::GzDecoder::new(buf))),
_=>Ok(ReaderType::Raw(buf)),
fn maybe_gzip_decode<R:Read+Seek>(input:&mut R)->AResult<ReaderType<R>>{
let mut first_2=[0u8;2];
if let (Ok(()),Ok(()))=(std::io::Read::read_exact(input,&mut first_2),std::io::Seek::rewind(input)){
match &first_2{
b"\x1f\x8b"=>Ok(ReaderType::GZip(flate2::read::GzDecoder::new(input))),
_=>Ok(ReaderType::Raw(input)),
}
}else{
Err(anyhow::Error::msg("failed to peek"))
}
}
@ -239,7 +216,11 @@ async fn download_list(cookie:String,asset_id_file_map:AssetIDFileMap)->AResult<
Ok(())
}
async fn get_version_history(client:&reqwest::Client,cookie:&str,asset_id:AssetID)->AResult<Vec<AssetVersion>>{
async fn download_history(cookie:String,asset_id:AssetID)->AResult<()>{
let client=reqwest::Client::new();
let asset_id_string=asset_id.to_string();
//poll paged list of all asset versions
let mut cursor:Option<String>=None;
let mut asset_list=Vec::new();
loop{
@ -257,7 +238,7 @@ async fn get_version_history(client:&reqwest::Client,cookie:&str,asset_id:AssetI
}
println!("page url={}",url);
let resp=client.get(url)
.header("Cookie",cookie)
.header("Cookie",cookie.clone())
.send().await?;
match resp.json::<VersionPage>().await{
Ok(mut page)=>{
@ -271,77 +252,60 @@ async fn get_version_history(client:&reqwest::Client,cookie:&str,asset_id:AssetI
}
}
asset_list.sort_by(|a,b|a.assetVersionNumber.cmp(&b.assetVersionNumber));
Ok(asset_list)
}
async fn download_asset_version(client:&reqwest::Client,cookie:&str,asset_id_str:&str,asset_version_str:&str)->AResult<reqwest::Response>{
let mut url=reqwest::Url::parse("https://assetdelivery.roblox.com/v1/asset/")?;
//url borrow scope
{
let mut query=url.query_pairs_mut();//borrow here
query.append_pair("ID",asset_id_str);
query.append_pair("version",asset_version_str);
}
println!("download url={}",url);
for i in 0..8{
let resp=client.get(url.clone())
.header("Cookie",cookie)
.send().await?;
if !resp.status().is_success(){
println!("request {} failed",i);
continue;
}
return Ok(resp);
}
Err(anyhow::Error::msg("all requests failed"))
}
struct DownloadHistoryConfig{
output_folder:std::path::PathBuf,
cookie:String,
asset_id:AssetID,
}
async fn download_history(config:DownloadHistoryConfig)->AResult<()>{
let client=reqwest::Client::new();
//poll paged list of all asset versions
let asset_list=get_version_history(&client,&config.cookie.as_str(),config.asset_id).await?;
let mut path=config.output_folder.clone();
let mut path=std::path::PathBuf::new();
path.set_file_name("versions.json");
tokio::fs::write(path,serde_json::to_string(&asset_list)?).await?;
//download all versions
let asset_id_string=config.asset_id.to_string();
futures::stream::iter(asset_list)
.map(|asset_version|{
let client=&client;
let cookie=config.cookie.as_str();
let cookie=cookie.as_str();
let asset_id_str=asset_id_string.as_str();
let output_folder=config.output_folder.clone();
async move{
let resp=download_asset_version(client,cookie,asset_id_str,asset_version.assetVersionNumber.to_string().as_str()).await?;
let contents=match maybe_gzip_decode(std::io::Cursor::new(resp.bytes().await?))?{
ReaderType::GZip(readable)=>read_readable(readable)?,
ReaderType::Raw(readable)=>read_readable(readable)?,
};
let mut url=reqwest::Url::parse("https://assetdelivery.roblox.com/v1/asset/")?;
//url borrow scope
{
let mut query=url.query_pairs_mut();//borrow here
query.append_pair("ID",asset_id_str);
query.append_pair("version",asset_version.assetVersionNumber.to_string().as_str());
}
println!("download url={}",url);
let mut result=Err(anyhow::Error::msg("all requests failed"));
for i in 1..=8{
let resp=client.get(url.clone())
.header("Cookie",cookie)
.send().await?;
let mut path=output_folder;
path.set_file_name(format!("{}_v{}.rbxl",config.asset_id,asset_version.assetVersionNumber));
if !resp.status().is_success(){
println!("request {} failed",i);
continue;
}
Ok((path,contents))
let mut path=std::path::PathBuf::new();
path.set_file_name(format!("{}_v{}.rbxl",asset_id,asset_version.assetVersionNumber));
result=Ok((path,resp.bytes().await?));
break;
}
result
}
})
.buffer_unordered(CONCURRENT_REQUESTS)
.for_each(|b:AResult<_>|async{
match b{
Ok((dest,data))=>{
match tokio::fs::write(dest,data).await{
Err(e)=>eprintln!("fs error: {}",e),
_=>(),
}
Ok((dest,body))=>{
let contents=match maybe_gzip_decode(&mut std::io::Cursor::new(body)){
Ok(ReaderType::GZip(readable))=>read_readable(readable),
Ok(ReaderType::Raw(readable))=>read_readable(readable),
Err(e)=>Err(e),
};
match contents{
Ok(data)=>match tokio::fs::write(dest,data).await{
Err(e)=>eprintln!("fs error: {}",e),
_=>(),
},
Err(e)=>eprintln!("gzip error: {}",e),
};
},
Err(e)=>eprintln!("dl error: {}",e),
}
@ -349,18 +313,21 @@ async fn download_history(config:DownloadHistoryConfig)->AResult<()>{
Ok(())
}
fn load_dom<R:Read>(input:R)->AResult<rbx_dom_weak::WeakDom>{
let mut buf=std::io::BufReader::new(input);
let peek=std::io::BufRead::fill_buf(&mut buf)?;
match &peek[0..4]{
b"<rob"=>{
match &peek[4..8]{
b"lox!"=>return rbx_binary::from_reader(buf).map_err(anyhow::Error::msg),
b"lox "=>return rbx_xml::from_reader_default(buf).map_err(anyhow::Error::msg),
other=>Err(anyhow::Error::msg(format!("Unknown Roblox file type {:?}",other))),
}
},
_=>Err(anyhow::Error::msg("unsupported file type")),
fn load_dom<R:Read+Seek>(input:&mut R)->AResult<rbx_dom_weak::WeakDom>{
let mut first_8=[0u8;8];
if let (Ok(()),Ok(()))=(std::io::Read::read_exact(input, &mut first_8),std::io::Seek::rewind(input)){
match &first_8[0..4]{
b"<rob"=>{
match &first_8[4..8]{
b"lox!"=>return rbx_binary::from_reader(input).map_err(anyhow::Error::msg),
b"lox "=>return rbx_xml::from_reader(input,rbx_xml::DecodeOptions::default()).map_err(anyhow::Error::msg),
other=>Err(anyhow::Error::msg(format!("Unknown Roblox file type {:?}",other))),
}
},
_=>Err(anyhow::Error::msg("unsupported file type")),
}
}else{
Err(anyhow::Error::msg("peek failed"))
}
}
@ -420,14 +387,11 @@ fn sanitize<'a>(s:&'a str)->std::borrow::Cow<'a,str>{
lazy_regex::regex!(r"[^a-zA-Z0-9._-]").replace_all(s,"_")
}
fn write_item(dom:&rbx_dom_weak::WeakDom,mut file:std::path::PathBuf,node:&TreeNode,node_name_override:&str,write_models:bool,write_scripts:bool)->AResult<()>{
fn write_item(dom:&rbx_dom_weak::WeakDom,mut file:std::path::PathBuf,node:&TreeNode,node_name_override:&str)->AResult<()>{
file.push(sanitize(node_name_override).as_ref());
match node.class{
Class::Folder=>(),
Class::ModuleScript|Class::LocalScript|Class::Script=>{
if !write_scripts{
return Ok(())
}
assert!(file.set_extension("lua"),"could not set extension");
assert!(dom.get_by_ref(node.referent).is_some_and(|item|{
//TODO: delete disabled scripts
@ -437,9 +401,6 @@ fn write_item(dom:&rbx_dom_weak::WeakDom,mut file:std::path::PathBuf,node:&TreeN
}),"no string property or file failed to write");
},
Class::Model=>{
if !write_models{
return Ok(())
}
assert!(file.set_extension("rbxmx"));
let output=std::io::BufWriter::new(std::fs::File::create(file)?);
rbx_xml::to_writer_default(output,dom,&[node.referent])?;
@ -448,13 +409,16 @@ fn write_item(dom:&rbx_dom_weak::WeakDom,mut file:std::path::PathBuf,node:&TreeN
Ok(())
}
struct DecompiledContext{
dom:rbx_dom_weak::WeakDom,
tree_refs:std::collections::HashMap<rbx_dom_weak::types::Ref,TreeNode>,
}
fn decompile(input_file:std::path::PathBuf,output_folder:std::path::PathBuf)->AResult<()>{
//rules:
//Class Script|LocalScript|ModuleScript->$Name.lua
//Class Model->$Name.rbxmx
//overrides.json per-folder [Override{name,class}]
//Everything else goes into template.rbxlx
fn generate_decompiled_context<R:Read>(input:R)->AResult<DecompiledContext>{
let dom=load_dom(input)?;
//read file
let mut input=std::io::BufReader::new(std::fs::File::open(input_file)?);
let mut dom=load_dom(&mut input)?;
let mut tree_refs=std::collections::HashMap::new();
tree_refs.insert(dom.root_ref(),TreeNode::new(
@ -484,7 +448,6 @@ fn generate_decompiled_context<R:Read>(input:R)->AResult<DecompiledContext>{
parent_node.children.push(referent);
tree_refs.insert(referent,node);
}
//look no further, turn this node and all its children into a model
if skip{
continue;
}
@ -544,28 +507,17 @@ fn generate_decompiled_context<R:Read>(input:R)->AResult<DecompiledContext>{
}
}
Ok(DecompiledContext{
dom,
tree_refs,
})
}
//generate folders, models, and scripts
//delete models and scripts from dom
struct WriteConfig{
output_folder:std::path::PathBuf,
write_template:bool,
write_models:bool,
write_scripts:bool,
}
fn write_files(config:WriteConfig,mut context:DecompiledContext)->AResult<()>{
let mut name_tally=std::collections::HashMap::<String,u32>::new();
let mut folder=config.output_folder.clone();
let mut stack=vec![WriteStackInstruction::Node(context.tree_refs.get(&context.dom.root_ref()).unwrap(),0)];
let mut folder=output_folder.clone();
let mut stack=vec![WriteStackInstruction::Node(tree_refs.get(&dom.root_ref()).unwrap(),0)];
while let Some(instruction)=stack.pop(){
match instruction{
WriteStackInstruction::PushFolder(component)=>folder.push(component),
WriteStackInstruction::PopFolder=>assert!(folder.pop(),"weirdness"),
WriteStackInstruction::Destroy(referent)=>context.dom.destroy(referent),
WriteStackInstruction::Destroy(referent)=>dom.destroy(referent),
WriteStackInstruction::Node(node,name_count)=>{
//properties.json to override class or other simple properties
let mut properties=PropertiesOverride::default();
@ -601,10 +553,10 @@ fn write_files(config:WriteConfig,mut context:DecompiledContext)->AResult<()>{
std::fs::write(file,serde_json::to_string(&properties)?)?
}
//write item in subfolder
write_item(&context.dom,subfolder,node,name_override.as_str(),config.write_models,config.write_scripts)?;
write_item(&dom,subfolder,node,name_override.as_str())?;
}else{
//write item
write_item(&context.dom,folder.clone(),node,name_override.as_str(),config.write_models,config.write_scripts)?;
write_item(&dom,folder.clone(),node,name_override.as_str())?;
}
//queue item to be deleted from dom after child objects are handled (stack is popped from the back)
match node.class{
@ -615,7 +567,7 @@ fn write_files(config:WriteConfig,mut context:DecompiledContext)->AResult<()>{
stack.push(WriteStackInstruction::PopFolder);
name_tally.clear();
for referent in &node.children{
if let Some(c)=context.tree_refs.get(referent){
if let Some(c)=tree_refs.get(referent){
let v=name_tally.entry(c.name.clone()).and_modify(|v|*v+=1).or_default();
stack.push(WriteStackInstruction::Node(c,*v));
}
@ -627,163 +579,17 @@ fn write_files(config:WriteConfig,mut context:DecompiledContext)->AResult<()>{
}
//write what remains in template.rbxlx
if config.write_template{
let mut file=config.output_folder.clone();
{
let mut file=output_folder.clone();
file.push("template");
assert!(file.set_extension("rbxlx"));
let output=std::io::BufWriter::new(std::fs::File::create(file)?);
rbx_xml::to_writer_default(output,&context.dom,&[context.dom.root_ref()])?;
rbx_xml::to_writer_default(output,&dom,&[dom.root_ref()])?;
}
Ok(())
}
struct DecompileConfig{
input_file:std::path::PathBuf,
output_folder:std::path::PathBuf,
write_template:bool,
write_models:bool,
write_scripts:bool,
}
fn decompile(config:DecompileConfig)->AResult<()>{
//rules:
//Class Script|LocalScript|ModuleScript->$Name.lua
//Class Model->$Name.rbxmx
//overrides.json per-folder [Override{name,class}]
//Everything else goes into template.rbxlx
//read file
let context=generate_decompiled_context(std::io::BufReader::new(std::fs::File::open(config.input_file)?))?;
//generate folders, models, and scripts
//delete models and scripts from dom
write_files(WriteConfig{
output_folder:config.output_folder,
write_template:config.write_template,
write_models:config.write_models,
write_scripts:config.write_scripts,
},context)?;
Ok(())
}
struct WriteCommitConfig{
output_folder:std::path::PathBuf,
write_template:bool,
write_models:bool,
write_scripts:bool,
}
fn write_commit(config:WriteCommitConfig,b:AResult<(AssetVersion,DecompiledContext)>,repo:&git2::Repository)->AResult<()>{
let (asset_version,context)=b?;
//clean output dir
{
let mut src=config.output_folder.clone();
src.push("src");
match std::fs::remove_dir_all(src){
Ok(())=>(),
Err(e)=>println!("remove_dir_all src failed {}",e),
}
let mut template=config.output_folder.clone();
template.push("template.rbxlx");
match std::fs::remove_file(template){
Ok(())=>(),
Err(e)=>println!("remove_file template.rbxlx failed {}",e),
}
}
//write files
write_files(WriteConfig{
output_folder:config.output_folder.clone(),
write_template:config.write_template,
write_models:config.write_models,
write_scripts:config.write_scripts,
},context)?;
let date=asset_version.created;
let sig=git2::Signature::new("roblox-git-history","git_history@strafes.net",&git2::Time::new(date.timestamp(),0)).unwrap();
let tree_id={
let mut tree_index = repo.index()?;
tree_index.add_all([config.output_folder].iter(),git2::IndexAddOption::DEFAULT,None)?;
tree_index.write_tree()?
};
let tree=repo.find_tree(tree_id)?;
match repo.head(){
Ok(reference)=>repo.commit(
Some("HEAD"),//update_ref
&sig,//author
&sig,//commiter
&format!("v{}", asset_version.assetVersionNumber),//message
&tree,//tree (basically files)
&[&reference.peel_to_commit()?],//parents
)?,
Err(_)=>repo.commit(
Some("HEAD"),//update_ref
&sig,//author
&sig,//commiter
&format!("v{}", asset_version.assetVersionNumber),//message
&tree,//tree (basically files)
&[],//parents
)?,
};
//commit
Ok(())
}
struct DecompileHistoryConfig{
cookie:String,
asset_id:AssetID,
output_folder:std::path::PathBuf,
write_template:bool,
write_models:bool,
write_scripts:bool,
}
async fn decompile_history_into_git(config:DecompileHistoryConfig)->AResult<()>{
let client=reqwest::Client::new();
//poll paged list of all asset versions
let asset_list=get_version_history(&client,&config.cookie.as_str(),config.asset_id).await?;
let repo=git2::Repository::init(config.output_folder.clone())?;
//download all versions
let asset_id_string=config.asset_id.to_string();
futures::stream::iter(asset_list)
.map(|asset_version|{
let client=&client;
let cookie=config.cookie.as_str();
let asset_id_str=asset_id_string.as_str();
async move{
let resp=download_asset_version(client,cookie,asset_id_str,asset_version.assetVersionNumber.to_string().as_str()).await?;
let contents=match maybe_gzip_decode(std::io::Cursor::new(resp.bytes().await?))?{
ReaderType::GZip(readable)=>generate_decompiled_context(readable)?,
ReaderType::Raw(readable)=>generate_decompiled_context(readable)?,
};
Ok((asset_version,contents))
}
})
.buffered(CONCURRENT_REQUESTS)
.for_each(|b:AResult<_>|async{
match write_commit(WriteCommitConfig{
output_folder:config.output_folder.clone(),
write_template:config.write_template,
write_models:config.write_models,
write_scripts:config.write_scripts,
},b,&repo){
Ok(())=>(),
Err(e)=>println!("download/unzip/decompile/write/commit error: {}",e),
}
}).await;
Ok(())
}
fn compile(_folder:std::path::PathBuf,_file:std::path::PathBuf)->AResult<()>{
Ok(())
}
}