Compare commits

...

7 Commits

Author SHA1 Message Date
19adf8205a only support one asset for now 2024-01-01 11:59:54 -08:00
56f222638d make kv less bad 2023-12-31 12:08:54 -08:00
c080634a53 CSRF challenge 2023-12-31 12:08:54 -08:00
5b68f23755 wip upload 2023-12-31 12:08:54 -08:00
e323becfbf redo asset stuff as asset id and file list 2023-12-31 12:08:54 -08:00
edb58b16b5 implement cookie and group as arg (bad) 2023-12-31 12:08:54 -08:00
9673bbbe8c release opti 2023-12-31 12:08:46 -08:00
2 changed files with 123 additions and 17 deletions

View File

@ -18,3 +18,8 @@ rbx_reflection_database = "0.2.7"
rbx_xml = "0.13.1" rbx_xml = "0.13.1"
reqwest = { version = "0.11.23", features = ["cookies"] } reqwest = { version = "0.11.23", features = ["cookies"] }
tokio = { version = "1.35.1", features = ["macros", "rt-multi-thread", "fs"] } tokio = { version = "1.35.1", features = ["macros", "rt-multi-thread", "fs"] }
[profile.release]
#lto = true
strip = true
codegen-units = 1

View File

@ -4,19 +4,47 @@ use anyhow::Result as AResult;
use futures::StreamExt; use futures::StreamExt;
type AssetID=u64; type AssetID=u64;
type AssetIDFileMap=Vec<(AssetID,std::path::PathBuf)>;
/// Parse a single key-value pair
fn parse_key_val<T,U>(s:&str)->AResult<(T,U)>
where
T:std::str::FromStr,
T::Err:std::error::Error+Send+Sync+'static,
U:std::str::FromStr,
U::Err:std::error::Error+Send+Sync+'static,
{
let pos=s
.find('=')
.ok_or_else(||anyhow::Error::msg(format!("invalid KEY=value: no `=` found in `{s}`")))?;
Ok((s[..pos].parse()?,s[pos+1..].parse()?))
}
#[derive(Parser)] #[derive(Parser)]
#[command(author,version,about,long_about=None)] #[command(author,version,about,long_about=None)]
#[command(propagate_version = true)] #[command(propagate_version = true)]
struct Cli{ struct Cli{
#[arg(short,long)]
group:Option<u64>,
//idk how to do this better
#[arg(long)]
cookie_literal:Option<String>,
#[arg(long)]
cookie_env:Option<String>,
#[arg(long)]
cookie_file:Option<std::path::PathBuf>,
#[arg(long,value_parser=parse_key_val::<AssetID,std::path::PathBuf>)]
asset_id:(AssetID,std::path::PathBuf),
#[command(subcommand)] #[command(subcommand)]
command:Commands, command:Commands,
} }
#[derive(Subcommand)] #[derive(Subcommand)]
enum Commands{ enum Commands{
Download(AssetIDList), Download,
Upload{path:std::path::PathBuf,asset_id:AssetID}, Upload,
} }
#[derive(Args)] #[derive(Args)]
@ -24,18 +52,44 @@ struct PathBufList{
paths:Vec<std::path::PathBuf> paths:Vec<std::path::PathBuf>
} }
#[derive(Args)]
struct AssetIDList{
asset_ids:Vec<AssetID>,
}
#[tokio::main] #[tokio::main]
async fn main()->AResult<()>{ async fn main()->AResult<()>{
let cli=Cli::parse(); let cli=Cli::parse();
match cli.command{
Commands::Download(asset_id_list)=>download_list(asset_id_list.asset_ids).await, let cookie_enum={
Commands::Upload{path,asset_id}=>upload_file(path,asset_id), match (cli.cookie_literal,cli.cookie_env,cli.cookie_file){
(Some(literal),None,None)=>Cookie::Literal(literal),
(None,Some(env_var),None)=>Cookie::Environment(env_var),
(None,None,Some(path))=>Cookie::File(path),
_=>return Err(anyhow::Error::msg("Cookie was not specified or was specified multiple times.")),
} }
};
let cookie=format!(".ROBLOSECURITY={}",match cookie_enum{
Cookie::Literal(s)=>s,
Cookie::Environment(var)=>std::env::var(var)?,
Cookie::File(path)=>tokio::fs::read_to_string(path).await?,
});
let group=match cli.group{
Some(group_id)=>Owner::Group(group_id),
None=>Owner::User,
};
match cli.command{
Commands::Download=>download_list(cookie,vec![cli.asset_id]).await,
Commands::Upload=>upload_list(cookie,group,vec![cli.asset_id]).await,
}
}
enum Owner{
Group(u64),
User
}
enum Cookie{
Literal(String),
Environment(String),
File(std::path::PathBuf),
} }
enum ReaderType<'a,R:Read+Seek>{ enum ReaderType<'a,R:Read+Seek>{
@ -55,7 +109,56 @@ fn maybe_gzip_decode<R:Read+Seek>(input:&mut R)->AResult<ReaderType<R>>{
} }
} }
fn upload_file(_path:std::path::PathBuf,_asset_id:AssetID)->AResult<()>{ async fn upload_list(cookie:String,owner:Owner,asset_id_file_map:AssetIDFileMap)->AResult<()>{
let client=reqwest::Client::new();
futures::stream::iter(asset_id_file_map)
.map(|(asset_id,file)|{
let client=&client;
let cookie=cookie.as_str();
let owner=&owner;
async move{
let mut url=reqwest::Url::parse("https://data.roblox.com/Data/Upload.ashx?json=1&type=Model&genreTypeId=1")?;
//url borrow scope
{
let mut query=url.query_pairs_mut();//borrow here
query.append_pair("assetid",asset_id.to_string().as_str());
match owner{
Owner::Group(group_id)=>{query.append_pair("groupId",group_id.to_string().as_str());},
Owner::User=>(),
}
}
let body=tokio::fs::read_to_string(file).await?;
let mut resp=client.post(url.clone())
.header("Cookie",cookie)
.body(body.clone())
.send().await?;
//This is called a CSRF challenge apparently
if resp.status()==reqwest::StatusCode::FORBIDDEN{
if let Some(csrf_token)=resp.headers().get("X-CSRF-Token"){
resp=client.post(url)
.header("X-CSRF-Token",csrf_token)
.header("Cookie",cookie)
.body(body)
.send().await?;
}else{
return Err(anyhow::Error::msg("Roblox returned 403 with no CSRF"));
}
}
Ok((asset_id,resp.bytes().await?))
}
})
.buffer_unordered(CONCURRENT_REQUESTS)
.for_each(|b:AResult<_>|async{
match b{
Ok((asset_id,body))=>{
println!("asset_id={} response.body={:?}",asset_id,body);
},
Err(e)=>eprintln!("ul error: {}",e),
}
}).await;
Ok(()) Ok(())
} }
@ -67,25 +170,23 @@ fn read_readable(mut readable:impl Read)->AResult<Vec<u8>>{
Ok(contents) Ok(contents)
} }
async fn download_list(asset_ids:Vec<AssetID>)->AResult<()>{ async fn download_list(cookie:String,asset_id_file_map:AssetIDFileMap)->AResult<()>{
let cookie=format!(".ROBLOSECURITY={}",std::env::var("RBXCOOKIE_PROJECTSLIME")?);
let client=reqwest::Client::new(); let client=reqwest::Client::new();
futures::stream::iter(asset_ids) futures::stream::iter(asset_id_file_map)
.map(|asset_id|{ .map(|(asset_id,file)|{
let client=&client; let client=&client;
let cookie=cookie.as_str(); let cookie=cookie.as_str();
async move{ async move{
let resp=client.get(format!("https://assetdelivery.roblox.com/v1/asset/?ID={}",asset_id)) let resp=client.get(format!("https://assetdelivery.roblox.com/v1/asset/?ID={}",asset_id))
.header("Cookie",cookie) .header("Cookie",cookie)
.send().await?; .send().await?;
Ok((asset_id,resp.bytes().await?)) Ok((file,resp.bytes().await?))
} }
}) })
.buffer_unordered(CONCURRENT_REQUESTS) .buffer_unordered(CONCURRENT_REQUESTS)
.for_each(|b:AResult<_>|async{ .for_each(|b:AResult<_>|async{
match b{ match b{
Ok((asset_id,body))=>{ Ok((dest,body))=>{
let dest=std::path::PathBuf::from(asset_id.to_string());
let contents=match maybe_gzip_decode(&mut std::io::Cursor::new(body)){ let contents=match maybe_gzip_decode(&mut std::io::Cursor::new(body)){
Ok(ReaderType::GZip(readable))=>read_readable(readable), Ok(ReaderType::GZip(readable))=>read_readable(readable),
Ok(ReaderType::Raw(readable))=>read_readable(readable), Ok(ReaderType::Raw(readable))=>read_readable(readable),