From ca81d0699827fb1935a9e16fa3d331d4d20f1fa7 Mon Sep 17 00:00:00 2001 From: Quaternions Date: Sat, 27 Apr 2024 23:36:21 -0700 Subject: [PATCH] use asset lib --- Cargo.lock | 308 ++++----------------------------------------------- Cargo.toml | 5 +- src/main.rs | 312 ++++++++-------------------------------------------- 3 files changed, 70 insertions(+), 555 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3f06925..c970f71 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -112,20 +112,17 @@ name = "asset-tool" version = "0.3.0" dependencies = [ "anyhow", - "chrono", "clap", - "flate2", "futures", "git2", "lazy-regex", "pollster", "rayon", + "rbx_asset", "rbx_binary", "rbx_dom_weak", "rbx_reflection_database", "rbx_xml", - "reqwest 0.11.27", - "serde", "serde_json", "tokio", ] @@ -157,12 +154,6 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" -[[package]] -name = "base64" -version = "0.21.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" - [[package]] name = "base64" version = "0.22.0" @@ -296,34 +287,6 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" -[[package]] -name = "cookie" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7efb37c3e1ccb1ff97164ad95ac1606e8ccd35b3fa0a7d99a304c7f4a428cc24" -dependencies = [ - "percent-encoding", - "time", - "version_check", -] - -[[package]] -name = "cookie_store" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "387461abbc748185c3a6e1673d826918b450b87ff22639429c694619a83b6cf6" -dependencies = [ - "cookie", - "idna 0.3.0", - "log", - "publicsuffix", - "serde", - "serde_derive", - "serde_json", - "time", - "url", -] - [[package]] name = "core-foundation" version = "0.9.4" @@ -374,15 +337,6 @@ version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" -[[package]] -name = "deranged" -version = "0.3.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" -dependencies = [ - "powerfmt", -] - [[package]] name = "either" version = "1.11.0" @@ -581,25 +535,6 @@ dependencies = [ "url", ] -[[package]] -name = "h2" -version = "0.3.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" -dependencies = [ - "bytes", - "fnv", - "futures-core", - "futures-sink", - "futures-util", - "http 0.2.12", - "indexmap", - "slab", - "tokio", - "tokio-util", - "tracing", -] - [[package]] name = "h2" version = "0.4.4" @@ -611,7 +546,7 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http 1.1.0", + "http", "indexmap", "slab", "tokio", @@ -637,17 +572,6 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" -[[package]] -name = "http" -version = "0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" -dependencies = [ - "bytes", - "fnv", - "itoa", -] - [[package]] name = "http" version = "1.1.0" @@ -659,17 +583,6 @@ dependencies = [ "itoa", ] -[[package]] -name = "http-body" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" -dependencies = [ - "bytes", - "http 0.2.12", - "pin-project-lite", -] - [[package]] name = "http-body" version = "1.0.0" @@ -677,7 +590,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" dependencies = [ "bytes", - "http 1.1.0", + "http", ] [[package]] @@ -688,8 +601,8 @@ checksum = "0475f8b2ac86659c21b64320d5d653f9efe42acd2a4e560073ec61a155a34f1d" dependencies = [ "bytes", "futures-core", - "http 1.1.0", - "http-body 1.0.0", + "http", + "http-body", "pin-project-lite", ] @@ -699,36 +612,6 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" -[[package]] -name = "httpdate" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" - -[[package]] -name = "hyper" -version = "0.14.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" -dependencies = [ - "bytes", - "futures-channel", - "futures-core", - "futures-util", - "h2 0.3.26", - "http 0.2.12", - "http-body 0.4.6", - "httparse", - "httpdate", - "itoa", - "pin-project-lite", - "socket2", - "tokio", - "tower-service", - "tracing", - "want", -] - [[package]] name = "hyper" version = "1.3.1" @@ -738,9 +621,9 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "h2 0.4.4", - "http 1.1.0", - "http-body 1.0.0", + "h2", + "http", + "http-body", "httparse", "itoa", "pin-project-lite", @@ -749,19 +632,6 @@ dependencies = [ "want", ] -[[package]] -name = "hyper-tls" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" -dependencies = [ - "bytes", - "hyper 0.14.28", - "native-tls", - "tokio", - "tokio-native-tls", -] - [[package]] name = "hyper-tls" version = "0.6.0" @@ -770,7 +640,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper 1.3.1", + "hyper", "hyper-util", "native-tls", "tokio", @@ -787,9 +657,9 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http 1.1.0", - "http-body 1.0.0", - "hyper 1.3.1", + "http", + "http-body", + "hyper", "pin-project-lite", "socket2", "tokio", @@ -821,16 +691,6 @@ dependencies = [ "cc", ] -[[package]] -name = "idna" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" -dependencies = [ - "unicode-bidi", - "unicode-normalization", -] - [[package]] name = "idna" version = "0.5.0" @@ -1038,12 +898,6 @@ dependencies = [ "tempfile", ] -[[package]] -name = "num-conv" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" - [[package]] name = "num-traits" version = "0.2.18" @@ -1178,12 +1032,6 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22686f4785f02a4fcc856d3b3bb19bf6c8160d103f7a99cc258bddd0251dc7f2" -[[package]] -name = "powerfmt" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" - [[package]] name = "ppv-lite86" version = "0.2.17" @@ -1218,22 +1066,6 @@ dependencies = [ "syn", ] -[[package]] -name = "psl-types" -version = "2.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33cb294fe86a74cbcf50d4445b37da762029549ebeea341421c7c70370f86cac" - -[[package]] -name = "publicsuffix" -version = "2.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96a8c1bda5ae1af7f99a2962e49df150414a43d62404644d98dd5c3a93d07457" -dependencies = [ - "idna 0.3.0", - "psl-types", -] - [[package]] name = "quote" version = "1.0.36" @@ -1299,7 +1131,7 @@ version = "0.1.0" dependencies = [ "chrono", "flate2", - "reqwest 0.12.4", + "reqwest", "serde", "url", ] @@ -1410,48 +1242,6 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" -[[package]] -name = "reqwest" -version = "0.11.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" -dependencies = [ - "base64 0.21.7", - "bytes", - "cookie", - "cookie_store", - "encoding_rs", - "futures-core", - "futures-util", - "h2 0.3.26", - "http 0.2.12", - "http-body 0.4.6", - "hyper 0.14.28", - "hyper-tls 0.5.0", - "ipnet", - "js-sys", - "log", - "mime", - "native-tls", - "once_cell", - "percent-encoding", - "pin-project-lite", - "rustls-pemfile 1.0.4", - "serde", - "serde_json", - "serde_urlencoded", - "sync_wrapper", - "system-configuration", - "tokio", - "tokio-native-tls", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", - "winreg 0.50.0", -] - [[package]] name = "reqwest" version = "0.12.4" @@ -1463,12 +1253,12 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "h2 0.4.4", - "http 1.1.0", - "http-body 1.0.0", + "h2", + "http", + "http-body", "http-body-util", - "hyper 1.3.1", - "hyper-tls 0.6.0", + "hyper", + "hyper-tls", "hyper-util", "ipnet", "js-sys", @@ -1478,7 +1268,7 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls-pemfile 2.1.2", + "rustls-pemfile", "serde", "serde_json", "serde_urlencoded", @@ -1491,7 +1281,7 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "winreg 0.52.0", + "winreg", ] [[package]] @@ -1535,15 +1325,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "rustls-pemfile" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" -dependencies = [ - "base64 0.21.7", -] - [[package]] name = "rustls-pemfile" version = "2.1.2" @@ -1742,37 +1523,6 @@ dependencies = [ "syn", ] -[[package]] -name = "time" -version = "0.3.36" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" -dependencies = [ - "deranged", - "itoa", - "num-conv", - "powerfmt", - "serde", - "time-core", - "time-macros", -] - -[[package]] -name = "time-core" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" - -[[package]] -name = "time-macros" -version = "0.2.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" -dependencies = [ - "num-conv", - "time-core", -] - [[package]] name = "tinyvec" version = "1.6.0" @@ -1922,7 +1672,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" dependencies = [ "form_urlencoded", - "idna 0.5.0", + "idna", "percent-encoding", ] @@ -1938,12 +1688,6 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" -[[package]] -name = "version_check" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" - [[package]] name = "want" version = "0.3.1" @@ -2183,16 +1927,6 @@ version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" -[[package]] -name = "winreg" -version = "0.50.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" -dependencies = [ - "cfg-if", - "windows-sys 0.48.0", -] - [[package]] name = "winreg" version = "0.52.0" diff --git a/Cargo.toml b/Cargo.toml index 5c8fb07..ebdb632 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,20 +8,17 @@ edition = "2021" [dependencies] anyhow = "1.0.75" -chrono = { version = "0.4.31", features = ["serde"] } clap = { version = "4.4.2", features = ["derive"] } -flate2 = "1.0.28" futures = "0.3.30" git2 = "0.18.1" lazy-regex = "3.1.0" pollster = "0.3.0" rayon = "1.8.0" +rbx_asset = { path = "rbx_asset" } rbx_binary = "0.7.4" rbx_dom_weak = "2.7.0" rbx_reflection_database = "0.2.10" rbx_xml = "0.13.3" -reqwest = { version = "0.11.23", features = ["cookies", "json"] } -serde = { version = "1.0.195", features = ["derive"] } serde_json = "1.0.111" tokio = { version = "1.35.1", features = ["macros", "rt-multi-thread", "fs"] } diff --git a/src/main.rs b/src/main.rs index 295f90a..cdf8a3a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,6 +4,7 @@ use anyhow::Result as AResult; use futures::StreamExt; use rbx_dom_weak::types::Ref; use tokio::io::AsyncReadExt; +use rbx_asset::context::{RobloxContext,InventoryItem,AssetVersion}; type AssetID=u64; type AssetIDFileMap=Vec<(AssetID,PathBuf)>; @@ -185,53 +186,6 @@ enum DecompileStyle{ RoxRojo, } -#[derive(serde::Deserialize)] -#[allow(nonstandard_style,dead_code)] -struct VersionPage{ - previousPageCursor:Option, - nextPageCursor:Option, - data:Vec, -} -#[derive(serde::Deserialize,serde::Serialize)] -#[allow(nonstandard_style,dead_code)] -struct AssetVersion{ - Id:u64, - assetId:AssetID, - assetVersionNumber:u64, - creatorType:String, - creatorTargetId:u64, - creatingUniverseId:Option, - created:chrono::DateTime, - isPublished:bool, -} - -#[derive(serde::Deserialize)] -#[allow(nonstandard_style,dead_code)] -struct InventoryPage{ - totalResults:u64,//up to 50 - filteredKeyword:Option,//"" - searchDebugInfo:Option,//null - spellCheckerResult:Option,//null - queryFacets:Option,//null - imageSearchStatus:Option,//null - previousPageCursor:Option, - nextPageCursor:Option, - data:Vec, -} -#[derive(serde::Deserialize,serde::Serialize)] -#[allow(nonstandard_style,dead_code)] -struct InventoryItem{ - id:u64, - name:String, -} - -#[derive(Debug,serde::Deserialize,serde::Serialize)] -#[allow(nonstandard_style,dead_code)] -struct UploadResponse{ - AssetId:u64, - AssetVersionId:u64, -} - #[tokio::main] async fn main()->AResult<()>{ let cli=Cli::parse(); @@ -323,20 +277,6 @@ impl Cookie{ } } -enum ReaderType{ - GZip(flate2::read::GzDecoder>), - Raw(std::io::BufReader), -} - -fn maybe_gzip_decode(input:R)->AResult>{ - let mut buf=std::io::BufReader::new(input); - let peek=std::io::BufRead::fill_buf(&mut buf)?; - match &peek[0..2]{ - b"\x1f\x8b"=>Ok(ReaderType::GZip(flate2::read::GzDecoder::new(buf))), - _=>Ok(ReaderType::Raw(buf)), - } -} - struct CreateConfig{ cookie:String, model_name:String, @@ -348,95 +288,33 @@ struct CreateConfig{ } async fn create(config:CreateConfig)->AResult<()>{ - let client=reqwest::Client::new(); - let client=&client; - let cookie=config.cookie.as_str(); - let group=&config.group; - let mut url=reqwest::Url::parse("https://data.roblox.com/Data/Upload.ashx?json=1&type=Model&genreTypeId=1")?; - //url borrow scope - { - let mut query=url.query_pairs_mut();//borrow here - //archaic roblox api uses 0 for new asset - query.append_pair("assetid","0"); - query.append_pair("name",config.model_name.as_str()); - query.append_pair("description",config.description.as_str()); - query.append_pair("ispublic",if config.free_model{"True"}else{"False"}); - query.append_pair("allowComments",if config.allow_comments{"True"}else{"False"}); - match group{ - Some(group_id)=>{query.append_pair("groupId",group_id.to_string().as_str());}, - None=>(), - } - } - - let body=tokio::fs::read(config.input_file).await?; - let mut resp=client.post(url.clone()) - .header("Cookie",cookie) - .body(body.clone()) - .send().await?; - - //This is called a CSRF challenge apparently - if resp.status()==reqwest::StatusCode::FORBIDDEN{ - if let Some(csrf_token)=resp.headers().get("X-CSRF-Token"){ - resp=client.post(url) - .header("X-CSRF-Token",csrf_token) - .header("Cookie",cookie) - .body(body) - .send().await?; - }else{ - Err(anyhow::Error::msg("Roblox returned 403 with no CSRF"))?; - } - } - let body=match resp.status(){ - reqwest::StatusCode::OK=>Ok(resp.json::().await?), - other=>Err(anyhow::Error::msg(other)), - }; - - println!("UploadResponse={:?}",body?); + let resp=RobloxContext::new(config.cookie) + .create(rbx_asset::context::CreateRequest{ + name:config.model_name, + description:config.description, + ispublic:config.free_model, + allowComments:config.allow_comments, + groupId:config.group, + },tokio::fs::read(config.input_file).await?).await?; + println!("UploadResponse={:?}",resp); Ok(()) } async fn upload_list(cookie:String,group:Option,asset_id_file_map:AssetIDFileMap)->AResult<()>{ - let client=reqwest::Client::new(); + let context=RobloxContext::new(cookie); //this is calling map on the vec because the closure produces an iterator of futures futures::stream::iter(asset_id_file_map.into_iter() .map(|(asset_id,file)|{ - let client=&client; - let cookie=cookie.as_str(); - let group=&group; + let context=&context; async move{ - let mut url=reqwest::Url::parse("https://data.roblox.com/Data/Upload.ashx?json=1&type=Model&genreTypeId=1")?; - //url borrow scope - { - let mut query=url.query_pairs_mut();//borrow here - query.append_pair("assetid",asset_id.to_string().as_str()); - match group{ - Some(group_id)=>{query.append_pair("groupId",group_id.to_string().as_str());}, - None=>(), - } - } - - let body=tokio::fs::read(file).await?; - let mut resp=client.post(url.clone()) - .header("Cookie",cookie) - .body(body.clone()) - .send().await?; - - //This is called a CSRF challenge apparently - if resp.status()==reqwest::StatusCode::FORBIDDEN{ - if let Some(csrf_token)=resp.headers().get("X-CSRF-Token"){ - resp=client.post(url) - .header("X-CSRF-Token",csrf_token) - .header("Cookie",cookie) - .body(body) - .send().await?; - }else{ - Err(anyhow::Error::msg("Roblox returned 403 with no CSRF"))?; - } - } - match resp.status(){ - reqwest::StatusCode::OK=>Ok((asset_id,resp.json::().await?)), - other=>Err(anyhow::Error::msg(other)), - } + Ok((asset_id,context.upload(rbx_asset::context::UploadRequest{ + assetid:asset_id, + name:None, + description:None, + ispublic:None, + allowComments:None, + groupId:group, + },tokio::fs::read(file).await?).await?)) } })) .buffer_unordered(CONCURRENT_REQUESTS) @@ -451,69 +329,35 @@ async fn upload_list(cookie:String,group:Option,asset_id_file_map:AssetIDFi Ok(()) } -fn read_readable(mut readable:impl Read)->AResult>{ - let mut contents=Vec::new(); - readable.read_to_end(&mut contents)?; - Ok(contents) -} - async fn download_list(cookie:String,asset_id_file_map:AssetIDFileMap)->AResult<()>{ - let client=reqwest::Client::new(); + let context=RobloxContext::new(cookie); futures::stream::iter(asset_id_file_map.into_iter() .map(|(asset_id,file)|{ - let client=&client; - let cookie=cookie.as_str(); + let context=&context; async move{ - let resp=client.get(format!("https://assetdelivery.roblox.com/v1/asset/?ID={}",asset_id)) - .header("Cookie",cookie) - .send().await?; - Ok((file,resp.bytes().await?)) + Ok((file,context.download(rbx_asset::context::DownloadRequest{asset_id,version:None}).await?)) } })) .buffer_unordered(CONCURRENT_REQUESTS) .for_each(|b:AResult<_>|async{ match b{ - Ok((dest,body))=>{ - let contents=match maybe_gzip_decode(&mut std::io::Cursor::new(body)){ - Ok(ReaderType::GZip(readable))=>read_readable(readable), - Ok(ReaderType::Raw(readable))=>read_readable(readable), - Err(e)=>Err(e), - }; - match contents{ - Ok(data)=>match tokio::fs::write(dest,data).await{ - Err(e)=>eprintln!("fs error: {}",e), - _=>(), - }, - Err(e)=>eprintln!("gzip error: {}",e), - }; + Ok((dest,data))=>{ + match tokio::fs::write(dest,data).await{ + Err(e)=>eprintln!("fs error: {}",e), + _=>(), + } }, Err(e)=>eprintln!("dl error: {}",e), } }).await; Ok(()) } -async fn download_inventory_page(client:&reqwest::Client,cookie:&str,group:u64,cursor:Option)->AResult{ - let mut url=reqwest::Url::parse(format!("https://apis.roblox.com/toolbox-service/v1/creations/group/{}/10?limit=50",group).as_str())?; - //url borrow scope - { - let mut query=url.query_pairs_mut();//borrow here - match cursor.as_deref(){ - Some(next_page)=>{query.append_pair("cursor",next_page);} - None=>(), - } - } - println!("page url={}",url); - let resp=client.get(url) - .header("Cookie",cookie) - .send().await?; - Ok(resp.json::().await?) -} -async fn get_inventory_pages(client:&reqwest::Client,cookie:&str,group:u64)->AResult>{ +async fn get_inventory_pages(context:&RobloxContext,group:u64)->AResult>{ let mut cursor:Option=None; let mut asset_list=Vec::new(); loop{ - let mut page=download_inventory_page(client,cookie,group,cursor).await?; + let mut page=context.inventory_page(rbx_asset::context::InventoryPageRequest{group,cursor}).await?; asset_list.append(&mut page.data); if page.nextPageCursor.is_none(){ break; @@ -524,8 +368,8 @@ async fn get_inventory_pages(client:&reqwest::Client,cookie:&str,group:u64)->ARe } async fn download_group_inventory_json(cookie:String,group:u64,output_folder:PathBuf)->AResult<()>{ - let client=reqwest::Client::new(); - let item_list=get_inventory_pages(&client,cookie.as_str(),group).await?; + let context=RobloxContext::new(cookie); + let item_list=get_inventory_pages(&context,group).await?; let mut path=output_folder.clone(); path.set_file_name("versions.json"); @@ -534,31 +378,11 @@ async fn download_group_inventory_json(cookie:String,group:u64,output_folder:Pat Ok(()) } -async fn download_page(client:&reqwest::Client,cookie:&str,asset_id:AssetID,cursor:Option)->AResult{ - let mut url=reqwest::Url::parse(format!("https://develop.roblox.com/v1/assets/{}/saved-versions",asset_id).as_str())?; - //url borrow scope - { - let mut query=url.query_pairs_mut();//borrow here - //query.append_pair("sortOrder","Asc"); - //query.append_pair("limit","100"); - //query.append_pair("count","100"); - match cursor.as_deref(){ - Some(next_page)=>{query.append_pair("cursor",next_page);} - None=>(), - } - } - println!("page url={}",url); - let resp=client.get(url) - .header("Cookie",cookie) - .send().await?; - Ok(resp.json::().await?) -} - -async fn get_version_history(client:&reqwest::Client,cookie:&str,asset_id:AssetID)->AResult>{ +async fn get_version_history(context:&RobloxContext,asset_id:AssetID)->AResult>{ let mut cursor:Option=None; let mut asset_list=Vec::new(); loop{ - let mut page=download_page(client,cookie,asset_id,cursor).await?; + let mut page=context.history_page(rbx_asset::context::HistoryPageRequest{asset_id,cursor}).await?; asset_list.append(&mut page.data); if page.nextPageCursor.is_none(){ break; @@ -569,30 +393,6 @@ async fn get_version_history(client:&reqwest::Client,cookie:&str,asset_id:AssetI Ok(asset_list) } -async fn download_asset_version(client:&reqwest::Client,cookie:&str,asset_id_str:&str,asset_version_str:&str)->AResult{ - let mut url=reqwest::Url::parse("https://assetdelivery.roblox.com/v1/asset/")?; - //url borrow scope - { - let mut query=url.query_pairs_mut();//borrow here - query.append_pair("ID",asset_id_str); - query.append_pair("version",asset_version_str); - } - println!("download url={}",url); - for i in 0..8{ - let resp=client.get(url.clone()) - .header("Cookie",cookie) - .send().await?; - - if !resp.status().is_success(){ - println!("request {} failed",i); - continue; - } - - return Ok(resp); - } - Err(anyhow::Error::msg("all requests failed")) -} - struct DownloadHistoryConfig{ continue_from_versions:bool, end_version:Option, @@ -639,9 +439,7 @@ async fn download_history(mut config:DownloadHistoryConfig)->AResult<()>{ None=>Err(anyhow::Error::msg("Cannot continue from versions.json - there are no previous versions"))?, } } - let client=reqwest::Client::new(); - - let asset_id_string=config.asset_id.to_string(); + let context=RobloxContext::new(config.cookie); //limit concurrent downloads let mut join_set=tokio::task::JoinSet::new(); @@ -649,10 +447,8 @@ async fn download_history(mut config:DownloadHistoryConfig)->AResult<()>{ //poll paged list of all asset versions let mut cursor:Option=None; loop{ - let mut page=download_page(&client,config.cookie.as_str(),config.asset_id,cursor).await?; - let client=&client; - let cookie=config.cookie.clone(); - let asset_id_str=asset_id_string.clone(); + let mut page=context.history_page(rbx_asset::context::HistoryPageRequest{asset_id:config.asset_id,cursor}).await?; + let context=&context; let output_folder=config.output_folder.clone(); let data=&page.data; let asset_list_contents=&asset_list_contents; @@ -677,19 +473,13 @@ async fn download_history(mut config:DownloadHistoryConfig)->AResult<()>{ while CONCURRENT_REQUESTS<=join_set.len(){ join_set.join_next().await.unwrap()??; } - let client=client.clone(); - let cookie=cookie.clone(); - let asset_id_str=asset_id_str.clone(); + let context=context.clone(); let mut path=output_folder.clone(); path.push(format!("{}_v{}.rbxl",config.asset_id,version_number)); join_set.spawn(async move{ - let resp=download_asset_version(&client,cookie.as_str(),asset_id_str.as_str(),version_number.to_string().as_str()).await?; - let contents=match maybe_gzip_decode(std::io::Cursor::new(resp.bytes().await?))?{ - ReaderType::GZip(readable)=>read_readable(readable)?, - ReaderType::Raw(readable)=>read_readable(readable)?, - }; + let file=context.download(rbx_asset::context::DownloadRequest{asset_id:config.asset_id,version:Some(version_number)}).await?; - tokio::fs::write(path,contents).await?; + tokio::fs::write(path,file).await?; Ok::<_,anyhow::Error>(()) }); @@ -738,8 +528,8 @@ fn load_dom(input:R)->AResult{ match &peek[0..4]{ b"{ match &peek[4..8]{ - b"lox!"=>return rbx_binary::from_reader(buf).map_err(anyhow::Error::msg), - b"lox "=>return rbx_xml::from_reader_default(buf).map_err(anyhow::Error::msg), + b"lox!"=>rbx_binary::from_reader(buf).map_err(anyhow::Error::msg), + b"lox "=>rbx_xml::from_reader_default(buf).map_err(anyhow::Error::msg), other=>Err(anyhow::Error::msg(format!("Unknown Roblox file type {:?}",other))), } }, @@ -1265,27 +1055,21 @@ struct DownloadAndDecompileHistoryConfig{ } async fn download_and_decompile_history_into_git(config:DownloadAndDecompileHistoryConfig)->AResult<()>{ - let client=reqwest::Client::new(); + let context=RobloxContext::new(config.cookie); //poll paged list of all asset versions - let asset_list=get_version_history(&client,&config.cookie.as_str(),config.asset_id).await?; + let asset_list=get_version_history(&context,config.asset_id).await?; let repo=git2::Repository::init(config.output_folder.clone())?; //download all versions - let asset_id_string=config.asset_id.to_string(); + let asset_id=config.asset_id; futures::stream::iter(asset_list.into_iter() .map(|asset_version|{ - let client=client.clone(); - let cookie=config.cookie.clone(); - let asset_id_str=asset_id_string.clone(); + let context=context.clone(); tokio::task::spawn(async move{ - let resp=download_asset_version(&client,cookie.as_str(),asset_id_str.as_str(),asset_version.assetVersionNumber.to_string().as_str()).await?; - let contents=match maybe_gzip_decode(std::io::Cursor::new(resp.bytes().await?))?{ - ReaderType::GZip(readable)=>generate_decompiled_context(readable)?, - ReaderType::Raw(readable)=>generate_decompiled_context(readable)?, - }; - Ok::<_,anyhow::Error>((asset_version,contents)) + let file=context.download(rbx_asset::context::DownloadRequest{asset_id,version:Some(asset_version.assetVersionNumber)}).await?; + Ok::<_,anyhow::Error>((asset_version,generate_decompiled_context(std::io::Cursor::new(file))?)) }) })) .buffered(CONCURRENT_DECODE)