diff --git a/rbx_asset/Cargo.toml b/rbx_asset/Cargo.toml index 2b0ad78..0120817 100644 --- a/rbx_asset/Cargo.toml +++ b/rbx_asset/Cargo.toml @@ -10,10 +10,14 @@ authors = ["Rhys Lloyd <krakow20@gmail.com>"] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[features] +default = ["gzip"] +gzip = ["dep:flate2"] + [dependencies] bytes = "1.10.1" chrono = { version = "0.4.38", features = ["serde"] } -flate2 = "1.0.29" +flate2 = { version = "1.0.29", optional = true } reqwest = { version = "0.12.4", features = ["json","multipart"] } serde = { version = "1.0.199", features = ["derive"] } serde_json = "1.0.111" diff --git a/rbx_asset/src/cloud.rs b/rbx_asset/src/cloud.rs index 93085d6..71f4a9e 100644 --- a/rbx_asset/src/cloud.rs +++ b/rbx_asset/src/cloud.rs @@ -1,5 +1,5 @@ -use crate::util::{serialize_u64,deserialize_u64,response_ok,maybe_gzip_decode}; -use crate::types::{ResponseError}; +use crate::util::{serialize_u64,deserialize_u64,response_ok}; +use crate::types::{ResponseError,MaybeGzippedBytes}; #[derive(Debug,serde::Deserialize,serde::Serialize)] #[allow(nonstandard_style,dead_code)] @@ -175,7 +175,6 @@ pub enum GetError{ ParseError(url::ParseError), Response(ResponseError), Reqwest(reqwest::Error), - IO(std::io::Error) } impl std::fmt::Display for GetError{ fn fmt(&self,f:&mut std::fmt::Formatter<'_>)->std::fmt::Result{ @@ -454,15 +453,15 @@ impl Context{ ).await.map_err(GetError::Response)? .json().await.map_err(GetError::Reqwest) } - pub async fn get_asset(&self,config:&AssetLocation)->Result<Vec<u8>,GetError>{ + pub async fn get_asset(&self,config:&AssetLocation)->Result<MaybeGzippedBytes,GetError>{ let url=reqwest::Url::parse(config.location()).map_err(GetError::ParseError)?; - let body=response_ok( + let bytes=response_ok( self.get(url).await.map_err(GetError::Reqwest)? ).await.map_err(GetError::Response)? .bytes().await.map_err(GetError::Reqwest)?; - maybe_gzip_decode(body).map_err(GetError::IO) + Ok(MaybeGzippedBytes::new(bytes)) } pub async fn get_asset_versions(&self,config:AssetVersionsRequest)->Result<AssetVersionsResponse,AssetVersionsError>{ let raw_url=format!("https://apis.roblox.com/assets/v1/assets/{}/versions",config.asset_id); diff --git a/rbx_asset/src/cookie.rs b/rbx_asset/src/cookie.rs index 8369965..efee3dc 100644 --- a/rbx_asset/src/cookie.rs +++ b/rbx_asset/src/cookie.rs @@ -1,5 +1,5 @@ -use crate::util::{response_ok,maybe_gzip_decode}; -use crate::types::ResponseError; +use crate::util::response_ok; +use crate::types::{ResponseError,MaybeGzippedBytes}; #[derive(Debug)] pub enum PostError{ @@ -92,7 +92,6 @@ pub enum GetError{ ParseError(url::ParseError), Response(ResponseError), Reqwest(reqwest::Error), - IO(std::io::Error) } impl std::fmt::Display for GetError{ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { @@ -450,7 +449,7 @@ impl Context{ }) } } - pub async fn get_asset(&self,config:GetAssetRequest)->Result<Vec<u8>,GetError>{ + pub async fn get_asset(&self,config:GetAssetRequest)->Result<MaybeGzippedBytes,GetError>{ let mut url=reqwest::Url::parse("https://assetdelivery.roblox.com/v1/asset/").map_err(GetError::ParseError)?; //url borrow scope { @@ -460,13 +459,13 @@ impl Context{ query.append_pair("version",version.to_string().as_str()); } } - let body=response_ok( + + let bytes=response_ok( self.get(url).await.map_err(GetError::Reqwest)? ).await.map_err(GetError::Response)? .bytes().await.map_err(GetError::Reqwest)?; - - maybe_gzip_decode(body).map_err(GetError::IO) + Ok(MaybeGzippedBytes::new(bytes)) } pub async fn get_asset_v2(&self,config:GetAssetRequest)->Result<GetAssetV2,GetAssetV2Error>{ let mut url=reqwest::Url::parse("https://assetdelivery.roblox.com/v2/asset").map_err(GetAssetV2Error::ParseError)?; @@ -498,15 +497,15 @@ impl Context{ info, }) } - pub async fn get_asset_v2_download(&self,config:&GetAssetV2Location)->Result<Vec<u8>,GetError>{ + pub async fn get_asset_v2_download(&self,config:&GetAssetV2Location)->Result<MaybeGzippedBytes,GetError>{ let url=reqwest::Url::parse(config.location.as_str()).map_err(GetError::ParseError)?; - let body=response_ok( + let bytes=response_ok( self.get(url).await.map_err(GetError::Reqwest)? ).await.map_err(GetError::Response)? .bytes().await.map_err(GetError::Reqwest)?; - maybe_gzip_decode(body).map_err(GetError::IO) + Ok(MaybeGzippedBytes::new(bytes)) } pub async fn get_asset_details(&self,config:GetAssetDetailsRequest)->Result<AssetDetails,GetError>{ let url=reqwest::Url::parse(format!("https://economy.roblox.com/v2/assets/{}/details",config.asset_id).as_str()).map_err(GetError::ParseError)?; diff --git a/rbx_asset/src/types.rs b/rbx_asset/src/types.rs index b9ef790..2aa64f5 100644 --- a/rbx_asset/src/types.rs +++ b/rbx_asset/src/types.rs @@ -16,3 +16,53 @@ impl std::fmt::Display for ResponseError{ } } impl std::error::Error for ResponseError{} + +#[cfg(feature="gzip")] +use std::io::Cursor; +#[cfg(feature="gzip")] +use flate2::read::GzDecoder; + +/// Some bytes that might be gzipped. Use the read_with or to_vec methods to transparently decode gzip. +pub struct MaybeGzippedBytes{ + bytes:bytes::Bytes, +} +impl MaybeGzippedBytes{ + pub(crate) fn new(bytes:bytes::Bytes)->Self{ + Self{bytes} + } + pub fn into_inner(self)->bytes::Bytes{ + self.bytes + } + /// get a reference to the bytes, ignoring gzip decoding + pub fn as_raw_ref(&self)->&[u8]{ + self.bytes.as_ref() + } + /// Transparently decode gzip data, if present (intermediate allocation) + #[cfg(feature="gzip")] + pub fn to_vec(&self)->std::io::Result<Vec<u8>>{ + use std::io::Read; + match self.bytes.get(0..2){ + Some(b"\x1f\x8b")=>{ + let mut buf=Vec::new(); + GzDecoder::new(Cursor::new(self.bytes.as_ref())).read_to_end(&mut buf)?; + Ok(buf) + }, + _=>Ok(self.bytes.to_vec()) + } + } + /// Read the bytes with the provided decoders. + /// The idea is to make a function that is generic over std::io::Read + /// and pass the same function to both closures. + /// This two closure hack must be done because of the different concrete types. + #[cfg(feature="gzip")] + pub fn read_with<'a,ReadGzip,ReadRaw,T>(&'a self,read_gzip:ReadGzip,read_raw:ReadRaw)->T + where + ReadGzip:Fn(GzDecoder<Cursor<&'a [u8]>>)->T, + ReadRaw:Fn(Cursor<&'a [u8]>)->T, + { + match self.bytes.get(0..2){ + Some(b"\x1f\x8b")=>read_gzip(GzDecoder::new(Cursor::new(self.bytes.as_ref()))), + _=>read_raw(Cursor::new(self.bytes.as_ref())) + } + } +} diff --git a/rbx_asset/src/util.rs b/rbx_asset/src/util.rs index 5cc08b7..9f8a785 100644 --- a/rbx_asset/src/util.rs +++ b/rbx_asset/src/util.rs @@ -17,18 +17,6 @@ pub(crate) async fn response_ok(response:reqwest::Response)->Result<reqwest::Res } } -pub(crate) fn maybe_gzip_decode(data:bytes::Bytes)->std::io::Result<Vec<u8>>{ - match data.get(0..2){ - Some(b"\x1f\x8b")=>{ - use std::io::Read; - let mut buf=Vec::new(); - flate2::read::GzDecoder::new(std::io::Cursor::new(data)).read_to_end(&mut buf)?; - Ok(buf) - }, - _=>Ok(data.to_vec()), - } -} - use serde::de::{Error,Unexpected}; use serde::{Deserializer,Serializer}; diff --git a/src/main.rs b/src/main.rs index b30aa28..e8703bf 100644 --- a/src/main.rs +++ b/src/main.rs @@ -903,11 +903,11 @@ async fn create_asset_medias(config:CreateAssetMediasConfig)->AResult<()>{ async move{(path, async move{ let asset_response=asset_response_result.map_err(DownloadDecalError::PollOperation)?; - let file=cookie_context.get_asset(rbx_asset::cookie::GetAssetRequest{ + let maybe_gzip=cookie_context.get_asset(rbx_asset::cookie::GetAssetRequest{ asset_id:asset_response.assetId, version:None, }).await.map_err(DownloadDecalError::Get)?; - let dom=load_dom(std::io::Cursor::new(file)).map_err(DownloadDecalError::LoadDom)?; + let dom=maybe_gzip.read_with(load_dom,load_dom).map_err(DownloadDecalError::LoadDom)?; let instance=dom.get_by_ref( *dom.root().children().first().ok_or(DownloadDecalError::NoFirstInstance)? ).ok_or(DownloadDecalError::NoFirstInstance)?; @@ -993,8 +993,8 @@ async fn asset_details(cookie:Cookie,asset_id:AssetID)->AResult<()>{ async fn download_version(cookie:Cookie,asset_id:AssetID,version:Option<u64>,dest:PathBuf)->AResult<()>{ let context=CookieContext::new(cookie); - let data=context.get_asset(rbx_asset::cookie::GetAssetRequest{asset_id,version}).await?; - tokio::fs::write(dest,data).await?; + let maybe_gzip=context.get_asset(rbx_asset::cookie::GetAssetRequest{asset_id,version}).await?; + tokio::fs::write(dest,maybe_gzip.to_vec()?).await?; Ok(()) } @@ -1006,9 +1006,9 @@ async fn download_version_v2(cookie:Cookie,asset_id:AssetID,version:Option<u64>, println!("version:{}",info.version); let location=info.info.locations.first().ok_or(anyhow::Error::msg("No locations"))?; - let data=context.get_asset_v2_download(location).await?; + let maybe_gzip=context.get_asset_v2_download(location).await?; - tokio::fs::write(dest,data).await?; + tokio::fs::write(dest,maybe_gzip.to_vec()?).await?; Ok(()) } @@ -1024,7 +1024,7 @@ async fn download_list(cookie:Cookie,asset_id_file_map:AssetIDFileMap)->AResult< .buffer_unordered(CONCURRENT_REQUESTS) .for_each(|b:AResult<_>|async{ match b{ - Ok((dest,data))=>if let Err(e)=tokio::fs::write(dest,data).await{ + Ok((dest,maybe_gzip))=>if let Err(e)=(async||{tokio::fs::write(dest,maybe_gzip.to_vec()?).await})().await{ eprintln!("fs error: {}",e); }, Err(e)=>eprintln!("dl error: {}",e), @@ -1228,9 +1228,9 @@ async fn download_history(mut config:DownloadHistoryConfig)->AResult<()>{ let mut path=output_folder.clone(); path.push(format!("{}_v{}.rbxl",config.asset_id,version_number)); join_set.spawn(async move{ - let file=context.get_asset(rbx_asset::cookie::GetAssetRequest{asset_id:config.asset_id,version:Some(version_number)}).await?; + let maybe_gzip=context.get_asset(rbx_asset::cookie::GetAssetRequest{asset_id:config.asset_id,version:Some(version_number)}).await?; - tokio::fs::write(path,file).await?; + tokio::fs::write(path,maybe_gzip.to_vec()?).await?; Ok::<_,anyhow::Error>(()) }); @@ -1350,9 +1350,9 @@ struct DownloadDecompileConfig{ async fn download_decompile(config:DownloadDecompileConfig)->AResult<()>{ let context=CookieContext::new(config.cookie); - let file=context.get_asset(rbx_asset::cookie::GetAssetRequest{asset_id:config.asset_id,version:None}).await?; + let maybe_gzip=context.get_asset(rbx_asset::cookie::GetAssetRequest{asset_id:config.asset_id,version:None}).await?; - let dom=load_dom(std::io::Cursor::new(file))?; + let dom=maybe_gzip.read_with(load_dom,load_dom)?; let context=rox_compiler::DecompiledContext::from_dom(dom); context.write_files(rox_compiler::WriteConfig{ @@ -1532,8 +1532,8 @@ async fn download_and_decompile_history_into_git(config:DownloadAndDecompileHist .map(|asset_version|{ let context=context.clone(); tokio::task::spawn(async move{ - let file=context.get_asset(rbx_asset::cookie::GetAssetRequest{asset_id,version:Some(asset_version.assetVersionNumber)}).await?; - let dom=load_dom(std::io::Cursor::new(file))?; + let maybe_gzip=context.get_asset(rbx_asset::cookie::GetAssetRequest{asset_id,version:Some(asset_version.assetVersionNumber)}).await?; + let dom=maybe_gzip.read_with(load_dom,load_dom)?; Ok::<_,anyhow::Error>((asset_version,rox_compiler::DecompiledContext::from_dom(dom))) }) }))