rbx_asset: change api to save intermediate allocation

rbx_asset/Cargo.toml

@@ -10,10 +10,14 @@ authors = ["Rhys Lloyd <krakow20@gmail.com>"]
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
+[features]
+default = ["gzip"]
+gzip = ["dep:flate2"]
+
 [dependencies]
 bytes = "1.10.1"
 chrono = { version = "0.4.38", features = ["serde"] }
-flate2 = "1.0.29"
+flate2 = { version = "1.0.29", optional = true }
 reqwest = { version = "0.12.4", features = ["json","multipart"] }
 serde = { version = "1.0.199", features = ["derive"] }
 serde_json = "1.0.111"

rbx_asset/src/cloud.rs

@@ -1,4 +1,4 @@
-use crate::util::{serialize_u64,deserialize_u64,response_ok,ResponseError,maybe_gzip_decode};
+use crate::util::{serialize_u64,deserialize_u64,response_ok,ResponseError,MaybeGzippedBytes};
 
 #[derive(Debug,serde::Deserialize,serde::Serialize)]
 #[allow(nonstandard_style,dead_code)]
@@ -174,7 +174,6 @@ pub enum GetError{
 	ParseError(url::ParseError),
 	Response(ResponseError),
 	Reqwest(reqwest::Error),
-	IO(std::io::Error)
 }
 impl std::fmt::Display for GetError{
 	fn fmt(&self,f:&mut std::fmt::Formatter<'_>)->std::fmt::Result{
@@ -453,15 +452,15 @@ impl Context{
 		).await.map_err(GetError::Response)?
 		.json().await.map_err(GetError::Reqwest)
 	}
-	pub async fn get_asset(&self,config:&AssetLocation)->Result<Vec<u8>,GetError>{
+	pub async fn get_asset(&self,config:&AssetLocation)->Result<MaybeGzippedBytes,GetError>{
 		let url=reqwest::Url::parse(config.location()).map_err(GetError::ParseError)?;
 
-		let body=response_ok(
+		let bytes=response_ok(
 			self.get(url).await.map_err(GetError::Reqwest)?
 		).await.map_err(GetError::Response)?
 		.bytes().await.map_err(GetError::Reqwest)?;
 
-		maybe_gzip_decode(body).map_err(GetError::IO)
+		Ok(MaybeGzippedBytes::new(bytes))
 	}
 	pub async fn get_asset_versions(&self,config:AssetVersionsRequest)->Result<AssetVersionsResponse,AssetVersionsError>{
 		let raw_url=format!("https://apis.roblox.com/assets/v1/assets/{}/versions",config.asset_id);

rbx_asset/src/cookie.rs

@@ -1,4 +1,4 @@
-use crate::util::{response_ok,ResponseError,maybe_gzip_decode};
+use crate::util::{response_ok,ResponseError,MaybeGzippedBytes};
 
 #[derive(Debug)]
 pub enum PostError{
@@ -91,7 +91,6 @@ pub enum GetError{
 	ParseError(url::ParseError),
 	Response(ResponseError),
 	Reqwest(reqwest::Error),
-	IO(std::io::Error)
 }
 impl std::fmt::Display for GetError{
 	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
@@ -449,7 +448,7 @@ impl Context{
 			})
 		}
 	}
-	pub async fn get_asset(&self,config:GetAssetRequest)->Result<Vec<u8>,GetError>{
+	pub async fn get_asset(&self,config:GetAssetRequest)->Result<MaybeGzippedBytes,GetError>{
 		let mut url=reqwest::Url::parse("https://assetdelivery.roblox.com/v1/asset/").map_err(GetError::ParseError)?;
 		//url borrow scope
 		{
@@ -459,13 +458,13 @@ impl Context{
 				query.append_pair("version",version.to_string().as_str());
 			}
 		}
-		let body=response_ok(
+
+		let bytes=response_ok(
 			self.get(url).await.map_err(GetError::Reqwest)?
 		).await.map_err(GetError::Response)?
 		.bytes().await.map_err(GetError::Reqwest)?;
 
-
-		maybe_gzip_decode(body).map_err(GetError::IO)
+		Ok(MaybeGzippedBytes::new(bytes))
 	}
 	pub async fn get_asset_v2(&self,config:GetAssetRequest)->Result<GetAssetV2,GetAssetV2Error>{
 		let mut url=reqwest::Url::parse("https://assetdelivery.roblox.com/v2/asset").map_err(GetAssetV2Error::ParseError)?;
@@ -497,15 +496,15 @@ impl Context{
 			info,
 		})
 	}
-	pub async fn get_asset_v2_download(&self,config:&GetAssetV2Location)->Result<Vec<u8>,GetError>{
+	pub async fn get_asset_v2_download(&self,config:&GetAssetV2Location)->Result<MaybeGzippedBytes,GetError>{
 		let url=reqwest::Url::parse(config.location.as_str()).map_err(GetError::ParseError)?;
 
-		let body=response_ok(
+		let bytes=response_ok(
 			self.get(url).await.map_err(GetError::Reqwest)?
 		).await.map_err(GetError::Response)?
 		.bytes().await.map_err(GetError::Reqwest)?;
 
-		maybe_gzip_decode(body).map_err(GetError::IO)
+		Ok(MaybeGzippedBytes::new(bytes))
 	}
 	pub async fn get_asset_details(&self,config:GetAssetDetailsRequest)->Result<AssetDetails,GetError>{
 		let url=reqwest::Url::parse(format!("https://economy.roblox.com/v2/assets/{}/details",config.asset_id).as_str()).map_err(GetError::ParseError)?;

rbx_asset/src/util.rs

@@ -33,15 +33,53 @@ pub(crate) async fn response_ok(response:reqwest::Response)->Result<reqwest::Res
 	}
 }
 
-pub(crate) fn maybe_gzip_decode(data:bytes::Bytes)->std::io::Result<Vec<u8>>{
-	match data.get(0..2){
-		Some(b"\x1f\x8b")=>{
-			use std::io::Read;
-			let mut buf=Vec::new();
-			flate2::read::GzDecoder::new(std::io::Cursor::new(data)).read_to_end(&mut buf)?;
-			Ok(buf)
-		},
-		_=>Ok(data.to_vec()),
+#[cfg(feature="gzip")]
+use std::io::Cursor;
+#[cfg(feature="gzip")]
+use flate2::read::GzDecoder;
+
+/// Some bytes that might be gzipped.  Use the read_with or to_vec methods to transparently decode gzip.
+pub struct MaybeGzippedBytes{
+	bytes:bytes::Bytes,
+}
+impl MaybeGzippedBytes{
+	pub(crate) fn new(bytes:bytes::Bytes)->Self{
+		Self{bytes}
+	}
+	pub fn into_inner(self)->bytes::Bytes{
+		self.bytes
+	}
+	/// get a reference to the bytes, ignoring gzip decoding
+	pub fn as_raw_ref(&self)->&[u8]{
+		self.bytes.as_ref()
+	}
+	/// Transparently decode gzip data, if present (intermediate allocation)
+	#[cfg(feature="gzip")]
+	pub fn to_vec(&self)->std::io::Result<Vec<u8>>{
+		use std::io::Read;
+		match self.bytes.get(0..2){
+			Some(b"\x1f\x8b")=>{
+				let mut buf=Vec::new();
+				GzDecoder::new(Cursor::new(self.bytes.as_ref())).read_to_end(&mut buf)?;
+				Ok(buf)
+			},
+			_=>Ok(self.bytes.to_vec())
+		}
+	}
+	/// Read the bytes with the provided decoders.
+	/// The idea is to make a function that is generic over std::io::Read
+	/// and pass the same function to both closures.
+	/// This two closure hack must be done because of the different concrete types.
+	#[cfg(feature="gzip")]
+	pub fn read_with<'a,ReadGzip,ReadRaw,T>(&'a self,read_gzip:ReadGzip,read_raw:ReadRaw)->T
+		where
+			ReadGzip:Fn(GzDecoder<Cursor<&'a [u8]>>)->T,
+			ReadRaw:Fn(Cursor<&'a [u8]>)->T,
+	{
+		match self.bytes.get(0..2){
+			Some(b"\x1f\x8b")=>read_gzip(GzDecoder::new(Cursor::new(self.bytes.as_ref()))),
+			_=>read_raw(Cursor::new(self.bytes.as_ref()))
+		}
 	}
 }
 

src/main.rs

@@ -903,11 +903,11 @@ async fn create_asset_medias(config:CreateAssetMediasConfig)->AResult<()>{
 		async move{(path,
 		async move{
 			let asset_response=asset_response_result.map_err(DownloadDecalError::PollOperation)?;
-			let file=cookie_context.get_asset(rbx_asset::cookie::GetAssetRequest{
+			let maybe_gzip=cookie_context.get_asset(rbx_asset::cookie::GetAssetRequest{
 				asset_id:asset_response.assetId,
 				version:None,
 			}).await.map_err(DownloadDecalError::Get)?;
-			let dom=load_dom(std::io::Cursor::new(file)).map_err(DownloadDecalError::LoadDom)?;
+			let dom=maybe_gzip.read_with(load_dom,load_dom).map_err(DownloadDecalError::LoadDom)?;
 			let instance=dom.get_by_ref(
 				*dom.root().children().first().ok_or(DownloadDecalError::NoFirstInstance)?
 			).ok_or(DownloadDecalError::NoFirstInstance)?;
@@ -993,8 +993,8 @@ async fn asset_details(cookie:Cookie,asset_id:AssetID)->AResult<()>{
 
 async fn download_version(cookie:Cookie,asset_id:AssetID,version:Option<u64>,dest:PathBuf)->AResult<()>{
 	let context=CookieContext::new(cookie);
-	let data=context.get_asset(rbx_asset::cookie::GetAssetRequest{asset_id,version}).await?;
-	tokio::fs::write(dest,data).await?;
+	let maybe_gzip=context.get_asset(rbx_asset::cookie::GetAssetRequest{asset_id,version}).await?;
+	tokio::fs::write(dest,maybe_gzip.to_vec()?).await?;
 	Ok(())
 }
 
@@ -1006,9 +1006,9 @@ async fn download_version_v2(cookie:Cookie,asset_id:AssetID,version:Option<u64>,
 	println!("version:{}",info.version);
 
 	let location=info.info.locations.first().ok_or(anyhow::Error::msg("No locations"))?;
-	let data=context.get_asset_v2_download(location).await?;
+	let maybe_gzip=context.get_asset_v2_download(location).await?;
 
-	tokio::fs::write(dest,data).await?;
+	tokio::fs::write(dest,maybe_gzip.to_vec()?).await?;
 	Ok(())
 }
 
@@ -1024,7 +1024,7 @@ async fn download_list(cookie:Cookie,asset_id_file_map:AssetIDFileMap)->AResult<
 	.buffer_unordered(CONCURRENT_REQUESTS)
 	.for_each(|b:AResult<_>|async{
 			match b{
-				Ok((dest,data))=>if let Err(e)=tokio::fs::write(dest,data).await{
+				Ok((dest,maybe_gzip))=>if let Err(e)=(async||{tokio::fs::write(dest,maybe_gzip.to_vec()?).await})().await{
 					eprintln!("fs error: {}",e);
 				},
 				Err(e)=>eprintln!("dl error: {}",e),
@@ -1228,9 +1228,9 @@ async fn download_history(mut config:DownloadHistoryConfig)->AResult<()>{
 			let mut path=output_folder.clone();
 			path.push(format!("{}_v{}.rbxl",config.asset_id,version_number));
 			join_set.spawn(async move{
-				let file=context.get_asset(rbx_asset::cookie::GetAssetRequest{asset_id:config.asset_id,version:Some(version_number)}).await?;
+				let maybe_gzip=context.get_asset(rbx_asset::cookie::GetAssetRequest{asset_id:config.asset_id,version:Some(version_number)}).await?;
 
-				tokio::fs::write(path,file).await?;
+				tokio::fs::write(path,maybe_gzip.to_vec()?).await?;
 
 				Ok::<_,anyhow::Error>(())
 			});
@@ -1350,9 +1350,9 @@ struct DownloadDecompileConfig{
 
 async fn download_decompile(config:DownloadDecompileConfig)->AResult<()>{
 	let context=CookieContext::new(config.cookie);
-	let file=context.get_asset(rbx_asset::cookie::GetAssetRequest{asset_id:config.asset_id,version:None}).await?;
+	let maybe_gzip=context.get_asset(rbx_asset::cookie::GetAssetRequest{asset_id:config.asset_id,version:None}).await?;
 
-	let dom=load_dom(std::io::Cursor::new(file))?;
+	let dom=maybe_gzip.read_with(load_dom,load_dom)?;
 	let context=rox_compiler::DecompiledContext::from_dom(dom);
 
 	context.write_files(rox_compiler::WriteConfig{
@@ -1532,8 +1532,8 @@ async fn download_and_decompile_history_into_git(config:DownloadAndDecompileHist
 	.map(|asset_version|{
 		let context=context.clone();
 		tokio::task::spawn(async move{
-			let file=context.get_asset(rbx_asset::cookie::GetAssetRequest{asset_id,version:Some(asset_version.assetVersionNumber)}).await?;
-			let dom=load_dom(std::io::Cursor::new(file))?;
+			let maybe_gzip=context.get_asset(rbx_asset::cookie::GetAssetRequest{asset_id,version:Some(asset_version.assetVersionNumber)}).await?;
+			let dom=maybe_gzip.read_with(load_dom,load_dom)?;
 			Ok::<_,anyhow::Error>((asset_version,rox_compiler::DecompiledContext::from_dom(dom)))
 		})
 	}))