From ca50bf35c25dc64b8e99ca73db869b39928dca0f Mon Sep 17 00:00:00 2001
From: Quaternions <krakow20@gmail.com>
Date: Tue, 4 Feb 2025 09:09:01 -0800
Subject: [PATCH] loader + tokio rewrite
---
Cargo.lock | 1166 ++++++++++++++++++++++++++++++++++++++++++++++---
Cargo.toml | 15 +-
src/main.rs | 9 +-
src/roblox.rs | 389 ++++++++++++-----
src/source.rs | 536 +++++++++++++----------
5 files changed, 1708 insertions(+), 407 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index 52092e5..5771838 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2,6 +2,15 @@
# It is not intended for manual editing.
version = 4
+[[package]]
+name = "addr2line"
+version = "0.24.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1"
+dependencies = [
+ "gimli",
+]
+
[[package]]
name = "adler2"
version = "2.0.0"
@@ -15,7 +24,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011"
dependencies = [
"cfg-if",
- "getrandom",
+ "getrandom 0.2.15",
"once_cell",
"version_check",
"zerocopy",
@@ -36,6 +45,21 @@ version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4aa90d7ce82d4be67b64039a3d588d38dbcc6736577de4a847025ce5b0c468d1"
+[[package]]
+name = "android-tzdata"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0"
+
+[[package]]
+name = "android_system_properties"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311"
+dependencies = [
+ "libc",
+]
+
[[package]]
name = "anstream"
version = "0.6.18"
@@ -72,7 +96,7 @@ version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c"
dependencies = [
- "windows-sys",
+ "windows-sys 0.59.0",
]
[[package]]
@@ -83,7 +107,7 @@ checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e"
dependencies = [
"anstyle",
"once_cell",
- "windows-sys",
+ "windows-sys 0.59.0",
]
[[package]]
@@ -136,6 +160,12 @@ version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
+[[package]]
+name = "atomic-waker"
+version = "1.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"
+
[[package]]
name = "autocfg"
version = "1.4.0"
@@ -165,6 +195,21 @@ dependencies = [
"arrayvec",
]
+[[package]]
+name = "backtrace"
+version = "0.3.74"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a"
+dependencies = [
+ "addr2line",
+ "cfg-if",
+ "libc",
+ "miniz_oxide",
+ "object",
+ "rustc-demangle",
+ "windows-targets",
+]
+
[[package]]
name = "base64"
version = "0.13.1"
@@ -172,10 +217,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
[[package]]
-name = "bcdec_rs"
-version = "0.1.2"
+name = "base64"
+version = "0.22.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c9934c2b68e46448d814db20e34a840ef9b4e7b3b7c8b1da91161481230f6350"
+checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
+
+[[package]]
+name = "bcdec_rs"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f09c37bc0e9f0924b7dae9988265ef3c76c88538f41a3b06caf4bed07cee5226"
[[package]]
name = "beef"
@@ -292,9 +343,9 @@ checksum = "c360505aed52b7ec96a3636c3f039d99103c37d1d9b4f7a8c743d3ea9ffcd03b"
[[package]]
name = "bumpalo"
-version = "3.16.0"
+version = "3.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c"
+checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf"
[[package]]
name = "bv"
@@ -344,10 +395,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495"
[[package]]
-name = "cc"
-version = "1.2.10"
+name = "bytes"
+version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "13208fcbb66eaeffe09b99fffbe1af420f00a7b35aa99ad683dfc1aa76145229"
+checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b"
+
+[[package]]
+name = "cc"
+version = "1.2.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e4730490333d58093109dc02c23174c3f4d490998c3fed3cc8e82d57afedb9cf"
dependencies = [
"jobserver",
"libc",
@@ -380,6 +437,21 @@ dependencies = [
"num-traits",
]
+[[package]]
+name = "chrono"
+version = "0.4.39"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825"
+dependencies = [
+ "android-tzdata",
+ "iana-time-zone",
+ "js-sys",
+ "num-traits",
+ "serde",
+ "wasm-bindgen",
+ "windows-targets",
+]
+
[[package]]
name = "clap"
version = "4.5.27"
@@ -438,6 +510,22 @@ version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6"
+[[package]]
+name = "core-foundation"
+version = "0.9.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
+[[package]]
+name = "core-foundation-sys"
+version = "0.8.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
+
[[package]]
name = "crc"
version = "3.2.1"
@@ -522,6 +610,15 @@ version = "1.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0"
+[[package]]
+name = "encoding_rs"
+version = "0.8.35"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3"
+dependencies = [
+ "cfg-if",
+]
+
[[package]]
name = "enum-primitive-derive"
version = "0.2.2"
@@ -553,6 +650,16 @@ dependencies = [
"synstructure 0.12.6",
]
+[[package]]
+name = "errno"
+version = "0.3.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d"
+dependencies = [
+ "libc",
+ "windows-sys 0.59.0",
+]
+
[[package]]
name = "exr"
version = "1.73.0"
@@ -568,6 +675,12 @@ dependencies = [
"zune-inflate",
]
+[[package]]
+name = "fastrand"
+version = "2.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
+
[[package]]
name = "fdeflate"
version = "0.3.7"
@@ -585,9 +698,9 @@ checksum = "835a3dc7d1ec9e75e2b5fb4ba75396837112d2060b03f7d43bc1897c7f7211da"
[[package]]
name = "fixed_wide"
-version = "0.1.1"
+version = "0.1.2"
source = "sparse+https://git.itzana.me/api/packages/strafesnet/cargo/"
-checksum = "d9c2cf115b3785ede870fada07e8b1aeba3378345b4ca86fe3c772ecabc05c0f"
+checksum = "a7e01a5b738e313c912fc41c425cf36e10c51647d3fd21d96db3d616344549fa"
dependencies = [
"arrayvec",
"bnum",
@@ -611,6 +724,21 @@ version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
+[[package]]
+name = "foreign-types"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
+dependencies = [
+ "foreign-types-shared",
+]
+
+[[package]]
+name = "foreign-types-shared"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
+
[[package]]
name = "form_urlencoded"
version = "1.2.1"
@@ -620,6 +748,95 @@ dependencies = [
"percent-encoding",
]
+[[package]]
+name = "futures"
+version = "0.3.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876"
+dependencies = [
+ "futures-channel",
+ "futures-core",
+ "futures-executor",
+ "futures-io",
+ "futures-sink",
+ "futures-task",
+ "futures-util",
+]
+
+[[package]]
+name = "futures-channel"
+version = "0.3.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10"
+dependencies = [
+ "futures-core",
+ "futures-sink",
+]
+
+[[package]]
+name = "futures-core"
+version = "0.3.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e"
+
+[[package]]
+name = "futures-executor"
+version = "0.3.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f"
+dependencies = [
+ "futures-core",
+ "futures-task",
+ "futures-util",
+]
+
+[[package]]
+name = "futures-io"
+version = "0.3.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6"
+
+[[package]]
+name = "futures-macro"
+version = "0.3.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.96",
+]
+
+[[package]]
+name = "futures-sink"
+version = "0.3.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7"
+
+[[package]]
+name = "futures-task"
+version = "0.3.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988"
+
+[[package]]
+name = "futures-util"
+version = "0.3.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81"
+dependencies = [
+ "futures-channel",
+ "futures-core",
+ "futures-io",
+ "futures-macro",
+ "futures-sink",
+ "futures-task",
+ "memchr",
+ "pin-project-lite",
+ "pin-utils",
+ "slab",
+]
+
[[package]]
name = "getrandom"
version = "0.2.15"
@@ -628,7 +845,19 @@ checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
dependencies = [
"cfg-if",
"libc",
- "wasi",
+ "wasi 0.11.0+wasi-snapshot-preview1",
+]
+
+[[package]]
+name = "getrandom"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "wasi 0.13.3+wasi-0.2.2",
+ "windows-targets",
]
[[package]]
@@ -641,12 +870,37 @@ dependencies = [
"weezl",
]
+[[package]]
+name = "gimli"
+version = "0.31.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f"
+
[[package]]
name = "glam"
version = "0.29.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc46dd3ec48fdd8e693a98d2b8bafae273a2d54c1de02a2a7e3d57d501f39677"
+[[package]]
+name = "h2"
+version = "0.4.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ccae279728d634d083c00f6099cb58f01cc99c145b84b8be2f6c74618d79922e"
+dependencies = [
+ "atomic-waker",
+ "bytes",
+ "fnv",
+ "futures-core",
+ "futures-sink",
+ "http",
+ "indexmap",
+ "slab",
+ "tokio",
+ "tokio-util",
+ "tracing",
+]
+
[[package]]
name = "half"
version = "2.4.1"
@@ -676,6 +930,141 @@ version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024"
+[[package]]
+name = "http"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f16ca2af56261c99fba8bac40a10251ce8188205a4c448fbb745a2e4daa76fea"
+dependencies = [
+ "bytes",
+ "fnv",
+ "itoa",
+]
+
+[[package]]
+name = "http-body"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184"
+dependencies = [
+ "bytes",
+ "http",
+]
+
+[[package]]
+name = "http-body-util"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f"
+dependencies = [
+ "bytes",
+ "futures-util",
+ "http",
+ "http-body",
+ "pin-project-lite",
+]
+
+[[package]]
+name = "httparse"
+version = "1.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2d708df4e7140240a16cd6ab0ab65c972d7433ab77819ea693fde9c43811e2a"
+
+[[package]]
+name = "hyper"
+version = "1.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cc2b571658e38e0c01b1fdca3bbbe93c00d3d71693ff2770043f8c29bc7d6f80"
+dependencies = [
+ "bytes",
+ "futures-channel",
+ "futures-util",
+ "h2",
+ "http",
+ "http-body",
+ "httparse",
+ "itoa",
+ "pin-project-lite",
+ "smallvec",
+ "tokio",
+ "want",
+]
+
+[[package]]
+name = "hyper-rustls"
+version = "0.27.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2d191583f3da1305256f22463b9bb0471acad48a4e534a5218b9963e9c1f59b2"
+dependencies = [
+ "futures-util",
+ "http",
+ "hyper",
+ "hyper-util",
+ "rustls",
+ "rustls-pki-types",
+ "tokio",
+ "tokio-rustls",
+ "tower-service",
+]
+
+[[package]]
+name = "hyper-tls"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0"
+dependencies = [
+ "bytes",
+ "http-body-util",
+ "hyper",
+ "hyper-util",
+ "native-tls",
+ "tokio",
+ "tokio-native-tls",
+ "tower-service",
+]
+
+[[package]]
+name = "hyper-util"
+version = "0.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4"
+dependencies = [
+ "bytes",
+ "futures-channel",
+ "futures-util",
+ "http",
+ "http-body",
+ "hyper",
+ "pin-project-lite",
+ "socket2",
+ "tokio",
+ "tower-service",
+ "tracing",
+]
+
+[[package]]
+name = "iana-time-zone"
+version = "0.1.61"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220"
+dependencies = [
+ "android_system_properties",
+ "core-foundation-sys",
+ "iana-time-zone-haiku",
+ "js-sys",
+ "wasm-bindgen",
+ "windows-core",
+]
+
+[[package]]
+name = "iana-time-zone-haiku"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
+dependencies = [
+ "cc",
+]
+
[[package]]
name = "icu_collections"
version = "1.5.0"
@@ -861,9 +1250,9 @@ dependencies = [
[[package]]
name = "image_dds"
-version = "0.6.2"
+version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "84c6d1a2d80bc7dd2928b2a72a46d71bccbb6becf8ce207522b0b92daf0a417f"
+checksum = "550c3f4c985c41d96aa133651aa1a3dbd54961c28a68aca574487fe8bbdda6fb"
dependencies = [
"bcdec_rs",
"bytemuck",
@@ -872,7 +1261,7 @@ dependencies = [
"image",
"intel_tex_2",
"strum",
- "thiserror",
+ "thiserror 1.0.69",
]
[[package]]
@@ -911,6 +1300,12 @@ dependencies = [
"syn 2.0.96",
]
+[[package]]
+name = "ipnet"
+version = "2.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130"
+
[[package]]
name = "is_terminal_polyfill"
version = "1.70.1"
@@ -945,6 +1340,12 @@ dependencies = [
"either",
]
+[[package]]
+name = "itoa"
+version = "1.0.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674"
+
[[package]]
name = "jobserver"
version = "0.1.32"
@@ -960,6 +1361,16 @@ version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f5d4a7da358eff58addd2877a45865158f0d78c911d43a5784ceb7bbf52833b0"
+[[package]]
+name = "js-sys"
+version = "0.3.77"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f"
+dependencies = [
+ "once_cell",
+ "wasm-bindgen",
+]
+
[[package]]
name = "lazy-regex"
version = "3.4.1"
@@ -1003,9 +1414,9 @@ checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a"
[[package]]
name = "libfuzzer-sys"
-version = "0.4.8"
+version = "0.4.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9b9569d2f74e257076d8c6bfa73fb505b46b851e51ddaecc825944aa3bed17fa"
+checksum = "cf78f52d400cf2d84a3a973a78a592b4adc535739e0a5597a0da6f0c357adc75"
dependencies = [
"arbitrary",
"cc",
@@ -1038,6 +1449,12 @@ dependencies = [
"ratio_ops",
]
+[[package]]
+name = "linux-raw-sys"
+version = "0.4.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab"
+
[[package]]
name = "litemap"
version = "0.7.4"
@@ -1104,9 +1521,9 @@ dependencies = [
[[package]]
name = "luau0-src"
-version = "0.11.2+luau653"
+version = "0.12.0+luau657"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "02313a53daf1fae25e82f7e7ca56180b72d1f08c514426672877cd957298201c"
+checksum = "3a4b4c16b82ddf60e0fa93ca5a6afc504a6db22310cc82ecec629cd2e405b2ca"
dependencies = [
"cc",
]
@@ -1156,17 +1573,22 @@ dependencies = [
"anyhow",
"clap",
"flate2",
+ "futures",
"image",
"image_dds",
"lazy-regex",
+ "rbx_asset",
"rbx_binary",
"rbx_dom_weak",
"rbx_reflection_database",
"rbx_xml",
+ "rbxassetid",
"strafesnet_bsp_loader",
"strafesnet_deferred_loader",
"strafesnet_rbx_loader",
"strafesnet_snf",
+ "thiserror 2.0.11",
+ "tokio",
"vbsp",
"vmdl",
"vmt-parser",
@@ -1198,7 +1620,7 @@ checksum = "317f146e2eb7021892722af37cf1b971f0a70c8406f487e24952667616192c64"
dependencies = [
"cfg-if",
"miette-derive",
- "thiserror",
+ "thiserror 1.0.69",
"unicode-width",
]
@@ -1213,6 +1635,22 @@ dependencies = [
"syn 2.0.96",
]
+[[package]]
+name = "mime"
+version = "0.3.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
+
+[[package]]
+name = "mime_guess"
+version = "2.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e"
+dependencies = [
+ "mime",
+ "unicase",
+]
+
[[package]]
name = "minimal-lexical"
version = "0.2.1"
@@ -1230,10 +1668,21 @@ dependencies = [
]
[[package]]
-name = "mlua"
-version = "0.10.2"
+name = "mio"
+version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9ea43c3ffac2d0798bd7128815212dd78c98316b299b7a902dabef13dc7b6b8d"
+checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd"
+dependencies = [
+ "libc",
+ "wasi 0.11.0+wasi-snapshot-preview1",
+ "windows-sys 0.52.0",
+]
+
+[[package]]
+name = "mlua"
+version = "0.10.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d3f763c1041eff92ffb5d7169968a327e1ed2ebfe425dac0ee5a35f29082534b"
dependencies = [
"bstr",
"either",
@@ -1246,9 +1695,9 @@ dependencies = [
[[package]]
name = "mlua-sys"
-version = "0.6.6"
+version = "0.6.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "63a11d485edf0f3f04a508615d36c7d50d299cf61a7ee6d3e2530651e0a31771"
+checksum = "1901c1a635a22fe9250ffcc4fcc937c16b47c2e9e71adba8784af8bca1f69594"
dependencies = [
"cc",
"cfg-if",
@@ -1256,6 +1705,23 @@ dependencies = [
"pkg-config",
]
+[[package]]
+name = "native-tls"
+version = "0.2.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0dab59f8e050d5df8e4dd87d9206fb6f65a483e20ac9fda365ade4fab353196c"
+dependencies = [
+ "libc",
+ "log",
+ "openssl",
+ "openssl-probe",
+ "openssl-sys",
+ "schannel",
+ "security-framework",
+ "security-framework-sys",
+ "tempfile",
+]
+
[[package]]
name = "new_debug_unreachable"
version = "1.0.6"
@@ -1359,12 +1825,65 @@ dependencies = [
"syn 2.0.96",
]
+[[package]]
+name = "object"
+version = "0.36.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87"
+dependencies = [
+ "memchr",
+]
+
[[package]]
name = "once_cell"
version = "1.20.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775"
+[[package]]
+name = "openssl"
+version = "0.10.69"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f5e534d133a060a3c19daec1eb3e98ec6f4685978834f2dbadfe2ec215bab64e"
+dependencies = [
+ "bitflags 2.8.0",
+ "cfg-if",
+ "foreign-types",
+ "libc",
+ "once_cell",
+ "openssl-macros",
+ "openssl-sys",
+]
+
+[[package]]
+name = "openssl-macros"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.96",
+]
+
+[[package]]
+name = "openssl-probe"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e"
+
+[[package]]
+name = "openssl-sys"
+version = "0.9.104"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "45abf306cbf99debc8195b66b7346498d7b10c210de50418b5ccd7ceba08c741"
+dependencies = [
+ "cc",
+ "libc",
+ "pkg-config",
+ "vcpkg",
+]
+
[[package]]
name = "owo-colors"
version = "3.5.0"
@@ -1504,6 +2023,12 @@ version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b"
+[[package]]
+name = "pin-utils"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
+
[[package]]
name = "pkg-config"
version = "0.3.31"
@@ -1644,7 +2169,7 @@ version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
dependencies = [
- "getrandom",
+ "getrandom 0.2.15",
]
[[package]]
@@ -1683,7 +2208,7 @@ dependencies = [
"rand_chacha",
"simd_helpers",
"system-deps",
- "thiserror",
+ "thiserror 1.0.69",
"v_frame",
"wasm-bindgen",
]
@@ -1723,6 +2248,20 @@ dependencies = [
"crossbeam-utils",
]
+[[package]]
+name = "rbx_asset"
+version = "0.2.5"
+source = "sparse+https://git.itzana.me/api/packages/strafesnet/cargo/"
+checksum = "dcf243f46bd41b3880a27278177a3f9996f95ab231d9a04345ad9dd381c3a54a"
+dependencies = [
+ "chrono",
+ "flate2",
+ "reqwest",
+ "serde",
+ "serde_json",
+ "url",
+]
+
[[package]]
name = "rbx_binary"
version = "0.7.4"
@@ -1735,7 +2274,7 @@ dependencies = [
"rbx_dom_weak",
"rbx_reflection",
"rbx_reflection_database",
- "thiserror",
+ "thiserror 1.0.69",
]
[[package]]
@@ -1750,9 +2289,9 @@ dependencies = [
[[package]]
name = "rbx_mesh"
-version = "0.1.2"
+version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "864ead0e98afce28c960f653d6203483834890d07f87b60e2f01415530a2fe9d"
+checksum = "36372fd7feb6d3c5780d2ada39d1397be9e196ddfbb23ba1d84e7a75cf790adb"
dependencies = [
"binrw 0.14.1",
"lazy-regex",
@@ -1766,7 +2305,7 @@ checksum = "c1b43fe592a4ce6fe54eb215fb82735efbb516d2cc045a94e3dc0234ff293620"
dependencies = [
"rbx_types",
"serde",
- "thiserror",
+ "thiserror 1.0.69",
]
[[package]]
@@ -1787,13 +2326,13 @@ version = "1.10.0"
source = "sparse+https://git.itzana.me/api/packages/strafesnet/cargo/"
checksum = "d7a390c44034fa448c53bd0983dfc2d70d8d6b2f65be4f164d4bec8b6a2a2d09"
dependencies = [
- "base64",
+ "base64 0.13.1",
"bitflags 1.3.2",
"blake3",
"lazy_static",
"rand",
"serde",
- "thiserror",
+ "thiserror 1.0.69",
]
[[package]]
@@ -1802,7 +2341,7 @@ version = "0.13.3"
source = "sparse+https://git.itzana.me/api/packages/strafesnet/cargo/"
checksum = "d6d1a15f58a1e4b4f578abe6eb5e1461cb16eea82fb4a147d5995c9b79f08d1f"
dependencies = [
- "base64",
+ "base64 0.13.1",
"log",
"rbx_dom_weak",
"rbx_reflection",
@@ -1810,6 +2349,15 @@ dependencies = [
"xml-rs",
]
+[[package]]
+name = "rbxassetid"
+version = "0.1.0"
+source = "sparse+https://git.itzana.me/api/packages/strafesnet/cargo/"
+checksum = "e6821fe9eaff54cd142932cb04c612b7599d9b8586973145b7ec1230ae84d184"
+dependencies = [
+ "url",
+]
+
[[package]]
name = "redox_syscall"
version = "0.5.8"
@@ -1848,12 +2396,72 @@ version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
+[[package]]
+name = "reqwest"
+version = "0.12.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "43e734407157c3c2034e0258f5e4473ddb361b1e85f95a66690d67264d7cd1da"
+dependencies = [
+ "base64 0.22.1",
+ "bytes",
+ "encoding_rs",
+ "futures-core",
+ "futures-util",
+ "h2",
+ "http",
+ "http-body",
+ "http-body-util",
+ "hyper",
+ "hyper-rustls",
+ "hyper-tls",
+ "hyper-util",
+ "ipnet",
+ "js-sys",
+ "log",
+ "mime",
+ "mime_guess",
+ "native-tls",
+ "once_cell",
+ "percent-encoding",
+ "pin-project-lite",
+ "rustls-pemfile",
+ "serde",
+ "serde_json",
+ "serde_urlencoded",
+ "sync_wrapper",
+ "system-configuration",
+ "tokio",
+ "tokio-native-tls",
+ "tower",
+ "tower-service",
+ "url",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+ "web-sys",
+ "windows-registry",
+]
+
[[package]]
name = "rgb"
version = "0.8.50"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57397d16646700483b67d2dd6511d79318f9d057fdbd21a4066aeac8b41d310a"
+[[package]]
+name = "ring"
+version = "0.17.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d"
+dependencies = [
+ "cc",
+ "cfg-if",
+ "getrandom 0.2.15",
+ "libc",
+ "spin",
+ "untrusted",
+ "windows-sys 0.52.0",
+]
+
[[package]]
name = "rmp"
version = "0.8.14"
@@ -1878,9 +2486,9 @@ dependencies = [
[[package]]
name = "roblox_emulator"
-version = "0.4.6"
+version = "0.4.7"
source = "sparse+https://git.itzana.me/api/packages/strafesnet/cargo/"
-checksum = "03e2535327bd9069b20caa9df0a5cac87fa886cd2418c7f174016502d584a488"
+checksum = "7fc98335ce4b8548b725d727c5b32bd0b38274606c48fce6b6e7e5807d94db6b"
dependencies = [
"glam",
"mlua",
@@ -1891,24 +2499,120 @@ dependencies = [
"rbx_types",
]
+[[package]]
+name = "rustc-demangle"
+version = "0.1.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f"
+
[[package]]
name = "rustc-hash"
version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c7fb8039b3032c191086b10f11f319a6e99e1e82889c5cc6046f515c9db1d497"
+[[package]]
+name = "rustix"
+version = "0.38.44"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154"
+dependencies = [
+ "bitflags 2.8.0",
+ "errno",
+ "libc",
+ "linux-raw-sys",
+ "windows-sys 0.59.0",
+]
+
+[[package]]
+name = "rustls"
+version = "0.23.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9fb9263ab4eb695e42321db096e3b8fbd715a59b154d5c88d82db2175b681ba7"
+dependencies = [
+ "once_cell",
+ "rustls-pki-types",
+ "rustls-webpki",
+ "subtle",
+ "zeroize",
+]
+
+[[package]]
+name = "rustls-pemfile"
+version = "2.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50"
+dependencies = [
+ "rustls-pki-types",
+]
+
+[[package]]
+name = "rustls-pki-types"
+version = "1.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c"
+
+[[package]]
+name = "rustls-webpki"
+version = "0.102.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9"
+dependencies = [
+ "ring",
+ "rustls-pki-types",
+ "untrusted",
+]
+
[[package]]
name = "rustversion"
version = "1.0.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7c45b9784283f1b2e7fb61b42047c2fd678ef0960d4f6f1eba131594cc369d4"
+[[package]]
+name = "ryu"
+version = "1.0.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ea1a2d0a644769cc99faa24c3ad26b379b786fe7c36fd3c546254801650e6dd"
+
+[[package]]
+name = "schannel"
+version = "0.1.27"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d"
+dependencies = [
+ "windows-sys 0.59.0",
+]
+
[[package]]
name = "scopeguard"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
+[[package]]
+name = "security-framework"
+version = "2.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02"
+dependencies = [
+ "bitflags 2.8.0",
+ "core-foundation",
+ "core-foundation-sys",
+ "libc",
+ "security-framework-sys",
+]
+
+[[package]]
+name = "security-framework-sys"
+version = "2.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
[[package]]
name = "serde"
version = "1.0.217"
@@ -1929,6 +2633,18 @@ dependencies = [
"syn 2.0.96",
]
+[[package]]
+name = "serde_json"
+version = "1.0.138"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d434192e7da787e94a6ea7e9670b26a036d0ca41e0b7efb2676dd32bae872949"
+dependencies = [
+ "itoa",
+ "memchr",
+ "ryu",
+ "serde",
+]
+
[[package]]
name = "serde_repr"
version = "0.1.19"
@@ -1949,6 +2665,18 @@ dependencies = [
"serde",
]
+[[package]]
+name = "serde_urlencoded"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd"
+dependencies = [
+ "form_urlencoded",
+ "itoa",
+ "ryu",
+ "serde",
+]
+
[[package]]
name = "shlex"
version = "1.3.0"
@@ -1976,12 +2704,37 @@ version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d"
+[[package]]
+name = "slab"
+version = "0.4.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67"
+dependencies = [
+ "autocfg",
+]
+
[[package]]
name = "smallvec"
version = "1.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67"
+[[package]]
+name = "socket2"
+version = "0.5.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8"
+dependencies = [
+ "libc",
+ "windows-sys 0.52.0",
+]
+
+[[package]]
+name = "spin"
+version = "0.9.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
+
[[package]]
name = "stable_deref_trait"
version = "1.2.0"
@@ -1996,21 +2749,23 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
[[package]]
name = "strafesnet_bsp_loader"
-version = "0.2.2"
+version = "0.3.0"
source = "sparse+https://git.itzana.me/api/packages/strafesnet/cargo/"
-checksum = "34f944637bc3b3ed4c430819c672174b3a3edfd51f79b6b87f4931e3714a398e"
+checksum = "3cc98773f2b98eb708b098946870b769975b63a396b84698b67e3d968029005d"
dependencies = [
"glam",
"strafesnet_common",
+ "strafesnet_deferred_loader",
"vbsp",
"vmdl",
+ "vpk",
]
[[package]]
name = "strafesnet_common"
-version = "0.5.2"
+version = "0.6.0"
source = "sparse+https://git.itzana.me/api/packages/strafesnet/cargo/"
-checksum = "91cc1f3699bd8248da18bf5d11273264396a257b5d47b8558acb2cb4e1761219"
+checksum = "0c1d7a83e1f6b579c6a9b4dc70c92373ab53b938601cd75928dd6795b5ffef21"
dependencies = [
"arrayvec",
"bitflags 2.8.0",
@@ -2023,20 +2778,18 @@ dependencies = [
[[package]]
name = "strafesnet_deferred_loader"
-version = "0.4.1"
+version = "0.5.0"
source = "sparse+https://git.itzana.me/api/packages/strafesnet/cargo/"
-checksum = "cb47034893e945c640063a6c0fb09c6186dcc9f0b221b8c41f5a22070fe430f4"
+checksum = "63d5d48e587d5f8bf5385bee3505ed790727fef68de855cf58247a08c5952bef"
dependencies = [
"strafesnet_common",
- "url",
- "vbsp",
]
[[package]]
name = "strafesnet_rbx_loader"
-version = "0.5.2"
+version = "0.6.0"
source = "sparse+https://git.itzana.me/api/packages/strafesnet/cargo/"
-checksum = "bb852ee329d26410daee50f1e583ea8286caf6a81a42ff887b78f21477c48731"
+checksum = "e4659a49128c8d12b9fbdb289969cae04bfc5c1750d4273897700c5c17730d8a"
dependencies = [
"bytemuck",
"glam",
@@ -2046,15 +2799,17 @@ dependencies = [
"rbx_mesh",
"rbx_reflection_database",
"rbx_xml",
+ "rbxassetid",
"roblox_emulator",
"strafesnet_common",
+ "strafesnet_deferred_loader",
]
[[package]]
name = "strafesnet_snf"
-version = "0.2.0"
+version = "0.3.0"
source = "sparse+https://git.itzana.me/api/packages/strafesnet/cargo/"
-checksum = "c6e8856d79c29bd5687b08bc1653370f7e242c84d5c06afa8629bd3e00c433bf"
+checksum = "fd24a22c484ca04213fa44b1d34bfbec385f0d176a2b5829cfa59ba7987b80d5"
dependencies = [
"binrw 0.14.1",
"id",
@@ -2112,6 +2867,12 @@ dependencies = [
"syn 2.0.96",
]
+[[package]]
+name = "subtle"
+version = "2.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
+
[[package]]
name = "syn"
version = "1.0.109"
@@ -2134,6 +2895,15 @@ dependencies = [
"unicode-ident",
]
+[[package]]
+name = "sync_wrapper"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263"
+dependencies = [
+ "futures-core",
+]
+
[[package]]
name = "synstructure"
version = "0.12.6"
@@ -2157,6 +2927,27 @@ dependencies = [
"syn 2.0.96",
]
+[[package]]
+name = "system-configuration"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b"
+dependencies = [
+ "bitflags 2.8.0",
+ "core-foundation",
+ "system-configuration-sys",
+]
+
+[[package]]
+name = "system-configuration-sys"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
[[package]]
name = "system-deps"
version = "6.2.2"
@@ -2176,6 +2967,20 @@ version = "0.12.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1"
+[[package]]
+name = "tempfile"
+version = "3.16.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "38c246215d7d24f48ae091a2902398798e05d978b24315d6efbc00ede9a8bb91"
+dependencies = [
+ "cfg-if",
+ "fastrand",
+ "getrandom 0.3.1",
+ "once_cell",
+ "rustix",
+ "windows-sys 0.59.0",
+]
+
[[package]]
name = "texpresso"
version = "2.0.1"
@@ -2191,7 +2996,16 @@ version = "1.0.69"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52"
dependencies = [
- "thiserror-impl",
+ "thiserror-impl 1.0.69",
+]
+
+[[package]]
+name = "thiserror"
+version = "2.0.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d452f284b73e6d76dd36758a0c8684b1d5be31f92b89d07fd5822175732206fc"
+dependencies = [
+ "thiserror-impl 2.0.11",
]
[[package]]
@@ -2205,6 +3019,17 @@ dependencies = [
"syn 2.0.96",
]
+[[package]]
+name = "thiserror-impl"
+version = "2.0.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.96",
+]
+
[[package]]
name = "tiff"
version = "0.9.1"
@@ -2226,6 +3051,66 @@ dependencies = [
"zerovec",
]
+[[package]]
+name = "tokio"
+version = "1.43.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3d61fa4ffa3de412bfea335c6ecff681de2b609ba3c77ef3e00e521813a9ed9e"
+dependencies = [
+ "backtrace",
+ "bytes",
+ "libc",
+ "mio",
+ "pin-project-lite",
+ "socket2",
+ "tokio-macros",
+ "windows-sys 0.52.0",
+]
+
+[[package]]
+name = "tokio-macros"
+version = "2.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.96",
+]
+
+[[package]]
+name = "tokio-native-tls"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2"
+dependencies = [
+ "native-tls",
+ "tokio",
+]
+
+[[package]]
+name = "tokio-rustls"
+version = "0.26.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5f6d0975eaace0cf0fcadee4e4aaa5da15b5c079146f2cffb67c113be122bf37"
+dependencies = [
+ "rustls",
+ "tokio",
+]
+
+[[package]]
+name = "tokio-util"
+version = "0.7.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d7fcaa8d55a2bdd6b83ace262b016eca0d79ee02818c5c1bcdf0305114081078"
+dependencies = [
+ "bytes",
+ "futures-core",
+ "futures-sink",
+ "pin-project-lite",
+ "tokio",
+]
+
[[package]]
name = "toml"
version = "0.8.19"
@@ -2249,9 +3134,9 @@ dependencies = [
[[package]]
name = "toml_edit"
-version = "0.22.22"
+version = "0.22.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5"
+checksum = "02a8b472d1a3d7c18e2d61a489aee3453fd9031c33e4f55bd533f4a7adca1bee"
dependencies = [
"indexmap",
"serde",
@@ -2260,6 +3145,33 @@ dependencies = [
"winnow",
]
+[[package]]
+name = "tower"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9"
+dependencies = [
+ "futures-core",
+ "futures-util",
+ "pin-project-lite",
+ "sync_wrapper",
+ "tokio",
+ "tower-layer",
+ "tower-service",
+]
+
+[[package]]
+name = "tower-layer"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e"
+
+[[package]]
+name = "tower-service"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3"
+
[[package]]
name = "tracing"
version = "0.1.41"
@@ -2292,10 +3204,22 @@ dependencies = [
]
[[package]]
-name = "unicode-ident"
-version = "1.0.15"
+name = "try-lock"
+version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "11cd88e12b17c6494200a9c1b683a04fcac9573ed74cd1b62aeb2727c5592243"
+checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
+
+[[package]]
+name = "unicase"
+version = "2.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539"
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a210d160f08b701c8721ba1c726c11662f877ea6b7094007e1ca9a1041945034"
[[package]]
name = "unicode-width"
@@ -2309,6 +3233,12 @@ version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
+[[package]]
+name = "untrusted"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
+
[[package]]
name = "url"
version = "2.5.4"
@@ -2366,11 +3296,17 @@ dependencies = [
"num_enum",
"serde",
"static_assertions",
- "thiserror",
+ "thiserror 1.0.69",
"vdf-reader",
"zip-lzma",
]
+[[package]]
+name = "vcpkg"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
+
[[package]]
name = "vdf-reader"
version = "0.2.0"
@@ -2381,7 +3317,7 @@ dependencies = [
"miette",
"parse-display 0.9.1",
"serde",
- "thiserror",
+ "thiserror 1.0.69",
]
[[package]]
@@ -2410,7 +3346,7 @@ dependencies = [
"itertools 0.13.0",
"num_enum",
"static_assertions",
- "thiserror",
+ "thiserror 1.0.69",
"tracing",
]
@@ -2423,7 +3359,7 @@ dependencies = [
"miette",
"serde",
"serde_repr",
- "thiserror",
+ "thiserror 1.0.69",
"vdf-reader",
]
@@ -2435,7 +3371,7 @@ checksum = "60ec10e731515f58d5494d472f027d9c6fc8500fcb790ff55751031bcad87b6b"
dependencies = [
"ahash",
"binrw 0.13.3",
- "thiserror",
+ "thiserror 1.0.69",
]
[[package]]
@@ -2452,12 +3388,30 @@ dependencies = [
"texpresso",
]
+[[package]]
+name = "want"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e"
+dependencies = [
+ "try-lock",
+]
+
[[package]]
name = "wasi"
version = "0.11.0+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
+[[package]]
+name = "wasi"
+version = "0.13.3+wasi-0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "26816d2e1a4a36a2940b96c5296ce403917633dff8f3440e9b236ed6f6bacad2"
+dependencies = [
+ "wit-bindgen-rt",
+]
+
[[package]]
name = "wasm-bindgen"
version = "0.2.100"
@@ -2484,6 +3438,19 @@ dependencies = [
"wasm-bindgen-shared",
]
+[[package]]
+name = "wasm-bindgen-futures"
+version = "0.4.50"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61"
+dependencies = [
+ "cfg-if",
+ "js-sys",
+ "once_cell",
+ "wasm-bindgen",
+ "web-sys",
+]
+
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.100"
@@ -2516,12 +3483,70 @@ dependencies = [
"unicode-ident",
]
+[[package]]
+name = "web-sys"
+version = "0.3.77"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2"
+dependencies = [
+ "js-sys",
+ "wasm-bindgen",
+]
+
[[package]]
name = "weezl"
version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "53a85b86a771b1c87058196170769dd264f66c0782acf1ae6cc51bfd64b39082"
+[[package]]
+name = "windows-core"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9"
+dependencies = [
+ "windows-targets",
+]
+
+[[package]]
+name = "windows-registry"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0"
+dependencies = [
+ "windows-result",
+ "windows-strings",
+ "windows-targets",
+]
+
+[[package]]
+name = "windows-result"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e"
+dependencies = [
+ "windows-targets",
+]
+
+[[package]]
+name = "windows-strings"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10"
+dependencies = [
+ "windows-result",
+ "windows-targets",
+]
+
+[[package]]
+name = "windows-sys"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
+dependencies = [
+ "windows-targets",
+]
+
[[package]]
name = "windows-sys"
version = "0.59.0"
@@ -2597,13 +3622,22 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
[[package]]
name = "winnow"
-version = "0.6.24"
+version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c8d71a593cc5c42ad7876e2c1fda56f314f3754c084128833e64f1345ff8a03a"
+checksum = "7e49d2d35d3fad69b39b94139037ecfb4f359f08958b9c11e7315ce770462419"
dependencies = [
"memchr",
]
+[[package]]
+name = "wit-bindgen-rt"
+version = "0.33.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c"
+dependencies = [
+ "bitflags 2.8.0",
+]
+
[[package]]
name = "write16"
version = "1.0.0"
@@ -2688,6 +3722,12 @@ dependencies = [
"synstructure 0.13.1",
]
+[[package]]
+name = "zeroize"
+version = "1.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde"
+
[[package]]
name = "zerovec"
version = "0.10.4"
diff --git a/Cargo.toml b/Cargo.toml
index 3089111..ac3282f 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -9,17 +9,22 @@ edition = "2021"
anyhow = "1.0.75"
clap = { version = "4.4.2", features = ["derive"] }
flate2 = "1.0.27"
+futures = "0.3.31"
image = "0.25.2"
-image_dds = "0.6.0"
+image_dds = "0.7.1"
lazy-regex = "3.1.0"
+rbx_asset = { version = "0.2.5", registry = "strafesnet" }
rbx_binary = { version = "0.7.4", registry = "strafesnet" }
rbx_dom_weak = { version = "2.7.0", registry = "strafesnet" }
rbx_reflection_database = { version = "0.2.10", registry = "strafesnet" }
rbx_xml = { version = "0.13.3", registry = "strafesnet" }
-strafesnet_bsp_loader = { version = "0.2.1", registry = "strafesnet" }
-strafesnet_deferred_loader = { version = "0.4.0", features = ["legacy"], registry = "strafesnet" }
-strafesnet_rbx_loader = { version = "0.5.1", registry = "strafesnet" }
-strafesnet_snf = { version = "0.2.0", registry = "strafesnet" }
+rbxassetid = { version = "0.1.0", registry = "strafesnet" }
+strafesnet_bsp_loader = { version = "0.3.0", registry = "strafesnet" }
+strafesnet_deferred_loader = { version = "0.5.0", registry = "strafesnet" }
+strafesnet_rbx_loader = { version = "0.6.0", registry = "strafesnet" }
+strafesnet_snf = { version = "0.3.0", registry = "strafesnet" }
+thiserror = "2.0.11"
+tokio = { version = "1.43.0", features = ["macros", "rt-multi-thread", "fs"] }
vbsp = "0.6.0"
vmdl = "0.2.0"
vmt-parser = "0.2.0"
diff --git a/src/main.rs b/src/main.rs
index 9b43f9e..da5352b 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -20,10 +20,11 @@ enum Commands{
Source(source::Commands),
}
-fn main() -> AResult<()> {
- let cli = Cli::parse();
+#[tokio::main]
+async fn main()->AResult<()>{
+ let cli=Cli::parse();
match cli.command{
- Commands::Roblox(commands)=>commands.run(),
- Commands::Source(commands)=>commands.run(),
+ Commands::Roblox(commands)=>commands.run().await,
+ Commands::Source(commands)=>commands.run().await,
}
}
diff --git a/src/roblox.rs b/src/roblox.rs
index b22b62f..da2c0d1 100644
--- a/src/roblox.rs
+++ b/src/roblox.rs
@@ -4,7 +4,11 @@ use std::collections::HashSet;
use clap::{Args,Subcommand};
use anyhow::Result as AResult;
use rbx_dom_weak::Instance;
-use strafesnet_deferred_loader::rbxassetid::RobloxAssetId;
+use strafesnet_deferred_loader::deferred_loader::LoadFailureMode;
+use rbxassetid::RobloxAssetId;
+use tokio::io::AsyncReadExt;
+
+const DOWNLOAD_LIMIT:usize=16;
#[derive(Subcommand)]
pub enum Commands{
@@ -21,27 +25,40 @@ pub struct RobloxToSNFSubcommand {
}
#[derive(Args)]
pub struct DownloadAssetsSubcommand{
- #[arg(long,required=true)]
- roblox_files:Vec<PathBuf>
+ #[arg(required=true)]
+ roblox_files:Vec<PathBuf>,
+ // #[arg(long)]
+ // cookie_file:Option<String>,
}
impl Commands{
- pub fn run(self)->AResult<()>{
+ pub async fn run(self)->AResult<()>{
match self{
- Commands::RobloxToSNF(subcommand)=>roblox_to_snf(subcommand.input_files,subcommand.output_folder),
- Commands::DownloadAssets(subcommand)=>download_assets(subcommand.roblox_files),
+ Commands::RobloxToSNF(subcommand)=>roblox_to_snf(subcommand.input_files,subcommand.output_folder).await,
+ Commands::DownloadAssets(subcommand)=>download_assets(
+ subcommand.roblox_files,
+ rbx_asset::cookie::Cookie::new("".to_string()),
+ ).await,
}
}
}
-fn load_dom<R:Read+Seek>(mut input:R)->AResult<rbx_dom_weak::WeakDom>{
+#[allow(unused)]
+#[derive(Debug)]
+enum LoadDomError{
+ IO(std::io::Error),
+ Binary(rbx_binary::DecodeError),
+ Xml(rbx_xml::DecodeError),
+ UnknownFormat,
+}
+fn load_dom<R:Read+Seek>(mut input:R)->Result<rbx_dom_weak::WeakDom,LoadDomError>{
let mut first_8=[0u8;8];
- input.read_exact(&mut first_8)?;
- input.rewind()?;
+ input.read_exact(&mut first_8).map_err(LoadDomError::IO)?;
+ input.rewind().map_err(LoadDomError::IO)?;
match &first_8{
- b"<roblox!"=>rbx_binary::from_reader(input).map_err(anyhow::Error::msg),
- b"<roblox "=>rbx_xml::from_reader(input,rbx_xml::DecodeOptions::default()).map_err(anyhow::Error::msg),
- _=>Err(anyhow::Error::msg("unsupported file type")),
+ b"<roblox!"=>rbx_binary::from_reader(input).map_err(LoadDomError::Binary),
+ b"<roblox "=>rbx_xml::from_reader(input,rbx_xml::DecodeOptions::default()).map_err(LoadDomError::Xml),
+ _=>Err(LoadDomError::UnknownFormat),
}
}
@@ -80,13 +97,13 @@ fn accumulate_content_id(content_list:&mut HashSet<RobloxAssetId>,object:&Instan
println!("Content failed to parse into AssetID: {:?}",content);
}
}else{
- println!("property={} does not exist for class={}",object.class.as_str(),property);
+ println!("property={} does not exist for class={}",property,object.class.as_str());
}
}
-fn read_entire_file(path:impl AsRef<Path>)->Result<Cursor<Vec<u8>>,std::io::Error>{
- let mut file=std::fs::File::open(path)?;
+async fn read_entire_file(path:impl AsRef<Path>)->Result<Cursor<Vec<u8>>,std::io::Error>{
+ let mut file=tokio::fs::File::open(path).await?;
let mut data=Vec::new();
- file.read_to_end(&mut data)?;
+ file.read_to_end(&mut data).await?;
Ok(Cursor::new(data))
}
#[derive(Default)]
@@ -123,30 +140,232 @@ impl UniqueAssets{
}
}
}
-fn unique_assets(path:&Path)->AResult<UniqueAssets>{
+
+#[allow(unused)]
+#[derive(Debug)]
+enum UniqueAssetError{
+ IO(std::io::Error),
+ LoadDom(LoadDomError),
+}
+async fn unique_assets(path:&Path)->Result<UniqueAssets,UniqueAssetError>{
// read entire file
let mut assets=UniqueAssets::default();
- let data=read_entire_file(path)?;
- let dom=load_dom(data)?;
+ let data=read_entire_file(path).await.map_err(UniqueAssetError::IO)?;
+ let dom=load_dom(data).map_err(UniqueAssetError::LoadDom)?;
for object in dom.into_raw().1.into_values(){
assets.collect(&object);
}
Ok(assets)
}
-struct UniqueAssetsResult{
- path:std::path::PathBuf,
- result:AResult<UniqueAssets>,
+enum DownloadType{
+ Texture(RobloxAssetId),
+ Mesh(RobloxAssetId),
+ Union(RobloxAssetId),
}
-fn do_thread(path:std::path::PathBuf,send:std::sync::mpsc::Sender<UniqueAssetsResult>){
- std::thread::spawn(move ||{
- let result=unique_assets(path.as_path());
- send.send(UniqueAssetsResult{
- path,
- result,
- }).unwrap();
+impl DownloadType{
+ fn path(&self)->PathBuf{
+ match self{
+ DownloadType::Texture(asset_id)=>format!("downloaded_textures/{}",asset_id.0.to_string()).into(),
+ DownloadType::Mesh(asset_id)=>format!("meshes/{}",asset_id.0.to_string()).into(),
+ DownloadType::Union(asset_id)=>format!("unions/{}",asset_id.0.to_string()).into(),
+ }
+ }
+ fn asset_id(&self)->u64{
+ match self{
+ DownloadType::Texture(asset_id)=>asset_id.0,
+ DownloadType::Mesh(asset_id)=>asset_id.0,
+ DownloadType::Union(asset_id)=>asset_id.0,
+ }
+ }
+}
+enum DownloadResult{
+ Cached(PathBuf),
+ Data(Vec<u8>),
+ Failed,
+}
+#[derive(Default,Debug)]
+struct Stats{
+ total_assets:u32,
+ cached_assets:u32,
+ downloaded_assets:u32,
+ failed_downloads:u32,
+ timed_out_downloads:u32,
+}
+async fn download_retry(stats:&mut Stats,context:&rbx_asset::cookie::CookieContext,download_instruction:DownloadType)->Result<DownloadResult,std::io::Error>{
+ stats.total_assets+=1;
+ let download_instruction=download_instruction;
+ // check if file exists on disk
+ let path=download_instruction.path();
+ if tokio::fs::try_exists(path.as_path()).await?{
+ stats.cached_assets+=1;
+ return Ok(DownloadResult::Cached(path));
+ }
+ let asset_id=download_instruction.asset_id();
+ // if not, download file
+ let mut retry=0;
+ const BACKOFF_MUL:f32=1.3956124250860895286;//exp(1/3)
+ let mut backoff=1000f32;
+ loop{
+ let asset_result=context.get_asset(rbx_asset::cookie::GetAssetRequest{
+ asset_id,
+ version:None,
+ }).await;
+ match asset_result{
+ Ok(asset_result)=>{
+ stats.downloaded_assets+=1;
+ tokio::fs::write(path,&asset_result).await?;
+ break Ok(DownloadResult::Data(asset_result));
+ },
+ Err(rbx_asset::cookie::GetError::Response(rbx_asset::ResponseError::StatusCodeWithUrlAndBody(scwuab)))=>{
+ if scwuab.status_code.as_u16()==429{
+ if retry==12{
+ println!("Giving up asset download {asset_id}");
+ stats.timed_out_downloads+=1;
+ break Ok(DownloadResult::Failed);
+ }
+ println!("Hit roblox rate limit, waiting {:.0}ms...",backoff);
+ tokio::time::sleep(std::time::Duration::from_millis(backoff as u64)).await;
+ backoff*=BACKOFF_MUL;
+ retry+=1;
+ }else{
+ stats.failed_downloads+=1;
+ println!("weird scuwab error: {scwuab:?}");
+ break Ok(DownloadResult::Failed);
+ }
+ },
+ Err(e)=>{
+ stats.failed_downloads+=1;
+ println!("sadly error: {e}");
+ break Ok(DownloadResult::Failed);
+ },
+ }
+ }
+}
+#[derive(Debug,thiserror::Error)]
+enum ConvertTextureError{
+ #[error("Io error {0:?}")]
+ Io(#[from]std::io::Error),
+ #[error("Image error {0:?}")]
+ Image(#[from]image::ImageError),
+ #[error("DDS create error {0:?}")]
+ DDS(#[from]image_dds::CreateDdsError),
+ #[error("DDS write error {0:?}")]
+ DDSWrite(#[from]image_dds::ddsfile::Error),
+}
+async fn convert_texture(asset_id:RobloxAssetId,download_result:DownloadResult)->Result<(),ConvertTextureError>{
+ let data=match download_result{
+ DownloadResult::Cached(path)=>tokio::fs::read(path).await?,
+ DownloadResult::Data(data)=>data,
+ DownloadResult::Failed=>return Ok(()),
+ };
+ // image::ImageFormat::Png
+ // image::ImageFormat::Jpeg
+ let image=image::load_from_memory(&data)?.to_rgba8();
+
+ // pick format
+ let format=if image.width()%4!=0||image.height()%4!=0{
+ image_dds::ImageFormat::Rgba8UnormSrgb
+ }else{
+ image_dds::ImageFormat::BC7RgbaUnormSrgb
+ };
+
+ //this fails if the image dimensions are not a multiple of 4
+ let dds=image_dds::dds_from_image(
+ &image,
+ format,
+ image_dds::Quality::Slow,
+ image_dds::Mipmaps::GeneratedAutomatic,
+ )?;
+
+ let file_name=format!("textures/{}.dds",asset_id.0);
+ let mut file=std::fs::File::create(file_name)?;
+ dds.write(&mut file)?;
+ Ok(())
+}
+async fn download_assets(paths:Vec<PathBuf>,cookie:rbx_asset::cookie::Cookie)->AResult<()>{
+ tokio::try_join!(
+ tokio::fs::create_dir_all("downloaded_textures"),
+ tokio::fs::create_dir_all("textures"),
+ tokio::fs::create_dir_all("meshes"),
+ tokio::fs::create_dir_all("unions"),
+ )?;
+ // use mpsc
+ let thread_limit=std::thread::available_parallelism()?.get();
+ let (send_assets,mut recv_assets)=tokio::sync::mpsc::channel(DOWNLOAD_LIMIT);
+ let (send_texture,mut recv_texture)=tokio::sync::mpsc::channel(thread_limit);
+ // map decode dispatcher
+ // read files multithreaded
+ // produce UniqueAssetsResult per file
+ tokio::spawn(async move{
+ // move send so it gets dropped when all maps have been decoded
+ // closing the channel
+ let mut it=paths.into_iter();
+ static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
+ SEM.add_permits(thread_limit);
+ while let (Ok(permit),Some(path))=(SEM.acquire().await,it.next()){
+ let send=send_assets.clone();
+ tokio::spawn(async move{
+ let result=unique_assets(path.as_path()).await;
+ _=send.send(result).await;
+ drop(permit);
+ });
+ }
});
-}
-fn download_assets(paths:Vec<PathBuf>)->AResult<()>{
+ // download manager
+ // insert into global unique assets guy
+ // add to download queue if the asset is globally unique and does not already exist on disk
+ let mut stats=Stats::default();
+ let context=rbx_asset::cookie::CookieContext::new(cookie);
+ let mut globally_unique_assets=UniqueAssets::default();
+ // pop a job = retry_queue.pop_front() or ingest(recv.recv().await)
+ // SLOW MODE:
+ // acquire all permits
+ // drop all permits
+ // pop one job
+ // if it succeeds go into fast mode
+ // FAST MODE:
+ // acquire one permit
+ // pop a job
+ let download_thread=tokio::spawn(async move{
+ while let Some(result)=recv_assets.recv().await{
+ let unique_assets=match result{
+ Ok(unique_assets)=>unique_assets,
+ Err(e)=>{
+ println!("error: {e:?}");
+ continue;
+ },
+ };
+ for texture_id in unique_assets.textures{
+ if globally_unique_assets.textures.insert(texture_id){
+ let data=download_retry(&mut stats,&context,DownloadType::Texture(texture_id)).await?;
+ send_texture.send((texture_id,data)).await?;
+ }
+ }
+ for mesh_id in unique_assets.meshes{
+ if globally_unique_assets.meshes.insert(mesh_id){
+ download_retry(&mut stats,&context,DownloadType::Mesh(mesh_id)).await?;
+ }
+ }
+ for union_id in unique_assets.unions{
+ if globally_unique_assets.unions.insert(union_id){
+ download_retry(&mut stats,&context,DownloadType::Union(union_id)).await?;
+ }
+ }
+ }
+ dbg!(stats);
+ Ok::<(),anyhow::Error>(())
+ });
+ static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
+ SEM.add_permits(thread_limit);
+ while let (Ok(permit),Some((asset_id,download_result)))=(SEM.acquire().await,recv_texture.recv().await){
+ tokio::spawn(async move{
+ let result=convert_texture(asset_id,download_result).await;
+ drop(permit);
+ result.unwrap();
+ });
+ }
+ download_thread.await??;
+ _=SEM.acquire_many(thread_limit as u32).await.unwrap();
Ok(())
}
@@ -155,7 +374,8 @@ fn download_assets(paths:Vec<PathBuf>)->AResult<()>{
enum ConvertError{
IO(std::io::Error),
SNFMap(strafesnet_snf::map::Error),
- RbxLoader(strafesnet_rbx_loader::ReadError),
+ RobloxRead(strafesnet_rbx_loader::ReadError),
+ RobloxLoad(strafesnet_rbx_loader::LoadError),
}
impl std::fmt::Display for ConvertError{
fn fmt(&self,f:&mut std::fmt::Formatter<'_>)->std::fmt::Result{
@@ -163,78 +383,49 @@ impl std::fmt::Display for ConvertError{
}
}
impl std::error::Error for ConvertError{}
+async fn convert_to_snf(path:&Path,output_folder:PathBuf)->AResult<()>{
+ let entire_file=tokio::fs::read(path).await?;
-type MapThread=std::thread::JoinHandle<Result<(),ConvertError>>;
+ let model=strafesnet_rbx_loader::read(
+ std::io::Cursor::new(entire_file)
+ ).map_err(ConvertError::RobloxRead)?;
-fn roblox_to_snf(pathlist:Vec<std::path::PathBuf>,output_folder:PathBuf)->AResult<()>{
- let n_paths=pathlist.len();
- let start = std::time::Instant::now();
- let mut threads:std::collections::VecDeque<MapThread>=std::collections::VecDeque::new();
- let mut i=0;
- let mut join_thread=|thread:MapThread|{
- i+=1;
- if let Err(e)=thread.join(){
- println!("thread error: {:?}",e);
- }else{
- println!("{}/{}",i,n_paths);
- }
- };
- for path in pathlist{
- if 32<=threads.len(){
- join_thread(threads.pop_front().unwrap());
- }
- let output_folder=output_folder.clone();
- threads.push_back(std::thread::spawn(move ||{
- let model=strafesnet_rbx_loader::read(
- std::fs::File::open(path.as_path())
- .map_err(ConvertError::IO)?
- ).map_err(ConvertError::RbxLoader)?;
+ let mut place=model.into_place();
+ place.run_scripts();
- let mut place=model.into_place();
- place.run_scripts();
+ let map=place.to_snf(LoadFailureMode::DefaultToNone).map_err(ConvertError::RobloxLoad)?;
- let mut loader=strafesnet_deferred_loader::roblox_legacy();
+ let mut dest=output_folder;
+ dest.push(path.file_stem().unwrap());
+ dest.set_extension("snfm");
+ let file=std::fs::File::create(dest).map_err(ConvertError::IO)?;
- let (texture_loader,mesh_loader)=loader.get_inner_mut();
+ strafesnet_snf::map::write_map(file,map).map_err(ConvertError::SNFMap)?;
- let map_step1=strafesnet_rbx_loader::convert(
- &place,
- |name|texture_loader.acquire_render_config_id(name),
- |name|mesh_loader.acquire_mesh_id(name),
- );
-
- let meshpart_meshes=mesh_loader.load_meshes().map_err(ConvertError::IO)?;
-
- let map_step2=map_step1.add_meshpart_meshes_and_calculate_attributes(
- meshpart_meshes.into_iter().map(|(mesh_id,loader_model)|
- (mesh_id,strafesnet_rbx_loader::data::RobloxMeshBytes::new(loader_model.get()))
- )
- );
-
- let (textures,render_configs)=loader.into_render_configs().map_err(ConvertError::IO)?.consume();
-
- let map=map_step2.add_render_configs_and_textures(
- render_configs.into_iter(),
- textures.into_iter().map(|(texture_id,texture)|
- (texture_id,match texture{
- strafesnet_deferred_loader::texture::Texture::ImageDDS(data)=>data,
- })
- )
- );
-
- let mut dest=output_folder.clone();
- dest.push(path.file_stem().unwrap());
- dest.set_extension("snfm");
- let file=std::fs::File::create(dest).map_err(ConvertError::IO)?;
-
- strafesnet_snf::map::write_map(file,map).map_err(ConvertError::SNFMap)?;
- Ok(())
- }));
- }
-
- for thread in threads{
- join_thread(thread);
- }
- println!("{:?}", start.elapsed());
+ Ok(())
+}
+
+async fn roblox_to_snf(paths:Vec<std::path::PathBuf>,output_folder:PathBuf)->AResult<()>{
+ let start=std::time::Instant::now();
+
+ let thread_limit=std::thread::available_parallelism()?.get();
+ let mut it=paths.into_iter();
+ static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
+ SEM.add_permits(thread_limit);
+
+ while let (Ok(permit),Some(path))=(SEM.acquire().await,it.next()){
+ let output_folder=output_folder.clone();
+ tokio::spawn(async move{
+ let result=convert_to_snf(path.as_path(),output_folder).await;
+ drop(permit);
+ match result{
+ Ok(())=>(),
+ Err(e)=>println!("Convert error: {e:?}"),
+ }
+ });
+ }
+ _=SEM.acquire_many(thread_limit as u32).await.unwrap();
+
+ println!("elapsed={:?}", start.elapsed());
Ok(())
}
diff --git a/src/source.rs b/src/source.rs
index 1d0f803..5fcecea 100644
--- a/src/source.rs
+++ b/src/source.rs
@@ -1,6 +1,11 @@
-use std::path::PathBuf;
+use std::path::{Path,PathBuf};
+use std::borrow::Cow;
use clap::{Args,Subcommand};
use anyhow::Result as AResult;
+use futures::StreamExt;
+use strafesnet_bsp_loader::loader::BspFinder;
+use strafesnet_deferred_loader::loader::Loader;
+use strafesnet_deferred_loader::deferred_loader::{LoadFailureMode,MeshDeferredLoader,RenderConfigDeferredLoader};
#[derive(Subcommand)]
pub enum Commands{
@@ -16,13 +21,15 @@ pub struct SourceToSNFSubcommand {
output_folder:PathBuf,
#[arg(required=true)]
input_files:Vec<PathBuf>,
+ #[arg(long)]
+ vpks:Vec<PathBuf>,
}
#[derive(Args)]
-pub struct ExtractTexturesSubcommand {
+pub struct ExtractTexturesSubcommand{
+ #[arg(required=true)]
+ bsp_files:Vec<PathBuf>,
#[arg(long)]
- bsp_file:PathBuf,
- #[arg(long)]
- vpk_dir_files:Vec<PathBuf>
+ vpks:Vec<PathBuf>,
}
#[derive(Args)]
pub struct VPKContentsSubcommand {
@@ -36,10 +43,10 @@ pub struct BSPContentsSubcommand {
}
impl Commands{
- pub fn run(self)->AResult<()>{
+ pub async fn run(self)->AResult<()>{
match self{
- Commands::SourceToSNF(subcommand)=>source_to_snf(subcommand.input_files,subcommand.output_folder),
- Commands::ExtractTextures(subcommand)=>extract_textures(vec![subcommand.bsp_file],subcommand.vpk_dir_files),
+ Commands::SourceToSNF(subcommand)=>source_to_snf(subcommand.input_files,subcommand.output_folder,subcommand.vpks).await,
+ Commands::ExtractTextures(subcommand)=>extract_textures(subcommand.bsp_files,subcommand.vpks).await,
Commands::VPKContents(subcommand)=>vpk_contents(subcommand.input_file),
Commands::BSPContents(subcommand)=>bsp_contents(subcommand.input_file),
}
@@ -63,9 +70,9 @@ impl VMTContent{
}
}
-fn get_some_texture(material:vmt_parser::material::Material)->AResult<VMTContent>{
+fn get_some_texture(material:vmt_parser::material::Material)->VMTContent{
//just grab some texture from somewhere for now
- Ok(match material{
+ match material{
vmt_parser::material::Material::LightMappedGeneric(mat)=>VMTContent::vtf(Some(mat.base_texture)),
vmt_parser::material::Material::VertexLitGeneric(mat)=>VMTContent::vtf(mat.base_texture.or(mat.decal_texture)),//this just dies if there is none
vmt_parser::material::Material::VertexLitGenericDx6(mat)=>VMTContent::vtf(mat.base_texture.or(mat.decal_texture)),
@@ -84,175 +91,258 @@ fn get_some_texture(material:vmt_parser::material::Material)->AResult<VMTContent
vmt_parser::material::Material::Sky(mat)=>VMTContent::vtf(Some(mat.base_texture)),
vmt_parser::material::Material::Replacements(_mat)=>VMTContent::Unsupported,
vmt_parser::material::Material::Patch(mat)=>VMTContent::Patch(mat),
- _=>return Err(anyhow::Error::msg("vmt failed to parse")),
- })
-}
-
-fn get_vmt<F:Fn(String)->AResult<Option<Vec<u8>>>>(find_stuff:&F,search_name:String)->AResult<vmt_parser::material::Material>{
- if let Some(stuff)=find_stuff(search_name)?{
- //decode vmt and then write
- let stuff=String::from_utf8(stuff)?;
- let material=vmt_parser::from_str(stuff.as_str())?;
- println!("vmt material={:?}",material);
- return Ok(material);
+ _=>unreachable!(),
}
- Err(anyhow::Error::msg("vmt not found"))
}
-fn recursive_vmt_loader<F:Fn(String)->AResult<Option<Vec<u8>>>>(find_stuff:&F,material:vmt_parser::material::Material)->AResult<Option<Vec<u8>>>{
- match get_some_texture(material)?{
- VMTContent::VMT(s)=>recursive_vmt_loader(find_stuff,get_vmt(find_stuff,s)?),
+#[derive(Debug,thiserror::Error)]
+enum GetVMTError{
+ #[error("Bsp error {0:?}")]
+ Bsp(#[from]vbsp::BspError),
+ #[error("Utf8 error {0:?}")]
+ Utf8(#[from]std::str::Utf8Error),
+ #[error("Vdf error {0:?}")]
+ Vdf(#[from]vmt_parser::VdfError),
+ #[error("Vmt not found")]
+ NotFound,
+}
+
+fn get_vmt(finder:BspFinder,search_name:&str)->Result<vmt_parser::material::Material,GetVMTError>{
+ let vmt_data=finder.find(search_name)?.ok_or(GetVMTError::NotFound)?;
+ //decode vmt and then write
+ let vmt_str=core::str::from_utf8(&vmt_data)?;
+ let material=vmt_parser::from_str(vmt_str)?;
+ //println!("vmt material={:?}",material);
+ Ok(material)
+}
+
+#[derive(Debug,thiserror::Error)]
+enum LoadVMTError{
+ #[error("Bsp error {0:?}")]
+ Bsp(#[from]vbsp::BspError),
+ #[error("GetVMT error {0:?}")]
+ GetVMT(#[from]GetVMTError),
+ #[error("FromUtf8 error {0:?}")]
+ FromUtf8(#[from]std::string::FromUtf8Error),
+ #[error("Vdf error {0:?}")]
+ Vdf(#[from]vmt_parser::VdfError),
+ #[error("Vmt unsupported")]
+ Unsupported,
+ #[error("Vmt unresolved")]
+ Unresolved,
+ #[error("Vmt not found")]
+ NotFound,
+}
+fn recursive_vmt_loader<'bsp,'vpk,'a>(finder:BspFinder<'bsp,'vpk>,material:vmt_parser::material::Material)->Result<Option<Cow<'a,[u8]>>,LoadVMTError>
+ where
+ 'bsp:'a,
+ 'vpk:'a,
+{
+ match get_some_texture(material){
+ VMTContent::VMT(s)=>recursive_vmt_loader(finder,get_vmt(finder,s.as_str())?),
VMTContent::VTF(s)=>{
let mut texture_file_name=PathBuf::from("materials");
texture_file_name.push(s);
texture_file_name.set_extension("vtf");
- find_stuff(texture_file_name.into_os_string().into_string().unwrap())
+ Ok(finder.find(texture_file_name.to_str().unwrap())?)
},
- VMTContent::Patch(mat)=>recursive_vmt_loader(find_stuff,
- mat.resolve(|search_name|{
- match find_stuff(search_name.to_string())?{
- Some(bytes)=>Ok(String::from_utf8(bytes)?),
- None=>Err(anyhow::Error::msg("could not find vmt")),
+ VMTContent::Patch(mat)=>recursive_vmt_loader(finder,
+ mat.resolve(|search_name|
+ match finder.find(search_name)?{
+ Some(bytes)=>Ok(String::from_utf8(bytes.into_owned())?),
+ None=>Err(LoadVMTError::NotFound),
}
- })?
+ )?
),
- VMTContent::Unsupported=>{println!("Unsupported vmt");Ok(None)},//print and move on
- VMTContent::Unresolved=>{println!("Unresolved vmt");Ok(None)},
+ VMTContent::Unsupported=>Err(LoadVMTError::Unsupported),
+ VMTContent::Unresolved=>Err(LoadVMTError::Unresolved),
}
}
-
-fn extract_textures(paths:Vec<PathBuf>,vpk_paths:Vec<PathBuf>)->AResult<()>{
- std::fs::create_dir_all("textures")?;
- let vpk_list:Vec<vpk::VPK>=vpk_paths.into_iter().map(|vpk_path|vpk::VPK::read(&vpk_path).expect("vpk file does not exist")).collect();
- for path in paths{
- let mut deduplicate=std::collections::HashSet::new();
- let bsp=vbsp::Bsp::read(std::fs::read(path)?.as_ref())?;
- for texture in bsp.textures(){
- deduplicate.insert(PathBuf::from(texture.name()));
- }
- //dedupe prop models
- let mut model_dedupe=std::collections::HashSet::new();
- for prop in bsp.static_props(){
- model_dedupe.insert(prop.model());
- }
-
- //grab texture names from props
- for model_name in model_dedupe{
- //.mdl, .vvd, .dx90.vtx
- let mut path=PathBuf::from(model_name);
- let file_name=PathBuf::from(path.file_stem().unwrap());
- path.pop();
- path.push(file_name);
- let mut vvd_path=path.clone();
- let mut vtx_path=path.clone();
- vvd_path.set_extension("vvd");
- vtx_path.set_extension("dx90.vtx");
- match (bsp.pack.get(model_name),bsp.pack.get(vvd_path.as_os_str().to_str().unwrap()),bsp.pack.get(vtx_path.as_os_str().to_str().unwrap())){
- (Ok(Some(mdl_file)),Ok(Some(vvd_file)),Ok(Some(vtx_file)))=>{
- match (vmdl::mdl::Mdl::read(mdl_file.as_ref()),vmdl::vvd::Vvd::read(vvd_file.as_ref()),vmdl::vtx::Vtx::read(vtx_file.as_ref())){
- (Ok(mdl),Ok(vvd),Ok(vtx))=>{
- let model=vmdl::Model::from_parts(mdl,vtx,vvd);
- for texture in model.textures(){
- for search_path in &texture.search_paths{
- let mut path=PathBuf::from(search_path.as_str());
- path.push(texture.name.as_str());
- deduplicate.insert(path);
- }
- }
- },
- _=>println!("model_name={} error",model_name),
- }
- },
- _=>println!("no model name={}",model_name),
- }
- }
-
- let pack=&bsp.pack;
- let vpk_list=&vpk_list;
- std::thread::scope(move|s|{
- let mut thread_handles=Vec::new();
- for texture_name in deduplicate{
- let mut found_texture=false;
- //LMAO imagine having to write type names
- let write_image=|mut stuff,write_file_name|{
- let image=vtf::from_bytes(&mut stuff)?.highres_image.decode(0)?.to_rgba8();
-
- let format=if image.width()%4!=0||image.height()%4!=0{
- image_dds::ImageFormat::Rgba8UnormSrgb
- }else{
- image_dds::ImageFormat::BC7RgbaUnormSrgb
- };
- //this fails if the image dimensions are not a multiple of 4
- let dds = image_dds::dds_from_image(
- &image,
- format,
- image_dds::Quality::Slow,
- image_dds::Mipmaps::GeneratedAutomatic,
- )?;
-
- //write dds
- let mut dest=PathBuf::from("textures");
- dest.push(write_file_name);
- dest.set_extension("dds");
- std::fs::create_dir_all(dest.parent().unwrap())?;
- let mut writer = std::io::BufWriter::new(std::fs::File::create(dest)?);
- dds.write(&mut writer)?;
- Ok::<(),anyhow::Error>(())
- };
- let find_stuff=|search_file_name:String|{
- println!("search_file_name={}",search_file_name);
- match pack.get(search_file_name.as_str())?{
- Some(file)=>return Ok(Some(file)),
- _=>(),
- }
- //search pak list
- for vpk_index in vpk_list{
- if let Some(vpk_entry)=vpk_index.tree.get(search_file_name.as_str()){
- return Ok(Some(match vpk_entry.get()?{
- std::borrow::Cow::Borrowed(bytes)=>bytes.to_vec(),
- std::borrow::Cow::Owned(bytes)=>bytes,
- }));
- }
- }
- Ok::<Option<Vec<u8>>,anyhow::Error>(None)
- };
- let loader=|texture_name:String|{
- let mut texture_file_name=PathBuf::from("materials");
- //lower case
- let texture_file_name_lowercase=texture_name.to_lowercase();
- texture_file_name.push(texture_file_name_lowercase.clone());
- //remove stem and search for both vtf and vmt files
- let stem=PathBuf::from(texture_file_name.file_stem().unwrap());
- texture_file_name.pop();
- texture_file_name.push(stem);
- //somehow search for both files
- let mut texture_file_name_vmt=texture_file_name.clone();
- texture_file_name.set_extension("vtf");
- texture_file_name_vmt.set_extension("vmt");
- if let Some(stuff)=find_stuff(texture_file_name.to_string_lossy().to_string())?{
- return Ok(Some(stuff))
- }
- recursive_vmt_loader(&find_stuff,get_vmt(&find_stuff,texture_file_name_vmt.to_string_lossy().to_string())?)
- };
- if let Some(stuff)=loader(texture_name.to_string_lossy().to_string())?{
- found_texture=true;
- let texture_name=texture_name.clone();
- thread_handles.push(s.spawn(move||write_image(stuff,texture_name)));
- }
- if !found_texture{
- println!("no data");
- }
- }
- for thread in thread_handles{
- match thread.join(){
- Ok(Err(e))=>println!("write error: {:?}",e),
- Err(e)=>println!("thread error: {:?}",e),
- Ok(_)=>(),
- }
- }
- Ok::<(),anyhow::Error>(())
- })?
+fn load_texture<'bsp,'vpk,'a>(finder:BspFinder<'bsp,'vpk>,texture_name:&str)->Result<Option<Cow<'a,[u8]>>,LoadVMTError>
+ where
+ 'bsp:'a,
+ 'vpk:'a,
+{
+ let mut texture_file_name=PathBuf::from("materials");
+ //lower case
+ let texture_file_name_lowercase=texture_name.to_lowercase();
+ texture_file_name.push(texture_file_name_lowercase.clone());
+ //remove stem and search for both vtf and vmt files
+ let stem=PathBuf::from(texture_file_name.file_stem().unwrap());
+ texture_file_name.pop();
+ texture_file_name.push(stem);
+ if let Some(stuff)=finder.find(texture_file_name.to_str().unwrap())?{
+ return Ok(Some(stuff))
}
+ //somehow search for both files
+ let mut texture_file_name_vmt=texture_file_name.clone();
+ texture_file_name.set_extension("vtf");
+ texture_file_name_vmt.set_extension("vmt");
+ recursive_vmt_loader(finder,get_vmt(finder,texture_file_name_vmt.to_str().unwrap())?)
+}
+#[derive(Debug,thiserror::Error)]
+enum ExtractTextureError{
+ #[error("Io error {0:?}")]
+ Io(#[from]std::io::Error),
+ #[error("Bsp error {0:?}")]
+ Bsp(#[from]vbsp::BspError),
+ #[error("MeshLoad error {0:?}")]
+ MeshLoad(#[from]strafesnet_bsp_loader::loader::MeshError),
+ #[error("Load VMT error {0:?}")]
+ LoadVMT(#[from]LoadVMTError),
+}
+async fn gimme_them_textures(path:&Path,vpk_list:&[vpk::VPK],send_texture:tokio::sync::mpsc::Sender<(Vec<u8>,String)>)->Result<(),ExtractTextureError>{
+ let bsp=vbsp::Bsp::read(tokio::fs::read(path).await?.as_ref())?;
+ let loader_bsp=strafesnet_bsp_loader::Bsp::new(bsp);
+ let bsp=loader_bsp.as_ref();
+
+ let mut texture_deferred_loader=RenderConfigDeferredLoader::new();
+ for texture in bsp.textures(){
+ texture_deferred_loader.acquire_render_config_id(Some(Cow::Borrowed(texture.name())));
+ }
+
+ let mut mesh_deferred_loader=MeshDeferredLoader::new();
+ for prop in bsp.static_props(){
+ mesh_deferred_loader.acquire_mesh_id(prop.model());
+ }
+
+ let finder=BspFinder{
+ bsp:&loader_bsp,
+ vpks:vpk_list
+ };
+
+ let mut mesh_loader=strafesnet_bsp_loader::loader::ModelLoader::new(finder);
+ // load models and collect requested textures
+ for model_path in mesh_deferred_loader.into_indices(){
+ let model:vmdl::Model=match mesh_loader.load(model_path){
+ Ok(model)=>model,
+ Err(e)=>{
+ println!("Model={model_path} Load model error: {e}");
+ continue;
+ },
+ };
+ for texture in model.textures(){
+ for search_path in &texture.search_paths{
+ let mut path=PathBuf::from(search_path.as_str());
+ path.push(texture.name.as_str());
+ let path=path.to_str().unwrap().to_owned();
+ texture_deferred_loader.acquire_render_config_id(Some(Cow::Owned(path)));
+ }
+ }
+ }
+
+ for texture_path in texture_deferred_loader.into_indices(){
+ match load_texture(finder,&texture_path){
+ Ok(Some(texture))=>send_texture.send(
+ (texture.into_owned(),texture_path.into_owned())
+ ).await.unwrap(),
+ Ok(None)=>(),
+ Err(e)=>println!("Texture={texture_path} Load error: {e}"),
+ }
+ }
+
+ Ok(())
+}
+
+
+#[derive(Debug,thiserror::Error)]
+enum ConvertTextureError{
+ #[error("Bsp error {0:?}")]
+ Bsp(#[from]vbsp::BspError),
+ #[error("Vtf error {0:?}")]
+ Vtf(#[from]vtf::Error),
+ #[error("DDS create error {0:?}")]
+ DDS(#[from]image_dds::CreateDdsError),
+ #[error("DDS write error {0:?}")]
+ DDSWrite(#[from]image_dds::ddsfile::Error),
+ #[error("Io error {0:?}")]
+ Io(#[from]std::io::Error),
+}
+
+async fn convert_texture(texture:Vec<u8>,write_file_name:impl AsRef<Path>)->Result<(),ConvertTextureError>{
+ let image=vtf::from_bytes(&texture)?.highres_image.decode(0)?.to_rgba8();
+
+ let format=if image.width()%4!=0||image.height()%4!=0{
+ image_dds::ImageFormat::Rgba8UnormSrgb
+ }else{
+ image_dds::ImageFormat::BC7RgbaUnormSrgb
+ };
+ //this fails if the image dimensions are not a multiple of 4
+ let dds = image_dds::dds_from_image(
+ &image,
+ format,
+ image_dds::Quality::Slow,
+ image_dds::Mipmaps::GeneratedAutomatic,
+ )?;
+
+ //write dds
+ let mut dest=PathBuf::from("textures");
+ dest.push(write_file_name);
+ dest.set_extension("dds");
+ std::fs::create_dir_all(dest.parent().unwrap())?;
+ let mut writer=std::io::BufWriter::new(std::fs::File::create(dest)?);
+ dds.write(&mut writer)?;
+
+ Ok(())
+}
+
+async fn read_vpks(vpk_paths:Vec<PathBuf>,thread_limit:usize)->Vec<vpk::VPK>{
+ futures::stream::iter(vpk_paths).map(|vpk_path|async{
+ // idk why it doesn't want to pass out the errors but this is fatal anyways
+ tokio::task::spawn_blocking(move||vpk::VPK::read(&vpk_path)).await.unwrap().unwrap()
+ })
+ .buffer_unordered(thread_limit)
+ .collect().await
+}
+
+async fn extract_textures(paths:Vec<PathBuf>,vpk_paths:Vec<PathBuf>)->AResult<()>{
+ tokio::try_join!(
+ tokio::fs::create_dir_all("extracted_textures"),
+ tokio::fs::create_dir_all("textures"),
+ tokio::fs::create_dir_all("meshes"),
+ )?;
+ let thread_limit=std::thread::available_parallelism()?.get();
+
+ // load vpk list
+ let vpk_list=read_vpks(vpk_paths,thread_limit).await;
+
+ // leak vpk_list for static lifetime?
+ let vpk_list:&[vpk::VPK]=vpk_list.leak();
+
+ let (send_texture,mut recv_texture)=tokio::sync::mpsc::channel(thread_limit);
+ let mut it=paths.into_iter();
+ let extract_thread=tokio::spawn(async move{
+ static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
+ SEM.add_permits(thread_limit);
+ while let (Ok(permit),Some(path))=(SEM.acquire().await,it.next()){
+ let send=send_texture.clone();
+ tokio::spawn(async move{
+ let result=gimme_them_textures(&path,vpk_list,send).await;
+ drop(permit);
+ match result{
+ Ok(())=>(),
+ Err(e)=>println!("Map={path:?} Decode error: {e:?}"),
+ }
+ });
+ }
+ });
+
+ // convert images
+ static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
+ SEM.add_permits(thread_limit);
+ while let (Ok(permit),Some((data,dest)))=(SEM.acquire().await,recv_texture.recv().await){
+ // TODO: dedup dest?
+ tokio::spawn(async move{
+ let result=convert_texture(data,dest).await;
+ drop(permit);
+ match result{
+ Ok(())=>(),
+ Err(e)=>println!("Convert error: {e:?}"),
+ }
+ });
+ }
+ extract_thread.await?;
+ _=SEM.acquire_many(thread_limit as u32).await?;
Ok(())
}
@@ -277,7 +367,8 @@ fn bsp_contents(path:PathBuf)->AResult<()>{
enum ConvertError{
IO(std::io::Error),
SNFMap(strafesnet_snf::map::Error),
- BspLoader(strafesnet_bsp_loader::ReadError),
+ BspRead(strafesnet_bsp_loader::ReadError),
+ BspLoad(strafesnet_bsp_loader::LoadError),
}
impl std::fmt::Display for ConvertError{
fn fmt(&self,f:&mut std::fmt::Formatter<'_>)->std::fmt::Result{
@@ -286,79 +377,52 @@ impl std::fmt::Display for ConvertError{
}
impl std::error::Error for ConvertError{}
-type MapThread=std::thread::JoinHandle<Result<(),ConvertError>>;
+async fn convert_to_snf(path:&Path,vpk_list:&[vpk::VPK],output_folder:PathBuf)->AResult<()>{
+ let entire_file=tokio::fs::read(path).await?;
-fn source_to_snf(pathlist:Vec<std::path::PathBuf>,output_folder:PathBuf)->AResult<()>{
- let n_paths=pathlist.len();
- let start = std::time::Instant::now();
- let mut threads:std::collections::VecDeque<MapThread>=std::collections::VecDeque::new();
- let mut i=0;
- let mut join_thread=|thread:MapThread|{
- i+=1;
- if let Err(e)=thread.join(){
- println!("thread error: {:?}",e);
- }else{
- println!("{}/{}",i,n_paths);
- }
- };
- for path in pathlist{
- if 32<=threads.len(){
- join_thread(threads.pop_front().unwrap());
- }
- let output_folder=output_folder.clone();
- threads.push_back(std::thread::spawn(move ||{
- let bsp=strafesnet_bsp_loader::read(
- std::fs::File::open(path.as_path())
- .map_err(ConvertError::IO)?
- ).map_err(ConvertError::BspLoader)?;
- let mut loader=strafesnet_deferred_loader::source_legacy();
+ let bsp=strafesnet_bsp_loader::read(
+ std::io::Cursor::new(entire_file)
+ ).map_err(ConvertError::BspRead)?;
- let (texture_loader,mesh_loader)=loader.get_inner_mut();
+ let map=bsp.to_snf(LoadFailureMode::DefaultToNone,vpk_list).map_err(ConvertError::BspLoad)?;
- let map_step1=strafesnet_bsp_loader::convert(
- &bsp,
- |name|texture_loader.acquire_render_config_id(name),
- |name|mesh_loader.acquire_mesh_id(name),
- );
+ let mut dest=output_folder;
+ dest.push(path.file_stem().unwrap());
+ dest.set_extension("snfm");
+ let file=std::fs::File::create(dest).map_err(ConvertError::IO)?;
- let prop_meshes=mesh_loader.load_meshes(&bsp.as_ref());
+ strafesnet_snf::map::write_map(file,map).map_err(ConvertError::SNFMap)?;
- let map_step2=map_step1.add_prop_meshes(
- //the type conflagulator 9000
- prop_meshes.into_iter().map(|(mesh_id,loader_model)|
- (mesh_id,strafesnet_bsp_loader::data::ModelData{
- mdl:strafesnet_bsp_loader::data::MdlData::new(loader_model.mdl.get()),
- vtx:strafesnet_bsp_loader::data::VtxData::new(loader_model.vtx.get()),
- vvd:strafesnet_bsp_loader::data::VvdData::new(loader_model.vvd.get()),
- })
- ),
- |name|texture_loader.acquire_render_config_id(name),
- );
-
- let (textures,render_configs)=loader.into_render_configs().map_err(ConvertError::IO)?.consume();
-
- let map=map_step2.add_render_configs_and_textures(
- render_configs.into_iter(),
- textures.into_iter().map(|(texture_id,texture)|
- (texture_id,match texture{
- strafesnet_deferred_loader::texture::Texture::ImageDDS(data)=>data,
- })
- ),
- );
-
- let mut dest=output_folder.clone();
- dest.push(path.file_stem().unwrap());
- dest.set_extension("snfm");
- let file=std::fs::File::create(dest).map_err(ConvertError::IO)?;
-
- strafesnet_snf::map::write_map(file,map).map_err(ConvertError::SNFMap)?;
- Ok(())
- }));
- }
-
- for thread in threads{
- join_thread(thread);
- }
- println!("{:?}", start.elapsed());
+ Ok(())
+}
+async fn source_to_snf(paths:Vec<std::path::PathBuf>,output_folder:PathBuf,vpk_paths:Vec<PathBuf>)->AResult<()>{
+ let start=std::time::Instant::now();
+
+ let thread_limit=std::thread::available_parallelism()?.get();
+
+ // load vpk list
+ let vpk_list=read_vpks(vpk_paths,thread_limit).await;
+
+ // leak vpk_list for static lifetime?
+ let vpk_list:&[vpk::VPK]=vpk_list.leak();
+
+ let mut it=paths.into_iter();
+ static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(0);
+ SEM.add_permits(thread_limit);
+
+ while let (Ok(permit),Some(path))=(SEM.acquire().await,it.next()){
+ let output_folder=output_folder.clone();
+ tokio::spawn(async move{
+ let result=convert_to_snf(path.as_path(),vpk_list,output_folder).await;
+ drop(permit);
+ match result{
+ Ok(())=>(),
+ Err(e)=>println!("Convert error: {e:?}"),
+ }
+ });
+ }
+ _=SEM.acquire_many(thread_limit as u32).await.unwrap();
+
+ println!("elapsed={:?}", start.elapsed());
Ok(())
}