Compare commits

..

2 Commits

15 changed files with 273 additions and 301 deletions

View File

@ -1,8 +0,0 @@
.PHONY: maps-service web validation
maps-service:
DOCKER_BUILDKIT=1 docker build . -f Containerfile -t maps-service \
--secret id=netrc,src=/home/quat/.netrc
web:
docker build web -f web/Containerfile -t maps-service-web
validation:
docker build validation -f validation/Containerfile -t maps-service-validation

View File

@ -3,23 +3,32 @@ networks:
maps-service-network:
driver: bridge
secrets:
netrc:
file: /home/quat/.netrc
services:
nats:
image: docker.io/nats:latest
container_name: nats
command: ["-js"] #"-DVV"
command: ["-js"]
networks:
- maps-service-network
ports:
- "4222:4222"
mapsservice:
image:
maps-service
container_name: mapsservice
maps-service:
build:
secrets:
- netrc
context: .
dockerfile: Containerfile
container_name: maps-service
command: [
# debug
"--debug","serve",
# http service port
"--port","8082",
"--port","8081",
# postgres
"--pg-host","10.0.0.29",
"--pg-port","5432",
@ -28,92 +37,38 @@ services:
"--pg-password","happypostgresuser",
# other hosts
"--nats-host","nats:4222",
"--auth-rpc-host","authrpc:8081"
"--auth-rpc-host","localhost:8090"
]
depends_on:
- authrpc
- nats
networks:
- maps-service-network
ports:
- "8082:8082"
- "8081:8081"
web:
image:
maps-service-web
build:
context: web
dockerfile: Containerfile
networks:
- maps-service-network
ports:
- "3000:3000"
validation:
image:
maps-service-validation
build:
context: validation
dockerfile: Containerfile
container_name: validation
environment:
- RBXCOOKIE
- API_HOST=http://mapsservice:8082
- RBXCOOKIE=RBXCOOKIE
- API_HOST=http://localhost:8081
- NATS_HOST=nats:4222
- DATA_HOST=http://dataservice:9000
- DATA_HOST=http://localhost:9000
depends_on:
- nats
# note: this races the mapsservice which creates a nats stream
# note: this races the maps-service which creates a nats stream
# the validation will panic if the nats stream is not created
- mapsservice
- dataservice
- maps-service
networks:
- maps-service-network
dataservice:
image: registry.itzana.me/strafesnet/data-service:master
container_name: dataservice
environment:
- DEBUG=true
- PG_HOST=10.0.0.29
- PG_PORT=5432
- PG_USER=quat
- PG_DB=data
- PG_PASS=happypostgresuser
networks:
- maps-service-network
authredis:
image: docker.io/redis:latest
container_name: authredis
volumes:
- redis-data:/data
command: ["redis-server", "--appendonly", "yes"]
networks:
- maps-service-network
authrpc:
image: registry.itzana.me/strafesnet/auth-service:master
container_name: authrpc
command: ["serve", "rpc"]
environment:
- REDIS_ADDR=authredis:6379
env_file:
- ../auth-compose/auth-service.env
depends_on:
- authredis
networks:
- maps-service-network
logging:
driver: "none"
auth-web:
image: registry.itzana.me/strafesnet/auth-service:master
command: ["serve", "web"]
environment:
- REDIS_ADDR=authredis:6379
env_file:
- ../auth-compose/auth-service.env
depends_on:
- authredis
networks:
- maps-service-network
ports:
- "8080:8080"
volumes:
redis-data:

View File

@ -313,7 +313,7 @@ func (svc *Service) ActionSubmissionTriggerPublish(ctx context.Context, params a
return err
}
svc.Nats.Publish("maptest.submissions.publishnew", []byte(j))
svc.Nats.Publish("maptest.submissions.publish.new", []byte(j))
} else {
// this is a map fix
publish_fix_request := model.PublishFixRequest{
@ -328,7 +328,7 @@ func (svc *Service) ActionSubmissionTriggerPublish(ctx context.Context, params a
return err
}
svc.Nats.Publish("maptest.submissions.publishfix", []byte(j))
svc.Nats.Publish("maptest.submissions.publish.fix", []byte(j))
}
return nil

138
validation/Cargo.lock generated
View File

@ -171,8 +171,8 @@ dependencies = [
"pin-project-lite",
"rustversion",
"serde",
"sync_wrapper",
"tower 0.5.2",
"sync_wrapper 1.0.2",
"tower 0.5.1",
"tower-layer",
"tower-service",
]
@ -192,7 +192,7 @@ dependencies = [
"mime",
"pin-project-lite",
"rustversion",
"sync_wrapper",
"sync_wrapper 1.0.2",
"tower-layer",
"tower-service",
]
@ -287,9 +287,9 @@ dependencies = [
[[package]]
name = "cc"
version = "1.2.4"
version = "1.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9157bbaa6b165880c27a4293a474c91cdcf265cc68cc829bf10be0964a391caf"
checksum = "f34d93e62b03caf570cccc334cbc6c2fceca82f39211051345108adcba3eebdc"
dependencies = [
"shlex",
]
@ -302,9 +302,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "chrono"
version = "0.4.39"
version = "0.4.38"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825"
checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401"
dependencies = [
"android-tzdata",
"iana-time-zone",
@ -510,9 +510,9 @@ dependencies = [
[[package]]
name = "fastrand"
version = "2.3.0"
version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
checksum = "486f806e73c5707928240ddc295403b1b93c96a02038563881c4a2fd84b81ac4"
[[package]]
name = "fiat-crypto"
@ -709,9 +709,9 @@ checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289"
[[package]]
name = "http"
version = "1.2.0"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f16ca2af56261c99fba8bac40a10251ce8188205a4c448fbb745a2e4daa76fea"
checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258"
dependencies = [
"bytes",
"fnv",
@ -1044,9 +1044,9 @@ checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674"
[[package]]
name = "js-sys"
version = "0.3.76"
version = "0.3.74"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6717b6b5b077764fb5966237269cb3c64edddde4b14ce42647430a78ced9e7b7"
checksum = "a865e038f7f6ed956f788f0d7d60c541fff74c7bd74272c5d4cf15c63743e705"
dependencies = [
"once_cell",
"wasm-bindgen",
@ -1060,9 +1060,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
[[package]]
name = "libc"
version = "0.2.168"
version = "0.2.167"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5aaeb2981e0606ca11d79718f8bb01164f1d6ed75080182d3abf017e6d244b6d"
checksum = "09d6582e104315a817dff97f75133544b2e094ee22447d2acf4a74e189ba06fc"
[[package]]
name = "linux-raw-sys"
@ -1477,9 +1477,9 @@ dependencies = [
[[package]]
name = "rbx_asset"
version = "0.2.5"
version = "0.2.3"
source = "sparse+https://git.itzana.me/api/packages/strafesnet/cargo/"
checksum = "dcf243f46bd41b3880a27278177a3f9996f95ab231d9a04345ad9dd381c3a54a"
checksum = "15542e7184e4b18e578308308cee72abcb2bfcb1d6c162f1eea50ec6cff7e3ac"
dependencies = [
"chrono",
"flate2",
@ -1627,7 +1627,7 @@ dependencies = [
"serde",
"serde_json",
"serde_urlencoded",
"sync_wrapper",
"sync_wrapper 1.0.2",
"system-configuration",
"tokio",
"tokio-native-tls",
@ -1705,22 +1705,22 @@ dependencies = [
[[package]]
name = "rustix"
version = "0.38.42"
version = "0.38.41"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f93dc38ecbab2eb790ff964bb77fa94faf256fd3e73285fd7ba0903b76bedb85"
checksum = "d7f649912bc1495e167a6edee79151c84b1bad49748cb4f1f1167f459f6224f6"
dependencies = [
"bitflags 2.6.0",
"errno",
"libc",
"linux-raw-sys",
"windows-sys 0.59.0",
"windows-sys 0.52.0",
]
[[package]]
name = "rustls"
version = "0.23.20"
version = "0.23.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5065c3f250cbd332cd894be57c40fa52387247659b14a2d6041d121547903b1b"
checksum = "934b404430bb06b3fae2cba809eb45a1ab1aecd64491213d7c3301b88393f8d1"
dependencies = [
"once_cell",
"ring",
@ -1766,9 +1766,9 @@ dependencies = [
[[package]]
name = "rustls-pki-types"
version = "1.10.1"
version = "1.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2bf47e6ff922db3825eb750c4e2ff784c6ff8fb9e13046ef6a1d1c5401b0b37"
checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b"
[[package]]
name = "rustls-webpki"
@ -1840,24 +1840,24 @@ dependencies = [
[[package]]
name = "semver"
version = "1.0.24"
version = "1.0.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3cb6eb87a131f756572d7fb904f6e7b68633f09cca868c5df1c4b8d1a694bbba"
checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b"
[[package]]
name = "serde"
version = "1.0.216"
version = "1.0.215"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e"
checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.216"
version = "1.0.215"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e"
checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0"
dependencies = [
"proc-macro2",
"quote",
@ -1925,15 +1925,6 @@ version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
[[package]]
name = "signal-hook-registry"
version = "1.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1"
dependencies = [
"libc",
]
[[package]]
name = "signatory"
version = "0.27.1"
@ -2026,6 +2017,12 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "sync_wrapper"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160"
[[package]]
name = "sync_wrapper"
version = "1.0.2"
@ -2102,9 +2099,9 @@ dependencies = [
[[package]]
name = "time"
version = "0.3.37"
version = "0.3.36"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "35e7868883861bd0e56d9ac6efcaaca0d6d5d82a2a7ec8209ff492c07cf37b21"
checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885"
dependencies = [
"deranged",
"itoa",
@ -2123,9 +2120,9 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3"
[[package]]
name = "time-macros"
version = "0.2.19"
version = "0.2.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2834e6017e3e5e4b9834939793b282bc03b37a3336245fa820e35e233e2a85de"
checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf"
dependencies = [
"num-conv",
"time-core",
@ -2143,16 +2140,15 @@ dependencies = [
[[package]]
name = "tokio"
version = "1.42.0"
version = "1.41.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5cec9b21b0450273377fc97bd4c33a8acffc8c996c987a7c5b319a0083707551"
checksum = "22cfb5bee7a6a52939ca9224d6ac897bb669134078daa8735560897f69de4d33"
dependencies = [
"backtrace",
"bytes",
"libc",
"mio",
"pin-project-lite",
"signal-hook-registry",
"socket2",
"tokio-macros",
"windows-sys 0.52.0",
@ -2181,11 +2177,12 @@ dependencies = [
[[package]]
name = "tokio-rustls"
version = "0.26.1"
version = "0.26.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f6d0975eaace0cf0fcadee4e4aaa5da15b5c079146f2cffb67c113be122bf37"
checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4"
dependencies = [
"rustls",
"rustls-pki-types",
"tokio",
]
@ -2202,9 +2199,9 @@ dependencies = [
[[package]]
name = "tokio-util"
version = "0.7.13"
version = "0.7.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d7fcaa8d55a2bdd6b83ace262b016eca0d79ee02818c5c1bcdf0305114081078"
checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a"
dependencies = [
"bytes",
"futures-core",
@ -2286,14 +2283,14 @@ dependencies = [
[[package]]
name = "tower"
version = "0.5.2"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9"
checksum = "2873938d487c3cfb9aed7546dc9f2711d867c9f90c46b889989a2cb84eba6b4f"
dependencies = [
"futures-core",
"futures-util",
"pin-project-lite",
"sync_wrapper",
"sync_wrapper 0.1.2",
"tower-layer",
"tower-service",
]
@ -2434,9 +2431,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
[[package]]
name = "wasm-bindgen"
version = "0.2.99"
version = "0.2.97"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a474f6281d1d70c17ae7aa6a613c87fce69a127e2624002df63dcb39d6cf6396"
checksum = "d15e63b4482863c109d70a7b8706c1e364eb6ea449b201a76c5b89cedcec2d5c"
dependencies = [
"cfg-if",
"once_cell",
@ -2445,12 +2442,13 @@ dependencies = [
[[package]]
name = "wasm-bindgen-backend"
version = "0.2.99"
version = "0.2.97"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f89bb38646b4f81674e8f5c3fb81b562be1fd936d84320f3264486418519c79"
checksum = "8d36ef12e3aaca16ddd3f67922bc63e48e953f126de60bd33ccc0101ef9998cd"
dependencies = [
"bumpalo",
"log",
"once_cell",
"proc-macro2",
"quote",
"syn",
@ -2459,9 +2457,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-futures"
version = "0.4.49"
version = "0.4.47"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "38176d9b44ea84e9184eff0bc34cc167ed044f816accfe5922e54d84cf48eca2"
checksum = "9dfaf8f50e5f293737ee323940c7d8b08a66a95a419223d9f41610ca08b0833d"
dependencies = [
"cfg-if",
"js-sys",
@ -2472,9 +2470,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.99"
version = "0.2.97"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2cc6181fd9a7492eef6fef1f33961e3695e4579b9872a6f7c83aee556666d4fe"
checksum = "705440e08b42d3e4b36de7d66c944be628d579796b8090bfa3471478a2260051"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
@ -2482,9 +2480,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro-support"
version = "0.2.99"
version = "0.2.97"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2"
checksum = "98c9ae5a76e46f4deecd0f0255cc223cfa18dc9b261213b8aa0c7b36f61b3f1d"
dependencies = [
"proc-macro2",
"quote",
@ -2495,15 +2493,15 @@ dependencies = [
[[package]]
name = "wasm-bindgen-shared"
version = "0.2.99"
version = "0.2.97"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6"
checksum = "6ee99da9c5ba11bd675621338ef6fa52296b76b83305e9b6e5c77d4c286d6d49"
[[package]]
name = "web-sys"
version = "0.3.76"
version = "0.3.74"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04dd7223427d52553d3702c004d3b2fe07c148165faa56313cb00211e31c12bc"
checksum = "a98bc3c33f0fe7e59ad7cd041b89034fa82a7c2d4365ca538dda6cdaf513863c"
dependencies = [
"js-sys",
"wasm-bindgen",
@ -2644,9 +2642,9 @@ checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51"
[[package]]
name = "xml-rs"
version = "0.8.24"
version = "0.8.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea8b391c9a790b496184c29f7f93b9ed5b16abb306c05415b68bcc16e4d06432"
checksum = "af310deaae937e48a26602b730250b4949e125f468f11e6990be3e5304ddd96f"
[[package]]
name = "yoke"

View File

@ -7,7 +7,7 @@ edition = "2021"
api = { path = "api" }
async-nats = "0.38.0"
futures = "0.3.31"
rbx_asset = { version = "0.2.5", registry = "strafesnet" }
rbx_asset = { version = "0.2.3", registry = "strafesnet" }
rbx_binary = { version = "0.7.4", registry = "strafesnet"}
rbx_dom_weak = { version = "2.9.0", registry = "strafesnet"}
rbx_reflection_database = { version = "0.2.12", registry = "strafesnet"}
@ -16,5 +16,5 @@ rust-grpc = { version = "1.0.3", registry = "strafesnet" }
serde = { version = "1.0.215", features = ["derive"] }
serde_json = "1.0.133"
siphasher = "1.0.1"
tokio = { version = "1.41.1", features = ["macros", "rt-multi-thread", "fs", "signal"] }
tokio = { version = "1.41.1", features = ["macros", "rt-multi-thread", "fs"] }
tonic = "0.12.3"

View File

@ -1,10 +1,8 @@
use futures::StreamExt;
mod types;
mod nats_types;
mod validator;
mod publish_new;
mod publish_fix;
mod message_handler;
#[allow(dead_code)]
#[derive(Debug)]
@ -12,8 +10,7 @@ pub enum StartupError{
API(api::ReqwestError),
NatsConnect(async_nats::ConnectError),
NatsGetStream(async_nats::jetstream::context::GetStreamError),
NatsConsumer(async_nats::jetstream::stream::ConsumerError),
NatsStream(async_nats::jetstream::consumer::StreamError),
NatsStartup(types::NatsStartupError),
GRPCConnect(tonic::transport::Error),
}
impl std::fmt::Display for StartupError{
@ -23,13 +20,8 @@ impl std::fmt::Display for StartupError{
}
impl std::error::Error for StartupError{}
// annoying mile-long type
pub type MapsServiceClient=rust_grpc::maps::maps_service_client::MapsServiceClient<tonic::transport::channel::Channel>;
pub const GROUP_STRAFESNET:u64=6980477;
pub const PARALLEL_REQUESTS:usize=16;
#[tokio::main]
async fn main()->Result<(),StartupError>{
// talk to roblox through STRAFESNET_CI2 account
@ -42,57 +34,29 @@ async fn main()->Result<(),StartupError>{
// nats
let nats_host=std::env::var("NATS_HOST").expect("NATS_HOST env required");
let nats_fut=async{
let nasty=async_nats::connect(nats_host).await.map_err(StartupError::NatsConnect)?;
// use nats jetstream
async_nats::jetstream::new(nasty)
.get_stream("maptest").await.map_err(StartupError::NatsGetStream)?
.get_or_create_consumer("validation",async_nats::jetstream::consumer::pull::Config{
name:Some("validation".to_owned()),
durable_name:Some("validation".to_owned()),
filter_subject:"maptest.submissions.>".to_owned(),
..Default::default()
}).await.map_err(StartupError::NatsConsumer)?
.messages().await.map_err(StartupError::NatsStream)
};
let nasty=async_nats::connect(nats_host).await.map_err(StartupError::NatsConnect)?; // use nats jetstream
let stream=async_nats::jetstream::new(nasty)
.get_stream("maptest").await.map_err(StartupError::NatsGetStream)?;
// data-service grpc for creating map entries
let data_host=std::env::var("DATA_HOST").expect("DATA_HOST env required");
let message_handler_fut=async{
let maps_grpc=crate::MapsServiceClient::connect(data_host).await.map_err(StartupError::GRPCConnect)?;
Ok(message_handler::MessageHandler::new(cookie_context,api,maps_grpc))
};
let maps_grpc=crate::types::MapsServiceClient::connect(data_host).await.map_err(StartupError::GRPCConnect)?;
// Create a signal listener for SIGTERM
let mut sig_term=tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()).expect("Failed to create SIGTERM signal listener");
// connect to nats
let (publish_new,publish_fix,validator)=tokio::try_join!(
publish_new::Publisher::new(stream.clone(),cookie_context.clone(),api.clone(),maps_grpc),
publish_fix::Publisher::new(stream.clone(),cookie_context.clone(),api.clone()),
// clone nats here because it's dropped within the function scope,
// meanining the last reference is dropped...
validator::Validator::new(stream.clone(),cookie_context,api)
).map_err(StartupError::NatsStartup)?;
// run futures
let (mut messages,message_handler)=tokio::try_join!(nats_fut,message_handler_fut)?;
// publisher threads
tokio::spawn(publish_new.run());
tokio::spawn(publish_fix.run());
// process up to PARALLEL_REQUESTS in parallel
let main_loop=async move{
static SEM:tokio::sync::Semaphore=tokio::sync::Semaphore::const_new(PARALLEL_REQUESTS);
// use memory leak to make static lifetime
let message_handler=Box::leak(Box::new(message_handler));
// acquire a permit before attempting to receive a message, exit if either fails
while let (Ok(permit),Some(message_result))=(SEM.acquire().await,messages.next().await){
// handle the message on a new thread (mainly to decode the model file)
tokio::spawn(async{
match message_handler.handle_message_result(message_result).await{
Ok(())=>println!("[Validation] Success, hooray!"),
Err(e)=>println!("[Validation] There was an error, oopsie! {e}"),
}
// explicitly call drop to make the move semantics and permit release more obvious
core::mem::drop(permit);
});
}
};
// race sigkill and main loop termination and then die
tokio::select!{
_=sig_term.recv()=>(),
_=main_loop=>(),
};
// run validator on the main thread indefinitely
validator.run().await;
Ok(())
}

View File

@ -1,48 +0,0 @@
#[allow(dead_code)]
#[derive(Debug)]
pub enum HandleMessageError{
Messages(async_nats::jetstream::consumer::pull::MessagesError),
DoubleAck(async_nats::Error),
UnknownSubject(String),
PublishNew(crate::publish_new::PublishError),
PublishFix(crate::publish_fix::PublishError),
Validation(crate::validator::ValidateError),
}
impl std::fmt::Display for HandleMessageError{
fn fmt(&self,f:&mut std::fmt::Formatter<'_>)->std::fmt::Result{
write!(f,"{self:?}")
}
}
impl std::error::Error for HandleMessageError{}
pub type MessageResult=Result<async_nats::jetstream::Message,async_nats::jetstream::consumer::pull::MessagesError>;
pub struct MessageHandler{
publish_new:crate::publish_new::Publisher,
publish_fix:crate::publish_fix::Publisher,
validator:crate::validator::Validator,
}
impl MessageHandler{
pub fn new(
cookie_context:rbx_asset::cookie::CookieContext,
api:api::Context,
maps_grpc:crate::MapsServiceClient,
)->Self{
Self{
publish_new:crate::publish_new::Publisher::new(cookie_context.clone(),api.clone(),maps_grpc),
publish_fix:crate::publish_fix::Publisher::new(cookie_context.clone(),api.clone()),
validator:crate::validator::Validator::new(cookie_context,api),
}
}
pub async fn handle_message_result(&self,message_result:MessageResult)->Result<(),HandleMessageError>{
let message=message_result.map_err(HandleMessageError::Messages)?;
message.double_ack().await.map_err(HandleMessageError::DoubleAck)?;
match message.subject.as_str(){
"maptest.submissions.publishnew"=>self.publish_new.publish(message).await.map_err(HandleMessageError::PublishNew),
"maptest.submissions.publishfix"=>self.publish_fix.publish(message).await.map_err(HandleMessageError::PublishFix),
"maptest.submissions.validate"=>self.validator.validate(message).await.map_err(HandleMessageError::Validation),
other=>Err(HandleMessageError::UnknownSubject(other.to_owned()))
}
}
}

View File

@ -1,8 +1,12 @@
use futures::StreamExt;
use crate::types::{MessageResult,NatsStartupError};
use crate::nats_types::PublishFixRequest;
#[allow(dead_code)]
#[derive(Debug)]
pub enum PublishError{
enum PublishError{
Messages(async_nats::jetstream::consumer::pull::MessagesError),
DoubleAck(async_nats::Error),
Get(rbx_asset::cookie::GetError),
Json(serde_json::Error),
Upload(rbx_asset::cookie::UploadError),
@ -16,21 +20,43 @@ impl std::fmt::Display for PublishError{
impl std::error::Error for PublishError{}
pub struct Publisher{
messages:async_nats::jetstream::consumer::pull::Stream,
roblox_cookie:rbx_asset::cookie::CookieContext,
api:api::Context,
}
impl Publisher{
pub const fn new(
pub async fn new(
stream:async_nats::jetstream::stream::Stream,
roblox_cookie:rbx_asset::cookie::CookieContext,
api:api::Context,
)->Self{
Self{
)->Result<Self,NatsStartupError>{
Ok(Self{
messages:stream.get_or_create_consumer("publish_fix",async_nats::jetstream::consumer::pull::Config{
name:Some("publish_fix".to_owned()),
durable_name:Some("publish_fix".to_owned()),
filter_subject:"maptest.submissions.publish.fix".to_owned(),
..Default::default()
}).await.map_err(NatsStartupError::Consumer)?
.messages().await.map_err(NatsStartupError::Stream)?,
roblox_cookie,
api,
})
}
pub async fn run(mut self){
while let Some(message_result)=self.messages.next().await{
self.publish_supress_error(message_result).await
}
}
pub async fn publish(&self,message:async_nats::jetstream::Message)->Result<(),PublishError>{
println!("publish_fix {:?}",message.message.payload);
async fn publish_supress_error(&self,message_result:MessageResult){
match self.publish(message_result).await{
Ok(())=>println!("[PublishFix] Published, hooray!"),
Err(e)=>println!("[PublishFix] There was an error, oopsie! {e}"),
}
}
async fn publish(&self,message_result:MessageResult)->Result<(),PublishError>{
println!("publish_fix {:?}",message_result);
let message=message_result.map_err(PublishError::Messages)?;
message.double_ack().await.map_err(PublishError::DoubleAck)?;
// decode json
let publish_info:PublishFixRequest=serde_json::from_slice(&message.payload).map_err(PublishError::Json)?;

View File

@ -1,8 +1,12 @@
use futures::StreamExt;
use crate::types::{MessageResult,NatsStartupError};
use crate::nats_types::PublishNewRequest;
#[allow(dead_code)]
#[derive(Debug)]
pub enum PublishError{
enum PublishError{
Messages(async_nats::jetstream::consumer::pull::MessagesError),
DoubleAck(async_nats::Error),
Get(rbx_asset::cookie::GetError),
Json(serde_json::Error),
Create(rbx_asset::cookie::CreateError),
@ -18,24 +22,46 @@ impl std::fmt::Display for PublishError{
impl std::error::Error for PublishError{}
pub struct Publisher{
messages:async_nats::jetstream::consumer::pull::Stream,
roblox_cookie:rbx_asset::cookie::CookieContext,
api:api::Context,
maps_grpc:crate::MapsServiceClient,
maps_grpc:crate::types::MapsServiceClient,
}
impl Publisher{
pub const fn new(
pub async fn new(
stream:async_nats::jetstream::stream::Stream,
roblox_cookie:rbx_asset::cookie::CookieContext,
api:api::Context,
maps_grpc:crate::MapsServiceClient,
)->Self{
Self{
maps_grpc:crate::types::MapsServiceClient,
)->Result<Self,NatsStartupError>{
Ok(Self{
messages:stream.get_or_create_consumer("publish_new",async_nats::jetstream::consumer::pull::Config{
name:Some("publish_new".to_owned()),
durable_name:Some("publish_new".to_owned()),
filter_subject:"maptest.submissions.publish.new".to_owned(),
..Default::default()
}).await.map_err(NatsStartupError::Consumer)?
.messages().await.map_err(NatsStartupError::Stream)?,
roblox_cookie,
api,
maps_grpc,
})
}
pub async fn run(mut self){
while let Some(message_result)=self.messages.next().await{
self.publish_supress_error(message_result).await
}
}
pub async fn publish(&self,message:async_nats::jetstream::Message)->Result<(),PublishError>{
println!("publish_new {:?}",message.message.payload);
async fn publish_supress_error(&self,message_result:MessageResult){
match self.publish(message_result).await{
Ok(())=>println!("[PublishNew] Published, hooray!"),
Err(e)=>println!("[PublishNew] There was an error, oopsie! {e}"),
}
}
async fn publish(&self,message_result:MessageResult)->Result<(),PublishError>{
println!("publish_new {:?}",message_result);
let message=message_result.map_err(PublishError::Messages)?;
message.double_ack().await.map_err(PublishError::DoubleAck)?;
// decode json
let publish_info:PublishNewRequest=serde_json::from_slice(&message.payload).map_err(PublishError::Json)?;

16
validation/src/types.rs Normal file
View File

@ -0,0 +1,16 @@
// annoying mile-long types
pub type MapsServiceClient=rust_grpc::maps::maps_service_client::MapsServiceClient<tonic::transport::channel::Channel>;
pub type MessageResult=Result<async_nats::jetstream::Message,async_nats::jetstream::consumer::pull::MessagesError>;
#[derive(Debug)]
pub enum NatsStartupError{
Consumer(async_nats::jetstream::stream::ConsumerError),
Stream(async_nats::jetstream::consumer::StreamError),
}
impl std::fmt::Display for NatsStartupError{
fn fmt(&self,f:&mut std::fmt::Formatter<'_>)->std::fmt::Result{
write!(f,"{self:?}")
}
}
impl std::error::Error for NatsStartupError{}

View File

@ -1,5 +1,5 @@
use futures::TryStreamExt;
use futures::{StreamExt,TryStreamExt};
use crate::types::{MessageResult,NatsStartupError};
use crate::nats_types::ValidateRequest;
const SCRIPT_CONCURRENCY:usize=16;
@ -13,9 +13,11 @@ enum Policy{
#[allow(dead_code)]
#[derive(Debug)]
pub enum ValidateError{
enum ValidateError{
Blocked,
NotAllowed,
Messages(async_nats::jetstream::consumer::pull::MessagesError),
DoubleAck(async_nats::Error),
Get(rbx_asset::cookie::GetError),
Json(serde_json::Error),
ReadDom(ReadDomError),
@ -35,22 +37,45 @@ impl std::fmt::Display for ValidateError{
impl std::error::Error for ValidateError{}
pub struct Validator{
messages:async_nats::jetstream::consumer::pull::Stream,
roblox_cookie:rbx_asset::cookie::CookieContext,
api:api::Context,
}
impl Validator{
pub const fn new(
pub async fn new(
stream:async_nats::jetstream::stream::Stream,
roblox_cookie:rbx_asset::cookie::CookieContext,
api:api::Context,
)->Self{
Self{
)->Result<Self,NatsStartupError>{
Ok(Self{
messages:stream.get_or_create_consumer("validation",async_nats::jetstream::consumer::pull::Config{
name:Some("validation".to_owned()),
durable_name:Some("validation".to_owned()),
ack_policy:async_nats::jetstream::consumer::AckPolicy::Explicit,
filter_subject:"maptest.submissions.validate".to_owned(),
..Default::default()
}).await.map_err(NatsStartupError::Consumer)?
.messages().await.map_err(NatsStartupError::Stream)?,
roblox_cookie,
api,
})
}
pub async fn run(mut self){
while let Some(message_result)=self.messages.next().await{
self.validate_supress_error(message_result).await
}
}
pub async fn validate(&self,message:async_nats::jetstream::Message)->Result<(),ValidateError>{
println!("validate {:?}",message.message.payload);
async fn validate_supress_error(&self,message_result:MessageResult){
match self.validate(message_result).await{
Ok(())=>println!("[Validation] Validated, hooray!"),
Err(e)=>println!("[Validation] There was an error, oopsie! {e}"),
}
}
async fn validate(&self,message_result:MessageResult)->Result<(),ValidateError>{
println!("validate {:?}",message_result);
let message=message_result.map_err(ValidateError::Messages)?;
message.double_ack().await.map_err(ValidateError::DoubleAck)?;
// decode json
let validate_info:ValidateRequest=serde_json::from_slice(&message.payload).map_err(ValidateError::Json)?;
@ -133,10 +158,10 @@ impl Validator{
if modified{
// serialize model (slow!)
let mut data=Vec::new();
rbx_binary::to_writer(&mut data,&dom,dom.root().children()).map_err(ValidateError::WriteDom)?;
rbx_binary::to_writer(&mut data,&dom,&[dom.root_ref()]).map_err(ValidateError::WriteDom)?;
// upload a model lol
let model_id=if let Some(model_id)=validate_info.ValidatedModelID{
let (model_id,model_version)=if let Some(model_id)=validate_info.ValidatedModelID{
// upload to existing id
let response=self.roblox_cookie.upload(rbx_asset::cookie::UploadRequest{
assetid:model_id,
@ -147,7 +172,7 @@ impl Validator{
groupId:None,
},data).await.map_err(ValidateError::Upload)?;
response.AssetId
(response.AssetId,response.AssetVersionId)
}else{
// create new model
let response=self.roblox_cookie.create(rbx_asset::cookie::CreateRequest{
@ -158,14 +183,14 @@ impl Validator{
groupId:None,
},data).await.map_err(ValidateError::Create)?;
response.AssetId
(response.AssetId,response.AssetVersionId)
};
// update the submission to use the validated model
self.api.update_submission_model(api::UpdateSubmissionModelRequest{
ID:validate_info.SubmissionID,
ModelID:model_id,
ModelVersion:1, //TODO
ModelVersion:model_version,
}).await.map_err(ValidateError::ApiUpdateSubmissionModel)?;
};
@ -180,7 +205,7 @@ impl Validator{
#[allow(dead_code)]
#[derive(Debug)]
pub enum ReadDomError{
enum ReadDomError{
Binary(rbx_binary::DecodeError),
Xml(rbx_xml::DecodeError),
Read(std::io::Error),

View File

@ -1,3 +1,15 @@
node_modules
build
bun.lockb
Dockerfile*
docker-compose*
.dockerignore
.git
.gitignore
README.md
LICENSE
.vscode
Makefile
helm-charts
.env
.editorconfig
.idea
coverage*

View File

@ -1,13 +1,19 @@
FROM oven/bun:latest
FROM oven/bun AS build
WORKDIR /app
COPY . .
EXPOSE 3000
ENV NODE_ENV=production
ENV NEXT_TELEMETRY_DISABLED=1
RUN bun install && bun run build
RUN bun install
RUN bun run build
ENTRYPOINT ["bun", "run", "start"]
FROM build AS release
WORKDIR /app
COPY --from=build /app/build ./.next
RUN bun add next
EXPOSE 3000
ENTRYPOINT ["bun", "run", "env", "--", "next", "start", "-p", "3000"]

View File

@ -2,12 +2,11 @@ import type { NextConfig } from "next";
const nextConfig: NextConfig = {
distDir: "build",
output: "standalone",
rewrites: async () => {
return [
{
source: "/v1/submissions/:submissionid/status/:statustype",
destination: "http://mapsservice:8082/v1/submissions/:submissionid/status/:statustype"
source: "/v1/submissions/1/status/:statustype",
destination: "http://localhost:8081/v1/submissions/:statustype"
}
]
}

View File

@ -1,6 +1,6 @@
import { Button, ButtonOwnProps } from "@mui/material";
type Review = "Completed" | "Submit" | "Reject" | "Revoke" | "Accept" | "Publish"
type Review = "Completed" | "Submit" | "Reject" | "Revoke" | "Validate" | "Publish"
type Action = "completed" | "submit" | "reject" | "revoke" | "trigger-validate" | "trigger-publish"
interface ReviewButton {
name: Review,
@ -13,6 +13,7 @@ function ReviewButtonClicked(action: Action) {
method: "POST",
headers: {
"Content-type": "application/json",
"Cookie": "session_id=c5191ddc-eee1-4010-900c-6b2c7b6780ab"
}
})
}
@ -24,12 +25,12 @@ function ReviewButton(props: ReviewButton) {
export default function ReviewButtons() {
return (
<section className="review-set">
<ReviewButton color="info" name="Submit" action="submit"/>
<ReviewButton color="info" name="Revoke" action="revoke"/>
<ReviewButton color="info" name="Accept" action="trigger-validate"/>
<ReviewButton color="error" name="Reject" action="reject"/>
<ReviewButton color="info" name="Revoke" action="revoke"/>
<ReviewButton color="info" name="Publish" action="trigger-publish"/>
<ReviewButton color="info" name="Completed" action="completed"/>
<ReviewButton color="info" name="Submit" action="submit"/>
<ReviewButton color="info" name="Validate" action="trigger-validate"/>
</section>
)
}