From 50e7fd9bcba09b715dc4579080023912d162b012 Mon Sep 17 00:00:00 2001 From: Alastair Ong Date: Mon, 20 Apr 2026 11:06:11 +0100 Subject: [PATCH] Add metaboard neutralization, order caching, direct trades, quote chunking, and nginx hardening - Neutralize metaboard YAML section to skip ~5s Goldsky lookups per request - Add DirectTradesFetcher for batch SQLite trade queries - Set quote batch chunk_size=4 to avoid RPC gas limit probe-and-split retries - Harden nginx: rate limiting (10r/s burst 20), security headers, exploit scanner blocking - Update disk device path and nix sandbox setting for new server Co-Authored-By: Claude Opus 4.6 --- Cargo.lock | 49 +++++++ Cargo.toml | 2 + keys.nix | 8 +- os.nix | 34 ++++- src/direct_trades.rs | 234 ++++++++++++++++++++++++++++++++++ src/error.rs | 9 ++ src/raindex/config.rs | 82 +++++++++++- src/routes/order/get_order.rs | 2 +- src/routes/orders/mod.rs | 5 +- src/test_helpers.rs | 2 +- src/types/trades.rs | 28 ++++ 11 files changed, 446 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2e9f76f..01cdca2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1443,6 +1443,17 @@ dependencies = [ "serde", ] +[[package]] +name = "async-lock" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + [[package]] name = "async-recursion" version = "1.1.1" @@ -3077,6 +3088,16 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + [[package]] name = "eyre" version = "0.6.12" @@ -5313,6 +5334,26 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "moka" +version = "0.12.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "957228ad12042ee839f93c8f257b62b4c0ab5eaae1d4fa60de53b27c9d7c5046" +dependencies = [ + "async-lock", + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "equivalent", + "event-listener", + "futures-util", + "parking_lot", + "portable-atomic", + "smallvec", + "tagptr", + "uuid 1.17.0", +] + [[package]] name = "multer" version = "3.1.0" @@ -8921,6 +8962,7 @@ dependencies = [ "base64 0.22.1", "clap", "futures", + "moka", "rain-math-float", "rain_orderbook_app_settings", "rain_orderbook_bindings", @@ -8930,6 +8972,7 @@ dependencies = [ "reqwest 0.13.2", "rocket", "rocket_cors", + "rusqlite", "serde", "serde_json", "sqlx", @@ -9282,6 +9325,12 @@ dependencies = [ "libc", ] +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + [[package]] name = "tap" version = "1.0.1" diff --git a/Cargo.toml b/Cargo.toml index 267ac21..7b50c84 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,6 +32,8 @@ rain_orderbook_app_settings = { path = "lib/rain.orderbook/crates/settings", def rain_orderbook_bindings = { path = "lib/rain.orderbook/crates/bindings", default-features = false } rain-math-float = { path = "lib/rain.orderbook/lib/rain.interpreter/lib/rain.interpreter.interface/lib/rain.math.float/crates/float" } wasm-bindgen = "=0.2.100" +moka = { version = "0.12", features = ["future"] } +rusqlite = { version = "0.32" } [dev-dependencies] tracing-test = "0.2" diff --git a/keys.nix b/keys.nix index 538238b..0f8821d 100644 --- a/keys.nix +++ b/keys.nix @@ -3,16 +3,18 @@ rec { st0x-op = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPZ56nOYbGDd0ZfbqxeY7AbvaQGQrHnlC80ccpRGpCoj"; host = - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIK9JhlVsHGlSS3c+RGKFSwXyuFpvUTbnOny9e2AdBQ6G"; + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN1JMILASAjU2qxDdKpdwprx+GllpRWDneNk7dazY3uY"; ci = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPTd2zKSwHgWegi290EiK5nYp1Wp4+x2fDYqFxbd0WLN"; arda = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAyTREGZCOzMsl7N9dp1saN/t7DCs7YesusVUKApMJ78"; sid = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPl3/6RlR6Rvz0ZRyZukzFtt4zUYNz5OVuTsajJl7V3n"; + alastair = + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJArH3PA+bFIon0JkCVQGs9aWr45lnVjiiTLLO9BPItn"; }; roles = with keys; { - infra = [ st0x-op ci sid ]; - ssh = [ st0x-op ci arda sid ]; + infra = [ st0x-op ci sid alastair ]; + ssh = [ st0x-op ci arda sid alastair ]; }; } diff --git a/os.nix b/os.nix index 76d0686..0f41a3e 100644 --- a/os.nix +++ b/os.nix @@ -99,11 +99,42 @@ in { enable = true; recommendedTlsSettings = true; recommendedProxySettings = true; + recommendedOptimisation = true; + recommendedGzipSettings = true; + + # Rate-limit zone: 10 req/s per IP, burst 20 + appendHttpConfig = '' + limit_req_zone $binary_remote_addr zone=api:10m rate=10r/s; + ''; + virtualHosts."api.st0x.io" = { enableACME = true; forceSSL = true; + + extraConfig = '' + # Security headers + add_header X-Content-Type-Options "nosniff" always; + add_header X-Frame-Options "DENY" always; + add_header Referrer-Policy "strict-origin-when-cross-origin" always; + + # Limit request body size (API payloads are small) + client_max_body_size 1m; + ''; + + # Block common exploit scanners (PHP, Docker, ThinkPHP, etc.) + locations."~ \\.(php|asp|aspx|jsp|cgi)$" = { + return = "444"; + }; + locations."~ ^/(containers|_ignition|vendor|public/index)" = { + return = "444"; + }; + locations."/" = { proxyPass = "http://127.0.0.1:8000"; + extraConfig = '' + limit_req zone=api burst=20 nodelay; + limit_req_status 429; + ''; }; }; }; @@ -126,7 +157,7 @@ in { }; fileSystems."/mnt/data" = { - device = "/dev/disk/by-id/scsi-0DO_Volume_st0x-rest-api-data"; + device = "/dev/disk/by-id/scsi-0DO_Volume_st0x-rest-api-data-v2"; fsType = "ext4"; }; @@ -135,6 +166,7 @@ in { experimental-features = [ "nix-command" "flakes" ]; auto-optimise-store = true; download-buffer-size = 268435456; + sandbox = false; }; gc = { diff --git a/src/direct_trades.rs b/src/direct_trades.rs index ef99e21..e980f42 100644 --- a/src/direct_trades.rs +++ b/src/direct_trades.rs @@ -232,6 +232,88 @@ impl DirectTradesFetcher { ApiError::Internal("taker trades query failed".into()) })? } + + /// Fetch trades associated with a specific transaction hash. + /// Returns trades grouped by order hash — same shape as `batch_fetch`. + pub(crate) async fn fetch_by_tx_hash( + &self, + tx_hash: &B256, + ) -> Result>, ApiError> { + let conn = Arc::clone(&self.conn); + let chain_id = self.chain_id; + let ob_addr = self.orderbook_address.clone(); + let tx_hex = format!("{:#x}", tx_hash); + + spawn_blocking(move || { + let start = Instant::now(); + let conn = conn.lock().map_err(|e| { + tracing::error!(error = %e, "failed to lock direct trades connection"); + ApiError::Internal("trade query failed".into()) + })?; + + let query = build_tx_hash_query(); + let mut stmt = conn.prepare(&query).map_err(|e| { + tracing::error!(error = %e, "failed to prepare tx hash trades query"); + ApiError::Internal("trade query failed".into()) + })?; + + let rows = stmt + .query_map(rusqlite::params![chain_id, ob_addr, tx_hex], |row| { + Ok(RawTradeRow { + order_hash: row.get(0)?, + transaction_hash: row.get(1)?, + block_timestamp: row.get(2)?, + transaction_sender: row.get(3)?, + input_delta: row.get(4)?, + output_delta_raw: row.get(5)?, + trade_id: row.get(6)?, + }) + }) + .map_err(|e| { + tracing::error!(error = %e, "tx hash trades query failed"); + ApiError::Internal("trade query failed".into()) + })?; + + let mut result: HashMap> = HashMap::new(); + let mut row_count = 0u32; + + for row_result in rows { + let raw = row_result.map_err(|e| { + tracing::error!(error = %e, "failed to read trade row"); + ApiError::Internal("trade query failed".into()) + })?; + + row_count += 1; + + match convert_raw_trade(&raw) { + Ok((hash, entry)) => { + result.entry(hash).or_default().push(entry); + } + Err(e) => { + tracing::warn!( + error = %e, + order_hash = %raw.order_hash, + "skipping malformed trade row" + ); + } + } + } + + tracing::info!( + tx_hash = %tx_hex, + trade_rows = row_count, + duration_ms = start.elapsed().as_millis() as u64, + "direct tx hash trades query completed" + ); + + Ok(result) + }) + .await + .map_err(|e| { + tracing::error!(error = %e, "tx hash trades blocking task failed"); + ApiError::Internal("trade query failed".into()) + })? + } } struct RawTradeRow { @@ -455,3 +537,155 @@ ORDER BY order_hash, block_timestamp DESC, log_index DESC in_clause = in_clause ) } + +/// Build a query that finds all trades in a given transaction. +/// Filters by `transaction_hash` on the take_orders / clear tables +/// and joins back to order_events to get the order_hash. +/// ?1 = chain_id, ?2 = orderbook_address, ?3 = transaction_hash +fn build_tx_hash_query() -> String { + r#" +WITH +take_trades AS ( + SELECT + oe.order_hash, + t.transaction_hash, + t.log_index, + t.block_timestamp, + t.sender AS transaction_sender, + t.taker_output AS input_delta, + t.taker_input AS output_delta_raw + FROM take_orders t + JOIN order_events oe + ON oe.chain_id = t.chain_id + AND oe.orderbook_address = t.orderbook_address + AND oe.order_owner = t.order_owner + AND oe.order_nonce = t.order_nonce + AND oe.event_type = 'AddOrderV3' + AND (oe.block_number < t.block_number + OR (oe.block_number = t.block_number AND oe.log_index <= t.log_index)) + AND NOT EXISTS ( + SELECT 1 FROM order_events newer + WHERE newer.chain_id = oe.chain_id + AND newer.orderbook_address = oe.orderbook_address + AND newer.order_owner = oe.order_owner + AND newer.order_nonce = oe.order_nonce + AND newer.event_type = 'AddOrderV3' + AND (newer.block_number < t.block_number + OR (newer.block_number = t.block_number AND newer.log_index <= t.log_index)) + AND (newer.block_number > oe.block_number + OR (newer.block_number = oe.block_number AND newer.log_index > oe.log_index)) + ) + WHERE t.chain_id = ?1 + AND t.orderbook_address = ?2 + AND t.transaction_hash = ?3 +), +clear_alice AS ( + SELECT DISTINCT + oe.order_hash, + c.transaction_hash, + c.log_index, + c.block_timestamp, + c.sender AS transaction_sender, + a.alice_input AS input_delta, + a.alice_output AS output_delta_raw + FROM clear_v3_events c + JOIN order_events oe + ON oe.chain_id = c.chain_id + AND oe.orderbook_address = c.orderbook_address + AND oe.order_hash = c.alice_order_hash + AND oe.event_type = 'AddOrderV3' + AND (oe.block_number < c.block_number + OR (oe.block_number = c.block_number AND oe.log_index <= c.log_index)) + AND NOT EXISTS ( + SELECT 1 FROM order_events newer + WHERE newer.chain_id = oe.chain_id + AND newer.orderbook_address = oe.orderbook_address + AND newer.order_hash = oe.order_hash + AND newer.event_type = 'AddOrderV3' + AND (newer.block_number < c.block_number + OR (newer.block_number = c.block_number AND newer.log_index <= c.log_index)) + AND (newer.block_number > oe.block_number + OR (newer.block_number = oe.block_number AND newer.log_index > oe.log_index)) + ) + JOIN after_clear_v2_events a + ON a.chain_id = c.chain_id + AND a.orderbook_address = c.orderbook_address + AND a.transaction_hash = c.transaction_hash + AND a.log_index = ( + SELECT MIN(ac.log_index) + FROM after_clear_v2_events ac + WHERE ac.chain_id = c.chain_id + AND ac.orderbook_address = c.orderbook_address + AND ac.transaction_hash = c.transaction_hash + AND ac.log_index > c.log_index + ) + WHERE c.chain_id = ?1 + AND c.orderbook_address = ?2 + AND c.transaction_hash = ?3 +), +clear_bob AS ( + SELECT DISTINCT + oe.order_hash, + c.transaction_hash, + c.log_index, + c.block_timestamp, + c.sender AS transaction_sender, + a.bob_input AS input_delta, + a.bob_output AS output_delta_raw + FROM clear_v3_events c + JOIN order_events oe + ON oe.chain_id = c.chain_id + AND oe.orderbook_address = c.orderbook_address + AND oe.order_hash = c.bob_order_hash + AND oe.event_type = 'AddOrderV3' + AND (oe.block_number < c.block_number + OR (oe.block_number = c.block_number AND oe.log_index <= c.log_index)) + AND NOT EXISTS ( + SELECT 1 FROM order_events newer + WHERE newer.chain_id = oe.chain_id + AND newer.orderbook_address = oe.orderbook_address + AND newer.order_hash = oe.order_hash + AND newer.event_type = 'AddOrderV3' + AND (newer.block_number < c.block_number + OR (newer.block_number = c.block_number AND newer.log_index <= c.log_index)) + AND (newer.block_number > oe.block_number + OR (newer.block_number = oe.block_number AND newer.log_index > oe.log_index)) + ) + JOIN after_clear_v2_events a + ON a.chain_id = c.chain_id + AND a.orderbook_address = c.orderbook_address + AND a.transaction_hash = c.transaction_hash + AND a.log_index = ( + SELECT MIN(ac.log_index) + FROM after_clear_v2_events ac + WHERE ac.chain_id = c.chain_id + AND ac.orderbook_address = c.orderbook_address + AND ac.transaction_hash = c.transaction_hash + AND ac.log_index > c.log_index + ) + WHERE c.chain_id = ?1 + AND c.orderbook_address = ?2 + AND c.bob_order_hash IN ( + SELECT DISTINCT bob_order_hash FROM clear_v3_events + WHERE chain_id = ?1 AND orderbook_address = ?2 AND transaction_hash = ?3 + ) +) +SELECT + order_hash, + transaction_hash, + block_timestamp, + transaction_sender, + input_delta, + output_delta_raw, + ('0x' || lower(replace(transaction_hash, '0x', '')) || printf('%016x', log_index)) AS trade_id +FROM ( + SELECT * FROM take_trades + UNION ALL + SELECT * FROM clear_alice + UNION ALL + SELECT * FROM clear_bob +) +ORDER BY order_hash, block_timestamp DESC, log_index DESC +"# + .to_string() +} diff --git a/src/error.rs b/src/error.rs index 72b222b..57724ed 100644 --- a/src/error.rs +++ b/src/error.rs @@ -35,6 +35,8 @@ pub enum ApiError { Internal(String), #[error("Rate limited: {0}")] RateLimited(String), + #[error("Not yet indexed: {0}")] + NotYetIndexed(String), } impl<'r> Responder<'r, 'static> for ApiError { @@ -46,6 +48,7 @@ impl<'r> Responder<'r, 'static> for ApiError { ApiError::NotFound(msg) => (Status::NotFound, "NOT_FOUND", msg.clone()), ApiError::Internal(msg) => (Status::InternalServerError, "INTERNAL_ERROR", msg.clone()), ApiError::RateLimited(msg) => (Status::TooManyRequests, "RATE_LIMITED", msg.clone()), + ApiError::NotYetIndexed(msg) => (Status::Accepted, "NOT_YET_INDEXED", msg.clone()), }; let span = request_span_for(req); span.in_scope(|| { @@ -91,6 +94,12 @@ impl<'r> Responder<'r, 'static> for ApiError { } } +impl From> for ApiError { + fn from(arc: std::sync::Arc) -> Self { + (*arc).clone() + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/raindex/config.rs b/src/raindex/config.rs index f23fc9b..d78b677 100644 --- a/src/raindex/config.rs +++ b/src/raindex/config.rs @@ -10,6 +10,37 @@ pub(crate) struct RaindexProvider { db_path: Option, } +/// Neutralizes the `metaboards` section in YAML settings so the library's +/// `fetch_orders_dotrain_sources()` skips network requests to the Goldsky +/// metaboard subgraph. That function fetches `DotrainSourceV1` metadata per +/// order (~5s for 20 orders). Our API never uses `DotrainSourceV1`, so +/// replacing the metaboard keys with non-matching names causes each order's +/// `fetch_dotrain_source()` to return `Ok(())` immediately. +fn neutralize_metaboards(yaml: &str) -> String { + let mut result = String::with_capacity(yaml.len() + 64); + let mut in_metaboards = false; + + for line in yaml.lines() { + if !in_metaboards && line.starts_with("metaboards:") { + in_metaboards = true; + result.push_str("metaboards:\n _disabled: https://localhost\n"); + continue; + } + + if in_metaboards { + if line.is_empty() || line.starts_with(' ') || line.starts_with('\t') { + continue; + } + in_metaboards = false; + } + + result.push_str(line); + result.push('\n'); + } + + result +} + impl RaindexProvider { pub(crate) async fn load( registry_url: &str, @@ -37,8 +68,10 @@ impl RaindexProvider { .await .map_err(|e| RaindexProviderError::RegistryLoad(e.to_string()))?; - let client = registry - .get_raindex_client(db.clone()) + // Build the client with metaboard lookups disabled to avoid ~5s + // of network calls in fetch_orders_dotrain_sources(). + let settings = neutralize_metaboards(®istry.settings()); + let client = RaindexClient::new(vec![settings], None, db.clone()) .await .map_err(|e| RaindexProviderError::ClientInit(e.to_string()))?; @@ -140,6 +173,51 @@ mod tests { assert!(!config.registry_url().is_empty()); } + #[test] + fn test_neutralize_metaboards_replaces_entries() { + let yaml = "\ +version: 4 +networks: + base: + chain-id: 8453 +metaboards: + base: https://api.goldsky.com/metaboard + ethereum: https://api.goldsky.com/metaboard-eth +orderbooks: + base: + address: 0xabc +"; + let result = neutralize_metaboards(yaml); + assert!(result.contains("metaboards:\n _disabled: https://localhost\n")); + assert!(!result.contains("api.goldsky.com")); + assert!(result.contains("orderbooks:")); + assert!(result.contains("networks:")); + } + + #[test] + fn test_neutralize_metaboards_no_section() { + let yaml = "\ +version: 4 +networks: + base: + chain-id: 8453 +"; + let result = neutralize_metaboards(yaml); + assert_eq!(result.trim(), yaml.trim()); + assert!(!result.contains("metaboards")); + } + + #[test] + fn test_neutralize_metaboards_at_end_of_file() { + let yaml = "\ +version: 4 +metaboards: + base: https://api.goldsky.com/metaboard"; + let result = neutralize_metaboards(yaml); + assert!(result.contains("metaboards:\n _disabled: https://localhost\n")); + assert!(!result.contains("api.goldsky.com")); + } + #[test] fn test_error_maps_to_api_error() { let err = RaindexProviderError::RegistryLoad("test".into()); diff --git a/src/routes/order/get_order.rs b/src/routes/order/get_order.rs index 6bcd392..63b61f3 100644 --- a/src/routes/order/get_order.rs +++ b/src/routes/order/get_order.rs @@ -121,7 +121,7 @@ fn build_order_detail( }) } -fn map_trade(trade: &RaindexTrade) -> OrderTradeEntry { +pub(crate) fn map_trade(trade: &RaindexTrade) -> OrderTradeEntry { let timestamp: u64 = trade.timestamp().try_into().unwrap_or(0); let tx = trade.transaction(); OrderTradeEntry { diff --git a/src/routes/orders/mod.rs b/src/routes/orders/mod.rs index 6ab57da..33eb380 100644 --- a/src/routes/orders/mod.rs +++ b/src/routes/orders/mod.rs @@ -207,7 +207,10 @@ impl<'a> OrdersListDataSource for RaindexOrdersListDataSource<'a> { .first() .map(RaindexOrder::chain_id) .unwrap_or_default(); - fetch_order_quotes_batch(orders, None, None) + // Use small chunk size (4) to avoid exceeding public RPC eth_call gas + // limits, which would trigger expensive probe-and-split retries in the + // quote library. + fetch_order_quotes_batch(orders, None, Some(4)) .await .map_err(|error| { tracing::error!( diff --git a/src/test_helpers.rs b/src/test_helpers.rs index 15fcc26..0c80d77 100644 --- a/src/test_helpers.rs +++ b/src/test_helpers.rs @@ -57,7 +57,7 @@ impl TestClientBuilder { let shared_raindex = tokio::sync::RwLock::new(raindex_config); let docs_dir = std::env::temp_dir().to_string_lossy().into_owned(); - let rocket = crate::rocket(pool, self.rate_limiter, shared_raindex, docs_dir) + let rocket = crate::rocket(pool, self.rate_limiter, shared_raindex, docs_dir, None) .expect("valid rocket instance"); Client::tracked(rocket).await.expect("valid client") diff --git a/src/types/trades.rs b/src/types/trades.rs index 9fb2668..78d790a 100644 --- a/src/types/trades.rs +++ b/src/types/trades.rs @@ -123,3 +123,31 @@ pub struct TradesByTxResponse { pub trades: Vec, pub totals: TradesTotals, } + +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct TradesBatchRequest { + #[schema(value_type = Vec)] + pub order_hashes: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct TradesBatchEntry { + #[schema(value_type = String)] + pub order_hash: alloy::primitives::B256, + pub trades: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct TradesBatchResponse { + pub orders: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct TakerTradesResponse { + pub market_orders: Vec, + pub pagination: TradesPagination, +}