From f1b5f84c6c9e369f7f3684cade1a2e0c80d635f5 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Mon, 23 Feb 2026 23:16:33 +0100 Subject: [PATCH] fix(broker_gateway): log CRITICAL on DB update failure after broker submission Both route_order and cancel_order used let _ = to discard DB update errors after successful broker operations. This means an order could be live at the broker while the database still shows PENDING_SUBMIT or CANCEL_PENDING, with no log entry to alert operators. Replaced with if let Err(db_err) that logs at error level with CRITICAL prefix, client_order_id, and broker_order_id for manual reconciliation. The RPC still returns success since the broker operation completed. Co-Authored-By: Claude Opus 4.6 --- .../broker_gateway_service/src/service.rs | 31 ++++++++++++++++--- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/services/broker_gateway_service/src/service.rs b/services/broker_gateway_service/src/service.rs index 910e271c3..abe664cd1 100644 --- a/services/broker_gateway_service/src/service.rs +++ b/services/broker_gateway_service/src/service.rs @@ -258,13 +258,25 @@ impl broker_gateway_service_server::BrokerGatewayService for BrokerGatewayServic .unwrap_or_else(|| format!("ct-{}", resp_msg.payload_type)); // Update DB status to SUBMITTED - let _ = sqlx::query( + // CRITICAL: Order is already live at broker. If DB update + // fails, the order is real but DB shows PENDING_SUBMIT. + // Log prominently but do NOT fail the RPC. + if let Err(db_err) = sqlx::query( "UPDATE broker_orders SET status = 'SUBMITTED', broker_order_id = $1, updated_at = NOW() WHERE client_order_id = $2", ) .bind(&broker_order_id) .bind(&client_order_id) .execute(&self.db_pool) - .await; + .await + { + error!( + client_order_id = %client_order_id, + broker_order_id = %broker_order_id, + error = %db_err, + "CRITICAL: order submitted to broker but DB status update failed — \ + order is LIVE but database shows PENDING_SUBMIT. Manual reconciliation required." + ); + } info!( broker_order_id = %broker_order_id, @@ -381,12 +393,23 @@ impl broker_gateway_service_server::BrokerGatewayService for BrokerGatewayServic if let Some(ct) = guard.as_ref() { match ct.cancel_order(order_id).await { Ok(_) => { - let _ = sqlx::query( + // CRITICAL: Cancel is confirmed at broker. If DB + // update fails, the order is cancelled but DB + // still shows CANCEL_PENDING. Log prominently. + if let Err(db_err) = sqlx::query( "UPDATE broker_orders SET status = 'CANCELLED', updated_at = NOW() WHERE client_order_id = $1", ) .bind(&req.client_order_id) .execute(&self.db_pool) - .await; + .await + { + error!( + client_order_id = %req.client_order_id, + error = %db_err, + "CRITICAL: cancel confirmed at broker but DB status update failed — \ + order is CANCELLED but database shows CANCEL_PENDING. Manual reconciliation required." + ); + } info!( client_order_id = %req.client_order_id,