gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(broker_gateway): log CRITICAL on DB update failure after broker submission

Browse Source 
Both route_order and cancel_order used let _ = to discard DB update
errors after successful broker operations. This means an order could be
live at the broker while the database still shows PENDING_SUBMIT or
CANCEL_PENDING, with no log entry to alert operators.

Replaced with if let Err(db_err) that logs at error level with CRITICAL
prefix, client_order_id, and broker_order_id for manual reconciliation.
The RPC still returns success since the broker operation completed.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-02-23 23:16:33 +01:00
parent a45aec5605
commit f1b5f84c6c
1 changed files with 27 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
services/broker_gateway_service/src/service.rs
View File

@@ -258,13 +258,25 @@ impl broker_gateway_service_server::BrokerGatewayService for BrokerGatewayServic
.unwrap_or_else(|| format!("ct-{}", resp_msg.payload_type));
// Update DB status to SUBMITTED
let _ = sqlx::query(
// CRITICAL: Order is already live at broker. If DB update
// fails, the order is real but DB shows PENDING_SUBMIT.
// Log prominently but do NOT fail the RPC.
if let Err(db_err) = sqlx::query(
"UPDATE broker_orders SET status = 'SUBMITTED', broker_order_id = $1, updated_at = NOW() WHERE client_order_id = $2",
)
.bind(&broker_order_id)
.bind(&client_order_id)
.execute(&self.db_pool)
.await;
.await
{
error!(
client_order_id = %client_order_id,
broker_order_id = %broker_order_id,
error = %db_err,
"CRITICAL: order submitted to broker but DB status update failed — \
order is LIVE but database shows PENDING_SUBMIT. Manual reconciliation required."
);
}
info!(
broker_order_id = %broker_order_id,
@@ -381,12 +393,23 @@ impl broker_gateway_service_server::BrokerGatewayService for BrokerGatewayServic
if let Some(ct) = guard.as_ref() {
match ct.cancel_order(order_id).await {
Ok(_) => {
let _ = sqlx::query(
// CRITICAL: Cancel is confirmed at broker. If DB
// update fails, the order is cancelled but DB
// still shows CANCEL_PENDING. Log prominently.
if let Err(db_err) = sqlx::query(
"UPDATE broker_orders SET status = 'CANCELLED', updated_at = NOW() WHERE client_order_id = $1",
)
.bind(&req.client_order_id)
.execute(&self.db_pool)
.await;
.await
{
error!(
client_order_id = %req.client_order_id,
error = %db_err,
"CRITICAL: cancel confirmed at broker but DB status update failed — \
order is CANCELLED but database shows CANCEL_PENDING. Manual reconciliation required."
);
}
info!(
client_order_id = %req.client_order_id,