foxhunt/services/backtesting_service/src/storage.rs

//! Storage layer for backtesting data persistence

use anyhow::{Context, Result};
use rust_decimal::{prelude::ToPrimitive, Decimal};
use sqlx::{PgPool, Row};
use std::collections::HashMap;
use tracing::{debug, info}; // For Decimal::from_f64

use crate::foxhunt::tli::BacktestStatus;
use crate::performance::PerformanceMetrics;
use crate::strategy_engine::BacktestTrade;
use common::database::DatabasePool;
use config::structures::BacktestingDatabaseConfig;

/// Backtest summary for listing
#[derive(Debug, Clone)]
pub struct BacktestSummary {
    /// Backtest ID
    pub backtest_id: String,
    /// Strategy name
    pub strategy_name: String,
    /// Symbols tested
    pub symbols: Vec<String>,
    /// Current status
    pub status: BacktestStatus,
    /// Total return percentage
    pub total_return: f64,
    /// Sharpe ratio
    pub sharpe_ratio: f64,
    /// Maximum drawdown percentage
    pub max_drawdown: f64,
    /// Creation timestamp
    pub created_at: chrono::DateTime<chrono::Utc>,
    /// Start date of backtest
    pub start_date: chrono::DateTime<chrono::Utc>,
    /// End date of backtest
    pub end_date: chrono::DateTime<chrono::Utc>,
    /// Description
    pub description: String,
}

/// Storage manager for backtesting data
#[derive(Debug)]
#[allow(dead_code)]
pub struct StorageManager {
    /// HFT-optimized Postgre`SQL` connection pool
    db_pool: DatabasePool,
    /// Raw PgPool for compatibility with existing queries
    pg_pool: PgPool,
    /// InfluxDB client (placeholder for now)
    _influxdb_client: Option<()>, // TODO: Implement InfluxDB client
}

impl StorageManager {
    /// Create a new storage manager
    pub async fn new(config: &BacktestingDatabaseConfig) -> Result<Self> {
        info!("Initializing storage manager with HFT optimizations");

        // Create backtesting-optimized database pool using config conversion
        // The From implementation handles all field mapping automatically
        let local_db_config: common::database::LocalDatabaseConfig = config.clone().into();

        let db_pool = DatabasePool::new(local_db_config)
            .await
            .context("Failed to create HFT-optimized database pool")?;

        let pg_pool = db_pool.pool().clone();

        // Run database migrations - simplified for now
        /*
        sqlx::migrate!("./migrations")
            .run(&pg_pool)
            .await
            .context("Failed to run database migrations")?;

        info!("Database migrations completed successfully");
        */

        // TODO: Initialize InfluxDB client
        let _influxdb_client = None;

        Ok(Self {
            db_pool,
            pg_pool,
            _influxdb_client,
        })
    }

    /// Save backtest results to storage
    pub async fn save_backtest_results(
        &self,
        backtest_id: &str,
        trades: &[BacktestTrade],
        metrics: &PerformanceMetrics,
    ) -> Result<()> {
        info!("Saving backtest results for {}", backtest_id);

        let mut tx = self.pg_pool.begin().await?;

        // Save individual trades
        for trade in trades {
            sqlx::query(
                r#"
                INSERT INTO backtest_trades (
                    backtest_id, trade_id, symbol, side, quantity, 
                    entry_price, exit_price, entry_time, exit_time,
                    pnl, return_percent, entry_signal, exit_signal
                ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
                "#,
            )
            .bind(backtest_id)
            .bind(&trade.trade_id)
            .bind(&trade.symbol)
            .bind(trade.side.to_string())
            .bind(trade.quantity.to_f64())
            .bind(trade.entry_price.to_f64())
            .bind(trade.exit_price.to_f64())
            .bind(trade.entry_time)
            .bind(trade.exit_time)
            .bind(trade.pnl.to_f64())
            .bind(trade.return_percent.to_f64())
            .bind(&trade.entry_signal)
            .bind(&trade.exit_signal)
            .execute(&mut *tx)
            .await?;
        }

        // Save detailed performance metrics
        sqlx::query(
            r#"
            INSERT INTO backtest_metrics (
                backtest_id, total_return, annualized_return, sharpe_ratio,
                sortino_ratio, max_drawdown, volatility, win_rate,
                profit_factor, total_trades, winning_trades, losing_trades,
                avg_win, avg_loss, largest_win, largest_loss, calmar_ratio,
                var_95, expected_shortfall
            ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19)
            "#,
        )
        .bind(backtest_id)
        .bind(metrics.total_return)
        .bind(metrics.annualized_return)
        .bind(metrics.sharpe_ratio)
        .bind(metrics.sortino_ratio)
        .bind(metrics.max_drawdown)
        .bind(metrics.volatility)
        .bind(metrics.win_rate)
        .bind(metrics.profit_factor)
        .bind(metrics.total_trades as i64)
        .bind(metrics.winning_trades as i64)
        .bind(metrics.losing_trades as i64)
        .bind(metrics.avg_win)
        .bind(metrics.avg_loss)
        .bind(metrics.largest_win)
        .bind(metrics.largest_loss)
        .bind(metrics.calmar_ratio)
        .bind(metrics.var_95.unwrap_or(0.0))
        .bind(metrics.expected_shortfall.unwrap_or(0.0))
        .execute(&mut *tx)
        .await?;

        tx.commit().await?;

        info!(
            "Successfully saved {} trades and metrics for backtest {}",
            trades.len(),
            backtest_id
        );

        Ok(())
    }

    /// Load backtest results from storage
    pub async fn load_backtest_results(
        &self,
        backtest_id: &str,
    ) -> Result<(Vec<BacktestTrade>, PerformanceMetrics)> {
        info!("Loading backtest results for {}", backtest_id);

        // Load trades
        let trade_rows = sqlx::query(
            r#"
            SELECT trade_id, symbol, side, quantity, entry_price, exit_price,
                   entry_time, exit_time, pnl, return_percent, entry_signal, exit_signal
            FROM backtest_trades 
            WHERE backtest_id = $1 
            ORDER BY entry_time
            "#,
        )
        .bind(backtest_id)
        .fetch_all(&self.pg_pool)
        .await?;

        let mut trades = Vec::new();
        for row in trade_rows {
            let side_str: String = row.try_get("side")?;
            let side = match side_str.as_str() {
                "Buy" => crate::strategy_engine::TradeSide::Buy,
                "Sell" => crate::strategy_engine::TradeSide::Sell,
                _ => continue, // Skip invalid trades
            };

            trades.push(BacktestTrade {
                trade_id: row.try_get("trade_id")?,
                symbol: row.try_get("symbol")?,
                side,
                quantity: Decimal::from_f64_retain(row.try_get::<f64, _>("quantity")?)
                    .unwrap_or(Decimal::ZERO),
                entry_price: Decimal::from_f64_retain(row.try_get::<f64, _>("entry_price")?)
                    .unwrap_or(Decimal::ZERO),
                exit_price: Decimal::from_f64_retain(row.try_get::<f64, _>("exit_price")?)
                    .unwrap_or(Decimal::ZERO),
                entry_time: row.try_get("entry_time")?,
                exit_time: row.try_get("exit_time")?,
                pnl: Decimal::from_f64_retain(row.try_get::<f64, _>("pnl")?)
                    .unwrap_or(Decimal::ZERO),
                return_percent: Decimal::from_f64_retain(row.try_get::<f64, _>("return_percent")?)
                    .unwrap_or(Decimal::ZERO),
                entry_signal: row.try_get("entry_signal")?,
                exit_signal: row.try_get("exit_signal")?,
            });
        }

        // Load metrics
        let metrics_row = sqlx::query(
            r#"
            SELECT total_return, annualized_return, sharpe_ratio, sortino_ratio,
                   max_drawdown, volatility, win_rate, profit_factor,
                   total_trades, winning_trades, losing_trades, avg_win, avg_loss,
                   largest_win, largest_loss, calmar_ratio, var_95, expected_shortfall
            FROM backtest_metrics 
            WHERE backtest_id = $1
            "#,
        )
        .bind(backtest_id)
        .fetch_one(&self.pg_pool)
        .await?;

        // Calculate backtest duration from trades
        let backtest_duration_nanos = if !trades.is_empty() {
            let earliest = trades
                .iter()
                .map(|t| t.entry_time)
                .min()
                .ok_or_else(|| anyhow::anyhow!("No trades found for earliest time"))?;
            let latest = trades
                .iter()
                .map(|t| t.exit_time)
                .max()
                .ok_or_else(|| anyhow::anyhow!("No trades found for latest time"))?;
            (latest - earliest).num_nanoseconds().unwrap_or(0) as u64
        } else {
            0
        };

        let metrics = PerformanceMetrics {
            total_return: metrics_row.try_get("total_return")?,
            annualized_return: metrics_row.try_get("annualized_return")?,
            sharpe_ratio: metrics_row.try_get("sharpe_ratio")?,
            sortino_ratio: metrics_row.try_get("sortino_ratio")?,
            max_drawdown: metrics_row.try_get("max_drawdown")?,
            volatility: metrics_row.try_get("volatility")?,
            win_rate: metrics_row.try_get("win_rate")?,
            profit_factor: metrics_row.try_get("profit_factor")?,
            total_trades: metrics_row.try_get::<i64, _>("total_trades")? as u64,
            winning_trades: metrics_row.try_get::<i64, _>("winning_trades")? as u64,
            losing_trades: metrics_row.try_get::<i64, _>("losing_trades")? as u64,
            avg_win: metrics_row.try_get("avg_win")?,
            avg_loss: metrics_row.try_get("avg_loss")?,
            largest_win: metrics_row.try_get("largest_win")?,
            largest_loss: metrics_row.try_get("largest_loss")?,
            calmar_ratio: metrics_row.try_get("calmar_ratio")?,
            backtest_duration_nanos: backtest_duration_nanos.try_into().unwrap_or(0),
            beta: None,
            alpha: None,
            information_ratio: None,
            var_95: Some(metrics_row.try_get("var_95")?),
            expected_shortfall: Some(metrics_row.try_get("expected_shortfall")?),
        };

        info!(
            "Loaded {} trades and metrics for backtest {}",
            trades.len(),
            backtest_id
        );

        Ok((trades, metrics))
    }

    /// List backtests with optional filtering
    pub async fn list_backtests(
        &self,
        limit: u32,
        offset: u32,
        _strategy_name: Option<String>,
        _status_filter: Option<BacktestStatus>,
    ) -> Result<Vec<BacktestSummary>> {
        info!("Listing backtests with limit={}, offset={}", limit, offset);

        // Simplified query without dynamic parameters for now
        let rows = sqlx::query(
            r#"
            SELECT backtest_id, strategy_name, symbols, status, total_return,
                   sharpe_ratio, max_drawdown, created_at, start_date, end_date,
                   description
            FROM backtests
            ORDER BY created_at DESC
            LIMIT $1 OFFSET $2
            "#,
        )
        .bind(limit as i64)
        .bind(offset as i64)
        .fetch_all(&self.pg_pool)
        .await?;

        let mut summaries = Vec::new();
        for row in rows {
            let status_str: String = row.try_get("status")?;
            let status = match status_str.as_str() {
                "queued" => BacktestStatus::Queued,
                "running" => BacktestStatus::Running,
                "completed" => BacktestStatus::Completed,
                "failed" => BacktestStatus::Failed,
                "cancelled" => BacktestStatus::Cancelled,
                "paused" => BacktestStatus::Paused,
                _ => BacktestStatus::Unspecified,
            };

            // Parse symbols JSON array (simplified)
            let symbols_str: String = row.try_get("symbols")?;
            let symbols: Vec<String> =
                serde_json::from_str(&symbols_str).unwrap_or_else(|_| vec![symbols_str.clone()]);

            summaries.push(BacktestSummary {
                backtest_id: row.try_get("backtest_id")?,
                strategy_name: row.try_get("strategy_name")?,
                symbols,
                status,
                total_return: row
                    .try_get::<Option<f64>, _>("total_return")?
                    .unwrap_or(0.0),
                sharpe_ratio: row
                    .try_get::<Option<f64>, _>("sharpe_ratio")?
                    .unwrap_or(0.0),
                max_drawdown: row
                    .try_get::<Option<f64>, _>("max_drawdown")?
                    .unwrap_or(0.0),
                created_at: row.try_get("created_at")?,
                start_date: row.try_get("start_date")?,
                end_date: row.try_get("end_date")?,
                description: row
                    .try_get::<Option<String>, _>("description")?
                    .unwrap_or_default(),
            });
        }

        info!("Found {} backtest summaries", summaries.len());
        Ok(summaries)
    }

    /// Create a new backtest record
    #[allow(dead_code)]
    pub async fn create_backtest_record(
        &self,
        backtest_id: &str,
        strategy_name: &str,
        symbols: &[String],
        start_date: chrono::DateTime<chrono::Utc>,
        end_date: chrono::DateTime<chrono::Utc>,
        initial_capital: f64,
        parameters: &HashMap<String, String>,
        description: &str,
    ) -> Result<()> {
        info!("Creating backtest record for {}", backtest_id);

        let symbols_json = serde_json::to_string(symbols).context("Failed to serialize symbols")?;

        let parameters_json =
            serde_json::to_string(parameters).context("Failed to serialize parameters")?;

        sqlx::query(
            r#"
            INSERT INTO backtests (
                backtest_id, strategy_name, symbols, start_date, end_date,
                initial_capital, parameters, description, status, created_at
            ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, 'queued', NOW())
            "#,
        )
        .bind(backtest_id)
        .bind(strategy_name)
        .bind(symbols_json)
        .bind(start_date)
        .bind(end_date)
        .bind(initial_capital)
        .bind(parameters_json)
        .bind(description)
        .execute(&self.pg_pool)
        .await?;

        info!("Created backtest record for {}", backtest_id);
        Ok(())
    }

    /// Update backtest status
    #[allow(dead_code)]
    pub async fn update_backtest_status(
        &self,
        backtest_id: &str,
        status: BacktestStatus,
        error_message: Option<&str>,
    ) -> Result<()> {
        let status_str = match status {
            BacktestStatus::Queued => "queued",
            BacktestStatus::Running => "running",
            BacktestStatus::Completed => "completed",
            BacktestStatus::Failed => "failed",
            BacktestStatus::Cancelled => "cancelled",
            BacktestStatus::Paused => "paused",
            _ => "unknown",
        };

        sqlx::query(
            r#"
            UPDATE backtests 
            SET status = $2, error_message = $3, updated_at = NOW()
            WHERE backtest_id = $1
            "#,
        )
        .bind(backtest_id)
        .bind(status_str)
        .bind(error_message)
        .execute(&self.pg_pool)
        .await?;

        Ok(())
    }

    /// Store time-series performance data in InfluxDB (placeholder)
    #[allow(dead_code)]
    pub async fn store_time_series_data(
        &self,
        _backtest_id: &str,
        _timestamp: chrono::DateTime<chrono::Utc>,
        _equity: f64,
        _drawdown: f64,
    ) -> Result<()> {
        // TODO: Implement InfluxDB storage for high-frequency performance data
        debug!("Time-series data storage not yet implemented");
        Ok(())
    }

    /// Get database health status and pool statistics
    #[allow(dead_code)]
    pub async fn get_health_status(&self) -> Result<serde_json::Value> {
        // Perform health check
        self.db_pool
            .health_check()
            .await
            .context("Database health check failed")?;

        // Get pool statistics
        let pool_stats = self.db_pool.pool_stats();

        Ok(serde_json::json!({
            "database": {
                "status": "healthy",
                "connection_pool": "HFT-optimized",
                "pool_stats": {
                    "size": pool_stats.size,
                    "idle": pool_stats.idle,
                    "active": pool_stats.active,
                    "max_size": pool_stats.max_size,
                    "utilization_percent": pool_stats.utilization_percentage(),
                    "is_healthy": pool_stats.is_healthy()
                },
                "performance_config": {
                    "query_timeout_micros": 5000,
                    "connection_prewarming": "enabled",
                    "prepared_statements": "enabled",
                    "slow_query_threshold_micros": 10000
                }
            }
        }))
    }
}

impl From<BacktestSummary> for crate::foxhunt::tli::BacktestSummary {
    fn from(summary: BacktestSummary) -> Self {
        Self {
            backtest_id: summary.backtest_id,
            strategy_name: summary.strategy_name,
            symbols: summary.symbols,
            status: summary.status as i32,
            total_return: summary.total_return,
            sharpe_ratio: summary.sharpe_ratio,
            max_drawdown: summary.max_drawdown,
            created_at_unix_nanos: summary.created_at.timestamp_nanos_opt().unwrap_or(0),
            start_date_unix_nanos: summary.start_date.timestamp_nanos_opt().unwrap_or(0),
            end_date_unix_nanos: summary.end_date.timestamp_nanos_opt().unwrap_or(0),
            description: summary.description,
        }
    }
}