gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: clean up examples, update ML binaries and risk tests

Browse Source 
- Delete 14 unused example files (-3,543 lines): config, adaptive-strategy,
  data, storage, trading_engine, api_gateway, backtesting, trading_service, chaos
- Update ML training/eval binaries: improved CLI args, completion tracking,
  CUDA test cleanup, hyperopt enhancements
- Fix KAN network and TFT module adjustments
- Update risk test assertions for consistency
- Fix backtesting repositories and promotion manager
- Update .serena project config and Cargo dependencies

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-02-27 01:33:18 +01:00
parent 1de1769c4f
commit afd85b2f8f
46 changed files with 403 additions and 3544 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
crates/data/Cargo.toml
View File

@@ -108,9 +108,6 @@ common = { path = "../common" }
storage = { workspace = true }
dbn = "0.42.0"
# CLI argument parsing (needed for examples)
clap = { version = "4", features = ["derive"] }
[dev-dependencies]
tokio-test = { workspace = true }
# proptest = { workspace = true } # REMOVED from workspace
@@ -130,9 +127,5 @@ icmarkets = []
ib = []
mock = []
[[example]]
name = "broker_connection"
path = "examples/broker_connection.rs"
[lints]
workspace = true

182
crates/data/examples/broker_connection.rs
View File

@@ -1,182 +0,0 @@
//! # Broker Connection Example
//!
//! Demonstrates how to connect to different brokers using the data module.
//! This example shows connection setup for Interactive Brokers.
#![allow(unused_crate_dependencies)]
use common::{Order, OrderSide, OrderType, Price, Quantity, Symbol, TimeInForce};
use data::brokers::interactive_brokers::{IBConfig, InteractiveBrokersAdapter};
use tokio::time::{timeout, Duration};
use tracing::{info, warn};
// use trading_engine::prelude::*; // REMOVED - prelude does not exist
#[tokio::main]
async fn main() -> anyhow::Result<()> {
// Initialize logging
tracing_subscriber::fmt::init();
info!("Starting broker connection example");
// Example 1: Interactive Brokers Connection
if let Err(e) = interactive_brokers_example().await {
warn!("Interactive Brokers example failed: {}", e);
}
// Example 2: Data Manager with multiple providers
// Note: DataManager and DataConfig not yet implemented
// if let Err(e) = data_manager_example().await {
// warn!("Data manager example failed: {}", e);
// }
info!("Broker connection examples completed");
Ok(())
}
/// Example of connecting to Interactive Brokers TWS
async fn interactive_brokers_example() -> anyhow::Result<()> {
info!("=== Interactive Brokers Connection Example ===");
// Create IB configuration
let config = IBConfig {
host: "127.0.0.1".to_string(),
port: 7497, // Paper trading port
client_id: 1,
account_id: "DU123456".to_string(), // Demo account
connection_timeout: 30,
max_reconnect_attempts: 3,
heartbeat_interval: 60,
request_timeout: 10,
};
// Create adapter
let mut adapter = InteractiveBrokersAdapter::new(config);
info!("Created Interactive Brokers adapter");
// Attempt connection with timeout
match timeout(Duration::from_secs(10), adapter.connect()).await {
Ok(Ok(())) => {
info!("✓ Successfully connected to Interactive Brokers TWS");
// Test basic functionality
info!("Connection status: {}", adapter.is_connected());
// info!("Adapter name: {}", adapter.name()); // DISABLED: name() method removed
// Subscribe to market data for a test symbol
// DISABLED: Methods removed from adapter
/*
let symbol = Symbol::from("AAPL");
match adapter.subscribe_market_data(symbol.clone()).await {
Ok(()) => {
info!("✓ Successfully subscribed to market data for {}", symbol);
// Wait for some data
tokio::time::sleep(Duration::from_secs(5)).await;
// Unsubscribe
if let Err(e) = adapter.unsubscribe_market_data(symbol).await {
warn!("Failed to unsubscribe from market data: {}", e);
}
},
Err(e) => warn!("Failed to subscribe to market data: {}", e),
}
*/
// Disconnect
info!("Disconnecting from Interactive Brokers...");
if let Err(e) = adapter.disconnect().await {
warn!("Error during disconnect: {}", e);
} else {
info!("✓ Successfully disconnected");
}
},
Ok(Err(e)) => {
warn!("Failed to connect to Interactive Brokers: {}", e);
warn!("Make sure TWS or IB Gateway is running on port 7497");
return Err(e.into());
},
Err(_) => {
warn!("Connection to Interactive Brokers timed out");
warn!("Make sure TWS or IB Gateway is running and accepting connections");
return Err(anyhow::anyhow!("Connection timeout"));
},
}
Ok(())
}
/// Example of using the DataManager for coordinated broker and provider access
async fn data_manager_example() -> anyhow::Result<()> {
// NOTE: DataManager and DataConfig not implemented yet
// This example is commented out until those types are available
println!("DataManager example not available - component not implemented");
Ok(())
}
/// Example of order submission (commented out for safety)
async fn order_submission_example(_adapter: &mut InteractiveBrokersAdapter) -> anyhow::Result<()> {
info!("=== Order Submission Example (DEMO ONLY) ===");
warn!("This example is for demonstration only - no actual orders will be submitted");
// Create a demo order (will not be submitted)
use rust_decimal_macros::dec;
let mut order = Order::new(
Symbol::from("AAPL"),
OrderSide::Buy,
Quantity::try_from(1.0)?,
Some(Price::from_decimal(dec!(100.0))), // Low price to avoid accidental fills
OrderType::Limit,
);
order.time_in_force = TimeInForce::Day;
info!("Demo order created:");
info!(" Symbol: {}", order.symbol);
info!(" Side: {:?}", order.side);
info!(" Quantity: {}", order.quantity);
info!(" Type: {:?}", order.order_type);
info!(" Price: {:?}", order.price);
// In a real application, you would submit the order like this:
// let result = adapter.submit_order(order).await;
// But for safety, we're just demonstrating the order structure
info!("Order submission example completed (no actual order submitted)");
Ok(())
}
/// Helper function to check if TWS is running
async fn check_tws_status() -> bool {
match tokio::net::TcpStream::connect("127.0.0.1:7497").await {
Ok(_) => {
info!("✓ TWS/IB Gateway appears to be running on port 7497");
true
},
Err(_) => {
warn!("✗ Cannot connect to port 7497 - TWS/IB Gateway may not be running");
warn!("To run this example successfully:");
warn!("1. Install and start TWS or IB Gateway");
warn!("2. Enable API connections in the configuration");
warn!("3. Set the socket port to 7497 (paper trading)");
false
},
}
}
/// Configuration helper
fn print_setup_instructions() {
info!("=== Setup Instructions ===");
info!("To run broker connection examples:");
info!("");
info!("Interactive Brokers:");
info!("1. Download and install TWS or IB Gateway");
info!("2. Start the application and log in");
info!("3. Go to Configure -> API -> Settings");
info!("4. Enable 'Enable ActiveX and Socket Clients'");
info!("5. Set Socket port to 7497 for paper trading");
info!("6. Add 127.0.0.1 to trusted IP addresses");
info!("");
info!("Environment Variables (optional):");
info!("- POLYGON_API_KEY: Your Polygon.io API key");
info!("- IB_TWS_HOST: TWS host (default: 127.0.0.1)");
info!("- IB_TWS_PORT: TWS port (default: 7497)");
info!("");
}

137
crates/data/examples/convert_dbn_to_parquet.rs
View File

@@ -1,137 +0,0 @@
//! CLI Tool: DBN to Parquet Converter
//!
//! Command-line tool for converting Databento binary format (DBN) files to Parquet format.
//!
//! ## Usage
//!
//! ```bash
//! # Convert a single DBN file
//! cargo run --example convert_dbn_to_parquet -- \
//! --input test_data/real/databento/ES.FUT_ohlcv-1m_2024-01-02.dbn \
//! --output market_data/converted
//!
//! # With custom configuration
//! cargo run --example convert_dbn_to_parquet -- \
//! --input data.dbn \
//! --output ./parquet \
//! --batch-size 5000 \
//! --compression snappy
//! ```
use anyhow::{Context, Result};
use clap::Parser;
use data::parquet_persistence::ParquetConfig;
use data::providers::databento::{ConversionReport, DbnToParquetConverter};
use std::path::PathBuf;
use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt};
#[derive(Parser, Debug)]
#[clap(name = "dbn-to-parquet")]
#[clap(about = "Convert Databento DBN files to Parquet format", long_about = None)]
struct Args {
/// Input DBN file path
#[clap(short, long)]
input: PathBuf,
/// Output directory for Parquet files
#[clap(short, long, default_value = "./market_data")]
output: PathBuf,
/// Batch size for processing (events per batch)
#[clap(short, long, default_value_t = 10000)]
batch_size: usize,
/// Compression algorithm (snappy, gzip, lz4, zstd, none)
#[clap(short, long, default_value = "snappy")]
compression: String,
/// Enable verbose logging
#[clap(short, long)]
verbose: bool,
}
#[tokio::main]
async fn main() -> Result<()> {
let args = Args::parse();
// Initialize tracing
let log_level = if args.verbose {
tracing::Level::DEBUG
} else {
tracing::Level::INFO
};
tracing_subscriber::registry()
.with(
tracing_subscriber::fmt::layer()
.with_target(false)
.with_level(true),
)
.with(tracing_subscriber::filter::LevelFilter::from_level(
log_level,
))
.init();
// Validate input file exists
if !args.input.exists() {
anyhow::bail!("Input file does not exist: {:?}", args.input);
}
// Create output directory if needed
if !args.output.exists() {
std::fs::create_dir_all(&args.output)
.with_context(|| format!("Failed to create output directory: {:?}", args.output))?;
}
// Parse compression algorithm
let compression = match args.compression.to_lowercase().as_str() {
"snappy" => parquet::basic::Compression::SNAPPY,
"gzip" => parquet::basic::Compression::GZIP(parquet::basic::GzipLevel::default()),
"lz4" => parquet::basic::Compression::LZ4,
"zstd" => parquet::basic::Compression::ZSTD(parquet::basic::ZstdLevel::default()),
"none" => parquet::basic::Compression::UNCOMPRESSED,
other => anyhow::bail!("Unknown compression algorithm: {}", other),
};
// Configure Parquet writer
let config = ParquetConfig {
base_path: args.output.to_string_lossy().to_string(),
batch_size: args.batch_size,
compression,
..Default::default()
};
// Create converter
tracing::info!("Initializing DBN to Parquet converter...");
let mut converter = DbnToParquetConverter::new(config).await?;
// Convert file
tracing::info!("Converting {:?}...", args.input);
let report = converter.convert_file(&args.input).await?;
// Print results
print_report(&report);
if !report.is_success() {
anyhow::bail!("Conversion completed with {} errors", report.events_failed);
}
tracing::info!("✓ Conversion successful!");
Ok(())
}
fn print_report(report: &ConversionReport) {
println!("\n{}", "=".repeat(60));
println!("Conversion Report");
println!("{}", "=".repeat(60));
println!("Events processed: {}", report.events_processed);
println!("Events skipped: {}", report.events_skipped);
println!("Events failed: {}", report.events_failed);
println!("Duration: {:?}", report.duration);
println!(
"Throughput: {} events/sec",
report.throughput_events_per_sec
);
println!("Success rate: {:.2}%", report.success_rate());
println!("{}", "=".repeat(60));
}

88
crates/data/examples/convert_es_fut_to_parquet.rs
View File

@@ -1,88 +0,0 @@
//! Convert ES.FUT DBN file to Parquet format
//!
//! This example demonstrates converting a real Databento DBN file containing
//! ES.FUT (E-mini S&P 500 Futures) OHLCV data to Parquet format for backtesting.
//!
//! Input: test_data/real/databento/ES.FUT_ohlcv-1m_2024-01-02.dbn (96 KB, ~390 bars)
//! Output: test_data/real/parquet/ES.FUT_ohlcv-1m_2024-01-02.parquet
//!
//! Schema: 11 columns (timestamp_ns, symbol, venue, event_type, price, quantity,
//! sequence, latency_ns, open, high, low)
//!
//! Example usage:
//! ```bash
//! cargo run --example convert_es_fut_to_parquet
//! ```
use anyhow::Result;
use data::parquet_persistence::ParquetConfig;
use data::providers::databento::DbnToParquetConverter;
#[tokio::main]
async fn main() -> Result<()> {
// Initialize tracing for logging
tracing_subscriber::fmt()
.with_env_filter(
tracing_subscriber::EnvFilter::from_default_env()
.add_directive(tracing::Level::INFO.into()),
)
.init();
println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
println!(" DBN to Parquet Converter - ES.FUT OHLCV Example");
println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
println!();
// Configure Parquet output
let config = ParquetConfig {
base_path: "test_data/real/parquet".to_string(),
batch_size: 10000,
..Default::default()
};
println!("📁 Input: test_data/real/databento/ES.FUT_ohlcv-1m_2024-01-02.dbn");
println!("📂 Output: test_data/real/parquet/");
println!();
// Create converter
let mut converter = DbnToParquetConverter::new(config).await?;
// Convert ES.FUT file
println!("⚙️ Converting DBN to Parquet...");
let report = converter
.convert_file("test_data/real/databento/ES.FUT_ohlcv-1m_2024-01-02.dbn")
.await?;
println!();
println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
println!(" Conversion Results");
println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
println!();
println!("✅ Events processed: {}", report.events_processed);
println!("⏭️ Events skipped: {}", report.events_skipped);
println!("❌ Events failed: {}", report.events_failed);
println!("📈 Success rate: {:.2}%", report.success_rate());
println!("⏱️ Duration: {:?}", report.duration);
println!(
"🚀 Throughput: {} events/sec",
report.throughput_events_per_sec
);
println!();
if report.is_success() {
println!("✅ SUCCESS! All events converted without errors.");
println!();
println!("📦 Output file ready for backtesting:");
println!(" test_data/real/parquet/ES.FUT_ohlcv-1m_2024-01-02.parquet");
} else {
println!(
"⚠️ WARNING: Some events failed to convert ({} failures)",
report.events_failed
);
}
println!();
println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
Ok(())
}

86
crates/data/examples/convert_nq_fut_to_parquet.rs
View File

@@ -1,86 +0,0 @@
//! Convert NQ.FUT DBN file to Parquet format
//!
//! This example converts the Databento DBN file containing
//! NQ.FUT (E-mini NASDAQ 100 Futures) OHLCV data to Parquet format for ML training.
//!
//! Input: test_data/NQ_FUT_180d.dbn (4.10 MB, ~262,442 bars)
//! Output: test_data/NQ_FUT_180d.parquet
//!
//! Example usage:
//! ```bash
//! cargo run -p data --example convert_nq_fut_to_parquet --release
//! ```
use anyhow::Result;
use data::parquet_persistence::ParquetConfig;
use data::providers::databento::DbnToParquetConverter;
#[tokio::main]
async fn main() -> Result<()> {
// Initialize tracing for logging
tracing_subscriber::fmt()
.with_env_filter(
tracing_subscriber::EnvFilter::from_default_env()
.add_directive(tracing::Level::INFO.into()),
)
.init();
println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
println!(" DBN to Parquet Converter - NQ.FUT 180d OHLCV");
println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
println!();
// Configure Parquet output
let config = ParquetConfig {
base_path: "test_data".to_string(),
batch_size: 10000,
compression: parquet::basic::Compression::SNAPPY,
..Default::default()
};
println!("📁 Input: test_data/NQ_FUT_180d_uncompressed.dbn");
println!("📂 Output: test_data/NQ_FUT_180d.parquet");
println!();
// Create converter
let mut converter = DbnToParquetConverter::new(config).await?;
// Convert NQ.FUT file
println!("⚙️ Converting DBN to Parquet...");
let report = converter
.convert_file("test_data/NQ_FUT_180d_uncompressed.dbn")
.await?;
println!();
println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
println!(" Conversion Results");
println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
println!();
println!("✅ Events processed: {}", report.events_processed);
println!("⏭️ Events skipped: {}", report.events_skipped);
println!("❌ Events failed: {}", report.events_failed);
println!("📈 Success rate: {:.2}%", report.success_rate());
println!("⏱️ Duration: {:?}", report.duration);
println!(
"🚀 Throughput: {} events/sec",
report.throughput_events_per_sec
);
println!();
if report.is_success() {
println!("✅ SUCCESS! All events converted without errors.");
println!();
println!("📦 Output file ready for ML training:");
println!(" test_data/NQ_FUT_180d.parquet");
} else {
println!(
"⚠️ WARNING: Some events failed to convert ({} failures)",
report.events_failed
);
}
println!();
println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
Ok(())
}

289
crates/data/examples/download_mbp10_data.rs
View File

@@ -1,289 +0,0 @@
//! Download MBP-10 (Market By Price, 10 levels) data from Databento
//!
//! This example downloads Level-2 order book data (MBP-10) for ES.FUT
//! to support TLOB neural network training.
//!
//! ## Usage
//!
//! ```bash
//! export DATABENTO_API_KEY="your_api_key_here"
//! cargo run --example download_mbp10_data --release
//! ```
//!
//! ## Data Specifications
//!
//! - **Dataset**: GLBX.MDP3 (CME Globex MDP 3.0)
//! - **Schema**: mbp-10 (Market By Price, 10 levels)
//! - **Symbols**: ES.c.0 (E-mini S&P 500 continuous front month)
//! - **Date Range**: 2024-01-02 to 2024-01-10 (7 trading days)
//! - **Encoding**: dbn (Databento Binary Encoding v2)
//! - **Compression**: zstd (Zstandard)
//! - **Expected Size**: ~50-70 GB compressed
use anyhow::{Context, Result};
use reqwest::Client;
use serde::{Deserialize, Serialize};
use std::env;
use std::fs::File;
use std::io::Write;
use std::path::PathBuf;
use std::time::Duration;
use tokio::time::sleep;
#[derive(Debug, Deserialize)]
struct BatchSubmitResponse {
id: String,
}
#[derive(Debug, Deserialize)]
struct BatchStatusResponse {
id: String,
state: String,
record_count: Option<u64>,
actual_size: Option<u64>,
}
#[derive(Debug, Deserialize)]
struct BatchFileUrls {
https: String,
ftp: String,
}
#[derive(Debug, Deserialize)]
struct BatchFileInfo {
filename: String,
size: u64,
hash: String,
urls: BatchFileUrls,
}
const DATABENTO_API_BASE: &str = "https://hist.databento.com/v0";
#[tokio::main]
async fn main() -> Result<()> {
tracing_subscriber::fmt()
.with_max_level(tracing::Level::INFO)
.init();
// Get API key from environment
let api_key =
env::var("DATABENTO_API_KEY").context("DATABENTO_API_KEY environment variable not set")?;
// Create output directory
let output_dir = PathBuf::from("test_data/mbp10");
std::fs::create_dir_all(&output_dir).context("Failed to create output directory")?;
println!("=== Databento MBP-10 Data Download ===\n");
println!("Dataset: GLBX.MDP3");
println!("Schema: mbp-10 (Market By Price, 10 levels)");
println!("Symbol: ES.c.0");
println!("Date Range: 2024-01-02 to 2024-01-10 (7 trading days)");
println!("Output: {}", output_dir.display());
println!("\n{}", "=".repeat(50));
// Step 1: Submit batch download job
println!("\n[1/4] Submitting batch download job...");
let job_id = submit_batch_job(&api_key).await?;
println!("✓ Job submitted successfully: {}", job_id);
// Step 2: Poll for job completion
println!("\n[2/4] Waiting for job to complete...");
poll_job_status(&api_key, &job_id).await?;
println!("✓ Job completed successfully");
// Step 3: Get file list and download all files
println!("\n[3/4] Getting file list...");
let files = list_batch_files(&api_key, &job_id).await?;
let data_files: Vec<_> = files
.iter()
.filter(|f| f.filename.ends_with(".dbn.zst"))
.collect();
println!("✓ Found {} data files", data_files.len());
println!("\n[4/4] Downloading data files...");
for (i, file) in data_files.iter().enumerate() {
println!("\n [{}/{}] {}", i + 1, data_files.len(), file.filename);
let output_file = output_dir.join(&file.filename);
download_file(&api_key, &file.urls.https, &output_file).await?;
}
println!("✓ Download completed successfully");
println!("\n{}", "=".repeat(50));
println!("SUCCESS: MBP-10 data downloaded to {}", output_dir.display());
println!("\nNext Steps:");
println!("1. Decompress files: for f in {}/*.dbn.zst; do zstd -d $f; done", output_dir.display());
println!("2. Validate schema: cargo run --example validate_dbn_schema");
println!("3. Implement MBP-10 parser in data/src/providers/databento/");
Ok(())
}
async fn submit_batch_job(api_key: &str) -> Result<String> {
let client = Client::new();
// Build form data (API expects application/x-www-form-urlencoded, not JSON)
let form_data = [
("dataset", "GLBX.MDP3"),
("schema", "mbp-10"),
("symbols", "ES.c.0"),
("stype_in", "continuous"),
("start", "2024-01-02"),
("end", "2024-01-10"),
("encoding", "dbn"),
("compression", "zstd"),
];
let url = format!("{}/batch.submit_job", DATABENTO_API_BASE);
let response = client
.post(&url)
.basic_auth(api_key, Some(""))
.form(&form_data)
.send()
.await
.context("Failed to submit batch job")?;
if !response.status().is_success() {
let status = response.status();
let body = response.text().await.unwrap_or_default();
anyhow::bail!("API request failed: {} - {}", status, body);
}
let result: BatchSubmitResponse = response
.json()
.await
.context("Failed to parse batch submit response")?;
Ok(result.id)
}
async fn poll_job_status(api_key: &str, job_id: &str) -> Result<()> {
let client = Client::new();
let url = format!("{}/batch.list_jobs", DATABENTO_API_BASE);
loop {
let response = client
.get(&url)
.basic_auth(api_key, Some(""))
.send()
.await
.context("Failed to check job status")?;
if !response.status().is_success() {
let status = response.status();
let body = response.text().await.unwrap_or_default();
anyhow::bail!("API request failed: {} - {}", status, body);
}
let jobs: Vec<BatchStatusResponse> = response
.json()
.await
.context("Failed to parse batch list response")?;
// Find our job in the list
let job = jobs
.iter()
.find(|j| j.id == job_id)
.context("Job not found in batch list")?;
match job.state.as_str() {
"done" => {
if let Some(size) = job.actual_size {
let size_mb = size as f64 / 1_048_576.0;
println!(" • Size: {:.2} MB", size_mb);
}
if let Some(records) = job.record_count {
println!(" • Records: {}", records);
}
return Ok(());
},
"error" => {
anyhow::bail!("Job failed with error state");
},
state => {
print!("\r • Status: {} ... ", state);
std::io::stdout().flush()?;
sleep(Duration::from_secs(5)).await;
},
}
}
}
async fn list_batch_files(api_key: &str, job_id: &str) -> Result<Vec<BatchFileInfo>> {
let client = Client::new();
let url = format!("{}/batch.list_files?job_id={}", DATABENTO_API_BASE, job_id);
let response = client
.get(&url)
.basic_auth(api_key, Some(""))
.send()
.await
.context("Failed to list batch files")?;
if !response.status().is_success() {
let status = response.status();
let body = response.text().await.unwrap_or_default();
anyhow::bail!("API request failed: {} - {}", status, body);
}
let files: Vec<BatchFileInfo> = response.json().await.context("Failed to parse file list")?;
Ok(files)
}
async fn download_file(api_key: &str, download_url: &str, output_path: &PathBuf) -> Result<()> {
let client = Client::new();
let response = client
.get(download_url)
.basic_auth(api_key, Some(""))
.send()
.await
.context("Failed to start download")?;
if !response.status().is_success() {
let status = response.status();
let body = response.text().await.unwrap_or_default();
anyhow::bail!("Download failed: {} - {}", status, body);
}
let total_size = response.content_length().unwrap_or(0);
let mut file = File::create(output_path).context("Failed to create output file")?;
let mut downloaded: u64 = 0;
// Use chunk() method for reqwest 0.12 (replaces bytes_stream())
let mut response = response;
while let Some(chunk) = response.chunk().await.context("Failed to read chunk")? {
file.write_all(&chunk)
.context("Failed to write chunk to file")?;
downloaded += chunk.len() as u64;
if total_size > 0 {
let percent = (downloaded as f64 / total_size as f64) * 100.0;
print!(
"\r • Progress: {:.2}% ({:.2} MB / {:.2} MB)",
percent,
downloaded as f64 / 1_048_576.0,
total_size as f64 / 1_048_576.0
);
std::io::stdout().flush()?;
}
}
println!();
Ok(())
}
fn validate_file(path: &PathBuf) -> Result<()> {
let metadata = std::fs::metadata(path).context("Failed to read file metadata")?;
let size_mb = metadata.len() as f64 / 1_048_576.0;
println!(" • File size: {:.2} MB", size_mb);
if size_mb < 1.0 {
anyhow::bail!("File is suspiciously small (< 1 MB), may be corrupted");
}
println!(" • File appears valid");
Ok(())
}