8ce7c52586
- Fixed feature dimension mismatch in evaluate_dqn_main_orchestrator.rs - Updated all 5 occurrences: state_dim, input comments, feature vector type - Aligned with Wave 16D training (128 features: 125 market + 3 portfolio) Issue: Validation backtest reveals 100% HOLD action collapse - requires reward system investigation and redesign per latest RL research.
186 lines
6.2 KiB
Python
186 lines
6.2 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
Download sequential validation data for DQN evaluation.
|
||
|
||
Training data ends: 2025-10-19 23:59:00+00:00
|
||
Unseen data should start: 2025-10-20 00:00:00+00:00
|
||
Today: 2025-11-08
|
||
|
||
We can download ~19 days of sequential data (Oct 20 - Nov 8).
|
||
"""
|
||
|
||
import os
|
||
import sys
|
||
from datetime import datetime, timezone
|
||
import databento as db
|
||
|
||
# Configuration
|
||
API_KEY = os.getenv("DATABENTO_API_KEY", "db-95LEt9gtDRPJfc55NVUB5KL3A3uf6")
|
||
OUTPUT_DIR = "test_data"
|
||
SYMBOL = "ESZ5" # December 2025 contract (frontmonth for Oct-Nov 2025)
|
||
SCHEMA = "ohlcv-1m"
|
||
DATASET = "GLBX.MDP3"
|
||
|
||
# Date range: Sequential after training (Oct 20 - Nov 7, 2025)
|
||
# Training ends: 2025-10-19 23:59:00
|
||
# Unseen starts: 2025-10-20 00:00:00
|
||
# End: 2025-11-07 23:59:59 (leave 1 day buffer)
|
||
START_DATE = "2025-10-20"
|
||
END_DATE = "2025-11-07"
|
||
OUTPUT_FILE_DBN = "ES_FUT_unseen_sequential.dbn"
|
||
|
||
|
||
def main():
|
||
"""Download sequential validation data."""
|
||
print("=" * 80)
|
||
print("ES.FUT Sequential Validation Data Download")
|
||
print("=" * 80)
|
||
print()
|
||
|
||
# Check API key
|
||
if not API_KEY or API_KEY == "your-key-here":
|
||
print("❌ ERROR: DATABENTO_API_KEY not found in environment!")
|
||
print("Set it with: export DATABENTO_API_KEY='your-key-here'")
|
||
sys.exit(1)
|
||
|
||
# Create output directory
|
||
os.makedirs(OUTPUT_DIR, exist_ok=True)
|
||
print(f"📁 Output directory: {OUTPUT_DIR}")
|
||
print(f"🎯 Symbol: {SYMBOL}")
|
||
print(f"📊 Schema: {SCHEMA}")
|
||
print(f"📦 Dataset: {DATASET}")
|
||
print(f"📅 Date range: {START_DATE} to {END_DATE}")
|
||
print()
|
||
print("📌 Context:")
|
||
print(" Training data ends: 2025-10-19 23:59:00+00:00")
|
||
print(" Unseen data starts: 2025-10-20 00:00:00+00:00")
|
||
print(" Duration: ~19 days (sequential continuation)")
|
||
print()
|
||
|
||
# Initialize Databento client
|
||
try:
|
||
client = db.Historical(API_KEY)
|
||
print("✅ Databento client initialized")
|
||
except Exception as e:
|
||
print(f"❌ Failed to initialize Databento client: {e}")
|
||
sys.exit(1)
|
||
|
||
print()
|
||
print("-" * 80)
|
||
print(f"📥 Downloading sequential validation data: {START_DATE} to {END_DATE}")
|
||
print("-" * 80)
|
||
|
||
try:
|
||
# Parse dates
|
||
start_dt = datetime.strptime(START_DATE, "%Y-%m-%d").replace(
|
||
hour=0, minute=0, second=0, tzinfo=timezone.utc
|
||
)
|
||
end_dt = datetime.strptime(END_DATE, "%Y-%m-%d").replace(
|
||
hour=23, minute=59, second=59, tzinfo=timezone.utc
|
||
)
|
||
|
||
# Build output path
|
||
output_path = os.path.join(OUTPUT_DIR, OUTPUT_FILE_DBN)
|
||
|
||
print(f" Start: {start_dt.isoformat()}")
|
||
print(f" End: {end_dt.isoformat()}")
|
||
print(f" Output: {output_path}")
|
||
print()
|
||
|
||
# Download data
|
||
days_count = (end_dt - start_dt).days + 1
|
||
print(f"⏳ Downloading... (this may take 1-3 minutes for {days_count} days)")
|
||
data = client.timeseries.get_range(
|
||
dataset=DATASET,
|
||
symbols=[SYMBOL],
|
||
schema=SCHEMA,
|
||
start=start_dt.isoformat(),
|
||
end=end_dt.isoformat(),
|
||
)
|
||
|
||
# Write to file
|
||
print("💾 Writing to DBN file...")
|
||
data.to_file(output_path)
|
||
|
||
# Get file size
|
||
file_size = os.path.getsize(output_path)
|
||
file_size_mb = file_size / (1024 * 1024)
|
||
|
||
print()
|
||
print("✅ Download complete!")
|
||
print(f" File: {output_path}")
|
||
print(f" Size: {file_size:,} bytes ({file_size_mb:.2f} MB)")
|
||
print()
|
||
|
||
# Verify data with databento
|
||
try:
|
||
store = db.DBNStore.from_file(output_path)
|
||
df = store.to_df()
|
||
record_count = len(df)
|
||
|
||
print("📊 Data Summary:")
|
||
print(f" Total bars: {record_count:,}")
|
||
print(f" Expected bars ({days_count} days × ~150 bars/day): ~{days_count * 150:,}")
|
||
|
||
if record_count > 0:
|
||
print(f" Date range: {df.index[0]} to {df.index[-1]}")
|
||
print(f" Price range: ${df['close'].min():.2f} - ${df['close'].max():.2f}")
|
||
print(f" Total volume: {df['volume'].sum():,.0f}")
|
||
|
||
# Market balance check
|
||
bullish_bars = (df['close'] > df['open']).sum()
|
||
bullish_pct = 100 * bullish_bars / record_count
|
||
trend_pct = 100 * (df['close'].iloc[-1] - df['close'].iloc[0]) / df['close'].iloc[0]
|
||
|
||
print(f" Bullish bars: {bullish_pct:.1f}%")
|
||
print(f" Overall trend: {trend_pct:+.2f}%")
|
||
|
||
# Assess balance
|
||
if 40 <= bullish_pct <= 60:
|
||
print(f" ✅ Market balance: GOOD (40-60% range)")
|
||
elif 30 <= bullish_pct <= 70:
|
||
print(f" ⚠️ Market balance: ACCEPTABLE (30-70% range)")
|
||
else:
|
||
print(f" ❌ Market balance: BIASED (outside 30-70% range)")
|
||
else:
|
||
print(" ⚠️ WARNING: No records in file!")
|
||
|
||
except Exception as e:
|
||
print(f"⚠️ Could not verify data with databento: {e}")
|
||
print(" (File downloaded but verification failed)")
|
||
|
||
# Estimate cost
|
||
estimated_cost = 0.10 * days_count # ~$0.10 per day
|
||
print()
|
||
print(f"💰 Estimated cost: ${estimated_cost:.2f}")
|
||
print()
|
||
|
||
print("=" * 80)
|
||
print("📋 NEXT STEPS")
|
||
print("=" * 80)
|
||
print()
|
||
print("The DBN file has been downloaded. It will be automatically")
|
||
print("converted to parquet format and renamed.")
|
||
print()
|
||
print("To manually convert (if needed):")
|
||
print(f" cargo run -p data --example convert_dbn_to_parquet --release -- \\")
|
||
print(f" --input {output_path} \\")
|
||
print(f" --output test_data")
|
||
print()
|
||
print("✅ SUCCESS: Sequential validation data downloaded!")
|
||
|
||
except Exception as e:
|
||
print()
|
||
print(f"❌ Download failed: {e}")
|
||
print()
|
||
print("Possible issues:")
|
||
print(" • API key invalid or expired")
|
||
print(" • Databento API rate limit exceeded")
|
||
print(" • Network connectivity issues")
|
||
print(" • Data not available for requested date range (future dates?)")
|
||
sys.exit(1)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|