e8a68ee39f
- Created data/examples/download_ml_training_data.rs using reqwest + Databento HTTP API - Downloaded 90 days × 4 symbols (ES.FUT, NQ.FUT, ZN.FUT, 6E.FUT) - Files saved to test_data/real/databento/ml_training/ - Total: 360 files, 15 MB compressed DBN format - Used existing Rust pattern from download_nq_fut.rs - API key loaded from .env file - 100% success rate (360/360 files) - Ready for ML training benchmarks Next: Create simplified training benchmark for RTX 3050 Ti GPU measurements
238 lines
7.7 KiB
Python
238 lines
7.7 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Detailed price action analysis for ES futures data.
|
|
Shows actual price movements to understand market regimes.
|
|
"""
|
|
|
|
import databento as db
|
|
import pandas as pd
|
|
import numpy as np
|
|
from pathlib import Path
|
|
|
|
FILES = [
|
|
{
|
|
"path": "test_data/real/databento/ES.FUT_ohlcv-1m_2024-01-02.dbn",
|
|
"date": "2024-01-02",
|
|
"label": "Baseline"
|
|
},
|
|
{
|
|
"path": "test_data/real/databento/ESH4_ohlcv-1m_2024-01-03.dbn",
|
|
"date": "2024-01-03",
|
|
"label": "Trending"
|
|
},
|
|
{
|
|
"path": "test_data/real/databento/ESH4_ohlcv-1m_2024-01-04.dbn",
|
|
"date": "2024-01-04",
|
|
"label": "Ranging"
|
|
},
|
|
{
|
|
"path": "test_data/real/databento/ESH4_ohlcv-1m_2024-01-05.dbn",
|
|
"date": "2024-01-05",
|
|
"label": "Volatile"
|
|
},
|
|
]
|
|
|
|
|
|
def analyze_price_action(df, date, label):
|
|
"""Analyze price action in detail."""
|
|
print(f"\n{'=' * 80}")
|
|
print(f"{date} ({label})")
|
|
print('=' * 80)
|
|
|
|
# Basic stats
|
|
prices = df['close']
|
|
open_price = df['open'].iloc[0]
|
|
close_price = df['close'].iloc[-1]
|
|
high_price = df['high'].max()
|
|
low_price = df['low'].min()
|
|
|
|
# Remove obvious outliers for better analysis
|
|
# (e.g., the $36.05 in 2024-01-02)
|
|
price_median = prices.median()
|
|
price_std = prices.std()
|
|
# Keep prices within 5 std devs of median
|
|
mask = np.abs(prices - price_median) <= (5 * price_std)
|
|
clean_prices = prices[mask]
|
|
|
|
if len(clean_prices) < len(prices) * 0.5:
|
|
print(f"⚠️ WARNING: {len(prices) - len(clean_prices)} outliers detected and filtered")
|
|
print(f" Original range: ${prices.min():.2f} - ${prices.max():.2f}")
|
|
print(f" Clean range: ${clean_prices.min():.2f} - ${clean_prices.max():.2f}")
|
|
|
|
# Recompute with clean data
|
|
open_price = clean_prices.iloc[0] if len(clean_prices) > 0 else open_price
|
|
close_price = clean_prices.iloc[-1] if len(clean_prices) > 0 else close_price
|
|
high_price = clean_prices.max()
|
|
low_price = clean_prices.min()
|
|
|
|
# Price movement
|
|
net_change = close_price - open_price
|
|
net_change_pct = (net_change / open_price) * 100
|
|
total_range = high_price - low_price
|
|
range_pct = (total_range / open_price) * 100
|
|
|
|
print(f"\n📊 Price Statistics:")
|
|
print(f" Open: ${open_price:.2f}")
|
|
print(f" Close: ${close_price:.2f}")
|
|
print(f" High: ${high_price:.2f}")
|
|
print(f" Low: ${low_price:.2f}")
|
|
print(f" Net change: ${net_change:+.2f} ({net_change_pct:+.2f}%)")
|
|
print(f" Range: ${total_range:.2f} ({range_pct:.2f}%)")
|
|
|
|
# Returns analysis
|
|
returns = prices.pct_change().dropna()
|
|
|
|
print(f"\n📈 Returns Analysis:")
|
|
print(f" Mean return: {returns.mean() * 100:.4f}%")
|
|
print(f" Std dev: {returns.std() * 100:.4f}%")
|
|
print(f" Skewness: {returns.skew():.4f}")
|
|
print(f" Max drawdown: {(returns.cumsum().min() * 100):.2f}%")
|
|
print(f" Max runup: {(returns.cumsum().max() * 100):.2f}%")
|
|
|
|
# Direction analysis
|
|
up_bars = (df['close'] > df['open']).sum()
|
|
down_bars = (df['close'] < df['open']).sum()
|
|
doji_bars = (df['close'] == df['open']).sum()
|
|
|
|
print(f"\n📊 Bar Direction:")
|
|
print(f" Up bars: {up_bars:,} ({up_bars/len(df)*100:.1f}%)")
|
|
print(f" Down bars: {down_bars:,} ({down_bars/len(df)*100:.1f}%)")
|
|
print(f" Doji bars: {doji_bars:,} ({doji_bars/len(df)*100:.1f}%)")
|
|
|
|
# Consecutive moves
|
|
direction = np.where(df['close'] > df['open'], 1, -1)
|
|
direction_changes = np.diff(direction) != 0
|
|
avg_consecutive = len(direction) / (direction_changes.sum() + 1)
|
|
|
|
print(f" Avg consecutive: {avg_consecutive:.1f} bars")
|
|
|
|
# Volume
|
|
total_volume = df['volume'].sum()
|
|
avg_volume = df['volume'].mean()
|
|
|
|
print(f"\n💹 Volume:")
|
|
print(f" Total: {total_volume:,.0f}")
|
|
print(f" Average: {avg_volume:,.0f}")
|
|
print(f" Max: {df['volume'].max():,.0f}")
|
|
|
|
# Price levels (quintiles)
|
|
quintiles = clean_prices.quantile([0.0, 0.25, 0.5, 0.75, 1.0])
|
|
|
|
print(f"\n📊 Price Distribution (Quintiles):")
|
|
print(f" Min (0%): ${quintiles.iloc[0]:.2f}")
|
|
print(f" Q1 (25%): ${quintiles.iloc[1]:.2f}")
|
|
print(f" Median: ${quintiles.iloc[2]:.2f}")
|
|
print(f" Q3 (75%): ${quintiles.iloc[3]:.2f}")
|
|
print(f" Max (100%): ${quintiles.iloc[4]:.2f}")
|
|
|
|
# Trading sessions (rough estimate based on volume patterns)
|
|
# High volume = regular session, low volume = extended hours
|
|
volume_threshold = avg_volume * 0.5
|
|
regular_hours = (df['volume'] > volume_threshold).sum()
|
|
extended_hours = (df['volume'] <= volume_threshold).sum()
|
|
|
|
print(f"\n⏰ Estimated Trading Hours:")
|
|
print(f" Regular hours: {regular_hours:,} bars ({regular_hours/len(df)*100:.1f}%)")
|
|
print(f" Extended hours: {extended_hours:,} bars ({extended_hours/len(df)*100:.1f}%)")
|
|
|
|
# Regime classification (detailed)
|
|
print(f"\n🎯 Regime Classification:")
|
|
|
|
# Trend strength
|
|
time_idx = np.arange(len(clean_prices))
|
|
trend_corr = np.corrcoef(time_idx, clean_prices)[0, 1]
|
|
|
|
if abs(trend_corr) > 0.7:
|
|
trend_str = "STRONG TREND" + (" UP" if trend_corr > 0 else " DOWN")
|
|
elif abs(trend_corr) > 0.3:
|
|
trend_str = "MODERATE TREND" + (" UP" if trend_corr > 0 else " DOWN")
|
|
else:
|
|
trend_str = "NO CLEAR TREND (Ranging)"
|
|
|
|
print(f" Trend: {trend_str} (corr: {trend_corr:.3f})")
|
|
|
|
# Volatility
|
|
volatility = returns.std() * np.sqrt(1440) # Annualized
|
|
if volatility > 0.20:
|
|
vol_str = "HIGH VOLATILITY"
|
|
elif volatility > 0.10:
|
|
vol_str = "MODERATE VOLATILITY"
|
|
else:
|
|
vol_str = "LOW VOLATILITY"
|
|
|
|
print(f" Volatility: {vol_str} ({volatility:.4f} annualized)")
|
|
|
|
# Range
|
|
if range_pct > 2.0:
|
|
range_str = "WIDE RANGE"
|
|
elif range_pct > 1.0:
|
|
range_str = "MODERATE RANGE"
|
|
else:
|
|
range_str = "NARROW RANGE"
|
|
|
|
print(f" Range: {range_str} ({range_pct:.2f}%)")
|
|
|
|
return {
|
|
'date': date,
|
|
'label': label,
|
|
'net_change_pct': net_change_pct,
|
|
'range_pct': range_pct,
|
|
'trend_correlation': trend_corr,
|
|
'volatility': volatility,
|
|
'up_bars_pct': up_bars/len(df)*100,
|
|
}
|
|
|
|
|
|
def main():
|
|
"""Analyze all files."""
|
|
print("=" * 80)
|
|
print("ES Futures Detailed Price Action Analysis")
|
|
print("=" * 80)
|
|
|
|
results = []
|
|
|
|
for file_config in FILES:
|
|
path = file_config["path"]
|
|
date = file_config["date"]
|
|
label = file_config["label"]
|
|
|
|
if not Path(path).exists():
|
|
print(f"\n❌ File not found: {path}")
|
|
continue
|
|
|
|
try:
|
|
store = db.DBNStore.from_file(path)
|
|
df = store.to_df()
|
|
|
|
result = analyze_price_action(df, date, label)
|
|
results.append(result)
|
|
|
|
except Exception as e:
|
|
print(f"\n❌ Error analyzing {date}: {e}")
|
|
|
|
# Summary table
|
|
if results:
|
|
print("\n" + "=" * 80)
|
|
print("SUMMARY TABLE")
|
|
print("=" * 80)
|
|
print()
|
|
|
|
df_results = pd.DataFrame(results)
|
|
|
|
print("Date | Label | Net Chg | Range | Trend | Volatility | Up Bars")
|
|
print("-" * 80)
|
|
for _, row in df_results.iterrows():
|
|
print(f"{row['date']} | {row['label']:8s} | {row['net_change_pct']:+6.2f}% | {row['range_pct']:5.2f}% | {row['trend_correlation']:+7.3f} | {row['volatility']:10.4f} | {row['up_bars_pct']:6.1f}%")
|
|
|
|
print()
|
|
print("💡 Regime Interpretation:")
|
|
print(" • Trending: |trend_correlation| > 0.7")
|
|
print(" • Ranging: range < 2% and |trend_correlation| < 0.3")
|
|
print(" • Volatile: volatility > 0.15")
|
|
print()
|
|
print("✅ Analysis complete!")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|