feat(ml): WAVE 29 DQN Codebase Cleanup & Refactoring Campaign
BREAKING CHANGES: - Removed orphaned dqn.rs monolithic trainer (4,975 lines) - Removed orphaned dqn_ensemble.rs module (816 lines) - Removed orphaned tft.rs and tft_complete_int8_integration_test.rs - TFT trainer split into modular directory structure DQN Module Refactoring: - Split trainers/dqn.rs into modular structure (config.rs, statistics.rs, trainer.rs) - Fixed hyperopt 39D search space (continuous params only) - Boolean flags (use_dueling, use_double_dqn, use_per, use_noisy_nets) are now FIXED architectural decisions - use_distributional defaults to false (Candle BUG #36 - scatter_add gradient issues) Clean Module Structure: - ml/src/trainers/dqn/ directory with proper mod.rs exports - ml/src/trainers/tft/ directory with config.rs, types.rs, model.rs, trainer.rs, tests.rs - All P0 features validated: TD-error clamping, batch diversity, LR scheduler, priority staleness Documentation: - Added comprehensive docs in docs/codebase-cleanup/ - ADR-001 for DQN refactoring decisions - Rainbow DQN component matrix and quick reference guides Build Status: Compiles with zero errors 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
jgrusewski
159
archive/temp/check_data_sequence.py
Normal file
159
archive/temp/check_data_sequence.py
Normal file
@@ -0,0 +1,159 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Verify chronological sequence of training and unseen validation data.
|
||||
Uses databento to read parquet files.
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
|
||||
def main():
|
||||
try:
|
||||
import databento as db
|
||||
from datetime import datetime
|
||||
|
||||
print("=" * 60)
|
||||
print("TRAINING DATA (ES_FUT_180d.parquet)")
|
||||
print("=" * 60)
|
||||
|
||||
# Read training data
|
||||
train_store = db.DBNStore.from_file('test_data/ES_FUT_180d.parquet')
|
||||
train_df = train_store.to_df()
|
||||
train_rows = len(train_df)
|
||||
|
||||
print(f"Rows: {train_rows:,}")
|
||||
print(f"File size: 2.9M")
|
||||
print(f"Columns: {list(train_df.columns)}")
|
||||
|
||||
train_start = train_df.index[0]
|
||||
train_end = train_df.index[-1]
|
||||
|
||||
print(f"\nDate range:")
|
||||
print(f" Start: {train_start}")
|
||||
print(f" End: {train_end}")
|
||||
|
||||
# Read unseen data
|
||||
print("\n" + "=" * 60)
|
||||
print("UNSEEN VALIDATION DATA (ES_FUT_unseen.parquet)")
|
||||
print("=" * 60)
|
||||
|
||||
unseen_store = db.DBNStore.from_file('test_data/ES_FUT_unseen.parquet')
|
||||
unseen_df = unseen_store.to_df()
|
||||
unseen_rows = len(unseen_df)
|
||||
|
||||
print(f"Rows: {unseen_rows:,}")
|
||||
print(f"File size: 224K")
|
||||
print(f"Columns: {list(unseen_df.columns)}")
|
||||
|
||||
unseen_start = unseen_df.index[0]
|
||||
unseen_end = unseen_df.index[-1]
|
||||
|
||||
print(f"\nDate range:")
|
||||
print(f" Start: {unseen_start}")
|
||||
print(f" End: {unseen_end}")
|
||||
|
||||
# Check chronological sequence
|
||||
print("\n" + "=" * 60)
|
||||
print("CHRONOLOGICAL SEQUENCE CHECK")
|
||||
print("=" * 60)
|
||||
|
||||
gap = unseen_start - train_end
|
||||
gap_seconds = gap.total_seconds()
|
||||
gap_days = gap_seconds / 86400
|
||||
|
||||
print(f"Training ends: {train_end}")
|
||||
print(f"Unseen starts: {unseen_start}")
|
||||
print(f"Gap: {gap} ({gap_seconds:,.0f} seconds = {gap_days:.1f} days)")
|
||||
|
||||
if gap_seconds < 0:
|
||||
print(f"⚠️ OVERLAP: Unseen data starts BEFORE training ends!")
|
||||
status = "FAILED"
|
||||
elif gap_seconds == 0:
|
||||
print(f"✅ PERFECT: No gap, immediate continuation")
|
||||
status = "EXISTS"
|
||||
elif gap_seconds <= 86400:
|
||||
print(f"✅ ACCEPTABLE: Gap is less than 1 day")
|
||||
status = "EXISTS"
|
||||
else:
|
||||
print(f"⚠️ GAP: {gap_days:.1f} days between training and unseen data")
|
||||
status = "FAILED"
|
||||
|
||||
# Calculate unseen duration
|
||||
unseen_duration = unseen_end - unseen_start
|
||||
days = unseen_duration.total_seconds() / 86400
|
||||
print(f"\nUnseen data duration: {days:.1f} days")
|
||||
|
||||
short_data = False
|
||||
if days < 30:
|
||||
print(f"⚠️ WARNING: Only {days:.1f} days (recommended: 30-90 days)")
|
||||
short_data = True
|
||||
else:
|
||||
print(f"✅ GOOD: {days:.1f} days of validation data")
|
||||
|
||||
# Data quality check
|
||||
print("\n" + "=" * 60)
|
||||
print("DATA QUALITY CHECK")
|
||||
print("=" * 60)
|
||||
|
||||
null_counts = unseen_df.isnull().sum()
|
||||
total_nulls = null_counts.sum()
|
||||
print(f"Null values: {dict(null_counts[null_counts > 0])}")
|
||||
print(f"Total nulls: {total_nulls}")
|
||||
|
||||
has_nulls = total_nulls > 0
|
||||
if has_nulls:
|
||||
print(f"⚠️ WARNING: {total_nulls} null values detected")
|
||||
else:
|
||||
print("✅ No null values")
|
||||
|
||||
print(f"\nPrice stats (close):")
|
||||
print(f" Min: ${unseen_df['close'].min():.2f}")
|
||||
print(f" Max: ${unseen_df['close'].max():.2f}")
|
||||
print(f" Mean: ${unseen_df['close'].mean():.2f}")
|
||||
|
||||
print(f"\nVolume stats:")
|
||||
print(f" Total: {unseen_df['volume'].sum():,.0f}")
|
||||
print(f" Mean: {unseen_df['volume'].mean():,.0f}")
|
||||
|
||||
# Final verdict
|
||||
print("\n" + "=" * 60)
|
||||
print("FINAL VERDICT")
|
||||
print("=" * 60)
|
||||
|
||||
if status == "FAILED":
|
||||
print(f"❌ FAILED: Need to re-download - chronological gap/overlap issue")
|
||||
ready = "NO"
|
||||
elif has_nulls:
|
||||
print(f"❌ FAILED: Null values detected - need to re-download")
|
||||
ready = "NO"
|
||||
elif short_data:
|
||||
print(f"⚠️ WARNING: Data exists but only {days:.1f} days (recommend 30-90)")
|
||||
ready = "PARTIAL"
|
||||
else:
|
||||
print(f"✅ PASSED: Data is valid and ready for backtest")
|
||||
ready = "YES"
|
||||
|
||||
print(f"\nStatus: {status}")
|
||||
print(f"Ready for backtest: {ready}")
|
||||
|
||||
return 0 if ready in ["YES", "PARTIAL"] else 1
|
||||
|
||||
except ImportError as e:
|
||||
print(f"ERROR: {e}")
|
||||
print("\nDatabento is not installed. Installing...")
|
||||
os.system("pip3 install --user databento --break-system-packages")
|
||||
print("\nPlease run this script again after installation.")
|
||||
return 2
|
||||
except FileNotFoundError as e:
|
||||
print(f"ERROR: File not found: {e}")
|
||||
print("\nThe unseen validation data doesn't exist.")
|
||||
print("Will need to download it from Databento.")
|
||||
return 3
|
||||
except Exception as e:
|
||||
print(f"ERROR: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return 4
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user