433af5c25d
- Docker: Delete 23 deprecated Dockerfiles, fix CI/CD to use Dockerfile.foxhunt-build - Config: Remove 36 .env files, keep 4 essential, delete config/environments/ - Docs: Archive 614 Wave D files to docs/archive/wave_d/, 95% reduction in root - Scripts: Delete 56 deprecated scripts, keep 58 production-critical (49% reduction) - Python: Organize 37 scripts into scripts/python/ subdirectories, delete ml/python/ - Build: Remove 1GB artifacts, delete old venvs, clean Python cache from git - Migrations: Delete deprecated directory (4,432 lines), remove duplicate database/migrations/ - Infrastructure: Delete deployment/ (61 files), docs/scripts/ (8 files) Total impact: ~2,500 files cleaned, 750MB+ space freed, zero production impact All deleted scripts backed up to archives. runpod/ and tests/runpod/ preserved. data_acquisition_service retained per user request.
89 lines
2.5 KiB
Python
Executable File
89 lines
2.5 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
Convert NQ.FUT DBN file to Parquet format.
|
|
|
|
Agent: W12-08
|
|
Input: test_data/NQ_FUT_180d.dbn (Zstandard compressed)
|
|
Output: test_data/NQ_FUT_180d.parquet (Snappy compressed)
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
import databento as db
|
|
import pyarrow.parquet as pq
|
|
|
|
INPUT_FILE = "test_data/NQ_FUT_180d.dbn"
|
|
OUTPUT_FILE = "test_data/NQ_FUT_180d.parquet"
|
|
|
|
def main():
|
|
"""Convert NQ.FUT DBN to Parquet."""
|
|
print("=" * 80)
|
|
print("NQ.FUT DBN → Parquet Converter (Agent W12-08)")
|
|
print("=" * 80)
|
|
print()
|
|
|
|
# Check input file
|
|
if not os.path.exists(INPUT_FILE):
|
|
print(f"❌ ERROR: Input file not found: {INPUT_FILE}")
|
|
sys.exit(1)
|
|
|
|
input_size = os.path.getsize(INPUT_FILE)
|
|
print(f"📁 Input: {INPUT_FILE} ({input_size / (1024*1024):.2f} MB)")
|
|
print(f"📂 Output: {OUTPUT_FILE}")
|
|
print()
|
|
|
|
# Load DBN file
|
|
try:
|
|
print("⚙️ Loading DBN file...")
|
|
store = db.DBNStore.from_file(INPUT_FILE)
|
|
|
|
# Convert to DataFrame
|
|
print("⚙️ Converting to DataFrame...")
|
|
df = store.to_df()
|
|
|
|
bar_count = len(df)
|
|
print(f"✅ Loaded {bar_count:,} bars")
|
|
print()
|
|
|
|
# Show sample data
|
|
print("📊 Data Preview:")
|
|
print(f" Columns: {list(df.columns)}")
|
|
print(f" First timestamp: {df.index[0]}")
|
|
print(f" Last timestamp: {df.index[-1]}")
|
|
print(f" Price range: ${df['close'].min():.2f} - ${df['close'].max():.2f}")
|
|
print(f" Total volume: {df['volume'].sum():,.0f}")
|
|
print()
|
|
|
|
# Convert to Parquet
|
|
print("⚙️ Writing Parquet file...")
|
|
df.to_parquet(OUTPUT_FILE, compression='snappy', engine='pyarrow')
|
|
|
|
output_size = os.path.getsize(OUTPUT_FILE)
|
|
compression_ratio = (1 - output_size / input_size) * 100
|
|
|
|
print()
|
|
print("=" * 80)
|
|
print("✅ SUCCESS: Conversion complete!")
|
|
print("=" * 80)
|
|
print()
|
|
print(f"📊 Conversion Results:")
|
|
print(f" Bar count: {bar_count:,}")
|
|
print(f" Input size: {input_size / (1024*1024):.2f} MB")
|
|
print(f" Output size: {output_size / (1024*1024):.2f} MB")
|
|
print(f" Compression: {compression_ratio:.1f}% smaller")
|
|
print()
|
|
print(f"📦 Output file ready for ML training:")
|
|
print(f" {OUTPUT_FILE}")
|
|
print()
|
|
|
|
except Exception as e:
|
|
print()
|
|
print(f"❌ Conversion failed: {e}")
|
|
import traceback
|
|
traceback.print_exc()
|
|
sys.exit(1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|