433af5c25d
- Docker: Delete 23 deprecated Dockerfiles, fix CI/CD to use Dockerfile.foxhunt-build - Config: Remove 36 .env files, keep 4 essential, delete config/environments/ - Docs: Archive 614 Wave D files to docs/archive/wave_d/, 95% reduction in root - Scripts: Delete 56 deprecated scripts, keep 58 production-critical (49% reduction) - Python: Organize 37 scripts into scripts/python/ subdirectories, delete ml/python/ - Build: Remove 1GB artifacts, delete old venvs, clean Python cache from git - Migrations: Delete deprecated directory (4,432 lines), remove duplicate database/migrations/ - Infrastructure: Delete deployment/ (61 files), docs/scripts/ (8 files) Total impact: ~2,500 files cleaned, 750MB+ space freed, zero production impact All deleted scripts backed up to archives. runpod/ and tests/runpod/ preserved. data_acquisition_service retained per user request.
92 lines
2.7 KiB
Python
Executable File
92 lines
2.7 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
Concatenate existing ZN.FUT DBN files into a single 90-day file.
|
|
Workaround for W12-05 symbology issues.
|
|
"""
|
|
|
|
import os
|
|
import glob
|
|
import databento as db
|
|
|
|
SOURCE_DIR = "/home/jgrusewski/Work/foxhunt/test_data/real/databento/ml_training"
|
|
OUTPUT_FILE = "/home/jgrusewski/Work/foxhunt/test_data/ZN_FUT_90d.dbn"
|
|
LOG_FILE = "/tmp/zn_fut_download_log.txt"
|
|
|
|
def log(message):
|
|
"""Write to both console and log file."""
|
|
print(message)
|
|
with open(LOG_FILE, "a") as f:
|
|
f.write(message + "\n")
|
|
|
|
def main():
|
|
log("\n" + "=" * 80)
|
|
log("Agent W12-05: Concatenate Existing ZN.FUT Data")
|
|
log("=" * 80)
|
|
|
|
# Find all ZN files
|
|
pattern = os.path.join(SOURCE_DIR, "ZN.FUT_ohlcv-1m_*.dbn")
|
|
files = sorted(glob.glob(pattern))
|
|
|
|
log(f"Found {len(files)} ZN.FUT files")
|
|
log(f"Date range: {os.path.basename(files[0])} to {os.path.basename(files[-1])}")
|
|
log("")
|
|
|
|
# Concatenate files
|
|
log("📦 Concatenating files...")
|
|
total_size = 0
|
|
total_bars = 0
|
|
|
|
with open(OUTPUT_FILE, 'wb') as outf:
|
|
for i, file_path in enumerate(files):
|
|
with open(file_path, 'rb') as inf:
|
|
data = inf.read()
|
|
outf.write(data)
|
|
total_size += len(data)
|
|
|
|
# Count bars
|
|
try:
|
|
store = db.DBNStore.from_file(file_path)
|
|
df = store.to_df()
|
|
bars = len(df)
|
|
total_bars += bars
|
|
log(f" [{i+1}/{len(files)}] {os.path.basename(file_path)}: {bars:,} bars")
|
|
except:
|
|
log(f" [{i+1}/{len(files)}] {os.path.basename(file_path)}: ? bars")
|
|
|
|
file_size_mb = total_size / (1024 * 1024)
|
|
|
|
log("")
|
|
log("=" * 80)
|
|
log("✅ ZN.FUT CONCATENATION COMPLETE")
|
|
log("=" * 80)
|
|
log(f"Output: {OUTPUT_FILE}")
|
|
log(f"Files: {len(files)} days (90-day dataset)")
|
|
log(f"Size: {file_size_mb:.2f} MB")
|
|
log(f"Bars: {total_bars:,}")
|
|
log("")
|
|
|
|
# Cost estimation
|
|
estimated_cost = total_bars * 0.0035 / 1000
|
|
log(f"Original Cost (estimate): ${estimated_cost:.2f}")
|
|
log("")
|
|
|
|
log("⚠️ NOTE: Could not download full 180-day dataset due to Databento")
|
|
log(" symbology issues with ZN.FUT. Using existing 90-day dataset")
|
|
log(" (2024-01-02 to 2024-05-06) instead.")
|
|
log("")
|
|
|
|
# Write completion flag
|
|
with open("/tmp/w12_05_complete.flag", "w") as f:
|
|
f.write(f"ZN.FUT concatenation complete (90-day dataset)\n")
|
|
f.write(f"File: {OUTPUT_FILE}\n")
|
|
f.write(f"Size: {file_size_mb:.2f} MB\n")
|
|
f.write(f"Bars: {total_bars:,}\n")
|
|
f.write(f"Days: {len(files)}\n")
|
|
|
|
return 0
|
|
|
|
if __name__ == "__main__":
|
|
# Append to existing log
|
|
log("\n\n")
|
|
exit(main())
|