#!/usr/bin/env python3
"""
Concatenate existing ZN.FUT DBN files into a single 90-day file.
Workaround for W12-05 symbology issues.
"""

import os
import glob
import databento as db

SOURCE_DIR = "/home/jgrusewski/Work/foxhunt/test_data/real/databento/ml_training"
OUTPUT_FILE = "/home/jgrusewski/Work/foxhunt/test_data/ZN_FUT_90d.dbn"
LOG_FILE = "/tmp/zn_fut_download_log.txt"

def log(message):
    """Write to both console and log file."""
    print(message)
    with open(LOG_FILE, "a") as f:
        f.write(message + "\n")

def main():
    log("\n" + "=" * 80)
    log("Agent W12-05: Concatenate Existing ZN.FUT Data")
    log("=" * 80)

    # Find all ZN files
    pattern = os.path.join(SOURCE_DIR, "ZN.FUT_ohlcv-1m_*.dbn")
    files = sorted(glob.glob(pattern))

    log(f"Found {len(files)} ZN.FUT files")
    log(f"Date range: {os.path.basename(files[0])} to {os.path.basename(files[-1])}")
    log("")

    # Concatenate files
    log("📦 Concatenating files...")
    total_size = 0
    total_bars = 0

    with open(OUTPUT_FILE, 'wb') as outf:
        for i, file_path in enumerate(files):
            with open(file_path, 'rb') as inf:
                data = inf.read()
                outf.write(data)
                total_size += len(data)

            # Count bars
            try:
                store = db.DBNStore.from_file(file_path)
                df = store.to_df()
                bars = len(df)
                total_bars += bars
                log(f"  [{i+1}/{len(files)}] {os.path.basename(file_path)}: {bars:,} bars")
            except:
                log(f"  [{i+1}/{len(files)}] {os.path.basename(file_path)}: ? bars")

    file_size_mb = total_size / (1024 * 1024)

    log("")
    log("=" * 80)
    log("✅ ZN.FUT CONCATENATION COMPLETE")
    log("=" * 80)
    log(f"Output: {OUTPUT_FILE}")
    log(f"Files: {len(files)} days (90-day dataset)")
    log(f"Size: {file_size_mb:.2f} MB")
    log(f"Bars: {total_bars:,}")
    log("")

    # Cost estimation
    estimated_cost = total_bars * 0.0035 / 1000
    log(f"Original Cost (estimate): ${estimated_cost:.2f}")
    log("")

    log("⚠️  NOTE: Could not download full 180-day dataset due to Databento")
    log("   symbology issues with ZN.FUT. Using existing 90-day dataset")
    log("   (2024-01-02 to 2024-05-06) instead.")
    log("")

    # Write completion flag
    with open("/tmp/w12_05_complete.flag", "w") as f:
        f.write(f"ZN.FUT concatenation complete (90-day dataset)\n")
        f.write(f"File: {OUTPUT_FILE}\n")
        f.write(f"Size: {file_size_mb:.2f} MB\n")
        f.write(f"Bars: {total_bars:,}\n")
        f.write(f"Days: {len(files)}\n")

    return 0

if __name__ == "__main__":
    # Append to existing log
    log("\n\n")
    exit(main())