fxhnt/tests/integration/test_paper_backfill_cli.py

import datetime as dt

import duckdb
from typer.testing import CliRunner

import fxhnt.cli as cli
from fxhnt.adapters.persistence.paper_repo import PaperRepo

runner = CliRunner()

_EPOCH = dt.date(1970, 1, 1)


class _FakeSleeve:
    def current_weights(self, as_of):
        return {"BTCUSDT": 1.0}


def test_paper_backfill_cli(tmp_path, monkeypatch):
    dsn = f"sqlite:///{tmp_path / 'cockpit.db'}"

    monkeypatch.setattr(cli, "get_settings", lambda: _settings(dsn))
    # Stub the heavy live-sleeve assembly (equal-weight sizing is internal to backfill_paper_book now).
    monkeypatch.setattr(cli, "_paper_backfill_sleeves",
                        lambda s, panel: {"crypto_tstrend": _FakeSleeve()})
    # Stub the warehouse panel read to a couple of crypto symbols on a small date axis.
    axis = [(dt.date.fromisoformat(d) - _EPOCH).days for d in ("2026-06-19", "2026-06-20", "2026-06-21")]
    panel = {"BTCUSDT": {day: 100.0 for day in axis}, "ETHUSDT": {day: 50.0 for day in axis}}
    monkeypatch.setattr(cli, "_paper_backfill_warehouse_panel", lambda s, days: panel)

    result = runner.invoke(cli.app, ["paper-backfill", "--days", "3"])
    assert result.exit_code == 0, result.output

    repo = PaperRepo(dsn)
    hist = repo.nav_history()
    assert hist  # non-empty equity series
    assert len(hist) >= 1


def test_paper_backfill_incremental_default_skips_done_dates(tmp_path, monkeypatch):
    """Default (no --rebuild) is INCREMENTAL: a second run with no new warehouse dates computes nothing
    (already current); --rebuild recomputes the whole window. The persisted curve is unchanged either way."""
    dsn = f"sqlite:///{tmp_path / 'cockpit.db'}"
    monkeypatch.setattr(cli, "get_settings", lambda: _settings(dsn))
    monkeypatch.setattr(cli, "_paper_backfill_sleeves",
                        lambda s, panel: {"crypto_tstrend": _FakeSleeve()})
    axis = [(dt.date.fromisoformat(d) - _EPOCH).days
            for d in ("2026-06-19", "2026-06-20", "2026-06-21")]
    panel = {"BTCUSDT": {day: 100.0 for day in axis}}
    monkeypatch.setattr(cli, "_paper_backfill_warehouse_panel", lambda s, days: panel)

    # First run builds the whole window.
    r1 = runner.invoke(cli.app, ["paper-backfill", "--days", "3"])
    assert r1.exit_code == 0, r1.output
    hist1 = [(h.run_date, h.equity) for h in PaperRepo(dsn).nav_history()]
    assert len(hist1) == 3

    # Second run (default incremental) sees no new dates → no-op, curve unchanged.
    r2 = runner.invoke(cli.app, ["paper-backfill", "--days", "3"])
    assert r2.exit_code == 0, r2.output
    assert "already current" in r2.output or "incremental" in r2.output
    hist2 = [(h.run_date, h.equity) for h in PaperRepo(dsn).nav_history()]
    assert hist2 == hist1, "incremental no-op changed the persisted curve"

    # --rebuild recomputes the whole window; idempotent → identical curve.
    r3 = runner.invoke(cli.app, ["paper-backfill", "--days", "3", "--rebuild"])
    assert r3.exit_code == 0, r3.output
    hist3 = [(h.run_date, h.equity) for h in PaperRepo(dsn).nav_history()]
    assert hist3 == hist1, "rebuild diverged from the original (should be idempotent)"


def test_paper_backfill_incremental_appends_new_date(tmp_path, monkeypatch):
    """Incremental append: after building a 2-date window, a 3rd warehouse date is appended by the default
    incremental run, producing the SAME 3-date curve as a from-scratch full rebuild over all 3 dates."""
    dsn_inc = f"sqlite:///{tmp_path / 'inc.db'}"
    dsn_full = f"sqlite:///{tmp_path / 'full.db'}"
    monkeypatch.setattr(cli, "_paper_backfill_sleeves",
                        lambda s, panel: {"crypto_tstrend": _FakeSleeve()})
    all_axis = [(dt.date.fromisoformat(d) - _EPOCH).days
                for d in ("2026-06-19", "2026-06-20", "2026-06-21")]

    # Incremental repo: build first 2 dates, then default-incremental over all 3 (appends the 3rd).
    monkeypatch.setattr(cli, "get_settings", lambda: _settings(dsn_inc))
    monkeypatch.setattr(cli, "_paper_backfill_warehouse_panel",
                        lambda s, days: {"BTCUSDT": {day: 100.0 for day in all_axis[:2]}})
    assert runner.invoke(cli.app, ["paper-backfill", "--days", "5"]).exit_code == 0
    monkeypatch.setattr(cli, "_paper_backfill_warehouse_panel",
                        lambda s, days: {"BTCUSDT": {day: 100.0 for day in all_axis}})
    assert runner.invoke(cli.app, ["paper-backfill", "--days", "5"]).exit_code == 0

    # Full rebuild repo: all 3 dates from scratch.
    monkeypatch.setattr(cli, "get_settings", lambda: _settings(dsn_full))
    monkeypatch.setattr(cli, "_paper_backfill_warehouse_panel",
                        lambda s, days: {"BTCUSDT": {day: 100.0 for day in all_axis}})
    assert runner.invoke(cli.app, ["paper-backfill", "--days", "5", "--rebuild"]).exit_code == 0

    inc = [(h.run_date, h.equity, h.realized, h.unrealized) for h in PaperRepo(dsn_inc).nav_history()]
    full = [(h.run_date, h.equity, h.realized, h.unrealized) for h in PaperRepo(dsn_full).nav_history()]
    assert len(inc) == 3
    assert inc == full, "incremental-append curve diverged from full rebuild"


def _seed_warehouse(path: str, panel: dict[str, dict[int, float]]) -> None:
    """Seed a real DuckDB `features` table (symbol, ts, feature, value) with daily crypto closes."""
    con = duckdb.connect(path)
    con.execute("CREATE TABLE IF NOT EXISTS features (symbol VARCHAR, ts BIGINT, feature VARCHAR, value DOUBLE)")
    rows = [(sym, day * 86_400, "close", float(c)) for sym, series in panel.items() for day, c in series.items()]
    con.executemany("INSERT INTO features VALUES (?, ?, ?, ?)", rows)
    con.close()


def test_paper_backfill_warehouse_driven(tmp_path, monkeypatch):
    """The universe + date axis come from the warehouse `features` table (consistent with the nightly
    book), NOT from sleeve.current_weights sampling — and the real ts-trend sleeve, wired to the same
    warehouse panel, surfaces non-empty weights so nav_history is populated over the warehouse axis."""
    dsn = f"sqlite:///{tmp_path / 'cockpit.db'}"
    wh = str(tmp_path / "warehouse.duckdb")

    # Two crypto symbols with >130 daily closes on a clean uptrend so ts-trend goes long/flat.
    base_day = (dt.date(2026, 1, 1) - _EPOCH).days
    n = 140
    panel = {
        "BTCUSDT": {base_day + i: 100.0 * (1.003 ** i) for i in range(n)},
        "ETHUSDT": {base_day + i: 50.0 * (1.002 ** i) for i in range(n)},
    }
    _seed_warehouse(wh, panel)

    expected_axis = sorted((_EPOCH + dt.timedelta(days=base_day + i)).isoformat() for i in range(n))

    monkeypatch.setattr(cli, "get_settings", lambda: _settings(dsn, warehouse_path=wh))

    # cap window wide enough to keep the whole seeded axis
    result = runner.invoke(cli.app, ["paper-backfill", "--days", "400"])
    assert result.exit_code == 0, result.output

    repo = PaperRepo(dsn)
    hist = repo.nav_history()
    assert hist, "nav_history must be non-empty (warehouse-driven backfill)"
    nav_dates = {p.run_date for p in hist}
    # the nav dates come from the warehouse axis, not from any current_weights probe
    assert nav_dates.issubset(set(expected_axis))
    assert max(nav_dates) == expected_axis[-1]

    # ts-trend actually surfaced weights → at least one date holds a position
    assert any(repo.positions_at(d) for d in sorted(nav_dates))

    # xsfunding (carry) is a TRACKED sleeve (its return is persisted forward) but NOT in the TRUSTED book
    # (REPLAYABLE_SLEEVES == 2-edge) — the decoupling.
    assert "xsfunding" in cli._TRACKED_SLEEVES
    assert "xsfunding" not in cli._REPLAYABLE_SLEEVES


def test_paper_backfill_costs_flag_lowers_equity(tmp_path, monkeypatch):
    """--costs yields a NET curve strictly below the GROSS (--no-costs default) curve; default is gross."""
    dsn_gross = f"sqlite:///{tmp_path / 'gross.db'}"
    dsn_net = f"sqlite:///{tmp_path / 'net.db'}"
    wh = str(tmp_path / "warehouse.duckdb")
    base_day = (dt.date(2026, 1, 1) - _EPOCH).days
    n = 140
    closes = {
        "BTCUSDT": {base_day + i: 100.0 * (1.003 ** i) for i in range(n)},
        "ETHUSDT": {base_day + i: 50.0 * (1.002 ** i) for i in range(n)},
    }
    funding = {
        "BTCUSDT": {base_day + i: 0.0001 for i in range(n)},
        "ETHUSDT": {base_day + i: 0.0001 for i in range(n)},
    }
    _seed_warehouse(wh, closes)
    _seed_funding(wh, funding)

    monkeypatch.setattr(cli, "get_settings", lambda: _settings(dsn_gross, warehouse_path=wh))
    assert runner.invoke(cli.app, ["paper-backfill", "--days", "400"]).exit_code == 0   # default gross

    monkeypatch.setattr(cli, "get_settings", lambda: _settings(dsn_net, warehouse_path=wh))
    res = runner.invoke(cli.app, ["paper-backfill", "--days", "400", "--costs"])
    assert res.exit_code == 0, res.output
    assert "NET" in res.output

    gross = {p.run_date: p.equity for p in PaperRepo(dsn_gross).nav_history()}
    net = {p.run_date: p.equity for p in PaperRepo(dsn_net).nav_history()}
    # Net is strictly below gross on the later dates (turnover + funding drag accumulate).
    assert net[max(net)] < gross[max(gross)]


def _seed_funding(path: str, panel: dict[str, dict[int, float]]) -> None:
    """Append daily 'funding' rows to an existing DuckDB `features` table (same shape as closes)."""
    con = duckdb.connect(path)
    con.execute("CREATE TABLE IF NOT EXISTS features (symbol VARCHAR, ts BIGINT, feature VARCHAR, value DOUBLE)")
    rows = [(sym, day * 86_400, "funding", float(v)) for sym, series in panel.items() for day, v in series.items()]
    con.executemany("INSERT INTO features VALUES (?, ?, ?, ?)", rows)
    con.close()


def test_paper_backfill_builds_xsfunding_carry_sleeve(tmp_path, monkeypatch):
    """xsfunding (carry) is now built as a replayable return-defined sleeve: with funding seeded in the
    warehouse, _paper_backfill_sleeves builds the XsFundingReplay carry sleeve (return_mode == 'carry')
    alongside the directional sleeves."""
    from fxhnt.application.xsfunding_replay import XsFundingReplay

    wh = str(tmp_path / "warehouse.duckdb")
    base_day = (dt.date(2026, 1, 1) - _EPOCH).days
    n = 140
    closes = {
        "BTCUSDT": {base_day + i: 100.0 * (1.003 ** i) for i in range(n)},
        "ETHUSDT": {base_day + i: 50.0 * (1.002 ** i) for i in range(n)},
    }
    funding = {
        "BTCUSDT": {base_day + i: 0.0001 for i in range(n)},
        "ETHUSDT": {base_day + i: -0.0002 for i in range(n)},
    }
    _seed_warehouse(wh, closes)
    _seed_funding(wh, funding)

    s = _settings(f"sqlite:///{tmp_path / 'cockpit.db'}", warehouse_path=wh)
    sleeves = cli._paper_backfill_sleeves(s, closes)
    assert "xsfunding" in sleeves
    assert isinstance(sleeves["xsfunding"], XsFundingReplay)
    assert getattr(sleeves["xsfunding"], "return_mode", "price") == "carry"
    assert "crypto_tstrend" in sleeves


def test_funding_by_date_inversion():
    """The command's funding panel -> {iso_date: {symbol: funding}} inversion is correct."""
    base_day = (dt.date(2026, 6, 20) - _EPOCH).days
    funding_panel = {
        "BTCUSDT": {base_day: 0.0001, base_day + 1: 0.0003},
        "ETHUSDT": {base_day + 1: -0.0002},
    }
    funding_by_date: dict[str, dict[str, float]] = {}
    for fsym, fseries in funding_panel.items():
        for fday, fval in fseries.items():
            fiso = (_EPOCH + dt.timedelta(days=int(fday))).isoformat()
            funding_by_date.setdefault(fiso, {})[fsym] = float(fval)

    assert funding_by_date == {
        "2026-06-20": {"BTCUSDT": 0.0001},
        "2026-06-21": {"BTCUSDT": 0.0003, "ETHUSDT": -0.0002},
    }


class _settings:
    def __init__(self, dsn, warehouse_path=":memory:"):
        self.operational_dsn = dsn
        self.paper_capital = 100_000.0
        self.paper_enabled = True
        self.warehouse_path = warehouse_path
        self.feature_store = "duckdb"
        self.crypto_pit_dir = "/nonexistent/crypto_pit"