fxhnt/src/fxhnt/cli.py

"""CLI — the composition root. The ONLY place that wires concrete adapters to the application services.
Swap an adapter here (Yahoo→Databento, SQLite→Postgres) without touching any other layer."""
from __future__ import annotations

import os

import typer

from fxhnt.adapters.data.crypto_fetch import fetch_crypto_pit
from fxhnt.adapters.data.futures_fetch import fetch_futures
from fxhnt.adapters.broker import IbkrBroker
from fxhnt.adapters.data import DatabentoDataProvider, YahooDataProvider
from fxhnt.adapters.llm import build_chat_model
from fxhnt.adapters.persistence import DuckDbAnalyticalStore, SqlOperationalRepository, SqlStrategyStore
from fxhnt.application import (
    DiscoveryService,
    ExecutionService,
    FactoryRuntime,
    FleetOrchestrator,
    GridGenerator,
    PortfolioEvaluationService,
    RegimeConditionalExecutor,
    ResearchService,
    Territory,
)
from fxhnt.domain.factory import StrategyStatus
from fxhnt.adapters.exchange.binance_ccxt import BinanceUsdmExchange
from fxhnt.application.crypto_execution import CryptoExecutionService, ExecConfig


def _default_territories(markets: list[str], asset_class: AssetClass) -> list[Territory]:
    """The standing hunters: a trend specialist and a mean-reversion specialist over the universe."""
    return [
        Territory(name="trend-hunter", kind="trend", markets=markets, asset_class=asset_class,
                  param_grid=[{"window": 100.0}, {"window": 200.0}]),
        Territory(name="meanrev-hunter", kind="mean_reversion", markets=markets, asset_class=asset_class,
                  param_grid=[{"window": 20.0, "z": 1.0}]),
    ]
from fxhnt.config import Settings, get_settings
from fxhnt.domain.models import AssetClass, Market, StrategySpec
from fxhnt.domain.portfolio import Book, StrategyAllocation, book_from_runs
from fxhnt.domain.strategies import available

app = typer.Typer(help="fxhnt — agentic strategy research & multi-strategy execution", no_args_is_help=True)


def _data_provider(data_source: str, settings: Settings) -> object:
    if data_source == "warehouse":
        from fxhnt.adapters.warehouse.duckdb_feature_store import DuckDbFeatureStore
        from fxhnt.adapters.warehouse.price_provider import WarehousePriceProvider
        return WarehousePriceProvider(DuckDbFeatureStore(settings.warehouse_path))
    if data_source == "databento":
        d = settings.databento
        return DatabentoDataProvider(max_cost_usd=d.max_cost_usd, equity_dataset=d.equity_dataset,
                                     future_dataset=d.future_dataset, default_start=d.default_start)
    return YahooDataProvider()


def _build_service(settings: Settings) -> ResearchService:
    return ResearchService(
        data=YahooDataProvider(),
        operational=SqlOperationalRepository(settings.operational_dsn),
        analytical=DuckDbAnalyticalStore(settings.analytical_path),
        settings=settings,
    )


@app.command()
def research(
    symbol: str = typer.Argument(..., help="ticker, e.g. SPY"),
    kind: str = typer.Option("trend", help="strategy kind"),
    asset_class: AssetClass = typer.Option(AssetClass.ETF),
    window: int = typer.Option(200, help="trend MA window"),
    n_trials: int = typer.Option(1, help="full search size (for the deflated-Sharpe correction)"),
) -> None:
    """Run one (market × strategy) candidate through the gauntlet and persist the verdict."""
    settings = get_settings()
    svc = _build_service(settings)
    market = Market(symbol=symbol.upper(), asset_class=asset_class)
    spec = StrategySpec(kind=kind, params={"window": float(window)})
    run = svc.evaluate_candidate(market, spec, n_trials=n_trials)
    s, v = run.stats, run.verdict
    typer.echo(f"\n{market} | {spec.key()}")
    typer.echo(f"  backtest: CAGR {100*s.cagr:+.1f}%  vol {100*s.ann_vol:.0f}%  Sharpe {s.sharpe:+.2f}  maxDD {100*s.max_drawdown:+.0f}%  (n={s.n_obs})")
    typer.echo(f"  gauntlet: {v.summary()}")
    for r in v.reasons:
        typer.echo(f"     - {r}")
    typer.echo(f"  -> {'PASS ✅' if v.passed else 'REJECT ❌'}  (run_id {run.run_id})\n")


@app.command("list")
def list_runs(passed_only: bool = typer.Option(False, "--passed-only")) -> None:
    """List persisted research runs (the survivor library when --passed-only)."""
    svc = _build_service(get_settings())
    runs = svc._op.list_runs(passed_only=passed_only)  # noqa: SLF001 (composition root may reach in)
    if not runs:
        typer.echo("no runs yet.")
        return
    for run in runs:
        flag = "PASS" if run.verdict.passed else "REJECT"
        typer.echo(f"  [{flag}] {run.market} {run.spec.key()} | Sharpe {run.stats.sharpe:+.2f} DSR {run.verdict.dsr:.2f} | {run.run_id}")


@app.command()
def strategies() -> None:
    """List available strategy kinds."""
    typer.echo("available strategies: " + ", ".join(available()))


@app.command()
def hunt(
    symbols: str = typer.Option("CL,GC,ZN,6E,ES,NQ", help="market universe the hunters search"),
    asset_class: AssetClass = typer.Option(AssetClass.FUTURE),
    data_source: str = typer.Option("databento", help="yahoo | databento | warehouse"),
) -> None:
    """Run the fleet: specialist hunters search the universe, evaluate every candidate REGIME-conditionally,
    and write the regime-specialists they find into the lifecycle store."""
    settings = get_settings()
    markets = [s.strip().upper() for s in symbols.split(",") if s.strip()]
    fleet = FleetOrchestrator(_data_provider(data_source, settings), DuckDbAnalyticalStore(settings.analytical_path),  # type: ignore[arg-type]
                              SqlStrategyStore(settings.operational_dsn), settings)
    report = fleet.hunt(_default_territories(markets, asset_class))
    typer.echo(f"hunted {report.territories} territories, tested {report.candidates_tested} candidates")
    typer.echo(f"specialists found: {report.specialists_found}  | global hypotheses tested: {report.global_trials}")
    typer.echo(f"by regime: {report.by_regime or '(none)'}")
    typer.echo("→ run `fxhnt factory` to see the population.")


@app.command()
def run(
    symbols: str = typer.Option("CL,GC,ZN,6E,ES,NQ", help="market universe"),
    asset_class: AssetClass = typer.Option(AssetClass.FUTURE),
    data_source: str = typer.Option("databento", help="yahoo | databento | warehouse"),
    hunters: str = typer.Option("fixed", help="fixed (grids) | agent (LLM proposes what to hunt)"),
    book_status: str = typer.Option("discovered", help="which specialists the book may deploy (discovered until layer-4 forward-validation promotes to deployed)"),
    interval_minutes: int = typer.Option(0, help="0 = one cycle (cron-friendly); >0 = loop forever as a daemon"),
) -> None:
    """One factory cycle: hunt new regime-specialists + assemble today's regime-adaptive book. Schedule it
    (cron) for the standing 'constantly seeking' force, or pass --interval-minutes for a foreground daemon."""
    import time

    settings = get_settings()
    markets = [s.strip().upper() for s in symbols.split(",") if s.strip()]
    store = SqlStrategyStore(settings.operational_dsn)

    if hunters == "agent":
        from fxhnt.application.agent import AgentHunter
        hunter = AgentHunter(build_chat_model(settings), asset_class=asset_class.value)

        def territory_provider() -> list[Territory]:
            population = "\n".join(f"- {r.spec.key()} on {','.join(r.markets)} → {r.specialist_regimes()}"
                                   for r in store.list())
            return hunter.propose_territories(universe=markets, kinds=list(available()), population=population)
    else:
        def territory_provider() -> list[Territory]:
            return _default_territories(markets, asset_class)

    runtime = FactoryRuntime(
        _data_provider(data_source, settings), DuckDbAnalyticalStore(settings.analytical_path), store, settings,  # type: ignore[arg-type]
        territory_provider=territory_provider, book_status=StrategyStatus(book_status),
    )
    while True:
        rep = runtime.cycle()
        typer.echo(f"[hunt] {rep.fleet.candidates_tested} new candidates → {rep.fleet.specialists_found} specialists "
                   f"| global hypotheses {rep.fleet.global_trials} | by regime {rep.fleet.by_regime or '{}'}")
        typer.echo(f"[book] as of {rep.book.as_of} | {len(rep.book.active)} active, {len(rep.book.dormant)} dormant "
                   f"| weights {rep.book.target_weights or '(flat)'}")
        if interval_minutes <= 0:
            break
        typer.echo(f"... sleeping {interval_minutes}m")
        time.sleep(interval_minutes * 60)


@app.command()
def book(
    status: str = typer.Option("deployed", help="which specialists to assemble: deployed | discovered"),
    data_source: str = typer.Option("databento", help="yahoo | databento | warehouse"),
    max_gross: float = typer.Option(1.0, help="book gross-leverage budget"),
) -> None:
    """Assemble TODAY's regime-adaptive multi-strategy book: activate specialists whose regime is live now.
    Legs are sized by HRP weights from the latest factory-cycle allocation; falls back to equal-weight."""
    from fxhnt.adapters.persistence.factory_store import FactoryStore
    settings = get_settings()
    fs = FactoryStore(settings.operational_dsn)
    hrp = fs.latest_weights()  # None if no cycle has ever run → equal-weight fallback in executor
    ex = RegimeConditionalExecutor(_data_provider(data_source, settings), DuckDbAnalyticalStore(settings.analytical_path),  # type: ignore[arg-type]
                                   SqlStrategyStore(settings.operational_dsn), settings, hrp_weights=hrp)
    rb = ex.assemble(status=StrategyStatus(status), max_gross_leverage=max_gross)
    typer.echo(f"as of {rb.as_of} | {len(rb.active)} active leg(s), {len(rb.dormant)} dormant"
               f" | sizing={'hrp' if hrp is not None else 'equal-weight'}")
    for a in rb.active:
        typer.echo(f"  ACTIVE  {a.spec} on {a.market}  [{a.current_regime}]  signal {a.position:+.0f}")
    for d in rb.dormant:
        typer.echo(f"  dormant {d.market} (now {d.current_regime}; specialist in {d.works_in})")
    typer.echo(f"target weights: {rb.target_weights or '(flat — no specialist active in the current regime)'}")


_FUTURES_UNIVERSE = [
    "6A", "6B", "6C", "6E", "6J", "6M", "6N", "6S", "CL", "NG", "HO", "RB", "GC", "SI", "HG", "PA", "PL",
    "ZC", "ZL", "ZM", "ZS", "ZO", "ZR", "KE", "GF", "HE", "LE", "ZB", "UB", "ZN", "ZF", "ZT",
    "ES", "NQ", "YM", "RTY", "EMD", "NKD",
]


@app.command("forward-track")
def forward_track(
    data_dir: str = typer.Option("/home/jgrusewski/Work/foxhunt/data/surfer", help="raw data dir (.dbn + crypto)"),
    state: str = typer.Option("/home/jgrusewski/Work/foxhunt/data/surfer/fxhnt_combined_forward.json"),
) -> None:
    """Forward-validate the combined book: book today's realized return into the paper track. Run daily on
    a cron alongside the data fetch — the forward NAV/Sharpe is the only ground truth before capital."""
    from fxhnt.application.combined_book import CombinedBook
    from fxhnt.application.combined_book_factory import make_combined_book_source
    from fxhnt.application.forward_tracker import CombinedBookForwardTracker

    _s = get_settings()
    book = CombinedBook(data_dir, _FUTURES_UNIVERSE,
                        source=make_combined_book_source(_s, data_dir))
    st = CombinedBookForwardTracker(book, state, kill_dd=_s.combined_book_kill_dd,
                                    reenter_dd=_s.combined_book_reenter_dd).step()
    typer.echo(f"forward track since {st.inception} (last {st.last_date}): {st.forward_days} forward days, "
               f"booked {st.booked_today} today")
    typer.echo(f"  forward return {st.forward_return_pct:+.2f}%  Sharpe {st.forward_sharpe:+.2f}  "
               f"maxDD {100*st.forward_max_drawdown:+.0f}%")
    if st.forward_days == 0:
        typer.echo("  (initialized — forward record starts as new data arrives; pair with the daily data fetch)")


@app.command()
def factory() -> None:
    """Show the strategy-lifecycle population: status counts, specialists per regime, global trial count."""
    from collections import Counter

    store = SqlStrategyStore(get_settings().operational_dsn)
    records = store.list()
    by_status = Counter(r.status.value for r in records)
    typer.echo(f"strategies: {len(records)} | {dict(by_status) or '(empty)'} | global hypotheses tested: {store.total_trials()}")
    for r in records:
        regimes = ", ".join(r.specialist_regimes()) or "(no passing regime)"
        typer.echo(f"  [{r.status.value:16}] {r.spec.key()} on {','.join(r.markets)}  | specialist in: {regimes}")


@app.command()
def ingest(
    symbols: str = typer.Option(..., help="comma-separated tickers to fetch + cache"),
    asset_class: AssetClass = typer.Option(AssetClass.ETF),
    data_source: str = typer.Option("yahoo", help="yahoo | databento | warehouse"),
) -> None:
    """Pre-fetch + cache market data into the analytical store. Decouples slow ingestion from fast search."""
    settings = get_settings()
    provider = _data_provider(data_source, settings)
    store = DuckDbAnalyticalStore(settings.analytical_path)
    for s in (x.strip().upper() for x in symbols.split(",") if x.strip()):
        market = Market(symbol=s, asset_class=asset_class)
        ps = provider.fetch(market)  # type: ignore[attr-defined]
        store.save_prices(ps)
        typer.echo(f"cached {market}: {len(ps)} bars  {ps.dates[0]}..{ps.dates[-1]}")


@app.command()
def discover(
    symbols: str = typer.Option("SPY,QQQ,IEF,GLD", help="comma-separated tickers to search"),
    asset_class: AssetClass = typer.Option(AssetClass.ETF),
    data_source: str = typer.Option("yahoo", help="data provider: yahoo | databento"),
) -> None:
    """Sweep strategies × markets through the gauntlet (deflated over the FULL search) and persist survivors."""
    settings = get_settings()
    markets = [Market(symbol=s.strip().upper(), asset_class=asset_class) for s in symbols.split(",") if s.strip()]
    grids = {
        "trend": [{"window": float(w)} for w in (100, 150, 200, 250)],
        "mean_reversion": [{"window": float(w), "z": 1.0} for w in (10, 20, 40)],
        "buy_hold": [{}],
    }
    svc = DiscoveryService(
        data=_data_provider(data_source, settings),  # type: ignore[arg-type]
        operational=SqlOperationalRepository(settings.operational_dsn),
        analytical=DuckDbAnalyticalStore(settings.analytical_path),
        settings=settings,
    )
    report = svc.search(GridGenerator(markets, grids))
    typer.echo(f"searched {report.n_candidates} candidates  (effective independent trials "
               f"{report.n_effective_trials:.1f}, sr_var={report.sr_variance:.5f})")
    typer.echo(f"passed the gauntlet: {report.n_passed_gauntlet}  →  survivors after FDR over the search: {report.n_survivors}")
    for run in sorted(report.survivors, key=lambda r: -r.verdict.dsr):
        typer.echo(f"  [PASS] {run.market} {run.spec.key():28} | Sharpe {run.stats.sharpe:+.2f}  DSR {run.verdict.dsr:.3f}  OOS {run.verdict.oos_sharpe:+.2f}")


@app.command()
def agent(
    goal: str = typer.Option("Find the most robust diversified futures book you can (try CL, GC, ZN, 6E, ES, NQ).",
                             help="the research goal for the agent"),
    data_source: str = typer.Option("databento", help="yahoo | databento | warehouse"),
    mode: str = typer.Option("structured", help="structured (robust proposal-loop, local-friendly) | react"),
) -> None:
    """Run the research agent: it proposes diversified books, tests them via the gauntlet, and reports the
    most robust survivor. The LLM proposes; the deterministic gauntlet disposes."""
    from fxhnt.application.agent import ResearchAgent, StructuredResearchAgent, build_tools

    settings = get_settings()
    provider = _data_provider(data_source, settings)
    analytical = DuckDbAnalyticalStore(settings.analytical_path)  # ONE shared store (DuckDB single-writer)
    research = ResearchService(provider, SqlOperationalRepository(settings.operational_dsn), analytical, settings)  # type: ignore[arg-type]
    portfolio = PortfolioEvaluationService(provider, settings, analytical)  # type: ignore[arg-type]
    llm = build_chat_model(settings)
    if mode == "react":
        researcher: object = ResearchAgent(llm, build_tools(research, portfolio), recursion_limit=settings.agent.recursion_limit)
    else:
        researcher = StructuredResearchAgent(llm, portfolio, settings)
    typer.echo(f"agent [{settings.agent.provider}:{settings.agent.model}, mode={mode}] researching:\n  {goal}\n")
    typer.echo(researcher.run(goal))  # type: ignore[attr-defined]


@app.command("validate-book")
def validate_book(
    symbols: str = typer.Option(..., help="markets in the book (comma-separated)"),
    kind: str = typer.Option("trend", help="strategy kind for every sleeve"),
    window: int = typer.Option(200),
    asset_class: AssetClass = typer.Option(AssetClass.FUTURE),
    data_source: str = typer.Option("databento", help="yahoo | databento | warehouse"),
    max_gross: float = typer.Option(1.0, help="book gross-leverage budget"),
    n_book_trials: int = typer.Option(1, help="how many book RECIPES you tried (1 if pre-specified)"),
) -> None:
    """Evaluate a diversified BOOK (combined sleeves) through the gauntlet — the unit of alpha."""
    settings = get_settings()
    markets = [Market(symbol=s.strip().upper(), asset_class=asset_class) for s in symbols.split(",") if s.strip()]
    weight = max_gross / len(markets)
    allocations = [StrategyAllocation(spec=StrategySpec(kind=kind, params={"window": float(window)}),
                                      market=m, weight=weight) for m in markets]
    book = Book(name=f"{kind}-book", allocations=allocations, max_gross_leverage=max_gross)
    svc = PortfolioEvaluationService(_data_provider(data_source, settings), settings,  # type: ignore[arg-type]
                                     DuckDbAnalyticalStore(settings.analytical_path))
    ev = svc.evaluate_book(book, n_book_trials=n_book_trials)
    s, v = ev.stats, ev.verdict
    typer.echo(f"book '{book.name}' — {len(markets)} markets × {kind}(window={window}), gross {max_gross}")
    typer.echo(f"  combined: CAGR {100*s.cagr:+.1f}%  vol {100*s.ann_vol:.0f}%  Sharpe {s.sharpe:+.2f}  maxDD {100*s.max_drawdown:+.0f}%  (n={ev.n_obs})")
    typer.echo(f"  gauntlet: {v.summary()}")
    for r in v.reasons:
        typer.echo(f"     - {r}")
    typer.echo(f"  -> {'PASS ✅' if v.passed else 'REJECT ❌'}")


@app.command()
def execute(
    book_name: str = typer.Option("survivors", help="name for the built book"),
    do_execute: bool = typer.Option(False, "--execute", help="place orders (default: dry-run plan only)"),
    live: bool = typer.Option(False, "--live", help="allow a non-paper account (requires a DU-free account)"),
    max_gross: float = typer.Option(1.0, help="book gross-leverage budget"),
) -> None:
    """Build a book from the survivor library and rebalance it on IBKR (dry-run by default)."""
    settings = get_settings()
    if live:
        settings.execution.allow_live = True
    runs = SqlOperationalRepository(settings.operational_dsn).list_runs(passed_only=True)
    try:
        book = book_from_runs(runs, book_name, max_gross_leverage=max_gross)
    except ValueError as e:
        typer.echo(f"cannot build book: {e}  (run `fxhnt research ...` first)")
        raise typer.Exit(1) from e
    typer.echo(f"book '{book.name}': {len(book.allocations)} sleeves over {len(book.markets())} markets")
    try:
        with IbkrBroker(settings.ibkr.host, settings.ibkr.port, settings.ibkr.client_id, settings.ibkr.timeout) as broker:
            plan = ExecutionService(YahooDataProvider(), broker, settings).rebalance(book, execute=do_execute)
    except Exception as e:  # connection / runtime — keep the message clean
        typer.echo(f"broker error: {type(e).__name__}: {str(e)[:100]}")
        typer.echo(f"  is IB Gateway/TWS running at {settings.ibkr.host}:{settings.ibkr.port}?")
        raise typer.Exit(1) from e
    typer.echo(f"NLV ${plan.nlv:,.0f}  | targets {plan.target_weights}")
    for n in plan.notes:
        typer.echo(f"  - {n}")
    for o in plan.orders:
        typer.echo(f"  {o.side} {o.quantity:g} {o.symbol}  (~${o.est_value:,.0f})")
    status = "BLOCKED" if plan.blocked else ("EXECUTED" if plan.executed else "DRY-RUN")
    typer.echo(f"-> {status}")


@app.command("migrate-cockpit")
def migrate_cockpit() -> None:
    """Create the cockpit tables (+ TimescaleDB hypertable on Postgres) and seed the strategy registry."""
    from fxhnt.adapters.persistence.forward_nav import ForwardNavRepo
    dsn = get_settings().operational_dsn
    ForwardNavRepo(dsn).migrate()
    typer.echo(f"cockpit migrated: {dsn.split('@')[-1]}")


@app.command("ingest-forward")
def ingest_forward(
    state_dir: str = typer.Option(..., "--state-dir", help="dir holding the *_state.json tracker files"),
) -> None:
    """Normalize every present tracker state file and upsert rows + summary into the cockpit DB."""
    from fxhnt.application.forward_ingest import ingest_forward_state
    ingest_forward_state(get_settings().operational_dsn, state_dir)
    typer.echo(f"ingested tracker(s) from {state_dir}")


@app.command()
def serve(
    host: str = typer.Option("0.0.0.0", "--host"),
    port: int = typer.Option(8080, "--port"),
) -> None:
    """Run the cockpit web app (uvicorn). Reads the operational DSN from settings."""
    import uvicorn

    from fxhnt.adapters.web.app import create_app_from_settings
    uvicorn.run(create_app_from_settings(), host=host, port=port)


@app.command()
def fetch(
    data_dir: str = typer.Option(..., "--data-dir", help="dir for crypto_pit/ and <root>.dbn"),
    crypto: bool = typer.Option(True, "--crypto/--no-crypto"),
    futures: bool = typer.Option(True, "--futures/--no-futures"),
) -> None:
    """Fetch the combined-book data into data_dir: crypto (Binance public, free) and/or futures (Databento)."""
    if crypto:
        n, total = fetch_crypto_pit(data_dir)
        typer.echo(f"crypto: {n}/{total} symbols -> {data_dir}/crypto_pit")
    if futures:
        if not os.environ.get("DATABENTO_API_KEY"):
            typer.echo("error: --futures requires DATABENTO_API_KEY in the environment", err=True)
            raise typer.Exit(1)
        s = get_settings().databento
        import datetime as _dt
        start = (_dt.date.today() - _dt.timedelta(days=365 * s.fetch_years)).isoformat()   # rolling window
        recs, ok, spent = fetch_futures(data_dir, start=start, max_cost_usd=s.max_cost_usd)
        typer.echo(f"futures: {ok} roots, {recs} records, ${spent:.2f} (since {start}) -> {data_dir}")


@app.command("warehouse-ingest")
def warehouse_ingest(
    data_dir: str = typer.Option(..., "--data-dir", help="dir with crypto_pit/ and/or <symbol>.dbn bronze data"),
    source: str = typer.Option(..., "--source", help="futures | crypto"),
    symbol: str = typer.Option("", "--symbol", help="futures: root for <data_dir>/<symbol>.dbn (required for futures)"),
) -> None:
    """Bronze -> silver -> gold: normalize raw bars and write them into the warehouse SSOT."""
    import glob
    import os

    import numpy as np

    from fxhnt.adapters.warehouse.duckdb_feature_store import DuckDbFeatureStore
    from fxhnt.adapters.warehouse.silver_crypto import normalize_crypto
    from fxhnt.adapters.warehouse.silver_futures import normalize_futures
    from fxhnt.application.warehouse_ingest import WarehouseIngest

    settings = get_settings()
    settings.ensure_dirs()

    if source not in ("futures", "crypto"):
        typer.echo("error: --source must be 'futures' or 'crypto'", err=True)
        raise typer.Exit(1)
    if source == "futures" and not symbol:
        typer.echo("error: --symbol is required for --source futures", err=True)
        raise typer.Exit(1)

    store = DuckDbFeatureStore(settings.warehouse_path)
    ingest = WarehouseIngest(store)
    try:
        if source == "futures":
            from fxhnt.adapters.data.dbn_local import read_front_arrays
            days, inst, close, high, low = read_front_arrays(os.path.join(data_dir, f"{symbol}.dbn"), symbol)
            n = ingest.ingest_bars(symbol, normalize_futures(symbol, days, inst, close, high, low))
            typer.echo(f"futures {symbol}: {n} bars -> warehouse")
        else:  # crypto (already validated)
            total = 0
            for path in sorted(glob.glob(os.path.join(data_dir, "crypto_pit", "*.npz"))):
                sym = os.path.splitext(os.path.basename(path))[0]
                with np.load(path) as z:                      # allow_pickle defaults False — numeric arrays only
                    bars = normalize_crypto(sym, z["day"], z["close"])
                total += ingest.ingest_bars(sym, bars)
            typer.echo(f"crypto: {total} bars -> warehouse")
    finally:
        store.close()


@app.command("ingest-historical")
def ingest_historical(
    limit: int = typer.Option(
        0, "--limit",
        help="ingest only the first N survivorship-free tickers (0 = full set; use a small N to validate)",
    ),
) -> None:
    """B3a: resumable, survivorship-free historical daily-EOD ingest into the DEDICATED backtest warehouse
    (settings.backtest_warehouse_path — its OWN PVC, never the live cockpit warehouse). Re-running is safe
    and cheap (already-processed tickers are skipped). Pass --limit N for a small-batch validation run."""
    from pathlib import Path

    from fxhnt.adapters.data.tiingo_daily import TiingoDailyClient
    from fxhnt.adapters.data.tiingo_universe import TiingoUniverseSource
    from fxhnt.adapters.warehouse.duckdb_feature_store import DuckDbFeatureStore
    from fxhnt.application.historical_eod_ingest import HistoricalEodIngest

    settings = get_settings()
    Path(settings.backtest_warehouse_path).parent.mkdir(parents=True, exist_ok=True)

    tickers: list[str] | None = None
    if limit > 0:
        tickers = [t[0] for t in TiingoUniverseSource().survivorship_free()[:limit]]
        typer.echo(f"ingest-historical: small-batch validation — first {len(tickers)} tickers")

    store = DuckDbFeatureStore(settings.backtest_warehouse_path)
    try:
        ingest = HistoricalEodIngest(TiingoDailyClient(), store, TiingoUniverseSource())
        counts = ingest.ingest(tickers=tickers)
    finally:
        store.close()
    typer.echo(
        f"ingest-historical -> {settings.backtest_warehouse_path}: "
        f"{counts['tickers']} ingested, {counts['skipped']} skipped, {counts['empty']} empty, "
        f"{counts['rows']} rows, {counts['members']} membership rows"
    )


@app.command("backtest-equity")
def backtest_equity(
    n: int = typer.Option(150, "--n", help="PIT universe size (top-N by trailing dollar-volume)."),
    cost_bps: float = typer.Option(15.0, "--cost-bps", help="Cost in bps per unit (two-way) turnover."),
    borrow_annual: float = typer.Option(0.0, "--borrow-annual", help="Annual borrow cost on short notional."),
    out: str = typer.Option("/backtest-data/equity_backtest_report.json", "--out", help="Report JSON path."),
    pit: bool = typer.Option(True, "--pit/--no-pit", help="POINT-IN-TIME universe: rank each month by TRAILING dollar-volume (data <= rebalance date). Removes peak-full-history lookahead. Default. --no-pit reverts to the legacy candidate/membership path."),
    trailing_window_months: int = typer.Option(12, "--trailing-window-months", help="PIT trailing window (months) for the dollar-volume rank/alive gate."),
    candidate_top_k: int = typer.Option(2000, help="Bounded candidate universe: top-K most-liquid names (ONLY used with --no-pit)."),
    candidate_min_history: int = typer.Option(300, help="Min daily bars (close+volume) a name needs to be a candidate (ONLY used with --no-pit)."),
    delisting_return: float = typer.Option(-0.30, "--delisting-return", help="Terminal return booked when a held name delists (data series ends mid-hold). -0.30 = Shumway canonical; 0.0 = none (biased); -1.0 = worst case."),
    persist_cockpit: bool = typer.Option(False, "--persist-cockpit", help="Also upsert each construction's verdict into the cockpit DB (operational_dsn) for the dashboard."),
) -> None:
    """B3b: walk-forward backtest of the 3 price-only equity-factor constructions against the DEDICATED
    survivorship-free backtest warehouse (settings.backtest_warehouse_path) -> gauntlet verdict + JSON report."""
    import json

    from fxhnt.adapters.warehouse.duckdb_feature_store import DuckDbFeatureStore
    from fxhnt.application.equity_backtest_runner import EquityBacktestRunner, evaluate_constructions

    s = get_settings()
    store = DuckDbFeatureStore(s.backtest_warehouse_path)
    try:
        result = EquityBacktestRunner(
            store, n=n, cost_bps_per_turnover=cost_bps, borrow_annual=borrow_annual,
            pit=pit, trailing_window_months=trailing_window_months,
            candidate_top_k=candidate_top_k if not pit else None,
            candidate_min_history=candidate_min_history,
            delisting_return=delisting_return,
        ).run()
        report = evaluate_constructions(
            result,
            oos_fraction=s.gauntlet.oos_fraction, dsr_min=s.gauntlet.dsr_min,
            oos_min_sharpe=s.gauntlet.oos_min_sharpe, max_is_oos_decay=s.gauntlet.max_is_oos_decay,
        )
    finally:
        store.close()
    with open(out, "w") as fh:
        json.dump(report, fh, indent=2)
    if persist_cockpit:
        import datetime as dt

        from fxhnt.adapters.persistence.forward_nav import ForwardNavRepo
        from fxhnt.application.backtest_ingest import backtest_summaries_from_report
        at = dt.datetime.now(dt.timezone.utc).replace(tzinfo=None)  # naive-UTC, matches forward ingest
        repo = ForwardNavRepo(s.operational_dsn)
        repo.migrate()
        for sm in backtest_summaries_from_report(report):
            repo.upsert_backtest_summary(sm, at=at)
        typer.echo(f"backtest-equity: persisted {len(report['constructions'])} verdict(s) to cockpit")
    for c, block in report["constructions"].items():
        v = block["verdict"]
        typer.echo(f"backtest-equity {c}: passed={v['passed']} dsr={v['dsr']:.3f} "
                   f"is_sharpe={v['is_sharpe']:.2f} oos_sharpe={v['oos_sharpe']:.2f} "
                   f"sharpe={block['stats']['sharpe']:.2f} -> {out}")


@app.command("backtest-funding")
def backtest_funding(
    floors: list[float] = typer.Option([1e6, 5e6, 2e7], "--floors", help="Liquidity floors ($/day) to sweep."),
    lookback_days: int = typer.Option(7, "--lookback-days", help="Trailing window for the funding score."),
    quantile: float = typer.Option(0.2, "--quantile", help="Top/bottom quantile for construction."),
    cost_bps: float = typer.Option(8.0, "--cost-bps", help="Taker round-trip cost (both legs), bps."),
    slip_coef: float = typer.Option(0.0005, "--slip-coef", help="Liquidity-scaled slippage coefficient."),
    spot_dir: str = typer.Option("", "--spot-dir", help="Spot npz panel dir; when set, models true spot-perp basis P&L."),
    out: str = typer.Option("/backtest-data/funding_backtest_report.json", "--out", help="Report JSON path."),
) -> None:
    """Cross-sectional crypto funding-dispersion backtest: verdict matrix (modes x liquidity floors)
    over the survivorship-free crypto_pit panel."""
    import json

    from fxhnt.adapters.data.crypto_pit_panel import CryptoPitPanelSource
    from fxhnt.application.funding_backtest_runner import evaluate_funding_matrix

    s = get_settings()
    panel = CryptoPitPanelSource(s.crypto_pit_dir).panel()
    spot = CryptoPitPanelSource.load_spot_panel(spot_dir) if spot_dir else None
    report = evaluate_funding_matrix(panel, spot=spot, floors=floors, lookback_days=lookback_days, quantile=quantile,
                                     cost_bps=cost_bps, slip_coef=slip_coef,
                                     oos_fraction=s.gauntlet.oos_fraction, dsr_min=s.gauntlet.dsr_min,
                                     oos_min_sharpe=s.gauntlet.oos_min_sharpe,
                                     max_is_oos_decay=s.gauntlet.max_is_oos_decay)
    with open(out, "w") as fh:
        json.dump(report, fh, indent=2)
    for key, c in sorted(report["cells"].items()):
        v, st = c["verdict"], c["stats"]
        typer.echo(f"backtest-funding {key}: passed={v['passed']} dsr={v['dsr']:.3f} "
                   f"sharpe={st['sharpe']:.2f} cagr={st['cagr']*100:.1f}% maxDD={st['max_drawdown']*100:.1f}% -> {out}")
    if report["basis_modeled"]:
        typer.echo("NOTE: spot-perp basis MODELED (true delta-neutral return = spot-perp basis + funding minus cost) "
                   "for coins with a spot leg; coins without spot fall back to funding-only. Counterparty risk not modeled.")
    else:
        typer.echo("NOTE: Sharpe/CAGR are FUNDING-CARRY-ONLY (no spot-perp basis vol or counterparty risk); "
                   "realistic deployable Sharpe ~2.5x lower with worse drawdowns. Pass --spot-dir to model basis.")


@app.command("build-basis-panel")
def build_basis_panel(
    spot_dir: str = typer.Option(..., "--spot-dir", help="Output dir for the cleaned spot-leg npz panel."),
    max_basis: float = typer.Option(0.30, "--max-basis", help="Reject |basis| above this (decoupling/relist guard)."),
) -> None:
    """Fetch + clean the SPOT leg (perp-window + |basis| guard) for every coin in the crypto_pit panel,
    writing a spot npz panel for the basis-aware funding backtest."""
    from fxhnt.adapters.data.crypto_pit_panel import CryptoPitPanelSource
    from fxhnt.adapters.data.binance_spot_history import BinanceSpotHistory
    from fxhnt.application.basis_panel_builder import build_spot_panel
    s = get_settings()
    panel = CryptoPitPanelSource(s.crypto_pit_dir).panel()
    spot = build_spot_panel(panel, BinanceSpotHistory(), max_basis=max_basis)
    CryptoPitPanelSource.write_spot_panel(spot, spot_dir)
    typer.echo(f"build-basis-panel: wrote spot leg for {len(spot)}/{len(panel)} coins -> {spot_dir}")


@app.command("warehouse-catalog")
def warehouse_catalog() -> None:
    """Show the warehouse SSOT inventory: per-symbol bar count + date range (freshness)."""
    from fxhnt.adapters.warehouse.duckdb_feature_store import DuckDbFeatureStore
    from fxhnt.application.warehouse_catalog import format_catalog

    settings = get_settings()
    store = DuckDbFeatureStore(settings.warehouse_path)
    try:
        typer.echo(format_catalog(store.catalog()))
    finally:
        store.close()


def _crypto_exec_service():
    from fxhnt.adapters.persistence.exec_store import ExecStore
    s = get_settings()
    b = s.binance
    ex = BinanceUsdmExchange(b.api_key, b.api_secret, testnet=b.testnet)
    cfg = ExecConfig(testnet=b.testnet, allow_live=b.allow_live, kill_switch=b.kill_switch, leverage=b.leverage,
                     gross_cap=b.gross_cap, per_symbol_cap=b.per_symbol_cap, participation_rate=b.participation_rate,
                     depth_cap=b.depth_cap, max_slippage_bps=b.max_slippage_bps, hysteresis=b.hysteresis,
                     kelly_cap=b.kelly_cap, maker_wait_s=b.maker_wait_s)
    return CryptoExecutionService(ex, ExecStore(s.operational_dsn), cfg), ex


@app.command("crypto-rebalance")
def crypto_rebalance(
    data_dir: str = typer.Option(..., "--data-dir", help="dir with crypto_pit/ for target weights"),
    lookback: int = typer.Option(30, "--lookback"),
    execute: bool = typer.Option(False, "--execute/--dry-run"),
) -> None:
    """Rebalance the crypto momentum book on the exchange (testnet unless live gates are open). Dry-run by default."""
    import numpy as np

    from fxhnt.application.crypto_momentum_research import latest_target_weights, load_crypto_panel
    days, syms, close, _ = load_crypto_panel(f"{data_dir}/crypto_pit")
    weights = latest_target_weights(close, syms, lookback)
    # ADV (USD) per symbol from the last 30 rows of close × a volume proxy is not in the panel; use NLV-neutral
    # large ADV so capacity falls back to per_symbol_cap unless a real ADV map is supplied (calibrated in M5).
    adv = {s: 1e12 for s in weights}
    svc, _ = _crypto_exec_service()
    rid = f"reb{int(days[-1])}" if days else "reb0"   # stable per data-date -> idempotent re-runs
    plan = svc.rebalance(weights, adv_usd=adv, edge_theta=1.0, kelly_fraction=1.0, rebalance_id=rid,
                         execute=execute)
    typer.echo(f"[{rid}] {'EXECUTED' if execute else 'DRY-RUN'} orders={plan.n_orders} "
               f"halted={plan.halted} reason={plan.block_reason} drift={plan.drift} "
               f"rate_limited={plan.rate_limited}")


@app.command("crypto-positions")
def crypto_positions() -> None:
    """Show current exchange positions + account NLV."""
    _, ex = _crypto_exec_service()
    acct = ex.account()
    typer.echo(f"NLV ${acct.nlv:,.2f}  avail ${acct.available_balance:,.2f}  positions={acct.positions}")


@app.command("crypto-flatten")
def crypto_flatten() -> None:
    """Kill-switch: close ALL crypto positions with reduce-only market orders."""
    from fxhnt.adapters.exchange.binance_ccxt import client_id
    from fxhnt.ports.crypto_exchange import ChildOrder
    _, ex = _crypto_exec_service()
    for p in ex.positions():
        side = "SELL" if p.qty > 0 else "BUY"
        o = ChildOrder(p.symbol, side, abs(p.qty), "MARKET", None, True, False,
                       client_id("flatten", p.symbol, "mkt"), "IOC")
        ex.place(o)
        typer.echo(f"flattened {p.symbol} {side} {abs(p.qty)}")


@app.command("factory-cycle")
def factory_cycle(
    execute: bool = typer.Option(False, "--execute/--dry-run"),
    symbols: str = typer.Option("CL,GC,ZN,6E,ES,NQ", help="market universe for the analyst fleet"),
    asset_class: AssetClass = typer.Option(AssetClass.FUTURE),
    data_source: str = typer.Option("databento", help="yahoo | databento | warehouse"),
    specialties: str = typer.Option("momentum,carry,mean_reversion", help="analyst specialties (comma-separated)"),
) -> None:
    """Full factory cycle: GENERATE (analyst proposals) → JUDGE (hunt+forward-validate) → PROMOTE
    (diversification-gated) → ALLOCATE (HRP).  Dry-run by default — pass --execute to write status + records."""
    import datetime as dt

    from fxhnt.adapters.persistence import SqlStrategyStore
    from fxhnt.adapters.persistence.factory_store import FactoryStore
    from fxhnt.application.factory.analyst_fleet import AnalystFleet
    from fxhnt.application.factory.analysts import LlmAnalyst
    from fxhnt.application.factory.factory_loop import FactoryLoop, LoopConfig, StrategyStoreLoopAdapter
    from fxhnt.application.fleet import FleetOrchestrator
    from fxhnt.application.forward import ForwardValidationService

    s = get_settings()
    f = s.factory
    now = dt.datetime.now(dt.UTC).replace(tzinfo=None)
    cyc = "fc" + dt.datetime.now(dt.UTC).strftime("%Y%m%d")

    markets = [m.strip().upper() for m in symbols.split(",") if m.strip()]
    data = _data_provider(data_source, s)
    analytical = DuckDbAnalyticalStore(s.analytical_path)
    store = SqlStrategyStore(s.operational_dsn)
    fs = FactoryStore(s.operational_dsn)

    if execute:
        # ── 1. GENERATE: analyst fleet proposes hypotheses ───────────────────────────────────
        specialty_list = [sp.strip() for sp in specialties.split(",") if sp.strip()]
        try:
            chat = build_chat_model(s)
            kinds = list(available())   # constrain analysts to backtestable strategy kinds (no hallucinated kinds)
            analysts = [LlmAnalyst(sp, chat, kinds=kinds) for sp in specialty_list]
        except Exception as _llm_err:  # noqa: BLE001 — no LLM available (offline / no provider configured)
            analysts = []
        fleet_store_adapter = store  # AnalystFleet duck-types StrategyStore (is_seen — read-only)
        analyst_fleet = AnalystFleet(analysts, fleet_store_adapter)
        proposals = analyst_fleet.gather(universe=markets)
        typer.echo(f"[generate] {len(proposals)} fresh proposals from {len(analysts)} analysts")

        # ── 2. Bridge proposals → Territories ──────────────────────────────────────────
        # Group proposals by kind; each group becomes one Territory with that kind's param grids.
        from collections import defaultdict
        by_kind: dict[str, list] = defaultdict(list)
        for p in proposals:
            by_kind[p.kind].append(p)

        analyst_territories: list[Territory] = []
        for kind, props in by_kind.items():
            analyst_territories.append(Territory(
                name=f"analyst-{kind}",
                kind=kind,
                markets=sorted({p.market for p in props}),
                asset_class=asset_class,
                param_grid=[p.params for p in props],
            ))

        # Merge with standing fixed-grid hunters so the cycle always searches the baseline universe.
        territories = _default_territories(markets, asset_class) + analyst_territories

        # ── 3. JUDGE: hunt new specialists + advance forward-validation ─────────────────
        hunter = FleetOrchestrator(data, analytical, store, s)  # type: ignore[arg-type]
        fleet_report = hunter.hunt(territories)
        typer.echo(f"[hunt] tested={fleet_report.candidates_tested} found={fleet_report.specialists_found} "
                   f"global_trials={fleet_report.global_trials} by_regime={fleet_report.by_regime or '{}'}")

        fwd_svc = ForwardValidationService(data, analytical, store, s,  # type: ignore[arg-type]
                                           slots=f.min_forward_days, min_forward_days=f.min_forward_days)
        fwd_report = fwd_svc.run()
        typer.echo(f"[forward] marked={fwd_report.marked} promoted={fwd_report.promoted} "
                   f"deployed={fwd_report.deployed} retired={fwd_report.retired} "
                   f"tracking={fwd_report.forward_tracking}")

        # ── 4. PROMOTE + ALLOCATE: diversification-gated FORWARD→DEPLOYED + HRP ──────────
        cfg = LoopConfig(max_tail_corr=f.max_tail_corr, min_marginal=f.min_marginal_sharpe,
                         tail_frac=f.tail_frac, min_forward_days=f.min_forward_days,
                         max_deployed=f.max_deployed_sleeves,
                         max_new_per_cycle=f.max_new_deploys_per_cycle,
                         kill_switch=f.kill_switch or fs.is_killed())
        loop = FactoryLoop(StrategyStoreLoopAdapter(store), cfg)
        res = loop.allocate_and_promote()
        typer.echo(f"[promote] forward_pool={res.forward_count} promoted={res.promoted} "
                   f"deployed_before={res.deployed_before} deployed_after={res.deployed_after} "
                   f"weights={ {k: round(v, 3) for k, v in res.weights.items()} }")

        # ── 5. PERSIST the real funnel counts ────────────────────────────────────────────────
        # Read forward count AFTER promotion so promoted sleeves are not counted in both
        # forward and deployed (M2: subtract promotions from the pre-promotion forward_tracking).
        fwd_after_promotion = fwd_report.forward_tracking - len(res.promoted)
        fs.record_cycle(
            cycle_id=cyc,
            proposed=len(proposals),
            tested=fleet_report.candidates_tested,
            survived=fleet_report.specialists_found,
            forward=fwd_after_promotion,
            deployed=res.deployed_after,
            retired=fwd_report.retired,
            global_trials=store.total_trials(),
            at=now,
        )
        fs.record_allocation(cyc, res.weights, now)

        typer.echo(f"[{cyc}] EXECUTED")
    else:
        # ── DRY-RUN: read-only preview — no discovery, no store writes ──────────────────────
        # Reports the CURRENT funnel/allocation without running the discovery chain.
        # Pass --execute to run GENERATE → JUDGE → PROMOTE → ALLOCATE and persist results.
        typer.echo(
            f"[{cyc}] DRY-RUN: previewing current funnel/allocation over existing pool — "
            "discovery (generate+hunt+forward) does NOT run; no store writes."
        )
        # Run the loop in kill-gated mode (kill_switch=True) so it promotes nothing but
        # reports the current pool state and what the allocation would look like.
        cfg = LoopConfig(max_tail_corr=f.max_tail_corr, min_marginal=f.min_marginal_sharpe,
                         tail_frac=f.tail_frac, min_forward_days=f.min_forward_days,
                         max_deployed=f.max_deployed_sleeves,
                         max_new_per_cycle=f.max_new_deploys_per_cycle, kill_switch=True)
        loop = FactoryLoop(StrategyStoreLoopAdapter(store), cfg)
        res = loop.allocate_and_promote()
        typer.echo(f"[promote] forward_pool={res.forward_count} promoted={res.promoted} "
                   f"deployed_before={res.deployed_before} deployed_after={res.deployed_after} "
                   f"weights={ {k: round(v, 3) for k, v in res.weights.items()} }")
        typer.echo(f"[{cyc}] DRY-RUN complete (pass --execute to run the full cycle)")


@app.command("factory-halt")
def factory_halt() -> None:
    """Kill-switch: persist halt flag to DB so all subsequent factory-cycle runs block promotions immediately."""
    from fxhnt.adapters.persistence.factory_store import FactoryStore
    fs = FactoryStore(get_settings().operational_dsn)
    fs.set_kill(True)
    typer.echo("factory halted (persisted kill=1). Run `fxhnt factory-resume` to re-enable.")


@app.command("factory-resume")
def factory_resume() -> None:
    """Clear the persisted kill flag so factory-cycle promotions are re-enabled."""
    from fxhnt.adapters.persistence.factory_store import FactoryStore
    fs = FactoryStore(get_settings().operational_dsn)
    fs.set_kill(False)
    typer.echo("factory resumed (kill=0). Promotions enabled on next factory-cycle --execute.")


@app.command("factory-status")
def factory_status() -> None:
    """Print the persisted kill state and the latest cycle summary."""
    from fxhnt.adapters.persistence.factory_store import FactoryStore
    fs = FactoryStore(get_settings().operational_dsn)
    killed = fs.is_killed()
    env_kill = get_settings().factory.kill_switch
    typer.echo(f"kill_switch: db={killed}  env={env_kill}  effective={killed or env_kill}")
    cyc = fs.latest_cycle()
    if cyc:
        typer.echo(f"latest cycle: {cyc.cycle_id}  proposed={cyc.proposed}  forward={cyc.forward}  "
                   f"survived={cyc.survived}  deployed={cyc.deployed}  at={cyc.at}")
    else:
        typer.echo("latest cycle: (none yet)")


if __name__ == "__main__":
    app()