"""DuckDB feature store — the gold-layer warehouse. Long-format (symbol, ts, feature, value) so new
features extend the schema without migrations, indexed by (symbol, ts) for fast point-in-time reads.
Single persistent connection (connect-per-op checkpoints and is far too slow on a growing file)."""
from __future__ import annotations

import duckdb

from fxhnt.ports.warehouse import FeatureRow

_SCHEMA = "CREATE TABLE IF NOT EXISTS features (symbol VARCHAR, ts BIGINT, feature VARCHAR, value DOUBLE)"
# ticker_membership: the survivorship-free listing index — each ticker's exchange + listing
# start/end dates, so a backtest reconstructs the point-in-time universe (incl. delisted names)
# straight from the warehouse, no live re-fetch. start_date/end_date kept as ISO TEXT (the
# supported-tickers file's native form; no parse/precision loss).
_MEMBERSHIP_SCHEMA = (
    "CREATE TABLE IF NOT EXISTS ticker_membership "
    "(symbol TEXT PRIMARY KEY, exchange TEXT, start_date TEXT, end_date TEXT)"
)


class DuckDbFeatureStore:
    def __init__(self, path: str) -> None:
        self._con = duckdb.connect(path)
        self._con.execute(_SCHEMA)
        self._con.execute(_MEMBERSHIP_SCHEMA)

    def write_features(self, symbol: str, rows: list[FeatureRow]) -> None:
        flat = [(symbol, ts, name, float(value)) for ts, feats in rows for name, value in feats.items()]
        if not flat:
            return
        # idempotent: clear only the (symbol, ts-range, feature) combinations being written so that
        # different features stored at the same ts coexist without clobbering each other.
        ts_values = sorted({ts for ts, _ in rows})
        feature_names = sorted({name for _, feats in rows for name in feats})
        placeholders = ",".join("?" for _ in feature_names)
        self._con.execute(
            f"DELETE FROM features WHERE symbol = ? AND ts >= ? AND ts <= ? AND feature IN ({placeholders})",
            [symbol, ts_values[0], ts_values[-1], *feature_names],
        )
        self._con.executemany("INSERT INTO features VALUES (?, ?, ?, ?)", flat)

    def write_features_bulk(self, items: list[tuple[str, list[FeatureRow]]]) -> None:
        """Write many symbols' rows in ONE transaction — one fsync/checkpoint instead of one per symbol
        (the per-symbol path is ~50x slower on a network PVC). Per-symbol idempotent (clears each
        (symbol, ts-range, feature-set) before insert), same result as write_features per symbol."""
        flat: list[tuple[str, int, str, float]] = []
        # (symbol, lo, hi, feature_names) — scoped delete preserves other features at same ts
        scoped_ranges: list[tuple[str, int, int, list[str]]] = []
        for symbol, rows in items:
            if not rows:
                continue
            ts_values = sorted({ts for ts, _ in rows})
            feature_names = sorted({name for _, feats in rows for name in feats})
            scoped_ranges.append((symbol, ts_values[0], ts_values[-1], feature_names))
            flat.extend((symbol, ts, name, float(value)) for ts, feats in rows for name, value in feats.items())
        if not flat:
            return
        self._con.execute("BEGIN TRANSACTION")
        try:
            for symbol, lo, hi, feature_names in scoped_ranges:
                placeholders = ",".join("?" for _ in feature_names)
                self._con.execute(
                    f"DELETE FROM features WHERE symbol = ? AND ts >= ? AND ts <= ? AND feature IN ({placeholders})",
                    [symbol, lo, hi, *feature_names],
                )
            self._con.executemany("INSERT INTO features VALUES (?, ?, ?, ?)", flat)
            self._con.execute("COMMIT")
        except Exception:
            self._con.execute("ROLLBACK")
            raise

    def read_features(self, symbol: str, *, start_ts: int | None = None, end_ts: int | None = None,
                      features: list[str] | None = None) -> list[FeatureRow]:
        clauses: list[str] = ["symbol = ?"]
        params: list[str | int] = [symbol]
        if start_ts is not None:
            clauses.append("ts >= ?")
            params.append(start_ts)
        if end_ts is not None:
            clauses.append("ts <= ?")
            params.append(end_ts)
        if features is not None:
            clauses.append(f"feature IN ({','.join('?' for _ in features)})")
            params.extend(features)
        rows = self._con.execute(
            f"SELECT ts, feature, value FROM features WHERE {' AND '.join(clauses)} ORDER BY ts",
            params,
        ).fetchall()
        out: dict[int, dict[str, float]] = {}
        for ts, feature, value in rows:
            out.setdefault(int(ts), {})[feature] = float(value)
        return [(ts, out[ts]) for ts in sorted(out)]

    def as_of(self, symbol: str, ts: int, features: list[str] | None = None) -> dict[str, float]:
        rows = self.read_features(symbol, end_ts=ts, features=features)
        result: dict[str, float] = {}
        for _, feats in rows:           # rows are ts-ordered; later rows overwrite → latest-known wins
            result.update(feats)
        return result

    def catalog(self) -> list[tuple[str, int, int, int]]:
        """Per symbol: (symbol, bar_count, first_ts, last_ts) over the 'close' feature — the data
        inventory + freshness view of the SSOT."""
        return self._con.execute(
            "SELECT symbol, count(*), min(ts), max(ts) FROM features WHERE feature = 'close' "
            "GROUP BY symbol ORDER BY symbol"
        ).fetchall()

    def crypto_close_panel(self, *, suffix: str = "USDT") -> dict[str, dict[int, float]]:
        """{symbol: {epoch_day: close}} for every crypto symbol (those whose name ends in `suffix`,
        e.g. BTCUSDT/ETHUSDT) that has a 'close' feature — the same USDT-perp universe the nightly
        crypto book reads. Read-only; ONE DuckDB query, ts (epoch-seconds) folded to epoch-days
        (ts // 86400). Empty {} when the warehouse holds no matching closes."""
        rows = self._con.execute(
            "SELECT symbol, ts, value FROM features WHERE feature = 'close' AND symbol LIKE ?",
            [f"%{suffix}"],
        ).fetchall()
        out: dict[str, dict[int, float]] = {}
        for symbol, ts, value in rows:
            out.setdefault(str(symbol), {})[int(ts) // 86_400] = float(value)
        return out

    def crypto_funding_panel(self, *, suffix: str = "USDT") -> dict[str, dict[int, float]]:
        """{symbol: {epoch_day: daily_funding}} for every crypto symbol ending in `suffix` that has a
        'funding' feature (legacy-venue perp funding history, ingested daily). Mirror of `crypto_close_panel`;
        ts (epoch-seconds) folded to epoch-days (ts // 86400). Empty {} when no funding ingested yet."""
        rows = self._con.execute(
            "SELECT symbol, ts, value FROM features WHERE feature = 'funding' AND symbol LIKE ?",
            [f"%{suffix}"],
        ).fetchall()
        out: dict[str, dict[int, float]] = {}
        for symbol, ts, value in rows:
            out.setdefault(str(symbol), {})[int(ts) // 86_400] = float(value)
        return out

    def crypto_spot_panel(self, *, suffix: str = "USDT") -> dict[str, dict[int, float]]:
        """{symbol: {epoch_day: spot_close}} for every crypto symbol ending in `suffix` that has a
        'spot_close' feature (legacy-venue SPOT daily closes — the long leg of the delta-neutral carry).
        Mirror of crypto_close_panel/crypto_funding_panel; ts folded to epoch-days. {} when none ingested."""
        rows = self._con.execute(
            "SELECT symbol, ts, value FROM features WHERE feature = 'spot_close' AND symbol LIKE ?",
            [f"%{suffix}"],
        ).fetchall()
        out: dict[str, dict[int, float]] = {}
        for symbol, ts, value in rows:
            out.setdefault(str(symbol), {})[int(ts) // 86_400] = float(value)
        return out

    def read_panel(self, symbols: list[str], feature: str) -> dict[str, dict[int, float]]:
        """{symbol: {epoch_day: value}} for one feature across symbols — the cross-sectional panel read.
        ts (epoch-seconds) is converted to epoch-days (ts // 86400)."""
        if not symbols:
            return {}
        placeholders = ",".join("?" for _ in symbols)
        rows = self._con.execute(
            f"SELECT symbol, ts, value FROM features WHERE feature = ? AND symbol IN ({placeholders})",
            [feature, *symbols],
        ).fetchall()
        out: dict[str, dict[int, float]] = {s: {} for s in symbols}
        for symbol, ts, value in rows:
            out[symbol][int(ts) // 86_400] = float(value)
        return out

    def read_latest(self, symbols: list[str], feature: str) -> dict[str, float]:
        """{symbol: latest value} — the last `feature` per symbol (the live-mark needs only this, not the
        whole panel). Mirrors TimescaleFeatureStore.read_latest; duckdb data is small so panel+max is fine."""
        if not symbols:
            return {}
        panel = self.read_panel(list(symbols), feature)
        return {s: float(series[max(series)]) for s, series in panel.items() if series}

    def select_candidate_universe(
        self, *, min_history: int, top_k: int, min_price: float = 1.0
    ) -> dict[str, float]:
        """Top-`top_k` most-liquid symbols (by PEAK calendar-year avg dollar-volume,
        close*volume) having at least `min_history` bars where BOTH close and volume
        are present. Returns {symbol: peak_yearly_avg_dollar_volume}. Survivorship-safe
        (peak, not lifetime-avg, so delisted-but-once-liquid names are included).

        `min_history` counts bars where BOTH close and volume are present (the
        self-joined set), NOT adjclose-only bars. Ranking is by peak calendar-year
        average dollar-volume; ties broken deterministically by symbol ascending.
        ONE DuckDB query, all aggregation pushed into the engine (no rows pulled into
        Python). Self-join `features` on (symbol, ts): one side close, other volume →
        per-bar dollar-volume. Bucket by approximate calendar year, take the per-year
        mean dollar-volume, then the PEAK year per symbol.

        Data-hygiene exclusions (FIX A):
        - NASDAQ test tickers: any symbol ending in 'ZZT' (the *ZZT family —
          ZXZZT/ZWZZT/ZVZZT/ZBZZT/ZJZZT/ZAZZT/ZEZZT etc.) or beginning with 'NTEST'
          carry fake prices (0.0001 -> 200000 daily) and are dropped by symbol
          pattern. No real ticker ends in 'ZZT'.
        - Penny/garbage names: the symbol's MEDIAN close must be >= `min_price`
          (default 1.0). Drops penny stocks and names whose price stream is junk,
          which otherwise inject implausible per-name returns into the verdict."""
        sql = """
            WITH dv AS (
                SELECT c.symbol AS symbol,
                       c.ts AS ts,
                       c.value AS close_val,
                       c.value * v.value AS dollar_vol
                FROM features c
                JOIN features v
                  ON c.symbol = v.symbol AND c.ts = v.ts
                WHERE c.feature = 'close' AND v.feature = 'volume'
            ),
            yearly AS (
                SELECT symbol,
                       CAST(ts / (86400 * 365.25) AS INTEGER) AS yr,
                       avg(dollar_vol) AS yr_avg_dv,
                       count(*) AS bars
                FROM dv
                GROUP BY symbol, yr
            ),
            med AS (
                SELECT symbol, median(close_val) AS med_close
                FROM dv
                GROUP BY symbol
            ),
            agg AS (
                SELECT y.symbol AS symbol,
                       max(y.yr_avg_dv) AS peak_yearly_dv,
                       sum(y.bars) AS total_bars
                FROM yearly y
                GROUP BY y.symbol
            )
            SELECT a.symbol, a.peak_yearly_dv
            FROM agg a
            JOIN med m ON a.symbol = m.symbol
            WHERE a.total_bars >= ? AND a.peak_yearly_dv > 0.0
              AND m.med_close >= ?
              AND a.symbol NOT LIKE '%ZZT'
              AND a.symbol NOT ILIKE 'NTEST%'
            ORDER BY a.peak_yearly_dv DESC, a.symbol ASC
            LIMIT ?
        """
        rows = self._con.execute(sql, [min_history, min_price, top_k]).fetchall()
        return {str(sym): float(dv) for sym, dv in rows}

    def monthly_dollar_volume(self) -> list[tuple[str, int, float, int, float]]:
        """Per (symbol, calendar-month): (symbol, ym, avg_dollar_vol, bars, avg_close).

        `ym` is the integer bucket `year*12 + (month-1)` derived from the bar's
        timestamp via a CORRECT calendar conversion (`to_timestamp(ts)` then
        `year(...)*12 + month(...) - 1`), NOT a ts/(86400*365.25) approximation.
        `avg_dollar_vol` = avg(close*volume) over that (symbol, month); `bars` =
        day count; `avg_close` = avg(close) that month. close & volume are
        self-joined on (symbol, ts). NASDAQ test tickers (*ZZT / NTEST*) excluded.
        ALL aggregation pushed into ONE DuckDB query (no raw rows pulled to Python).
        Ordered by symbol, ym — the PIT (point-in-time) universe builder's input.
        """
        sql = """
            WITH dv AS (
                SELECT c.symbol AS symbol,
                       year(to_timestamp(c.ts)) * 12 + month(to_timestamp(c.ts)) - 1 AS ym,
                       c.value AS close_val,
                       c.value * v.value AS dollar_vol
                FROM features c
                JOIN features v
                  ON c.symbol = v.symbol AND c.ts = v.ts
                WHERE c.feature = 'close' AND v.feature = 'volume'
                  AND c.symbol NOT LIKE '%ZZT'
                  AND c.symbol NOT ILIKE 'NTEST%'
            )
            SELECT symbol,
                   CAST(ym AS BIGINT) AS ym,
                   avg(dollar_vol) AS avg_dollar_vol,
                   count(*) AS bars,
                   avg(close_val) AS avg_close
            FROM dv
            GROUP BY symbol, ym
            ORDER BY symbol, ym
        """
        rows = self._con.execute(sql).fetchall()
        return [
            (str(sym), int(ym), float(avg_dv), int(bars), float(avg_close))
            for sym, ym, avg_dv, bars, avg_close in rows
        ]

    def trading_days_epoch(self) -> list[int]:
        """Sorted, distinct adjclose trading days as epoch-DAYS (ts // 86400) —
        the trading calendar for the rebalance schedule. ts is epoch-seconds."""
        rows = self._con.execute(
            "SELECT DISTINCT ts FROM features WHERE feature = 'adjclose' ORDER BY ts"
        ).fetchall()
        return [int(r[0]) // 86_400 for r in rows]

    # --- ticker membership (survivorship-free listing index) ---------------------------------

    def upsert_membership(self, rows: list[tuple[str, str, str, str]]) -> None:
        """Upsert (symbol, exchange, start_date, end_date) rows — idempotent on symbol (PK).
        Re-running overwrites a symbol's row in place; no duplicates."""
        if not rows:
            return
        self._con.executemany(
            "INSERT INTO ticker_membership VALUES (?, ?, ?, ?) "
            "ON CONFLICT (symbol) DO UPDATE SET "
            "exchange = excluded.exchange, start_date = excluded.start_date, "
            "end_date = excluded.end_date",
            [(s, e, sd, ed) for s, e, sd, ed in rows],
        )

    def read_membership(self) -> list[tuple[str, str, str, str]]:
        """All membership rows: (symbol, exchange, start_date, end_date), symbol-ordered."""
        return self._con.execute(
            "SELECT symbol, exchange, start_date, end_date FROM ticker_membership ORDER BY symbol"
        ).fetchall()

    def close(self) -> None:
        self._con.close()
