"""urllib UniverseSource — the N most-liquid US common stocks by trailing dollar-volume.

Two cheap reads (one snapshot per run):

  candidates:  Tiingo publishes a supported-tickers CSV (zipped) listing every symbol it carries.
                 GET https://apimedia.tiingo.com/docs/tiingo/daily/supported_tickers.zip
                 → one CSV, columns: ticker,exchange,assetType,priceCurrency,startDate,endDate
               We download+parse it once (cached on the instance) and filter to US common stock:
                 assetType == "Stock", priceCurrency == "USD",
                 exchange ∈ {NYSE, NASDAQ, NYSE ARCA, AMEX, BATS},
                 and ACTIVE — endDate within ~7d of the file's max endDate (drops delisted names).
               This yields thousands of candidates. It is NOT auth'd (public docs asset).

  volumes:     Tiingo's IEX batch endpoint prices many tickers in one call.
                 GET {base}/iex/?tickers=t1,t2,...   (comma-separated, chunked ~90/call)
                 → [{ticker, last/tngoLast, volume, ...}]
               dollar-volume ≈ last × volume (today's IEX volume — a consistent cross-sectional
               liquidity proxy, fine for RANKING). One snapshot, summed across chunks.

The FULL filtered candidate set is fed to the (auth'd, paid) IEX volume rank — no alphabetical
pre-slice — so "top-N by dollar-volume" ranks every liquid name, not just early-alphabet ones.
`max_candidates` (default 30000) is a runaway safety guard ONLY: it comfortably covers the entire
filtered US-common-stock set (~22k names), so it never triggers in practice. If the filtered set
ever exceeds it we LOG a WARNING naming how many were dropped — never a silent truncation. We then
return the top-`n` by dollar-volume, descending.

Auth: header `Authorization: Token <key>` with TIINGO_API_KEY (env). The key is used ONLY in the
IEX request header — never logged or interpolated into a URL. The supported-tickers zip is public
and is fetched WITHOUT the key.

NOTE: the supported-tickers CSV and IEX shapes above are coded to the documented Tiingo shapes;
this build env had no TIINGO_API_KEY, so the first live run will confirm. Rows missing
last/volume (or yielding zero dollar-volume) are skipped.
"""
from __future__ import annotations

import csv
import io
import json
import logging
import os
import time
import urllib.parse
import urllib.request
import zipfile
from typing import Any

log = logging.getLogger(__name__)

_SUPPORTED_TICKERS_URL = "https://apimedia.tiingo.com/docs/tiingo/daily/supported_tickers.zip"
_IEX = "{base}/iex/?tickers={tickers}"

# US common-stock exchanges we keep (everything else — OTC, foreign boards — is dropped).
# Verified live: Stock rows only ever match NYSE / NASDAQ / AMEX; NYSE ARCA / BATS never appear
# among Stock rows (harmless to keep). OTC strings (PINK, OTCGREY, OTCMKTS, …) + foreign (ASX)
# are excluded by not being in this set.
_US_EXCHANGES = {"NYSE", "NASDAQ", "NYSE ARCA", "AMEX", "BATS"}
# A name is "active" if its endDate is within this many days of the file's max endDate.
_ACTIVE_TOL_DAYS = 7
# IEX batch size — comma-separated tickers per call.
_CHUNK = 90


class TiingoUniverseSource:
    def __init__(
        self,
        api_key: str | None = None,
        base_url: str = "https://api.tiingo.com",
        max_candidates: int = 30000,
    ) -> None:
        self._key = api_key if api_key is not None else os.environ["TIINGO_API_KEY"]
        self._base = base_url.rstrip("/")
        self._max_candidates = max_candidates

    # --- HTTP layer (monkeypatched in tests) -------------------------------------------------

    def _fetch_candidates(self) -> list[dict[str, str]]:
        """Download+unzip the public supported-tickers CSV → list of row dicts. NO api key."""
        for a in range(4):
            try:
                raw = urllib.request.urlopen(_SUPPORTED_TICKERS_URL, timeout=60).read()
                break
            except Exception:  # noqa: BLE001 — transient HTTP; retry then re-raise
                if a == 3:
                    raise
                time.sleep(2 * (a + 1))
        with zipfile.ZipFile(io.BytesIO(raw)) as zf:
            name = zf.namelist()[0]
            text = zf.read(name).decode("utf-8", errors="replace")
        return list(csv.DictReader(io.StringIO(text)))

    def _fetch_volumes(self, tickers: list[str]) -> list[dict[str, Any]]:
        """IEX batch snapshot for `tickers` (chunked). Returns concatenated row dicts."""
        out: list[dict[str, Any]] = []
        for i in range(0, len(tickers), _CHUNK):
            chunk = tickers[i : i + _CHUNK]
            url = _IEX.format(base=self._base, tickers=urllib.parse.quote(",".join(chunk)))
            out.extend(self._get(url) or [])
        return out

    def _get(self, url: str, tries: int = 4) -> Any:
        for a in range(tries):
            try:
                # The key lives ONLY in this header — never in the URL or any log line.
                req = urllib.request.Request(url, headers={"Authorization": f"Token {self._key}"})
                return json.loads(urllib.request.urlopen(req, timeout=30).read())
            except Exception:  # noqa: BLE001 — transient HTTP; retry then re-raise
                if a == tries - 1:
                    raise
                time.sleep(2 * (a + 1))
        raise RuntimeError("unreachable")

    # --- filtering + ranking -----------------------------------------------------------------

    @staticmethod
    def _is_us_common_stock(r: dict[str, str]) -> bool:
        """The survivorship-free filter: US common stock in USD on a kept exchange (NO active gate —
        delisted names pass). Listing-status (active vs delisted) is a SEPARATE concern applied by
        callers that want only live names; the row itself is in-scope for the universe either way."""
        if not (r.get("ticker") or "").strip():
            return False
        if (r.get("assetType") or "") != "Stock":
            return False
        if (r.get("priceCurrency") or "") != "USD":
            return False
        return (r.get("exchange") or "") in _US_EXCHANGES

    def survivorship_free(self) -> list[tuple[str, str, str, str]]:
        """The FULL survivorship-free candidate set WITH dates: (ticker, exchange, startDate, endDate)
        for every US common stock (Stock/USD/NYSE+NASDAQ+AMEX+…) the file carries, INCLUDING DELISTED
        names. No active gate, no dollar-volume rank — this is the point-in-time-universe backbone a
        walk-forward backtest reconstructs membership from. Reuses the supported-tickers
        download/parse (`_fetch_candidates`); no zip logic duplicated."""
        out: list[tuple[str, str, str, str]] = []
        for r in self._fetch_candidates():
            if not self._is_us_common_stock(r):
                continue
            out.append(
                (
                    (r.get("ticker") or "").strip(),
                    (r.get("exchange") or ""),
                    str(r.get("startDate") or "")[:10],
                    str(r.get("endDate") or "")[:10],
                )
            )
        log.info("TiingoUniverseSource: %d survivorship-free US common-stock candidates (incl. delisted).",
                 len(out))
        return out

    def _candidates(self) -> list[str]:
        """Active US common-stock tickers from the supported-tickers file, in file order."""
        rows = self._fetch_candidates()
        end_dates = [str(r.get("endDate") or "") for r in rows if r.get("endDate")]
        max_end = max(end_dates) if end_dates else ""
        # "active" cutoff: max endDate minus tolerance (lexical ISO compare; cheap, no parse).
        cutoff = ""
        if max_end:
            import datetime as dt

            try:
                cutoff = (
                    dt.date.fromisoformat(max_end[:10]) - dt.timedelta(days=_ACTIVE_TOL_DAYS)
                ).isoformat()
            except ValueError:
                cutoff = ""

        out: list[str] = []
        for r in rows:
            if not self._is_us_common_stock(r):
                continue
            end = str(r.get("endDate") or "")
            if cutoff and end[:10] < cutoff:
                continue  # delisted
            out.append((r.get("ticker") or "").strip())
        return out

    @staticmethod
    def _dollar_volume(row: dict[str, Any]) -> float:
        last = row.get("last")
        if last is None:
            last = row.get("tngoLast")
        vol = row.get("volume")
        try:
            return float(last) * float(vol)  # type: ignore[arg-type]
        except (TypeError, ValueError):
            return 0.0

    def top_n(self, n: int) -> list[str]:
        candidates = self._candidates()
        n_filtered = len(candidates)
        # Runaway guard only: the filtered US-common-stock set (~22k) sits well under the default
        # 30000, so this never trips in practice. If it ever does, WARN (never silent) so the
        # dropped names are visible — but DO NOT pre-slice by file order before this point.
        if n_filtered > self._max_candidates:
            dropped = n_filtered - self._max_candidates
            log.warning(
                "TiingoUniverseSource: %d filtered candidates exceed runaway guard "
                "max_candidates=%d; dropping %d by file order (raise max_candidates to widen).",
                n_filtered,
                self._max_candidates,
                dropped,
            )
            candidates = candidates[: self._max_candidates]

        # Rank the FULL filtered set by dollar-volume — no alphabetical pre-slice bias.
        rows = self._fetch_volumes(candidates)
        ranked = [
            (str(r.get("ticker") or ""), self._dollar_volume(r))
            for r in rows
            if r.get("ticker")
        ]
        ranked = [(t, dv) for t, dv in ranked if dv > 0.0]
        ranked.sort(key=lambda tv: tv[1], reverse=True)
        result = [t for t, _ in ranked[:n]]
        log.info(
            "TiingoUniverseSource: %d filtered candidates, %d priced with dollar-volume>0, "
            "returning top %d.",
            n_filtered,
            len(ranked),
            len(result),
        )
        return result
