diff --git a/src/fxhnt/adapters/data/tiingo_universe.py b/src/fxhnt/adapters/data/tiingo_universe.py index b1b7f03..a751dbd 100644 --- a/src/fxhnt/adapters/data/tiingo_universe.py +++ b/src/fxhnt/adapters/data/tiingo_universe.py @@ -17,9 +17,12 @@ Two cheap reads (one snapshot per run): dollar-volume ≈ last × volume (today's IEX volume — a consistent cross-sectional liquidity proxy, fine for RANKING). One snapshot, summed across chunks. -To bound cost the candidate set fed to the (auth'd, paid) IEX volume rank is capped at -`max_candidates` (default 1500): we take an initial slice and LOG how many were dropped — never a -silent truncation. We then return the top-`n` by dollar-volume, descending. +The FULL filtered candidate set is fed to the (auth'd, paid) IEX volume rank — no alphabetical +pre-slice — so "top-N by dollar-volume" ranks every liquid name, not just early-alphabet ones. +`max_candidates` (default 30000) is a runaway safety guard ONLY: it comfortably covers the entire +filtered US-common-stock set (~22k names), so it never triggers in practice. If the filtered set +ever exceeds it we LOG a WARNING naming how many were dropped — never a silent truncation. We then +return the top-`n` by dollar-volume, descending. Auth: header `Authorization: Token ` with TIINGO_API_KEY (env). The key is used ONLY in the IEX request header — never logged or interpolated into a URL. The supported-tickers zip is public @@ -48,6 +51,9 @@ _SUPPORTED_TICKERS_URL = "https://apimedia.tiingo.com/docs/tiingo/daily/supporte _IEX = "{base}/iex/?tickers={tickers}" # US common-stock exchanges we keep (everything else — OTC, foreign boards — is dropped). +# Verified live: Stock rows only ever match NYSE / NASDAQ / AMEX; NYSE ARCA / BATS never appear +# among Stock rows (harmless to keep). OTC strings (PINK, OTCGREY, OTCMKTS, …) + foreign (ASX) +# are excluded by not being in this set. _US_EXCHANGES = {"NYSE", "NASDAQ", "NYSE ARCA", "AMEX", "BATS"} # A name is "active" if its endDate is within this many days of the file's max endDate. _ACTIVE_TOL_DAYS = 7 @@ -60,7 +66,7 @@ class TiingoUniverseSource: self, api_key: str | None = None, base_url: str = "https://api.tiingo.com", - max_candidates: int = 1500, + max_candidates: int = 30000, ) -> None: self._key = api_key if api_key is not None else os.environ["TIINGO_API_KEY"] self._base = base_url.rstrip("/") @@ -153,18 +159,22 @@ class TiingoUniverseSource: def top_n(self, n: int) -> list[str]: candidates = self._candidates() - if len(candidates) > self._max_candidates: - dropped = len(candidates) - self._max_candidates - log.info( - "TiingoUniverseSource: %d candidates exceed max_candidates=%d; " - "ranking the first %d, dropping %d (raise max_candidates to widen).", - len(candidates), - self._max_candidates, + n_filtered = len(candidates) + # Runaway guard only: the filtered US-common-stock set (~22k) sits well under the default + # 30000, so this never trips in practice. If it ever does, WARN (never silent) so the + # dropped names are visible — but DO NOT pre-slice by file order before this point. + if n_filtered > self._max_candidates: + dropped = n_filtered - self._max_candidates + log.warning( + "TiingoUniverseSource: %d filtered candidates exceed runaway guard " + "max_candidates=%d; dropping %d by file order (raise max_candidates to widen).", + n_filtered, self._max_candidates, dropped, ) candidates = candidates[: self._max_candidates] + # Rank the FULL filtered set by dollar-volume — no alphabetical pre-slice bias. rows = self._fetch_volumes(candidates) ranked = [ (str(r.get("ticker") or ""), self._dollar_volume(r)) @@ -173,4 +183,12 @@ class TiingoUniverseSource: ] ranked = [(t, dv) for t, dv in ranked if dv > 0.0] ranked.sort(key=lambda tv: tv[1], reverse=True) - return [t for t, _ in ranked[:n]] + result = [t for t, _ in ranked[:n]] + log.info( + "TiingoUniverseSource: %d filtered candidates, %d priced with dollar-volume>0, " + "returning top %d.", + n_filtered, + len(ranked), + len(result), + ) + return result diff --git a/tests/integration/test_tiingo_universe.py b/tests/integration/test_tiingo_universe.py index a7f37fa..45af165 100644 --- a/tests/integration/test_tiingo_universe.py +++ b/tests/integration/test_tiingo_universe.py @@ -5,8 +5,10 @@ IEX snapshot) are monkeypatched to return fixed fixtures. Invariants proven here: (1) the supported-tickers candidate filter drops non-Stock assetType, non-US/foreign-currency, and delisted (stale endDate) rows; valid active US stocks are kept. - (2) candidates are ranked by dollar-volume (last × volume), descending — highest first. + (2) the FULL filtered candidate set is ranked by dollar-volume (last × volume), descending — + no alphabetical/file-order pre-slice (a late-in-file liquid name beats an early illiquid one). (3) top_n(n) returns at most n tickers. + (4) max_candidates is a runaway safety guard only — it never trips for a normal set. """ from __future__ import annotations @@ -97,9 +99,10 @@ def test_top_n_returns_at_most_n() -> None: assert len(src.top_n(99)) == 3 -def test_max_candidates_caps_volume_set() -> None: - # 5 candidates but max_candidates=2 → only the first 2 are priced (no silent over-fetch). - src = TiingoUniverseSource(api_key="x", max_candidates=2) +def test_full_filtered_set_is_volume_ranked_not_pre_sliced() -> None: + # With the default (high) runaway guard, ALL filtered candidates are priced — no file-order + # pre-slice. 5 candidates, guard untouched → all 5 reach the IEX volume fetch. + src = TiingoUniverseSource(api_key="x") rows = [ {"ticker": t, "exchange": "NYSE", "assetType": "Stock", "priceCurrency": "USD", "startDate": "2000-01-01", "endDate": _RECENT} @@ -115,4 +118,83 @@ def test_max_candidates_caps_volume_set() -> None: src._fetch_volumes = fake_volumes # type: ignore[method-assign] src.top_n(2) - assert len(captured["tickers"]) == 2 + # all 5 priced — the guard did not pre-slice + assert set(captured["tickers"]) == {"A", "B", "C", "D", "E"} + + +def test_runaway_guard_only_trips_above_threshold() -> None: + # The guard is a safety-only cap: it must NOT trip for a normal-sized filtered set, and it + # DOES trip (file-order drop) only when the filtered set exceeds max_candidates. + rows = [ + {"ticker": t, "exchange": "NYSE", "assetType": "Stock", + "priceCurrency": "USD", "startDate": "2000-01-01", "endDate": _RECENT} + for t in ("A", "B", "C", "D", "E") + ] + + # guard above the set size → all priced + src_ok = TiingoUniverseSource(api_key="x", max_candidates=10) + src_ok._fetch_candidates = lambda: rows # type: ignore[method-assign] # noqa: E731 + captured_ok: dict[str, list[str]] = {} + + def fake_ok(tickers: list[str]) -> list[dict[str, Any]]: + captured_ok["tickers"] = tickers + return [{"ticker": t, "last": 1.0, "volume": 1} for t in tickers] + + src_ok._fetch_volumes = fake_ok # type: ignore[method-assign] + src_ok.top_n(2) + assert len(captured_ok["tickers"]) == 5 # guard did not trip + + # guard below the set size → trips, drops by file order down to the cap + src_cap = TiingoUniverseSource(api_key="x", max_candidates=2) + src_cap._fetch_candidates = lambda: rows # type: ignore[method-assign] # noqa: E731 + captured_cap: dict[str, list[str]] = {} + + def fake_cap(tickers: list[str]) -> list[dict[str, Any]]: + captured_cap["tickers"] = tickers + return [{"ticker": t, "last": 1.0, "volume": 1} for t in tickers] + + src_cap._fetch_volumes = fake_cap # type: ignore[method-assign] + src_cap.top_n(2) + assert len(captured_cap["tickers"]) == 2 # guard tripped + + +def test_late_alphabet_high_volume_beats_early_low_volume() -> None: + # ANTI-BIAS GUARANTEE: a late-in-file high-volume ticker (ZM) must out-rank an early-in-file + # low-volume ticker (AAA). The old file-order pre-slice would have ranked only the early + # names and missed ZM; the fix ranks the full filtered set by dollar-volume. + order = ["AAA", "AAB", "TSLA", "WMT", "ZM"] # alphabetical file order + rows = [ + {"ticker": t, "exchange": "NASDAQ", "assetType": "Stock", + "priceCurrency": "USD", "startDate": "2000-01-01", "endDate": _RECENT} + for t in order + ] + # Default (high) runaway guard → full set priced. Under the OLD buggy file-order pre-slice + # the liquid late names (TSLA/WMT/ZM) could be dropped before ranking; the fix prices them. + src = TiingoUniverseSource(api_key="x") + src._fetch_candidates = lambda: rows # type: ignore[method-assign] # noqa: E731 + + # Late-in-order names carry the liquidity; early names are illiquid. + volumes = { + "AAA": (1.0, 100), # tiny dollar-volume + "AAB": (1.0, 100), + "TSLA": (250.0, 5_000_000), # huge + "WMT": (160.0, 3_000_000), + "ZM": (70.0, 4_000_000), + } + + captured: dict[str, list[str]] = {} + + def fake_volumes(tickers: list[str]) -> list[dict[str, Any]]: + captured["tickers"] = tickers + return [{"ticker": t, "last": volumes[t][0], "volume": volumes[t][1]} for t in tickers] + + src._fetch_volumes = fake_volumes # type: ignore[method-assign] + + top3 = src.top_n(3) + # the full filtered set was priced (no file-order pre-slice) + assert set(captured["tickers"]) == set(order) + # late-alphabet liquid names selected; early illiquid names excluded. + # dollar-volume: TSLA 1.25B > WMT 480M > ZM 280M. + assert top3 == ["TSLA", "WMT", "ZM"] + assert "AAA" not in top3 + assert "AAB" not in top3