fix(b2): rank full filtered universe by dollar-volume (drop alphabetical pre-slice bias)

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

src/fxhnt/adapters/data/tiingo_universe.py

@@ -17,9 +17,12 @@ Two cheap reads (one snapshot per run):
                dollar-volume ≈ last × volume (today's IEX volume — a consistent cross-sectional
                liquidity proxy, fine for RANKING). One snapshot, summed across chunks.
 
-To bound cost the candidate set fed to the (auth'd, paid) IEX volume rank is capped at
-`max_candidates` (default 1500): we take an initial slice and LOG how many were dropped — never a
-silent truncation. We then return the top-`n` by dollar-volume, descending.
+The FULL filtered candidate set is fed to the (auth'd, paid) IEX volume rank — no alphabetical
+pre-slice — so "top-N by dollar-volume" ranks every liquid name, not just early-alphabet ones.
+`max_candidates` (default 30000) is a runaway safety guard ONLY: it comfortably covers the entire
+filtered US-common-stock set (~22k names), so it never triggers in practice. If the filtered set
+ever exceeds it we LOG a WARNING naming how many were dropped — never a silent truncation. We then
+return the top-`n` by dollar-volume, descending.
 
 Auth: header `Authorization: Token <key>` with TIINGO_API_KEY (env). The key is used ONLY in the
 IEX request header — never logged or interpolated into a URL. The supported-tickers zip is public
@@ -48,6 +51,9 @@ _SUPPORTED_TICKERS_URL = "https://apimedia.tiingo.com/docs/tiingo/daily/supporte
 _IEX = "{base}/iex/?tickers={tickers}"
 
 # US common-stock exchanges we keep (everything else — OTC, foreign boards — is dropped).
+# Verified live: Stock rows only ever match NYSE / NASDAQ / AMEX; NYSE ARCA / BATS never appear
+# among Stock rows (harmless to keep). OTC strings (PINK, OTCGREY, OTCMKTS, …) + foreign (ASX)
+# are excluded by not being in this set.
 _US_EXCHANGES = {"NYSE", "NASDAQ", "NYSE ARCA", "AMEX", "BATS"}
 # A name is "active" if its endDate is within this many days of the file's max endDate.
 _ACTIVE_TOL_DAYS = 7
@@ -60,7 +66,7 @@ class TiingoUniverseSource:
         self,
         api_key: str | None = None,
         base_url: str = "https://api.tiingo.com",
-        max_candidates: int = 1500,
+        max_candidates: int = 30000,
     ) -> None:
         self._key = api_key if api_key is not None else os.environ["TIINGO_API_KEY"]
         self._base = base_url.rstrip("/")
@@ -153,18 +159,22 @@ class TiingoUniverseSource:
 
     def top_n(self, n: int) -> list[str]:
         candidates = self._candidates()
-        if len(candidates) > self._max_candidates:
-            dropped = len(candidates) - self._max_candidates
-            log.info(
-                "TiingoUniverseSource: %d candidates exceed max_candidates=%d; "
-                "ranking the first %d, dropping %d (raise max_candidates to widen).",
-                len(candidates),
-                self._max_candidates,
+        n_filtered = len(candidates)
+        # Runaway guard only: the filtered US-common-stock set (~22k) sits well under the default
+        # 30000, so this never trips in practice. If it ever does, WARN (never silent) so the
+        # dropped names are visible — but DO NOT pre-slice by file order before this point.
+        if n_filtered > self._max_candidates:
+            dropped = n_filtered - self._max_candidates
+            log.warning(
+                "TiingoUniverseSource: %d filtered candidates exceed runaway guard "
+                "max_candidates=%d; dropping %d by file order (raise max_candidates to widen).",
+                n_filtered,
                 self._max_candidates,
                 dropped,
             )
             candidates = candidates[: self._max_candidates]
 
+        # Rank the FULL filtered set by dollar-volume — no alphabetical pre-slice bias.
         rows = self._fetch_volumes(candidates)
         ranked = [
             (str(r.get("ticker") or ""), self._dollar_volume(r))
@@ -173,4 +183,12 @@ class TiingoUniverseSource:
         ]
         ranked = [(t, dv) for t, dv in ranked if dv > 0.0]
         ranked.sort(key=lambda tv: tv[1], reverse=True)
-        return [t for t, _ in ranked[:n]]
+        result = [t for t, _ in ranked[:n]]
+        log.info(
+            "TiingoUniverseSource: %d filtered candidates, %d priced with dollar-volume>0, "
+            "returning top %d.",
+            n_filtered,
+            len(ranked),
+            len(result),
+        )
+        return result

tests/integration/test_tiingo_universe.py

@@ -5,8 +5,10 @@ IEX snapshot) are monkeypatched to return fixed fixtures.
 Invariants proven here:
   (1) the supported-tickers candidate filter drops non-Stock assetType, non-US/foreign-currency,
       and delisted (stale endDate) rows; valid active US stocks are kept.
-  (2) candidates are ranked by dollar-volume (last × volume), descending — highest first.
+  (2) the FULL filtered candidate set is ranked by dollar-volume (last × volume), descending —
+      no alphabetical/file-order pre-slice (a late-in-file liquid name beats an early illiquid one).
   (3) top_n(n) returns at most n tickers.
+  (4) max_candidates is a runaway safety guard only — it never trips for a normal set.
 """
 from __future__ import annotations
 
@@ -97,9 +99,10 @@ def test_top_n_returns_at_most_n() -> None:
     assert len(src.top_n(99)) == 3
 
 
-def test_max_candidates_caps_volume_set() -> None:
-    # 5 candidates but max_candidates=2 → only the first 2 are priced (no silent over-fetch).
-    src = TiingoUniverseSource(api_key="x", max_candidates=2)
+def test_full_filtered_set_is_volume_ranked_not_pre_sliced() -> None:
+    # With the default (high) runaway guard, ALL filtered candidates are priced — no file-order
+    # pre-slice. 5 candidates, guard untouched → all 5 reach the IEX volume fetch.
+    src = TiingoUniverseSource(api_key="x")
     rows = [
         {"ticker": t, "exchange": "NYSE", "assetType": "Stock",
          "priceCurrency": "USD", "startDate": "2000-01-01", "endDate": _RECENT}
@@ -115,4 +118,83 @@ def test_max_candidates_caps_volume_set() -> None:
 
     src._fetch_volumes = fake_volumes  # type: ignore[method-assign]
     src.top_n(2)
-    assert len(captured["tickers"]) == 2
+    # all 5 priced — the guard did not pre-slice
+    assert set(captured["tickers"]) == {"A", "B", "C", "D", "E"}
+
+
+def test_runaway_guard_only_trips_above_threshold() -> None:
+    # The guard is a safety-only cap: it must NOT trip for a normal-sized filtered set, and it
+    # DOES trip (file-order drop) only when the filtered set exceeds max_candidates.
+    rows = [
+        {"ticker": t, "exchange": "NYSE", "assetType": "Stock",
+         "priceCurrency": "USD", "startDate": "2000-01-01", "endDate": _RECENT}
+        for t in ("A", "B", "C", "D", "E")
+    ]
+
+    # guard above the set size → all priced
+    src_ok = TiingoUniverseSource(api_key="x", max_candidates=10)
+    src_ok._fetch_candidates = lambda: rows  # type: ignore[method-assign]  # noqa: E731
+    captured_ok: dict[str, list[str]] = {}
+
+    def fake_ok(tickers: list[str]) -> list[dict[str, Any]]:
+        captured_ok["tickers"] = tickers
+        return [{"ticker": t, "last": 1.0, "volume": 1} for t in tickers]
+
+    src_ok._fetch_volumes = fake_ok  # type: ignore[method-assign]
+    src_ok.top_n(2)
+    assert len(captured_ok["tickers"]) == 5  # guard did not trip
+
+    # guard below the set size → trips, drops by file order down to the cap
+    src_cap = TiingoUniverseSource(api_key="x", max_candidates=2)
+    src_cap._fetch_candidates = lambda: rows  # type: ignore[method-assign]  # noqa: E731
+    captured_cap: dict[str, list[str]] = {}
+
+    def fake_cap(tickers: list[str]) -> list[dict[str, Any]]:
+        captured_cap["tickers"] = tickers
+        return [{"ticker": t, "last": 1.0, "volume": 1} for t in tickers]
+
+    src_cap._fetch_volumes = fake_cap  # type: ignore[method-assign]
+    src_cap.top_n(2)
+    assert len(captured_cap["tickers"]) == 2  # guard tripped
+
+
+def test_late_alphabet_high_volume_beats_early_low_volume() -> None:
+    # ANTI-BIAS GUARANTEE: a late-in-file high-volume ticker (ZM) must out-rank an early-in-file
+    # low-volume ticker (AAA). The old file-order pre-slice would have ranked only the early
+    # names and missed ZM; the fix ranks the full filtered set by dollar-volume.
+    order = ["AAA", "AAB", "TSLA", "WMT", "ZM"]  # alphabetical file order
+    rows = [
+        {"ticker": t, "exchange": "NASDAQ", "assetType": "Stock",
+         "priceCurrency": "USD", "startDate": "2000-01-01", "endDate": _RECENT}
+        for t in order
+    ]
+    # Default (high) runaway guard → full set priced. Under the OLD buggy file-order pre-slice
+    # the liquid late names (TSLA/WMT/ZM) could be dropped before ranking; the fix prices them.
+    src = TiingoUniverseSource(api_key="x")
+    src._fetch_candidates = lambda: rows  # type: ignore[method-assign]  # noqa: E731
+
+    # Late-in-order names carry the liquidity; early names are illiquid.
+    volumes = {
+        "AAA": (1.0, 100),          # tiny dollar-volume
+        "AAB": (1.0, 100),
+        "TSLA": (250.0, 5_000_000),  # huge
+        "WMT": (160.0, 3_000_000),
+        "ZM": (70.0, 4_000_000),
+    }
+
+    captured: dict[str, list[str]] = {}
+
+    def fake_volumes(tickers: list[str]) -> list[dict[str, Any]]:
+        captured["tickers"] = tickers
+        return [{"ticker": t, "last": volumes[t][0], "volume": volumes[t][1]} for t in tickers]
+
+    src._fetch_volumes = fake_volumes  # type: ignore[method-assign]
+
+    top3 = src.top_n(3)
+    # the full filtered set was priced (no file-order pre-slice)
+    assert set(captured["tickers"]) == set(order)
+    # late-alphabet liquid names selected; early illiquid names excluded.
+    # dollar-volume: TSLA 1.25B > WMT 480M > ZM 280M.
+    assert top3 == ["TSLA", "WMT", "ZM"]
+    assert "AAA" not in top3
+    assert "AAB" not in top3