From e1deecc70268fb16abb2bf0e044d58a06c2ea0c6 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Tue, 16 Jun 2026 17:41:27 +0200 Subject: [PATCH] feat(b2): TiingoUniverseSource (supported-tickers filter + dollar-volume top-N) Co-Authored-By: Claude Opus 4.8 (1M context) --- src/fxhnt/adapters/data/tiingo_universe.py | 176 +++++++++++++++++++++ tests/integration/test_tiingo_universe.py | 118 ++++++++++++++ 2 files changed, 294 insertions(+) create mode 100644 src/fxhnt/adapters/data/tiingo_universe.py create mode 100644 tests/integration/test_tiingo_universe.py diff --git a/src/fxhnt/adapters/data/tiingo_universe.py b/src/fxhnt/adapters/data/tiingo_universe.py new file mode 100644 index 0000000..b1b7f03 --- /dev/null +++ b/src/fxhnt/adapters/data/tiingo_universe.py @@ -0,0 +1,176 @@ +"""urllib UniverseSource — the N most-liquid US common stocks by trailing dollar-volume. + +Two cheap reads (one snapshot per run): + + candidates: Tiingo publishes a supported-tickers CSV (zipped) listing every symbol it carries. + GET https://apimedia.tiingo.com/docs/tiingo/daily/supported_tickers.zip + → one CSV, columns: ticker,exchange,assetType,priceCurrency,startDate,endDate + We download+parse it once (cached on the instance) and filter to US common stock: + assetType == "Stock", priceCurrency == "USD", + exchange ∈ {NYSE, NASDAQ, NYSE ARCA, AMEX, BATS}, + and ACTIVE — endDate within ~7d of the file's max endDate (drops delisted names). + This yields thousands of candidates. It is NOT auth'd (public docs asset). + + volumes: Tiingo's IEX batch endpoint prices many tickers in one call. + GET {base}/iex/?tickers=t1,t2,... (comma-separated, chunked ~90/call) + → [{ticker, last/tngoLast, volume, ...}] + dollar-volume ≈ last × volume (today's IEX volume — a consistent cross-sectional + liquidity proxy, fine for RANKING). One snapshot, summed across chunks. + +To bound cost the candidate set fed to the (auth'd, paid) IEX volume rank is capped at +`max_candidates` (default 1500): we take an initial slice and LOG how many were dropped — never a +silent truncation. We then return the top-`n` by dollar-volume, descending. + +Auth: header `Authorization: Token ` with TIINGO_API_KEY (env). The key is used ONLY in the +IEX request header — never logged or interpolated into a URL. The supported-tickers zip is public +and is fetched WITHOUT the key. + +NOTE: the supported-tickers CSV and IEX shapes above are coded to the documented Tiingo shapes; +this build env had no TIINGO_API_KEY, so the first live run will confirm. Rows missing +last/volume (or yielding zero dollar-volume) are skipped. +""" +from __future__ import annotations + +import csv +import io +import json +import logging +import os +import time +import urllib.parse +import urllib.request +import zipfile +from typing import Any + +log = logging.getLogger(__name__) + +_SUPPORTED_TICKERS_URL = "https://apimedia.tiingo.com/docs/tiingo/daily/supported_tickers.zip" +_IEX = "{base}/iex/?tickers={tickers}" + +# US common-stock exchanges we keep (everything else — OTC, foreign boards — is dropped). +_US_EXCHANGES = {"NYSE", "NASDAQ", "NYSE ARCA", "AMEX", "BATS"} +# A name is "active" if its endDate is within this many days of the file's max endDate. +_ACTIVE_TOL_DAYS = 7 +# IEX batch size — comma-separated tickers per call. +_CHUNK = 90 + + +class TiingoUniverseSource: + def __init__( + self, + api_key: str | None = None, + base_url: str = "https://api.tiingo.com", + max_candidates: int = 1500, + ) -> None: + self._key = api_key if api_key is not None else os.environ["TIINGO_API_KEY"] + self._base = base_url.rstrip("/") + self._max_candidates = max_candidates + + # --- HTTP layer (monkeypatched in tests) ------------------------------------------------- + + def _fetch_candidates(self) -> list[dict[str, str]]: + """Download+unzip the public supported-tickers CSV → list of row dicts. NO api key.""" + for a in range(4): + try: + raw = urllib.request.urlopen(_SUPPORTED_TICKERS_URL, timeout=60).read() + break + except Exception: # noqa: BLE001 — transient HTTP; retry then re-raise + if a == 3: + raise + time.sleep(2 * (a + 1)) + with zipfile.ZipFile(io.BytesIO(raw)) as zf: + name = zf.namelist()[0] + text = zf.read(name).decode("utf-8", errors="replace") + return list(csv.DictReader(io.StringIO(text))) + + def _fetch_volumes(self, tickers: list[str]) -> list[dict[str, Any]]: + """IEX batch snapshot for `tickers` (chunked). Returns concatenated row dicts.""" + out: list[dict[str, Any]] = [] + for i in range(0, len(tickers), _CHUNK): + chunk = tickers[i : i + _CHUNK] + url = _IEX.format(base=self._base, tickers=urllib.parse.quote(",".join(chunk))) + out.extend(self._get(url) or []) + return out + + def _get(self, url: str, tries: int = 4) -> Any: + for a in range(tries): + try: + # The key lives ONLY in this header — never in the URL or any log line. + req = urllib.request.Request(url, headers={"Authorization": f"Token {self._key}"}) + return json.loads(urllib.request.urlopen(req, timeout=30).read()) + except Exception: # noqa: BLE001 — transient HTTP; retry then re-raise + if a == tries - 1: + raise + time.sleep(2 * (a + 1)) + raise RuntimeError("unreachable") + + # --- filtering + ranking ----------------------------------------------------------------- + + def _candidates(self) -> list[str]: + """Active US common-stock tickers from the supported-tickers file, in file order.""" + rows = self._fetch_candidates() + end_dates = [str(r.get("endDate") or "") for r in rows if r.get("endDate")] + max_end = max(end_dates) if end_dates else "" + # "active" cutoff: max endDate minus tolerance (lexical ISO compare; cheap, no parse). + cutoff = "" + if max_end: + import datetime as dt + + try: + cutoff = ( + dt.date.fromisoformat(max_end[:10]) - dt.timedelta(days=_ACTIVE_TOL_DAYS) + ).isoformat() + except ValueError: + cutoff = "" + + out: list[str] = [] + for r in rows: + ticker = (r.get("ticker") or "").strip() + if not ticker: + continue + if (r.get("assetType") or "") != "Stock": + continue + if (r.get("priceCurrency") or "") != "USD": + continue + if (r.get("exchange") or "") not in _US_EXCHANGES: + continue + end = str(r.get("endDate") or "") + if cutoff and end[:10] < cutoff: + continue # delisted + out.append(ticker) + return out + + @staticmethod + def _dollar_volume(row: dict[str, Any]) -> float: + last = row.get("last") + if last is None: + last = row.get("tngoLast") + vol = row.get("volume") + try: + return float(last) * float(vol) # type: ignore[arg-type] + except (TypeError, ValueError): + return 0.0 + + def top_n(self, n: int) -> list[str]: + candidates = self._candidates() + if len(candidates) > self._max_candidates: + dropped = len(candidates) - self._max_candidates + log.info( + "TiingoUniverseSource: %d candidates exceed max_candidates=%d; " + "ranking the first %d, dropping %d (raise max_candidates to widen).", + len(candidates), + self._max_candidates, + self._max_candidates, + dropped, + ) + candidates = candidates[: self._max_candidates] + + rows = self._fetch_volumes(candidates) + ranked = [ + (str(r.get("ticker") or ""), self._dollar_volume(r)) + for r in rows + if r.get("ticker") + ] + ranked = [(t, dv) for t, dv in ranked if dv > 0.0] + ranked.sort(key=lambda tv: tv[1], reverse=True) + return [t for t, _ in ranked[:n]] diff --git a/tests/integration/test_tiingo_universe.py b/tests/integration/test_tiingo_universe.py new file mode 100644 index 0000000..a7f37fa --- /dev/null +++ b/tests/integration/test_tiingo_universe.py @@ -0,0 +1,118 @@ +"""TiingoUniverseSource — top-N most-liquid US common stocks by trailing dollar-volume. +NO live network: the two fetch helpers (`_fetch_candidates` supported-tickers, `_fetch_volumes` +IEX snapshot) are monkeypatched to return fixed fixtures. + +Invariants proven here: + (1) the supported-tickers candidate filter drops non-Stock assetType, non-US/foreign-currency, + and delisted (stale endDate) rows; valid active US stocks are kept. + (2) candidates are ranked by dollar-volume (last × volume), descending — highest first. + (3) top_n(n) returns at most n tickers. +""" +from __future__ import annotations + +import datetime as dt +from typing import Any + +from fxhnt.adapters.data.tiingo_universe import TiingoUniverseSource + +_TODAY = dt.date.today() +_RECENT = _TODAY.isoformat() +_STALE = (_TODAY - dt.timedelta(days=400)).isoformat() + + +def _candidate_rows() -> list[dict[str, str]]: + """supported_tickers.csv rows (active US stocks + a non-stock, a foreign-ccy, a delisted).""" + return [ + # valid active US common stocks (max endDate = _RECENT) + {"ticker": "A", "exchange": "NYSE", "assetType": "Stock", + "priceCurrency": "USD", "startDate": "2000-01-01", "endDate": _RECENT}, + {"ticker": "B", "exchange": "NASDAQ", "assetType": "Stock", + "priceCurrency": "USD", "startDate": "2000-01-01", "endDate": _RECENT}, + {"ticker": "C", "exchange": "NYSE ARCA", "assetType": "Stock", + "priceCurrency": "USD", "startDate": "2000-01-01", "endDate": _RECENT}, + # DROP: not a Stock (ETF) + {"ticker": "SPY", "exchange": "NYSE ARCA", "assetType": "ETF", + "priceCurrency": "USD", "startDate": "2000-01-01", "endDate": _RECENT}, + # DROP: foreign price currency + {"ticker": "FOREIGN", "exchange": "NYSE", "assetType": "Stock", + "priceCurrency": "EUR", "startDate": "2000-01-01", "endDate": _RECENT}, + # DROP: delisted (stale endDate, far before the file max) + {"ticker": "DEAD", "exchange": "NASDAQ", "assetType": "Stock", + "priceCurrency": "USD", "startDate": "2000-01-01", "endDate": _STALE}, + # DROP: unsupported exchange (OTC) + {"ticker": "OTCJUNK", "exchange": "OTC", "assetType": "Stock", + "priceCurrency": "USD", "startDate": "2000-01-01", "endDate": _RECENT}, + ] + + +def test_candidate_filter_drops_non_stock_foreign_and_delisted() -> None: + src = TiingoUniverseSource(api_key="x") + src._fetch_candidates = lambda: _candidate_rows() # type: ignore[method-assign] # noqa: E731 + + kept = src._candidates() + + assert set(kept) == {"A", "B", "C"} + assert "SPY" not in kept # non-Stock dropped + assert "FOREIGN" not in kept # non-USD dropped + assert "DEAD" not in kept # delisted dropped + assert "OTCJUNK" not in kept # unsupported exchange dropped + + +def test_top_n_ranks_by_dollar_volume_descending() -> None: + src = TiingoUniverseSource(api_key="x") + src._fetch_candidates = lambda: _candidate_rows() # type: ignore[method-assign] # noqa: E731 + + # C has highest last×volume, A medium, B lowest. + volumes = [ + {"ticker": "A", "last": 50.0, "volume": 1_000_000}, # 50M + {"ticker": "B", "tngoLast": 10.0, "volume": 1_000_000}, # 10M (uses tngoLast) + {"ticker": "C", "last": 100.0, "volume": 1_000_000}, # 100M + ] + + captured: dict[str, list[str]] = {} + + def fake_volumes(tickers: list[str]) -> list[dict[str, Any]]: + captured["tickers"] = tickers + return volumes + + src._fetch_volumes = fake_volumes # type: ignore[method-assign] + + assert src.top_n(2) == ["C", "A"] + # only the filtered candidates were priced + assert set(captured["tickers"]) == {"A", "B", "C"} + + +def test_top_n_returns_at_most_n() -> None: + src = TiingoUniverseSource(api_key="x") + src._fetch_candidates = lambda: _candidate_rows() # type: ignore[method-assign] # noqa: E731 + src._fetch_volumes = lambda tickers: [ # type: ignore[method-assign] # noqa: E731 + {"ticker": "A", "last": 50.0, "volume": 1_000_000}, + {"ticker": "B", "last": 10.0, "volume": 1_000_000}, + {"ticker": "C", "last": 100.0, "volume": 1_000_000}, + ] + + assert len(src.top_n(2)) == 2 + assert len(src.top_n(1)) == 1 + # n larger than the universe → at most the available count + assert len(src.top_n(99)) == 3 + + +def test_max_candidates_caps_volume_set() -> None: + # 5 candidates but max_candidates=2 → only the first 2 are priced (no silent over-fetch). + src = TiingoUniverseSource(api_key="x", max_candidates=2) + rows = [ + {"ticker": t, "exchange": "NYSE", "assetType": "Stock", + "priceCurrency": "USD", "startDate": "2000-01-01", "endDate": _RECENT} + for t in ("A", "B", "C", "D", "E") + ] + src._fetch_candidates = lambda: rows # type: ignore[method-assign] # noqa: E731 + + captured: dict[str, list[str]] = {} + + def fake_volumes(tickers: list[str]) -> list[dict[str, Any]]: + captured["tickers"] = tickers + return [{"ticker": t, "last": 1.0, "volume": 1} for t in tickers] + + src._fetch_volumes = fake_volumes # type: ignore[method-assign] + src.top_n(2) + assert len(captured["tickers"]) == 2