feat(b2): TiingoUniverseSource (supported-tickers filter + dollar-volume top-N)
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
jgrusewski
176
src/fxhnt/adapters/data/tiingo_universe.py
Normal file
176
src/fxhnt/adapters/data/tiingo_universe.py
Normal file
@@ -0,0 +1,176 @@
|
||||
"""urllib UniverseSource — the N most-liquid US common stocks by trailing dollar-volume.
|
||||
|
||||
Two cheap reads (one snapshot per run):
|
||||
|
||||
candidates: Tiingo publishes a supported-tickers CSV (zipped) listing every symbol it carries.
|
||||
GET https://apimedia.tiingo.com/docs/tiingo/daily/supported_tickers.zip
|
||||
→ one CSV, columns: ticker,exchange,assetType,priceCurrency,startDate,endDate
|
||||
We download+parse it once (cached on the instance) and filter to US common stock:
|
||||
assetType == "Stock", priceCurrency == "USD",
|
||||
exchange ∈ {NYSE, NASDAQ, NYSE ARCA, AMEX, BATS},
|
||||
and ACTIVE — endDate within ~7d of the file's max endDate (drops delisted names).
|
||||
This yields thousands of candidates. It is NOT auth'd (public docs asset).
|
||||
|
||||
volumes: Tiingo's IEX batch endpoint prices many tickers in one call.
|
||||
GET {base}/iex/?tickers=t1,t2,... (comma-separated, chunked ~90/call)
|
||||
→ [{ticker, last/tngoLast, volume, ...}]
|
||||
dollar-volume ≈ last × volume (today's IEX volume — a consistent cross-sectional
|
||||
liquidity proxy, fine for RANKING). One snapshot, summed across chunks.
|
||||
|
||||
To bound cost the candidate set fed to the (auth'd, paid) IEX volume rank is capped at
|
||||
`max_candidates` (default 1500): we take an initial slice and LOG how many were dropped — never a
|
||||
silent truncation. We then return the top-`n` by dollar-volume, descending.
|
||||
|
||||
Auth: header `Authorization: Token <key>` with TIINGO_API_KEY (env). The key is used ONLY in the
|
||||
IEX request header — never logged or interpolated into a URL. The supported-tickers zip is public
|
||||
and is fetched WITHOUT the key.
|
||||
|
||||
NOTE: the supported-tickers CSV and IEX shapes above are coded to the documented Tiingo shapes;
|
||||
this build env had no TIINGO_API_KEY, so the first live run will confirm. Rows missing
|
||||
last/volume (or yielding zero dollar-volume) are skipped.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import zipfile
|
||||
from typing import Any
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_SUPPORTED_TICKERS_URL = "https://apimedia.tiingo.com/docs/tiingo/daily/supported_tickers.zip"
|
||||
_IEX = "{base}/iex/?tickers={tickers}"
|
||||
|
||||
# US common-stock exchanges we keep (everything else — OTC, foreign boards — is dropped).
|
||||
_US_EXCHANGES = {"NYSE", "NASDAQ", "NYSE ARCA", "AMEX", "BATS"}
|
||||
# A name is "active" if its endDate is within this many days of the file's max endDate.
|
||||
_ACTIVE_TOL_DAYS = 7
|
||||
# IEX batch size — comma-separated tickers per call.
|
||||
_CHUNK = 90
|
||||
|
||||
|
||||
class TiingoUniverseSource:
|
||||
def __init__(
|
||||
self,
|
||||
api_key: str | None = None,
|
||||
base_url: str = "https://api.tiingo.com",
|
||||
max_candidates: int = 1500,
|
||||
) -> None:
|
||||
self._key = api_key if api_key is not None else os.environ["TIINGO_API_KEY"]
|
||||
self._base = base_url.rstrip("/")
|
||||
self._max_candidates = max_candidates
|
||||
|
||||
# --- HTTP layer (monkeypatched in tests) -------------------------------------------------
|
||||
|
||||
def _fetch_candidates(self) -> list[dict[str, str]]:
|
||||
"""Download+unzip the public supported-tickers CSV → list of row dicts. NO api key."""
|
||||
for a in range(4):
|
||||
try:
|
||||
raw = urllib.request.urlopen(_SUPPORTED_TICKERS_URL, timeout=60).read()
|
||||
break
|
||||
except Exception: # noqa: BLE001 — transient HTTP; retry then re-raise
|
||||
if a == 3:
|
||||
raise
|
||||
time.sleep(2 * (a + 1))
|
||||
with zipfile.ZipFile(io.BytesIO(raw)) as zf:
|
||||
name = zf.namelist()[0]
|
||||
text = zf.read(name).decode("utf-8", errors="replace")
|
||||
return list(csv.DictReader(io.StringIO(text)))
|
||||
|
||||
def _fetch_volumes(self, tickers: list[str]) -> list[dict[str, Any]]:
|
||||
"""IEX batch snapshot for `tickers` (chunked). Returns concatenated row dicts."""
|
||||
out: list[dict[str, Any]] = []
|
||||
for i in range(0, len(tickers), _CHUNK):
|
||||
chunk = tickers[i : i + _CHUNK]
|
||||
url = _IEX.format(base=self._base, tickers=urllib.parse.quote(",".join(chunk)))
|
||||
out.extend(self._get(url) or [])
|
||||
return out
|
||||
|
||||
def _get(self, url: str, tries: int = 4) -> Any:
|
||||
for a in range(tries):
|
||||
try:
|
||||
# The key lives ONLY in this header — never in the URL or any log line.
|
||||
req = urllib.request.Request(url, headers={"Authorization": f"Token {self._key}"})
|
||||
return json.loads(urllib.request.urlopen(req, timeout=30).read())
|
||||
except Exception: # noqa: BLE001 — transient HTTP; retry then re-raise
|
||||
if a == tries - 1:
|
||||
raise
|
||||
time.sleep(2 * (a + 1))
|
||||
raise RuntimeError("unreachable")
|
||||
|
||||
# --- filtering + ranking -----------------------------------------------------------------
|
||||
|
||||
def _candidates(self) -> list[str]:
|
||||
"""Active US common-stock tickers from the supported-tickers file, in file order."""
|
||||
rows = self._fetch_candidates()
|
||||
end_dates = [str(r.get("endDate") or "") for r in rows if r.get("endDate")]
|
||||
max_end = max(end_dates) if end_dates else ""
|
||||
# "active" cutoff: max endDate minus tolerance (lexical ISO compare; cheap, no parse).
|
||||
cutoff = ""
|
||||
if max_end:
|
||||
import datetime as dt
|
||||
|
||||
try:
|
||||
cutoff = (
|
||||
dt.date.fromisoformat(max_end[:10]) - dt.timedelta(days=_ACTIVE_TOL_DAYS)
|
||||
).isoformat()
|
||||
except ValueError:
|
||||
cutoff = ""
|
||||
|
||||
out: list[str] = []
|
||||
for r in rows:
|
||||
ticker = (r.get("ticker") or "").strip()
|
||||
if not ticker:
|
||||
continue
|
||||
if (r.get("assetType") or "") != "Stock":
|
||||
continue
|
||||
if (r.get("priceCurrency") or "") != "USD":
|
||||
continue
|
||||
if (r.get("exchange") or "") not in _US_EXCHANGES:
|
||||
continue
|
||||
end = str(r.get("endDate") or "")
|
||||
if cutoff and end[:10] < cutoff:
|
||||
continue # delisted
|
||||
out.append(ticker)
|
||||
return out
|
||||
|
||||
@staticmethod
|
||||
def _dollar_volume(row: dict[str, Any]) -> float:
|
||||
last = row.get("last")
|
||||
if last is None:
|
||||
last = row.get("tngoLast")
|
||||
vol = row.get("volume")
|
||||
try:
|
||||
return float(last) * float(vol) # type: ignore[arg-type]
|
||||
except (TypeError, ValueError):
|
||||
return 0.0
|
||||
|
||||
def top_n(self, n: int) -> list[str]:
|
||||
candidates = self._candidates()
|
||||
if len(candidates) > self._max_candidates:
|
||||
dropped = len(candidates) - self._max_candidates
|
||||
log.info(
|
||||
"TiingoUniverseSource: %d candidates exceed max_candidates=%d; "
|
||||
"ranking the first %d, dropping %d (raise max_candidates to widen).",
|
||||
len(candidates),
|
||||
self._max_candidates,
|
||||
self._max_candidates,
|
||||
dropped,
|
||||
)
|
||||
candidates = candidates[: self._max_candidates]
|
||||
|
||||
rows = self._fetch_volumes(candidates)
|
||||
ranked = [
|
||||
(str(r.get("ticker") or ""), self._dollar_volume(r))
|
||||
for r in rows
|
||||
if r.get("ticker")
|
||||
]
|
||||
ranked = [(t, dv) for t, dv in ranked if dv > 0.0]
|
||||
ranked.sort(key=lambda tv: tv[1], reverse=True)
|
||||
return [t for t, _ in ranked[:n]]
|
||||
118
tests/integration/test_tiingo_universe.py
Normal file
118
tests/integration/test_tiingo_universe.py
Normal file
@@ -0,0 +1,118 @@
|
||||
"""TiingoUniverseSource — top-N most-liquid US common stocks by trailing dollar-volume.
|
||||
NO live network: the two fetch helpers (`_fetch_candidates` supported-tickers, `_fetch_volumes`
|
||||
IEX snapshot) are monkeypatched to return fixed fixtures.
|
||||
|
||||
Invariants proven here:
|
||||
(1) the supported-tickers candidate filter drops non-Stock assetType, non-US/foreign-currency,
|
||||
and delisted (stale endDate) rows; valid active US stocks are kept.
|
||||
(2) candidates are ranked by dollar-volume (last × volume), descending — highest first.
|
||||
(3) top_n(n) returns at most n tickers.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
from typing import Any
|
||||
|
||||
from fxhnt.adapters.data.tiingo_universe import TiingoUniverseSource
|
||||
|
||||
_TODAY = dt.date.today()
|
||||
_RECENT = _TODAY.isoformat()
|
||||
_STALE = (_TODAY - dt.timedelta(days=400)).isoformat()
|
||||
|
||||
|
||||
def _candidate_rows() -> list[dict[str, str]]:
|
||||
"""supported_tickers.csv rows (active US stocks + a non-stock, a foreign-ccy, a delisted)."""
|
||||
return [
|
||||
# valid active US common stocks (max endDate = _RECENT)
|
||||
{"ticker": "A", "exchange": "NYSE", "assetType": "Stock",
|
||||
"priceCurrency": "USD", "startDate": "2000-01-01", "endDate": _RECENT},
|
||||
{"ticker": "B", "exchange": "NASDAQ", "assetType": "Stock",
|
||||
"priceCurrency": "USD", "startDate": "2000-01-01", "endDate": _RECENT},
|
||||
{"ticker": "C", "exchange": "NYSE ARCA", "assetType": "Stock",
|
||||
"priceCurrency": "USD", "startDate": "2000-01-01", "endDate": _RECENT},
|
||||
# DROP: not a Stock (ETF)
|
||||
{"ticker": "SPY", "exchange": "NYSE ARCA", "assetType": "ETF",
|
||||
"priceCurrency": "USD", "startDate": "2000-01-01", "endDate": _RECENT},
|
||||
# DROP: foreign price currency
|
||||
{"ticker": "FOREIGN", "exchange": "NYSE", "assetType": "Stock",
|
||||
"priceCurrency": "EUR", "startDate": "2000-01-01", "endDate": _RECENT},
|
||||
# DROP: delisted (stale endDate, far before the file max)
|
||||
{"ticker": "DEAD", "exchange": "NASDAQ", "assetType": "Stock",
|
||||
"priceCurrency": "USD", "startDate": "2000-01-01", "endDate": _STALE},
|
||||
# DROP: unsupported exchange (OTC)
|
||||
{"ticker": "OTCJUNK", "exchange": "OTC", "assetType": "Stock",
|
||||
"priceCurrency": "USD", "startDate": "2000-01-01", "endDate": _RECENT},
|
||||
]
|
||||
|
||||
|
||||
def test_candidate_filter_drops_non_stock_foreign_and_delisted() -> None:
|
||||
src = TiingoUniverseSource(api_key="x")
|
||||
src._fetch_candidates = lambda: _candidate_rows() # type: ignore[method-assign] # noqa: E731
|
||||
|
||||
kept = src._candidates()
|
||||
|
||||
assert set(kept) == {"A", "B", "C"}
|
||||
assert "SPY" not in kept # non-Stock dropped
|
||||
assert "FOREIGN" not in kept # non-USD dropped
|
||||
assert "DEAD" not in kept # delisted dropped
|
||||
assert "OTCJUNK" not in kept # unsupported exchange dropped
|
||||
|
||||
|
||||
def test_top_n_ranks_by_dollar_volume_descending() -> None:
|
||||
src = TiingoUniverseSource(api_key="x")
|
||||
src._fetch_candidates = lambda: _candidate_rows() # type: ignore[method-assign] # noqa: E731
|
||||
|
||||
# C has highest last×volume, A medium, B lowest.
|
||||
volumes = [
|
||||
{"ticker": "A", "last": 50.0, "volume": 1_000_000}, # 50M
|
||||
{"ticker": "B", "tngoLast": 10.0, "volume": 1_000_000}, # 10M (uses tngoLast)
|
||||
{"ticker": "C", "last": 100.0, "volume": 1_000_000}, # 100M
|
||||
]
|
||||
|
||||
captured: dict[str, list[str]] = {}
|
||||
|
||||
def fake_volumes(tickers: list[str]) -> list[dict[str, Any]]:
|
||||
captured["tickers"] = tickers
|
||||
return volumes
|
||||
|
||||
src._fetch_volumes = fake_volumes # type: ignore[method-assign]
|
||||
|
||||
assert src.top_n(2) == ["C", "A"]
|
||||
# only the filtered candidates were priced
|
||||
assert set(captured["tickers"]) == {"A", "B", "C"}
|
||||
|
||||
|
||||
def test_top_n_returns_at_most_n() -> None:
|
||||
src = TiingoUniverseSource(api_key="x")
|
||||
src._fetch_candidates = lambda: _candidate_rows() # type: ignore[method-assign] # noqa: E731
|
||||
src._fetch_volumes = lambda tickers: [ # type: ignore[method-assign] # noqa: E731
|
||||
{"ticker": "A", "last": 50.0, "volume": 1_000_000},
|
||||
{"ticker": "B", "last": 10.0, "volume": 1_000_000},
|
||||
{"ticker": "C", "last": 100.0, "volume": 1_000_000},
|
||||
]
|
||||
|
||||
assert len(src.top_n(2)) == 2
|
||||
assert len(src.top_n(1)) == 1
|
||||
# n larger than the universe → at most the available count
|
||||
assert len(src.top_n(99)) == 3
|
||||
|
||||
|
||||
def test_max_candidates_caps_volume_set() -> None:
|
||||
# 5 candidates but max_candidates=2 → only the first 2 are priced (no silent over-fetch).
|
||||
src = TiingoUniverseSource(api_key="x", max_candidates=2)
|
||||
rows = [
|
||||
{"ticker": t, "exchange": "NYSE", "assetType": "Stock",
|
||||
"priceCurrency": "USD", "startDate": "2000-01-01", "endDate": _RECENT}
|
||||
for t in ("A", "B", "C", "D", "E")
|
||||
]
|
||||
src._fetch_candidates = lambda: rows # type: ignore[method-assign] # noqa: E731
|
||||
|
||||
captured: dict[str, list[str]] = {}
|
||||
|
||||
def fake_volumes(tickers: list[str]) -> list[dict[str, Any]]:
|
||||
captured["tickers"] = tickers
|
||||
return [{"ticker": t, "last": 1.0, "volume": 1} for t in tickers]
|
||||
|
||||
src._fetch_volumes = fake_volumes # type: ignore[method-assign]
|
||||
src.top_n(2)
|
||||
assert len(captured["tickers"]) == 2
|
||||
Reference in New Issue
Block a user