gitadmin/fxhnt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
e00bac067f7de6ee7bf9d3354bb9f1e7cfa96c21
fxhnt/tests/unit/test_databento_options.py
jgrusewski 70708ca8a7 fix(vrp): freeze the LAST COMPLETE OPRA session via bounded step-back 
The vrp health axis flagged vrp=STALE because the OPRA freeze
(xsp_option_bars) was stuck: vrp_nav froze as_of=today, but today's
session is only partially available intraday, so the freeze failed with
`422 data_schema_not_fully_available` (distinct from the C1a clamp's
`data_end_after_available_end`). That made the freeze timing-dependent —
it could only succeed in a narrow post-settlement window and otherwise
silently degraded to recompute-off-stale-frozen.

Make it robust and timing-independent:
- databento: normalize the `data_schema_not_fully_available` 422 into the
  single catchable DatabentoRangeUnavailable (surgical — only that 422;
  any other client error propagates raw). Expose
  last_available_option_date() as the step-back's starting candidate.
- assets: new _freeze_latest_session entry point walks back one TRADING
  day (skip Sat/Sun) from OPRA's available-end until a COMPLETE session
  freezes, bounded to 5 steps; returns/logs the frozen session date.
  Never freezes a partial session's marks.
- No silent degradation: if no complete session is found in the bound it
  raises DatabentoRangeUnavailable; the existing C1b wrapper in vrp_nav
  catches it → recompute off frozen, and the health axis keeps vrp STALE
  (loud), never a quiet no-op.

TDD: step-back-past-partial, weekend-skip, bound-exhausted-raises, 422
normalization (both chain + bars paths), unrelated-error-propagates,
last_available_option_date caching. Full suite green (one pre-existing,
unrelated seed-dependent gauntlet flake).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-14 19:45:48 +02:00

491 lines
22 KiB
Python
Raw Blame History

import datetime as dt
import pandas as pd
from fxhnt.adapters.data.databento import _bars_from_ohlcv_df, _chain_from_definition_df
def test_chain_from_definition_df_parses_puts_and_calls():
df = pd.DataFrame(
{
"raw_symbol": ["XSP 240816P00470000", "XSP 240816C00470000"],
"expiration": [pd.Timestamp("2024-08-16", tz="UTC"), pd.Timestamp("2024-08-16", tz="UTC")],
"strike_price": [470.0, 470.0],
"instrument_class": ["P", "C"],
}
)
chain = _chain_from_definition_df(df)
assert len(chain) == 2
put = next(c for c in chain if c.right == "P")
assert put.strike == 470.0 and put.expiry == dt.date(2024, 8, 16)
assert put.osi_symbol == "XSP 240816P00470000"
def test_bars_from_ohlcv_df_groups_by_symbol():
idx = pd.DatetimeIndex([pd.Timestamp("2024-07-15", tz="UTC"), pd.Timestamp("2024-07-16", tz="UTC")])
df = pd.DataFrame(
{"symbol": ["XSP 240816P00470000", "XSP 240816P00470000"], "close": [1.20, 1.35]}, index=idx
)
bars = _bars_from_ohlcv_df(df)
assert bars["XSP 240816P00470000"] == {"2024-07-15": 1.20, "2024-07-16": 1.35}
def test_fetch_option_chain_range_dedupes_across_days(monkeypatch):
"""The batched backfill's monthly definition fetch: `definition` emits a record per contract per DAY over
the range, so fetch_option_chain_range must de-dupe by raw_symbol -> one OptionContract per contract."""
import databento
from fxhnt.adapters.data.databento import DatabentoDataProvider
captured = {}
class _FakeClient:
class metadata:
@staticmethod
def get_dataset_range(**kw):
return {"start": "2018-05-01", "end": "2030-01-01"} # far future -> never clamps here
@staticmethod
def get_cost(**kw):
captured["end"] = kw["end"]
return 0.01
class timeseries:
@staticmethod
def get_range(**kw):
# same contract emitted on 2 days -> a duplicate raw_symbol row
_df = pd.DataFrame({
"raw_symbol": ["XSP 240816P00470000", "XSP 240816P00470000", "XSP 240816C00470000"],
"expiration": [pd.Timestamp("2024-08-16", tz="UTC")] * 3,
"strike_price": [470.0, 470.0, 470.0],
"instrument_class": ["P", "P", "C"],
})
return type("_D", (), {"to_df": lambda self: _df})()
monkeypatch.setattr(databento, "Historical", lambda *a, **k: _FakeClient())
chain = DatabentoDataProvider(max_cost_usd=1.0).fetch_option_chain_range("XSP.OPT", "2024-08-01", "2024-08-02")
assert captured["end"] == "2024-08-03" # end made inclusive (+1 day)
assert {c.osi_symbol for c in chain} == {"XSP 240816P00470000", "XSP 240816C00470000"} # deduped
def test_fetch_option_bars_makes_end_inclusive(monkeypatch):
"""Databento's range end is EXCLUSIVE and 422s on start==end; a single-day slice (the freeze path) must
be queried as [start, end+1day). Regression for a live-caught bug (unit tests used a fake provider before)."""
import databento
from fxhnt.adapters.data.databento import DatabentoDataProvider
captured: dict[str, str] = {}
class _FakeClient:
class metadata:
@staticmethod
def get_dataset_range(**kw):
return {"start": "2018-05-01", "end": "2030-01-01"} # far future -> never clamps here
@staticmethod
def get_cost(**kw):
captured["cost_end"] = kw["end"]
return 0.001
class timeseries:
@staticmethod
def get_range(**kw):
captured["range_end"] = kw["end"]
idx = pd.DatetimeIndex([pd.Timestamp("2026-07-06", tz="UTC")])
_df = pd.DataFrame({"symbol": ["XSP 260807P00470000"], "close": [1.0]}, index=idx)
return type("_D", (), {"to_df": lambda self: _df})()
monkeypatch.setattr(databento, "Historical", lambda *a, **k: _FakeClient())
out = DatabentoDataProvider(max_cost_usd=1.0).fetch_option_bars(
["XSP 260807P00470000"], "2026-07-06", "2026-07-06")
assert captured["cost_end"] == "2026-07-07" # start==end single day -> queried as [start, end+1day)
assert captured["range_end"] == "2026-07-07"
assert out == {"XSP 260807P00470000": {"2026-07-06": 1.0}}
def test_fetch_option_bars_chunks_over_databento_symbol_limit(monkeypatch):
"""The wide freeze band has >2000 contracts; Databento 422s 'data_exceeded_maximum_number_of_symbols' on
a single request. Must chunk at 2000, sum cost across chunks (refuse before download), and merge results.
Regression for a live-caught bug."""
import databento
from fxhnt.adapters.data.databento import DatabentoDataProvider
range_call_sizes: list[int] = []
class _FakeClient:
class metadata:
@staticmethod
def get_dataset_range(**kw):
return {"start": "2018-05-01", "end": "2030-01-01"} # far future -> never clamps here
@staticmethod
def get_cost(**kw):
return 0.0001 * len(kw["symbols"]) # any chunk >2000 would 422 in real life
class timeseries:
@staticmethod
def get_range(**kw):
syms = list(kw["symbols"])
range_call_sizes.append(len(syms))
idx = pd.DatetimeIndex([pd.Timestamp("2026-07-06", tz="UTC")] * len(syms))
_df = pd.DataFrame({"symbol": syms, "close": [1.0] * len(syms)}, index=idx)
return type("_D", (), {"to_df": lambda self: _df})()
monkeypatch.setattr(databento, "Historical", lambda *a, **k: _FakeClient())
syms = [f"XSP 260807P{i:08d}" for i in range(2500)]
out = DatabentoDataProvider(max_cost_usd=100.0).fetch_option_bars(syms, "2026-07-06", "2026-07-06")
assert range_call_sizes == [2000, 500] # chunked at 2000 -> no single request over the limit
assert len(out) == 2500 # every symbol's bar merged across chunks
assert out[syms[0]] == {"2026-07-06": 1.0} and out[syms[-1]] == {"2026-07-06": 1.0}
def test_fetch_option_bars_cost_guard_sums_all_chunks(monkeypatch):
"""The estimate-then-refuse cost guard must sum ALL chunks' cost and refuse BEFORE any get_range."""
import databento
from fxhnt.adapters.data.databento import DatabentoCostExceeded, DatabentoDataProvider
downloaded = {"n": 0}
class _FakeClient:
class metadata:
@staticmethod
def get_dataset_range(**kw):
return {"start": "2018-05-01", "end": "2030-01-01"} # far future -> never clamps here
@staticmethod
def get_cost(**kw):
return 0.30 # 2 chunks * 0.30 = 0.60 > cap 0.50
class timeseries:
@staticmethod
def get_range(**kw):
downloaded["n"] += 1
return type("_D", (), {"to_df": lambda self: pd.DataFrame()})()
monkeypatch.setattr(databento, "Historical", lambda *a, **k: _FakeClient())
syms = [f"XSP 260807P{i:08d}" for i in range(2500)] # 2 chunks
try:
DatabentoDataProvider(max_cost_usd=0.50).fetch_option_bars(syms, "2026-07-06", "2026-07-06")
raise AssertionError("expected DatabentoCostExceeded")
except DatabentoCostExceeded as e:
assert "0.60" in str(e) # summed across both chunks (2 * 0.30)
assert downloaded["n"] == 0 # refused BEFORE any download
# --------------------------------------------------------------- C1a: clamp OPRA `end` to available range
class _ClampFakeClient:
"""Fake databento client whose dataset range ENDS at a fixed available-end (models OPRA's ~22:30-UTC
intraday tail at the nightly 23:30 freeze). Records every `end` the provider passes to the API."""
def __init__(self, available_end: str, captured: dict, download_count: dict) -> None:
self._available_end = available_end
self._captured = captured
self._download_count = download_count
outer = self
class _Meta:
@staticmethod
def get_dataset_range(**kw):
return {"start": "2018-05-01", "end": outer._available_end}
@staticmethod
def get_cost(**kw):
outer._captured["cost_end"] = kw["end"]
return 0.001
class _TS:
@staticmethod
def get_range(**kw):
outer._captured["range_end"] = kw["end"]
outer._download_count["n"] += 1
syms = list(kw["symbols"])
idx = pd.DatetimeIndex([pd.Timestamp("2026-07-13", tz="UTC")] * len(syms))
_df = pd.DataFrame({"symbol": syms, "close": [1.0] * len(syms)}, index=idx)
return type("_D", (), {"to_df": lambda self: _df})()
self.metadata = _Meta
self.timeseries = _TS
def test_fetch_option_bars_clamps_end_to_available_end(monkeypatch):
"""At the 23:30-UTC freeze, OPRA only has data to ~22:30 UTC today, so end=as_of+1day (tomorrow 00:00)
exceeds the available range → a raw 422 `data_end_after_available_end`. The provider must obtain the
dataset's available end and clamp the query `end` down to it (never past what OPRA has published)."""
import databento
from fxhnt.adapters.data.databento import DatabentoDataProvider
captured: dict[str, str] = {}
downloads = {"n": 0}
avail = "2026-07-13T22:30:00+00:00"
monkeypatch.setattr(databento, "Historical",
lambda *a, **k: _ClampFakeClient(avail, captured, downloads))
# multi-day window whose +1day end (2026-07-14) exceeds the available end
out = DatabentoDataProvider(max_cost_usd=1.0).fetch_option_bars(
["XSP 260714P00470000"], "2026-07-10", "2026-07-13")
assert captured["cost_end"] == avail # clamped down to available-end, NOT "2026-07-14"
assert captured["range_end"] == avail
assert downloads["n"] == 1 # clamped window still queried (not raised)
assert out # returned data
def test_fetch_option_chain_clamps_end_to_available_end(monkeypatch):
"""Same clamp for the `definition` chain fetch (end = as_of + 1 day)."""
import databento
from fxhnt.adapters.data.databento import DatabentoDataProvider
captured: dict[str, str] = {}
downloads = {"n": 0}
avail = "2026-07-13T22:30:00+00:00"
class _ChainClient(_ClampFakeClient):
def __init__(self, *a, **k):
super().__init__(*a, **k)
outer = self
class _TS:
@staticmethod
def get_range(**kw):
outer._captured["range_end"] = kw["end"]
outer._download_count["n"] += 1
_df = pd.DataFrame({
"raw_symbol": ["XSP 260815P00470000"],
"expiration": [pd.Timestamp("2026-08-15", tz="UTC")],
"strike_price": [470.0], "instrument_class": ["P"]})
return type("_D", (), {"to_df": lambda self: _df})()
self.timeseries = _TS
monkeypatch.setattr(databento, "Historical",
lambda *a, **k: _ChainClient(avail, captured, downloads))
chain = DatabentoDataProvider(max_cost_usd=1.0).fetch_option_chain("XSP.OPT", "2026-07-13")
assert captured["cost_end"] == avail # end (2026-07-14) clamped down to available-end
assert captured["range_end"] == avail
assert chain # returned, not raised
# --------------------------------------------------------------- Part A: robust get_dataset_range PARSE
def test_dataset_range_ends_parses_all_real_shapes():
"""The 0.79 `metadata.get_dataset_range` return is `dict[str, str | dict[str, str]]`; the parse must
handle every real form and only return [] for a genuinely-empty/malformed response."""
from fxhnt.adapters.data.databento import _dataset_range_ends
# top-level {"start","end"} ISO strings
assert _dataset_range_ends({"start": "2018-05-01", "end": "2026-07-13"}) == ["2026-07-13"]
# per-schema nested {schema: {"start","end"}} → one end per schema (caller takes the max)
nested = {"ohlcv-1d": {"start": "2018-05-01", "end": "2026-07-12"},
"definition": {"start": "2018-05-01", "end": "2026-07-13"}}
assert set(_dataset_range_ends(nested)) == {"2026-07-12", "2026-07-13"}
# DatasetRange-like OBJECT exposing .end/.start attributes
obj = type("_R", (), {"start": "2018-05-01", "end": "2026-07-13"})()
assert _dataset_range_ends(obj) == ["2026-07-13"]
# genuinely empty / malformed → [] (caller raises DatabentoRangeUnavailable)
assert _dataset_range_ends({}) == []
assert _dataset_range_ends({"start": "2018-05-01"}) == []
assert _dataset_range_ends(None) == []
def _range_client(rng):
"""A fake databento client whose metadata.get_dataset_range returns exactly `rng`."""
class _C:
class metadata:
@staticmethod
def get_dataset_range(**kw):
return rng
return _C()
def test_option_available_end_parses_per_schema_nested_shape():
"""A VALID per-schema-nested response (NO top-level 'end') must yield the latest end across schemas,
NOT spuriously raise DatabentoRangeUnavailable (the old parse only read a top-level 'end')."""
import pandas as pd
from fxhnt.adapters.data.databento import DatabentoDataProvider
p = DatabentoDataProvider()
ts = p._option_available_end_ts(_range_client(
{"ohlcv-1d": {"start": "2018-05-01", "end": "2026-07-12T22:30:00+00:00"},
"definition": {"start": "2018-05-01", "end": "2026-07-13T22:30:00+00:00"}}))
assert ts == pd.Timestamp("2026-07-13T22:30:00+00:00") # latest end across schemas, UTC
def test_option_available_end_parses_datasetrange_object():
"""A DatasetRange-like object (attribute access, not a dict) must parse via .end, not raise."""
import pandas as pd
from fxhnt.adapters.data.databento import DatabentoDataProvider
obj = type("_R", (), {"start": "2018-05-01", "end": "2026-07-13"})()
ts = DatabentoDataProvider()._option_available_end_ts(_range_client(obj))
assert ts == pd.Timestamp("2026-07-13", tz="UTC")
def test_option_available_end_raises_only_on_empty_malformed():
"""No end anywhere → catchable DatabentoRangeUnavailable (the genuine nothing-published signal)."""
from fxhnt.adapters.data.databento import DatabentoDataProvider, DatabentoRangeUnavailable
p = DatabentoDataProvider()
for bad in ({}, {"start": "2018-05-01"}, {"ohlcv-1d": {"start": "2018-05-01"}}):
try:
p._option_available_end = None # reset cache between shapes
p._option_available_end_ts(_range_client(bad))
raise AssertionError(f"expected DatabentoRangeUnavailable for {bad}")
except DatabentoRangeUnavailable:
pass
def test_clamp_option_end_accepts_tz_aware_input():
"""C1b gap: _clamp_option_end used pd.Timestamp(x).tz_localize('UTC'), which raises TypeError on a
tz-AWARE start/end. The tzinfo guard must accept a tz-aware input (localize if naive, convert if aware)."""
from fxhnt.adapters.data.databento import DatabentoDataProvider
p = DatabentoDataProvider()
client = _range_client({"start": "2018-05-01", "end": "2030-01-01"}) # far future → no clamp
# tz-AWARE start + end_excl — previously raised TypeError, must now clamp cleanly.
out = p._clamp_option_end(client, start="2026-07-10T00:00:00+00:00", end_excl="2026-07-13T00:00:00+00:00")
import pandas as pd
assert pd.Timestamp(out) == pd.Timestamp("2026-07-13T00:00:00+00:00")
# ------------------------------------------------- Phase 2b: last-complete-session freeze robustness
def test_last_available_option_date_returns_dataset_end_date(monkeypatch):
"""`last_available_option_date()` exposes OPRA's available-END as a DATE — the starting candidate for the
freeze step-back. It must reuse the cached `_option_available_end_ts` (one metadata call per provider)."""
import databento
from fxhnt.adapters.data.databento import DatabentoDataProvider
calls = {"n": 0}
class _C:
class metadata:
@staticmethod
def get_dataset_range(**kw):
calls["n"] += 1
return {"start": "2018-05-01", "end": "2026-07-14T22:30:00+00:00"} # partial today session
monkeypatch.setattr(databento, "Historical", lambda *a, **k: _C())
p = DatabentoDataProvider()
assert p.last_available_option_date() == dt.date(2026, 7, 14) # the available-end's calendar date, UTC
p.last_available_option_date() # second call reuses the instance cache
assert calls["n"] == 1 # only ONE metadata round-trip
def test_fetch_option_chain_normalizes_schema_not_available_422(monkeypatch):
"""A partial (still-settling) session raises `422 data_schema_not_fully_available` — a DIFFERENT 422 than
the clamp's `data_end_after_available_end`. The provider must normalize it into the single catchable
`DatabentoRangeUnavailable` so the freeze step-back has ONE type to catch."""
import databento
from databento.common.error import BentoClientError
from fxhnt.adapters.data.databento import DatabentoDataProvider, DatabentoRangeUnavailable
class _C:
class metadata:
@staticmethod
def get_dataset_range(**kw):
return {"start": "2018-05-01", "end": "2030-01-01"} # far future → no clamp
@staticmethod
def get_cost(**kw):
return 0.001
class timeseries:
@staticmethod
def get_range(**kw):
raise BentoClientError(http_status=422, message="data_schema_not_fully_available")
monkeypatch.setattr(databento, "Historical", lambda *a, **k: _C())
try:
DatabentoDataProvider(max_cost_usd=1.0).fetch_option_chain("XSP.OPT", "2026-07-14")
raise AssertionError("expected DatabentoRangeUnavailable")
except DatabentoRangeUnavailable:
pass
def test_fetch_option_bars_normalizes_schema_not_available_422(monkeypatch):
"""Same normalization on the ohlcv-1d marks path (the band freeze)."""
import databento
from databento.common.error import BentoClientError
from fxhnt.adapters.data.databento import DatabentoDataProvider, DatabentoRangeUnavailable
class _C:
class metadata:
@staticmethod
def get_dataset_range(**kw):
return {"start": "2018-05-01", "end": "2030-01-01"}
@staticmethod
def get_cost(**kw):
return 0.001
class timeseries:
@staticmethod
def get_range(**kw):
raise BentoClientError(http_status=422, message="data_schema_not_fully_available")
monkeypatch.setattr(databento, "Historical", lambda *a, **k: _C())
try:
DatabentoDataProvider(max_cost_usd=1.0).fetch_option_bars(
["XSP 260807P00470000"], "2026-07-14", "2026-07-14")
raise AssertionError("expected DatabentoRangeUnavailable")
except DatabentoRangeUnavailable:
pass
def test_fetch_option_chain_does_not_swallow_unrelated_client_error(monkeypatch):
"""The normalization is SURGICAL: only a 422 whose message is `data_schema_not_fully_available` becomes
DatabentoRangeUnavailable. An unrelated client error (e.g. 401 auth, or a different 422) must propagate
RAW — never be masked as a benign range-unavailable and silently degrade the freeze."""
import databento
from databento.common.error import BentoClientError
from fxhnt.adapters.data.databento import DatabentoDataProvider, DatabentoRangeUnavailable
class _C:
class metadata:
@staticmethod
def get_dataset_range(**kw):
return {"start": "2018-05-01", "end": "2030-01-01"}
@staticmethod
def get_cost(**kw):
return 0.001
class timeseries:
@staticmethod
def get_range(**kw):
raise BentoClientError(http_status=401, message="authentication_failed")
monkeypatch.setattr(databento, "Historical", lambda *a, **k: _C())
try:
DatabentoDataProvider(max_cost_usd=1.0).fetch_option_chain("XSP.OPT", "2026-07-14")
raise AssertionError("expected the raw BentoClientError to propagate")
except DatabentoRangeUnavailable:
raise AssertionError("must NOT normalize an unrelated 401 into DatabentoRangeUnavailable")
except BentoClientError as e:
assert e.http_status == 401
def test_fetch_option_bars_raises_catchable_error_when_nothing_available(monkeypatch):
"""If even the clamped window yields no data (OPRA has nothing through `as_of`: start >= clamped end),
raise a clear, CATCHABLE DatabentoRangeUnavailable — never a raw 422 — and never hit the download."""
import databento
from fxhnt.adapters.data.databento import DatabentoDataProvider, DatabentoRangeUnavailable
captured: dict[str, str] = {}
downloads = {"n": 0}
avail = "2026-07-12T22:30:00+00:00" # available ends BEFORE as_of=2026-07-13
monkeypatch.setattr(databento, "Historical",
lambda *a, **k: _ClampFakeClient(avail, captured, downloads))
try:
DatabentoDataProvider(max_cost_usd=1.0).fetch_option_bars(
["XSP 260714P00470000"], "2026-07-13", "2026-07-13")
raise AssertionError("expected DatabentoRangeUnavailable")
except DatabentoRangeUnavailable:
pass
assert downloads["n"] == 0 # refused BEFORE any download (no raw 422)
Reference in New Issue View Git Blame Copy Permalink