2e50b04eff
Root cause of slow /paper + /paper/live (profiled, not guessed): WarehouseLivePrice.get_prices called store.read_panel(book_symbols, 'close'), which reads the ENTIRE multi-year close history of every open-book symbol (EXPLAIN ANALYZE on prod: 21,359 rows / 502ms server) and then takes max() per symbol in Python — to produce ~26 numbers. New store.read_latest() returns one row per symbol via a per-symbol index seek (LATERAL ... ORDER BY ts DESC LIMIT 1): EXPLAIN ANALYZE 26 rows / 1.3ms, ~400x less server work and no 21k-row transfer/parse. NOT a cache — a correct query. (Postgres uses LATERAL; duckdb/sqlite fall back to panel+max for tests.) Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
47 lines
2.7 KiB
Python
47 lines
2.7 KiB
Python
"""read_panel returns per-symbol {epoch_day: value} dicts for one feature — the crypto-panel read path."""
|
|
from __future__ import annotations
|
|
|
|
from fxhnt.adapters.warehouse.duckdb_feature_store import DuckDbFeatureStore
|
|
from fxhnt.application.warehouse_ingest import WarehouseIngest
|
|
from fxhnt.domain.warehouse.bar import BarRecord
|
|
|
|
|
|
def test_read_panel_groups_by_symbol_and_day(tmp_path) -> None:
|
|
store = DuckDbFeatureStore(str(tmp_path / "wh.duckdb"))
|
|
WarehouseIngest(store).ingest_bars("BTCUSDT", [BarRecord("BTCUSDT", 19000 * 86400, 100.0, 100.0, 100.0),
|
|
BarRecord("BTCUSDT", 19001 * 86400, 110.0, 110.0, 110.0)])
|
|
WarehouseIngest(store).ingest_bars("ETHUSDT", [BarRecord("ETHUSDT", 19001 * 86400, 50.0, 50.0, 50.0)])
|
|
panel = store.read_panel(["BTCUSDT", "ETHUSDT"], "close")
|
|
assert panel["BTCUSDT"] == {19000: 100.0, 19001: 110.0}
|
|
assert panel["ETHUSDT"] == {19001: 50.0}
|
|
store.close()
|
|
|
|
|
|
def test_read_panel_empty_symbols(tmp_path) -> None:
|
|
store = DuckDbFeatureStore(str(tmp_path / "wh.duckdb"))
|
|
assert store.read_panel([], "close") == {}
|
|
store.close()
|
|
|
|
|
|
def test_read_latest_returns_last_value_per_symbol(tmp_path) -> None:
|
|
"""read_latest returns ONLY the last value per symbol — what the live cockpit mark needs — instead of
|
|
the whole multi-year panel (the /paper hotspot: read_panel read ~21k rows for ~26 numbers)."""
|
|
store = DuckDbFeatureStore(str(tmp_path / "wh.duckdb"))
|
|
WarehouseIngest(store).ingest_bars("BTCUSDT", [BarRecord("BTCUSDT", 19000 * 86400, 100.0, 100.0, 100.0),
|
|
BarRecord("BTCUSDT", 19001 * 86400, 110.0, 110.0, 110.0)])
|
|
WarehouseIngest(store).ingest_bars("ETHUSDT", [BarRecord("ETHUSDT", 19001 * 86400, 50.0, 50.0, 50.0)])
|
|
assert store.read_latest(["BTCUSDT", "ETHUSDT"], "close") == {"BTCUSDT": 110.0, "ETHUSDT": 50.0}
|
|
assert store.read_latest([], "close") == {}
|
|
store.close()
|
|
|
|
|
|
def test_warehouse_live_price_uses_read_latest(tmp_path) -> None:
|
|
"""WarehouseLivePrice.get_prices marks at the LATEST close per requested symbol (via read_latest), and
|
|
omits symbols with no warehouse rows (the view then marks those at entry)."""
|
|
from fxhnt.adapters.exchange.warehouse_live_price import WarehouseLivePrice
|
|
store = DuckDbFeatureStore(str(tmp_path / "wh.duckdb"))
|
|
WarehouseIngest(store).ingest_bars("BTCUSDT", [BarRecord("BTCUSDT", 19000 * 86400, 100.0, 100.0, 100.0),
|
|
BarRecord("BTCUSDT", 19005 * 86400, 123.0, 123.0, 123.0)])
|
|
assert WarehouseLivePrice(store).get_prices(["BTCUSDT", "UNSEENUSDT"]) == {"BTCUSDT": 123.0}
|
|
store.close()
|