Merge: /paper/sim perf fix (lazy-load + cache)

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

src/fxhnt/adapters/web/app.py

@@ -4,6 +4,7 @@ from __future__ import annotations
 
 import datetime as dt
 import logging
+import time
 from pathlib import Path
 from typing import TYPE_CHECKING, Any
 
@@ -165,13 +166,27 @@ def create_app(repo: ForwardNavRepo, svc: DashboardReadModel | None = None,
     _SIM_BOOKS = ("binance_combined", "bybit_4edge")
     _DEFAULT_BOOK = "binance_combined"
 
+    # In-process TTL cache for the per-sleeve return tables. These are PRECOMPUTED nightly (the sim never
+    # recomputes an edge on the request path), so re-reading the whole history from the DB on every request
+    # is pure waste. A short TTL keeps the page from going stale-forever: the nightly snapshot lands within
+    # one TTL window, and a manual book-switch is at most TTL seconds behind the DB. Keyed by book.
+    _SIM_RETURNS_TTL_S = 300.0
+    _sim_returns_cache: dict[str, tuple[float, dict[str, dict[int, float]]]] = {}
+
     def _sim_returns(book: str = _DEFAULT_BOOK) -> dict[str, dict[int, float]]:
         """All cached per-sleeve returns for `book`. binance_combined reads paper_sleeve_ret (a far-future
         cutoff = full history); bybit_4edge reads the precomputed bybit_sleeve_ret table — NEITHER recomputes
-        any edge on the request path."""
+        any edge on the request path. Memoized in-process with a 300s TTL (returns change only nightly)."""
+        now = time.monotonic()
+        hit = _sim_returns_cache.get(book)
+        if hit is not None and (now - hit[0]) < _SIM_RETURNS_TTL_S:
+            return hit[1]
         if book == "bybit_4edge":
-            return _paper_repo().bybit_sleeve_returns()
-        return _paper_repo().sleeve_returns(before="2999-01-01")
+            returns = _paper_repo().bybit_sleeve_returns()
+        else:
+            returns = _paper_repo().sleeve_returns(before="2999-01-01")
+        _sim_returns_cache[book] = (now, returns)
+        return returns
 
     def _forward_track(strategy_id: str) -> dict[str, Any] | None:
         """The live forward track view for `strategy_id` (T0, gate status/days, the NAV curve) — read from
@@ -250,14 +265,23 @@ def create_app(repo: ForwardNavRepo, svc: DashboardReadModel | None = None,
 
     @app.get("/paper/sim", response_class=HTMLResponse)
     def paper_sim(request: Request, book: str = _DEFAULT_BOOK) -> HTMLResponse:
+        """Render the config form + controls INSTANTLY, then let HTMX fetch the (expensive) sim result from
+        /paper/sim/run on load. The page no longer runs the full book overlay before first paint — that was
+        the ~7s cost. The default sim (all sleeves) matches what /paper/sim/run computes for an unconfigured
+        page, so the lazy-loaded result is identical to what the eager render produced."""
         from fxhnt.config import get_settings
         book = book if book in _SIM_BOOKS else _DEFAULT_BOOK
         returns = _sim_returns(book)
         avail = sorted(returns.keys())
-        # bybit_4edge defaults to all 4 sleeves (the whole book); binance defaults to all available too.
-        ctx = _sim_context(returns, capital=get_settings().paper_capital, sleeves=avail,
-                           start=None, end=None, kelly=1.0, controller="killswitch", cost_bps=0.0,
-                           book=book)
+        capital = get_settings().paper_capital
+        # The form's echoed config + the URL HTMX loads on first paint. Defaults mirror /paper/sim/run's
+        # whole-book default: all available sleeves, kelly=1.0, killswitch, cost 0.
+        cfg = {"capital": capital, "start": "", "end": "", "sleeves": avail, "kelly": 1.0,
+               "controller": "killswitch", "cost_bps": 0.0, "book": book}
+        result_url = (f"/paper/sim/run?capital={capital:g}&sleeves={','.join(avail)}"
+                      f"&kelly=1.0&controller=killswitch&cost_bps=0&book={book}")
+        ctx = {"cfg": cfg, "available_sleeves": avail, "presets": _SIM_PRESETS,
+               "books": _SIM_BOOKS, "result_url": result_url, "now": _now()}
         return templates.TemplateResponse(request, "sim.html", ctx)
 
     @app.get("/paper/sim/run", response_class=HTMLResponse)

src/fxhnt/adapters/web/templates/sim.html

@@ -70,8 +70,11 @@
           style="background:#193b1e; color:#3fb950; border:1px solid #2ea043; border-radius:6px; padding:6px 16px; cursor:pointer">Run</button>
 </form>
 
-<div id="sim-result-wrap">
-  {% include "_sim_result.html" %}
+{# The result is lazy-loaded: the form above paints INSTANTLY; HTMX fetches the (expensive) book sim from
+   /paper/sim/run on load and swaps it in here. Using htmx (not fetch+innerHTML) means the result fragment's
+   inline scripts — the timeline scrubber / ▶Play wiring — actually run on every swap. #}
+<div id="sim-result-wrap" hx-get="{{ result_url }}" hx-trigger="load" hx-swap="innerHTML">
+  <p class="muted">running sim…</p>
 </div>
 
 <script>
@@ -102,9 +105,8 @@
       p.set("controller", fd.get("controller") || "killswitch");
       p.set("cost_bps", fd.get("cost_bps") || "0");
       p.set("book", form.getAttribute("data-book") || "binance_combined");
-      fetch("/paper/sim/run?" + p.toString())
-        .then(function (r) { return r.text(); })
-        .then(function (html) { wrap.innerHTML = html; });
+      // htmx.ajax swaps the response AND runs its inline scripts (plain innerHTML would not).
+      htmx.ajax("GET", "/paper/sim/run?" + p.toString(), { target: "#sim-result-wrap", swap: "innerHTML" });
     }
 
     form.addEventListener("submit", function (e) { e.preventDefault(); run(); });

tests/integration/test_bybit_sim_web.py

@@ -72,13 +72,24 @@ def _sim_data(html: str) -> dict:
     return json.loads(m.group(1))
 
 
-def test_bybit_page_renders_selector_and_backtest_label() -> None:
+def test_bybit_page_renders_selector_and_lazy_loads_backtest() -> None:
     c = _client(_seeded_paper_repo())
     r = c.get("/paper/sim", params={"book": "bybit_4edge"})
     assert r.status_code == 200
-    assert "Bybit 4-edge" in r.text                       # book selector
+    assert "Bybit 4-edge" in r.text                       # book selector (painted instantly)
+    assert "naive equal-weight" in r.text                 # naive eq-wt note (form-side, not the overlay)
+    # The backtest curve/label is lazy-loaded for the bybit book too (hx-get carries book=bybit_4edge).
+    assert 'hx-get="/paper/sim/run' in r.text and "book=bybit_4edge" in r.text
+    assert "<svg" not in r.text
+
+
+def test_bybit_run_fragment_has_backtest_label_and_curve() -> None:
+    c = _client(_seeded_paper_repo())
+    r = c.get("/paper/sim/run",
+              params={"book": "bybit_4edge",
+                      "sleeves": "crypto_tstrend,unlock,xsfunding,positioning"})
+    assert r.status_code == 200
     assert "Backtest (configurable)" in r.text            # the backtest is labelled
-    assert "naive equal-weight" in r.text                 # naive eq-wt (not the overlay) is shown
     assert "<svg" in r.text
 
 
@@ -129,11 +140,13 @@ def test_bybit_run_does_not_recompute_the_heavy_edges(monkeypatch) -> None:
 
 
 def test_bybit_view_shows_live_forward_track_with_gate() -> None:
+    # The forward track renders in the lazy-loaded result fragment (same as the backtest curve).
     c = _client(_seeded_paper_repo(), _seeded_forward_repo(with_track=True))
-    r = c.get("/paper/sim", params={"book": "bybit_4edge"})
+    r = c.get("/paper/sim/run", params={"book": "bybit_4edge",
+                                        "sleeves": "crypto_tstrend,unlock,xsfunding,positioning"})
     assert r.status_code == 200
     assert "Live forward" in r.text                       # forward track labelled distinctly
-    assert "Backtest (configurable)" in r.text            # both present on the same view
+    assert "Backtest (configurable)" in r.text            # both present on the same fragment
     assert "15/60 forward days" in r.text                 # gate status / days
     assert "WAIT" in r.text
 
@@ -142,7 +155,8 @@ def test_bybit_view_handles_no_forward_data() -> None:
     fwd = ForwardNavRepo("sqlite://")
     fwd.migrate()                                         # registry seeds bybit_4edge but no nav rows
     c = _client(_seeded_paper_repo(), fwd)
-    r = c.get("/paper/sim", params={"book": "bybit_4edge"})
+    r = c.get("/paper/sim/run", params={"book": "bybit_4edge",
+                                        "sleeves": "crypto_tstrend,unlock,xsfunding,positioning"})
     assert r.status_code == 200
     assert "Live forward" in r.text
     assert "no live forward NAV recorded yet" in r.text

tests/integration/test_paper_sim_web.py

@@ -44,13 +44,31 @@ def _sim_data(html: str) -> dict:
     return json.loads(m.group(1))
 
 
-def test_sim_page_renders_config_form_and_curve() -> None:
+def test_sim_page_renders_config_form_instantly_and_lazy_loads_result() -> None:
+    """The page paints the config form WITHOUT running the (expensive) book sim — the result is fetched
+    lazily by HTMX from /paper/sim/run on load. So the form + controls are present, but the computed curve
+    /metrics are NOT inlined: they arrive via the hx-get trigger."""
     c = _client(_seeded_repo())
     r = c.get("/paper/sim")
     assert r.status_code == 200
     assert 'id="sim-form"' in r.text          # the config panel
+    assert 'type="range"' in r.text            # Kelly/cost sliders (form controls, painted instantly)
+    # The result is lazy-loaded, not computed inline: the wrap div triggers an hx-get on load.
+    assert 'id="sim-result-wrap"' in r.text
+    assert 'hx-get="/paper/sim/run' in r.text
+    assert 'hx-trigger="load"' in r.text
+    # The expensive sim output (curve SVG + metrics) is NOT inlined on the page itself.
+    assert "<svg" not in r.text
+    assert "Sharpe" not in r.text
+
+
+def test_sim_run_fragment_has_the_curve_and_metrics() -> None:
+    """The lazy-loaded fragment (same default config the page's hx-get uses) carries the curve + metrics."""
+    c = _client(_seeded_repo())
+    r = c.get("/paper/sim/run",
+              params={"sleeves": "crypto_tstrend,unlock,stablecoin_rotation,xsfunding"})
+    assert r.status_code == 200
     assert "<svg" in r.text                    # the equity curve
-    assert 'type="range"' in r.text            # Kelly/cost sliders + the timeline scrubber
     assert "Sharpe" in r.text                  # metrics row
 
 
@@ -126,3 +144,25 @@ def test_sim_page_exposes_book_configs_presets() -> None:
         assert f'data-sleeves="{",".join(sleeves)}"' in r.text, f"missing preset button for {name}"
     # xsfunding-solo (the carry comparison track) is now exposed too.
     assert 'data-sleeves="xsfunding"' in r.text
+
+
+def test_sleeve_returns_are_cached_not_reread_every_request() -> None:
+    """_sim_returns memoizes the per-sleeve return table (a nightly-changing read) in-process with a TTL,
+    so repeated sim requests don't re-read the whole history from the DB each time. Count the underlying
+    repo reads across several requests: the first populates the cache, the rest are served from it."""
+    repo = _seeded_repo()
+    calls = {"n": 0}
+    real = repo.sleeve_returns
+
+    def counting(*a, **k):
+        calls["n"] += 1
+        return real(*a, **k)
+
+    repo.sleeve_returns = counting  # type: ignore[method-assign]
+    c = _client(repo)
+    for _ in range(5):
+        assert c.get("/paper/sim").status_code == 200
+        assert c.get("/paper/sim/run",
+                     params={"sleeves": "crypto_tstrend,unlock"}).status_code == 200
+    # 10 requests that each need the table, but the read happens once (cache TTL still warm).
+    assert calls["n"] == 1, f"expected 1 cached DB read, got {calls['n']}"