Compare commits
252 Commits
feat/unifi
...
master
| Author | SHA1 | Date | |
|---|---|---|---|
| 0d8ce554a3 | |||
| 7da51863a1 | |||
| 05f6d624c0 | |||
| 40c0b187a7 | |||
| ee077a2cc1 | |||
| e00bac067f | |||
| 9d35df6bfc | |||
| da28c049f7 | |||
| 6677f73a64 | |||
| 330bb56b52 | |||
| 9f2b47a789 | |||
| 80bf243023 | |||
| 6cf19ec379 | |||
| 7d708b03b1 | |||
| 7318f77054 | |||
| 6d8bcc3ae7 | |||
| 0069b75d0e | |||
| 603aaaabff | |||
| 9b2b25433e | |||
| 6e9dfec067 | |||
| c98b8d4b49 | |||
| fc8b1b7849 | |||
| 5d15b0f801 | |||
| bf99ce2bd1 | |||
| 2f957bb6a6 | |||
| aa85ffd3a3 | |||
| baf6ddbe9e | |||
| c7a40b70ec | |||
| 92b0871050 | |||
| adf3edb054 | |||
| fc663215ef | |||
| 5b5fbb0e0b | |||
| eb6ecf8272 | |||
| b71cd6b3b0 | |||
| 59cc06e594 | |||
| a263b676de | |||
| fb14ddc75d | |||
| 8df02c2198 | |||
| 39b313800d | |||
| a16b341d90 | |||
| 9baf1f8115 | |||
| 420cffa93c | |||
| 90fcbfde5c | |||
| 9080dffed5 | |||
| 3a26b87f15 | |||
| f325a37070 | |||
| dd527e6658 | |||
| fb4cb49b18 | |||
| 5d4ddcfb39 | |||
| f2730d195d | |||
| 46401afbd1 | |||
| 3d1c325d2c | |||
| 8bbe7fb123 | |||
| 6a109bfec7 | |||
| 9e94b3b645 | |||
| a455dc0fea | |||
| f8831a7502 | |||
| 1ec5afa0c4 | |||
| dcf333d97e | |||
| 0ed2cfdff3 | |||
| a00001e826 | |||
| 4e7adf42f2 | |||
| 1ecf3983d9 | |||
| 06156bdb1a | |||
| 3461f0e331 | |||
| 5b016b9c4d | |||
| 04305a1c72 | |||
| 25e3d7ca85 | |||
| 18cdf3f704 | |||
| 22a1a404ba | |||
| fa6c0c11f3 | |||
| dce4a72c45 | |||
| 2986c0d940 | |||
| aacc7eff8f | |||
| 2438294f48 | |||
| 33c1f62918 | |||
| 6ff522cf7c | |||
| 1240113d33 | |||
| 05c209b64c | |||
| e67c6e1612 | |||
| ce1c978d4b | |||
| 9396f53669 | |||
| 0d3fca6b0c | |||
| 06d12fd4e3 | |||
| 30e62675fc | |||
| 17c36d3afe | |||
| edeabc611b | |||
| 3844a9774d | |||
| de6f738719 | |||
| bd8ef0fed8 | |||
| c182c0209a | |||
| 543129a248 | |||
|
|
d66d11325b | ||
|
|
38b8ff26d2 | ||
|
|
8d66f37981 | ||
|
|
713424a89d | ||
|
|
8e0016a302 | ||
|
|
f6f1c9082b | ||
|
|
2055d0366f | ||
|
|
ebd31d7c07 | ||
|
|
c71a1d8a06 | ||
|
|
9e3ff64e1d | ||
|
|
ecc51cb2db | ||
|
|
5d584c5dbb | ||
|
|
fd2316b1e8 | ||
|
|
b1ceee39a5 | ||
|
|
8bd87e8ab1 | ||
|
|
ce464f82f4 | ||
|
|
c5758e30bb | ||
|
|
bd5f820697 | ||
|
|
5f5c032f24 | ||
|
|
d5f871af0a | ||
|
|
a71737eeca | ||
|
|
048e5ca26d | ||
|
|
9564305920 | ||
|
|
07e6091647 | ||
|
|
70d584251f | ||
|
|
d96b1ea666 | ||
|
|
cd5203a33c | ||
|
|
ee8694dffb | ||
|
|
2a7492fd07 | ||
|
|
875294d679 | ||
|
|
515117de2c | ||
|
|
5f520165a4 | ||
|
|
0c4ffc11fc | ||
|
|
c2dd057ce1 | ||
|
|
1ad41f5145 | ||
|
|
201bd3f217 | ||
|
|
514f17df6a | ||
|
|
797462bdf7 | ||
|
|
dde51b8641 | ||
|
|
3f2c114241 | ||
|
|
1f260b9ddf | ||
|
|
1322706080 | ||
|
|
0d3e06bb53 | ||
|
|
90cc809456 | ||
|
|
e5c2ac003a | ||
|
|
a16ccc4406 | ||
|
|
86e0035d46 | ||
|
|
c4102c7319 | ||
|
|
bc83fbdbe7 | ||
|
|
6ac1325d36 | ||
|
|
f40c44c873 | ||
|
|
2bae84e8c6 | ||
|
|
8b887af153 | ||
|
|
98dda4e835 | ||
|
|
c1d1cb80fb | ||
|
|
d2353ced52 | ||
|
|
b5a2b48de0 | ||
|
|
756b3bab47 | ||
|
|
b1cfdf1a7f | ||
|
|
6abf2a8289 | ||
|
|
efc304e3f9 | ||
|
|
242277db30 | ||
|
|
5c5171efc5 | ||
|
|
a27fc717e6 | ||
|
|
41913da027 | ||
|
|
74ae34a28b | ||
|
|
c5f4405570 | ||
|
|
5874c2d279 | ||
|
|
20437d7f67 | ||
|
|
b6e3f24ed2 | ||
|
|
dd55f331c3 | ||
|
|
91f25215f3 | ||
|
|
d1cba19133 | ||
|
|
255d5baa81 | ||
|
|
65a9ae1f90 | ||
|
|
5019e9ffbb | ||
|
|
edf7e41527 | ||
|
|
4dbaf90609 | ||
|
|
19e669664b | ||
|
|
ec972017ce | ||
|
|
1958315344 | ||
|
|
595dc5da0e | ||
|
|
97b530de10 | ||
|
|
206cde3ee5 | ||
|
|
cd6e554534 | ||
|
|
516499aa17 | ||
|
|
bcf58e703d | ||
|
|
67ca804395 | ||
|
|
0d191badac | ||
|
|
c5a5a0ba24 | ||
|
|
09ebc4c4e0 | ||
|
|
c493516c8f | ||
|
|
29b6c19167 | ||
|
|
8697e1c0ba | ||
|
|
38f5dfa784 | ||
|
|
768ce4a068 | ||
|
|
0a3308c3a6 | ||
|
|
33318eca7c | ||
|
|
42b5fe322f | ||
|
|
8af24f96c2 | ||
|
|
edf7a659f6 | ||
|
|
35bf7a25de | ||
|
|
c3c627ebfd | ||
|
|
fe9509c189 | ||
|
|
55c8644b51 | ||
|
|
524c43eebd | ||
|
|
29f60a8584 | ||
|
|
d25ff9659a | ||
|
|
4a6b5b8f2f | ||
|
|
ff7a81ceeb | ||
|
|
784cd27949 | ||
|
|
3532671892 | ||
|
|
49b6a102d8 | ||
|
|
2cbd2736e5 | ||
|
|
8a99aff76a | ||
|
|
5c0b9ddb27 | ||
|
|
9c51b754ba | ||
|
|
86db827740 | ||
|
|
d399982cd1 | ||
|
|
bb61323904 | ||
|
|
39fa034f49 | ||
|
|
fd5d97bc94 | ||
|
|
43afd24062 | ||
|
|
129409a282 | ||
|
|
7e7a657090 | ||
|
|
70708ca8a7 | ||
|
|
de9beb89ac | ||
|
|
01e437816b | ||
|
|
d070ec9127 | ||
|
|
6c9d43774e | ||
|
|
515dec4b55 | ||
|
|
3302ca5d04 | ||
|
|
afe488e6aa | ||
|
|
95d47aaa05 | ||
|
|
2677bcd02a | ||
|
|
94ad91977e | ||
|
|
649ea25000 | ||
|
|
f38b5ad55b | ||
|
|
375cd108bc | ||
|
|
68970a793f | ||
|
|
ea66df5d7f | ||
|
|
62357d6cc2 | ||
|
|
badded008c | ||
|
|
0be7eb2fc6 | ||
|
|
5fc25a428b | ||
|
|
994bb59641 | ||
|
|
86aba147c9 | ||
|
|
4fd0f9454d | ||
|
|
af68419909 | ||
|
|
fb9564c798 | ||
|
|
49e30b17ea | ||
|
|
4cf6de17ee | ||
|
|
d7a9c32690 | ||
|
|
a213edb800 | ||
|
|
47419b398b | ||
|
|
14f9894482 | ||
|
|
5a3fca0de3 | ||
|
|
810ebdc105 | ||
|
|
9a9fc431a2 | ||
|
|
22e35b39af |
1
.playwright-mcp/console-2026-07-18T20-41-10-847Z.log
Normal file
1
.playwright-mcp/console-2026-07-18T20-41-10-847Z.log
Normal file
@@ -0,0 +1 @@
|
||||
[ 4296ms] [ERROR] Failed to load resource: the server responded with a status of 404 (Not Found) @ http://127.0.0.1:8801/favicon.ico:0
|
||||
171
.playwright-mcp/page-2026-07-18T20-41-15-196Z.yml
Normal file
171
.playwright-mcp/page-2026-07-18T20-41-15-196Z.yml
Normal file
@@ -0,0 +1,171 @@
|
||||
- generic [active] [ref=e1]:
|
||||
- banner [ref=e2]:
|
||||
- img "fxhnt" [ref=e3]
|
||||
- heading "Overview" [level=1] [ref=e4]
|
||||
- navigation [ref=e5]:
|
||||
- link "Overview" [ref=e6] [cursor=pointer]:
|
||||
- /url: /
|
||||
- link "Paper" [ref=e7] [cursor=pointer]:
|
||||
- /url: /paper
|
||||
- link "Replay" [ref=e8] [cursor=pointer]:
|
||||
- /url: /paper/replay
|
||||
- link "Backtest" [ref=e9] [cursor=pointer]:
|
||||
- /url: /paper/sim
|
||||
- generic [ref=e10]: 2026-07-18 20:41 UTC
|
||||
- main [ref=e11]:
|
||||
- generic [ref=e12]:
|
||||
- heading "Deploy — the fund" [level=2] [ref=e13]
|
||||
- generic [ref=e14]:
|
||||
- heading "Fund status — Bybit 4-edge book" [level=2] [ref=e15]
|
||||
- generic [ref=e17]:
|
||||
- generic [ref=e18]: $832,534
|
||||
- generic [ref=e19]: +738%
|
||||
- generic [ref=e20]: ·
|
||||
- generic [ref=e21]: Sharpe 1.74
|
||||
- generic [ref=e22]: ·
|
||||
- generic [ref=e23]: maxDD -13%
|
||||
- generic [ref=e24]: · since 2021
|
||||
- generic "live forward readiness — the gate reports, it never acts" [ref=e25]: "live: gate 11/21 · WAIT"
|
||||
- generic [ref=e29]:
|
||||
- generic [ref=e30]:
|
||||
- generic [ref=e31]: Capital deployed
|
||||
- generic [ref=e32]: $75,000 / $100,000
|
||||
- generic [ref=e35]:
|
||||
- generic [ref=e36]: 1 funded · 0 de-funded
|
||||
- generic [ref=e37]: 75% of fund capital
|
||||
- table [ref=e39]:
|
||||
- rowgroup [ref=e40]:
|
||||
- row [ref=e41]:
|
||||
- columnheader "edge" [ref=e42]
|
||||
- columnheader "metric" [ref=e43]
|
||||
- columnheader "maxDD" [ref=e44]
|
||||
- rowgroup [ref=e45]:
|
||||
- row [ref=e46]:
|
||||
- cell "individual edges · each shown on its own" [ref=e47]
|
||||
- row [ref=e48]:
|
||||
- cell "Crypto trend (long/flat, TS-momentum) PAPER" [ref=e49]
|
||||
- cell "Sh 0.72 / +13%" [ref=e50]
|
||||
- cell "-28%" [ref=e51]
|
||||
- row [ref=e52]:
|
||||
- cell [ref=e53]:
|
||||
- link "Token-unlock dilution shorts (market-neutral)" [ref=e54] [cursor=pointer]:
|
||||
- /url: /strategy/unlock
|
||||
- text: PAPER
|
||||
- cell "Sh 1.03 / +17%" [ref=e55]
|
||||
- cell "-30%" [ref=e56]
|
||||
- row [ref=e57]:
|
||||
- cell [ref=e58]:
|
||||
- link "Crypto funding harvest (cross-sectional, delta-neutral)" [ref=e59] [cursor=pointer]:
|
||||
- /url: /strategy/xsfunding
|
||||
- text: PAPER
|
||||
- cell "Sh 3.62 / +102%" [ref=e60]
|
||||
- cell "-18%" [ref=e61]
|
||||
- row [ref=e62]:
|
||||
- cell [ref=e63]:
|
||||
- link "Positioning (contrarian long/short ratio)" [ref=e64] [cursor=pointer]:
|
||||
- /url: /strategy/positioning
|
||||
- text: PAPER
|
||||
- cell "Sh 2.55 / +63%" [ref=e65]
|
||||
- cell "-23%" [ref=e66]
|
||||
- generic [ref=e67]:
|
||||
- heading "All forward tracks — every gate at a glance" [level=2] [ref=e68]
|
||||
- table [ref=e70]:
|
||||
- rowgroup [ref=e71]:
|
||||
- row [ref=e72]:
|
||||
- columnheader "track" [ref=e73]
|
||||
- columnheader "tier" [ref=e74]
|
||||
- columnheader "allocated" [ref=e75]
|
||||
- columnheader "funding" [ref=e76]
|
||||
- columnheader "backtest" [ref=e77]
|
||||
- columnheader "forward gate" [ref=e78]
|
||||
- rowgroup [ref=e79]:
|
||||
- row [ref=e80]:
|
||||
- cell [ref=e81]:
|
||||
- link "Bybit 4-edge book" [ref=e82] [cursor=pointer]:
|
||||
- /url: /strategy/bybit_4edge
|
||||
- text: · bybit-perp PAPER
|
||||
- cell "deploy" [ref=e83]
|
||||
- cell "$75,000 · 75%" [ref=e84]
|
||||
- cell [ref=e85]
|
||||
- cell "Sh 2.11 / +31%" [ref=e88]
|
||||
- cell "11/21 · WAIT" [ref=e89]
|
||||
- row [ref=e90]:
|
||||
- cell [ref=e91]:
|
||||
- link "Bybit 4-edge book (levered shadow, <=1.5x)" [ref=e92] [cursor=pointer]:
|
||||
- /url: /strategy/bybit_4edge_levered
|
||||
- text: · bybit-perp PAPER
|
||||
- cell "research" [ref=e93]
|
||||
- cell "—" [ref=e94]
|
||||
- cell "—" [ref=e95]
|
||||
- cell "Sh 2.55 / +40%" [ref=e96]
|
||||
- cell "11/21 · WAIT" [ref=e97]
|
||||
- row [ref=e98]:
|
||||
- cell [ref=e99]:
|
||||
- link "ETF Portfolio" [ref=e100] [cursor=pointer]:
|
||||
- /url: /strategy/multistrat
|
||||
- text: · ibkr-equity PAPER
|
||||
- cell "research" [ref=e101]
|
||||
- cell "—" [ref=e102]
|
||||
- cell "—" [ref=e103]
|
||||
- cell "Sh 1.63 / +11%" [ref=e104]
|
||||
- cell "9/20 · WAIT" [ref=e105]
|
||||
- row [ref=e106]:
|
||||
- cell "↳ real trades IBKR paper account" [ref=e107]
|
||||
- cell "real -0.77%" [ref=e108]
|
||||
- cell [ref=e109]:
|
||||
- link "9 trades · 5 holdings →" [ref=e110] [cursor=pointer]:
|
||||
- /url: /strategy/multistrat
|
||||
- cell [ref=e111]
|
||||
- cell "cost to trade 0.0%" [ref=e114]
|
||||
- row [ref=e115]:
|
||||
- cell [ref=e116]:
|
||||
- link "ETF Portfolio (levered shadow)" [ref=e117] [cursor=pointer]:
|
||||
- /url: /strategy/multistrat_levered
|
||||
- text: · ibkr-equity PAPER
|
||||
- cell "research" [ref=e118]
|
||||
- cell "—" [ref=e119]
|
||||
- cell "—" [ref=e120]
|
||||
- cell "—" [ref=e121]
|
||||
- cell "0/20 · WAIT" [ref=e122]
|
||||
- row [ref=e123]:
|
||||
- cell [ref=e124]:
|
||||
- link "Positioning (contrarian long/short ratio)" [ref=e125] [cursor=pointer]:
|
||||
- /url: /strategy/positioning
|
||||
- text: · bybit-perp PAPER
|
||||
- cell "research" [ref=e126]
|
||||
- cell "—" [ref=e127]
|
||||
- cell "—" [ref=e128]
|
||||
- cell "Sh 2.55 / +108%" [ref=e129]
|
||||
- cell "11/14 · WAIT" [ref=e130]
|
||||
- row [ref=e131]:
|
||||
- cell [ref=e132]:
|
||||
- link "60/40 baseline" [ref=e133] [cursor=pointer]:
|
||||
- /url: /strategy/sixtyforty
|
||||
- text: · glbx PAPER ref
|
||||
- cell "research" [ref=e134]
|
||||
- cell "—" [ref=e135]
|
||||
- cell "—" [ref=e136]
|
||||
- cell "—" [ref=e137]
|
||||
- cell "23/20 · GO" [ref=e138]
|
||||
- row [ref=e139]:
|
||||
- cell [ref=e140]:
|
||||
- link "Token-unlock dilution shorts (market-neutral)" [ref=e141] [cursor=pointer]:
|
||||
- /url: /strategy/unlock
|
||||
- text: · bybit-perp PAPER
|
||||
- cell "research" [ref=e142]
|
||||
- cell "—" [ref=e143]
|
||||
- cell "—" [ref=e144]
|
||||
- cell "Sh 1.03 / +20%" [ref=e145]
|
||||
- cell "26/14 · PASS" [ref=e146]
|
||||
- row [ref=e147]:
|
||||
- cell [ref=e148]:
|
||||
- link "Crypto funding harvest (cross-sectional, delta-neutral)" [ref=e149] [cursor=pointer]:
|
||||
- /url: /strategy/xsfunding
|
||||
- text: · bybit-perp PAPER
|
||||
- cell "research" [ref=e150]
|
||||
- cell "—" [ref=e151]
|
||||
- cell "—" [ref=e152]
|
||||
- cell "Sh 3.62 / +16%" [ref=e153]
|
||||
- cell "11/14 · WAIT" [ref=e154]
|
||||
- group [ref=e156]:
|
||||
- generic "Research — not the fund (experimental edges)" [ref=e157] [cursor=pointer]
|
||||
35
.playwright-mcp/page-2026-07-18T20-41-31-788Z.yml
Normal file
35
.playwright-mcp/page-2026-07-18T20-41-31-788Z.yml
Normal file
@@ -0,0 +1,35 @@
|
||||
- generic [active] [ref=f1e1]:
|
||||
- banner [ref=f1e2]:
|
||||
- img "fxhnt" [ref=f1e3]
|
||||
- heading "Paper books" [level=1] [ref=f1e4]
|
||||
- navigation [ref=f1e5]:
|
||||
- link "Overview" [ref=f1e6] [cursor=pointer]:
|
||||
- /url: /
|
||||
- link "Paper" [ref=f1e7] [cursor=pointer]:
|
||||
- /url: /paper
|
||||
- link "Replay" [ref=f1e8] [cursor=pointer]:
|
||||
- /url: /paper/replay
|
||||
- link "Backtest" [ref=f1e9] [cursor=pointer]:
|
||||
- /url: /paper/sim
|
||||
- generic [ref=f1e10]: 2026-07-18 20:41 UTC
|
||||
- main [ref=f1e11]:
|
||||
- heading "Paper books" [level=1] [ref=f1e12]
|
||||
- generic [ref=f1e13]: the fund's live paper accounts — proving the edges with simulated money before real capital
|
||||
- generic [ref=f1e14]:
|
||||
- link "Bybit 4-edge book PAPER account value $832,534 +738% Sharpe 1.74 · maxDD -13% · since 2021 Simulated 4-edge crypto book, live on Bybit → holdings & trades" [ref=f1e15] [cursor=pointer]:
|
||||
- /url: /paper/crypto
|
||||
- generic [ref=f1e16]:
|
||||
- heading "Bybit 4-edge book PAPER" [level=3] [ref=f1e17]
|
||||
- generic [ref=f1e18]: account value
|
||||
- generic [ref=f1e19]: $832,534 +738%
|
||||
- generic [ref=f1e20]: Sharpe 1.74 · maxDD -13% · since 2021
|
||||
- generic [ref=f1e24]: Simulated 4-edge crypto book, live on Bybit → holdings & trades
|
||||
- link "IBKR paper account PAPER account value $1,029,015 1 strategy trading · 5 holdings 0.9% behind plan Real IBKR paper account (US equities/options) → holdings & per-strategy breakdown" [ref=f1e25] [cursor=pointer]:
|
||||
- /url: /strategy/multistrat
|
||||
- generic [ref=f1e26]:
|
||||
- heading "IBKR paper account PAPER" [level=3] [ref=f1e27]
|
||||
- generic [ref=f1e28]: account value
|
||||
- generic [ref=f1e29]: $1,029,015
|
||||
- generic [ref=f1e30]: 1 strategy trading · 5 holdings
|
||||
- generic [ref=f1e31]: 0.9% behind plan
|
||||
- generic [ref=f1e38]: Real IBKR paper account (US equities/options) → holdings & per-strategy breakdown
|
||||
BIN
cockpit-overview-bizworx.png
Normal file
BIN
cockpit-overview-bizworx.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 173 KiB |
@@ -0,0 +1,119 @@
|
||||
# fxhnt — Fund / Tax / Accounting Research Brief (2026-07-20)
|
||||
|
||||
**> This is researched context, NOT legal, tax, or financial advice.** The Dutch tax rules that govern this the most (Box 3 wealth taxation and the FGR fund regime) are in **active reform** as of mid-2026: Box 3's actual-return system is legislatively unsettled (Eerste Kamer voting paused) and the FGR definition was rewritten Jan 2025 with further consultation open. Every rate, threshold, and classification below must be confirmed with a licensed Dutch belastingadviseur / fiscaal jurist against the finalized Belastingdienst publications for your actual filing year before you rely on it. The single highest-stakes item — whether your systematic/algorithmic trading is Box 3 or Box 1 — is genuinely unsettled case law and needs a written advisor opinion (and possibly a vooroverleg with the Belastingdienst) *before* you scale real capital.
|
||||
|
||||
**> STATE NOTE (2026-07-20): the fund is STILL PAPER — no live book yet** (IBKR DU9600528 is a paper account; Bybit exec never armed). This is go-live planning from euro zero, and the Bybit ~1 Jul 2026 EEA change (§5.15) is **not** a live-position emergency — just a reason to open the right Bybit-EU (EUR-not-USDT) account before funding crypto.
|
||||
|
||||
**> RESOLVED STRUCTURE (2026-07-20) — supersedes §1's "start personal, then decide" and §3/§4's BV-from-zero modelling.** The operator surfaced that they **already have an IT/tech-BV + holding + a DGA-salary**. That collapses the whole Box-1-vs-Box-3 gamble into a clean, already-in-place structure:
|
||||
|
||||
> **Tech-BV = ALL the arbeid** (build/run the system, buy/develop technology) — already taxed there (Vpb 19% + the existing DGA loon, so *no extra €58k gebruikelijk-loon is needed*). → BV pays **dividend** (Vpb 19% + Box 2 24.5%, cleanly taxed) to private. → that al-belaste private money is invested in the **fund as passive capital → Box 3** (~6% forfait × 36% ≈ 2.2%/yr of assets, regardless of real return).
|
||||
|
||||
This **removes the Box-1 reclassification risk entirely** — the only real threat — because the danger was ever *arbeid + winst in one hand* (what sank GHARL:2024:4613). Here the labour is taxed in the BV and the private capital does nothing but *renderen*: exactly the "normaal vermogensbeheer" the daytrader cases won on. **No new entity, no fund wrapper, no AIFMD (own money). Dividend is an option the operator HAS, not a forced cost.** Two design guards remain (not risks): keep the fund genuinely passive → price any tech-BV→fund service **at arm's length**; and time the dividend for the Box 2 bracket. So §1's staged "personal → BV → fund" path and §3/§4's break-even math are **background theory** for this operator — the resolved route above is what applies. The advisor conversation shifts from *"which box?"* to *"how do I price the tech-BV→fund service so the fund stays demonstrably passive?"*
|
||||
|
||||
**> ACCOUNTING NOTE: EUR is the functional currency, no conversion pipeline.** IBKR is EUR-native and Bybit-EU is EUR too → there is no USD leg. §2's FX-revaluation subsystem (realized/unrealized-FX split, ECB-rate join, USDT proxy) is **downgraded from core to dormant insurance**: keep a `currency` + `fx_rate_to_eur` column per posting (default EUR/1.0) so a stray non-EUR line is representable, but build no conversion pipeline. Pure-EUR book = no FX P&L = cleaner books and cleaner tax.
|
||||
|
||||
---
|
||||
|
||||
# Decision brief: bookkeeping, profit extraction, and structure for an NL-resident agentic quant fund
|
||||
|
||||
## 1. Bottom line up front
|
||||
|
||||
**Start personal (Box 3), keep your own capital, and build the accounting system now — but get a written advisor opinion on the Box 1 reclassification risk before you go live with real money.** At ~$35k pilot scale, a BV's fixed costs (incorporation, annual accounts, Vpb filing, and especially the ~€58k gebruikelijk-loon obligation that an actively-run trading BV likely cannot escape) are disproportionate, and every pooled-fund wrapper (FGR/VBI/FBI/RAIF/QIAIF) is either closed off post-2025 reform or needs tens of millions in AUM to be economical. The one serious caveat: your setup — a continuously-running, code-driven, systematic strategy — sits squarely on the fact pattern (ECLI:NL:GHARL:2024:4613, the self-built crypto bot with a stable 75–80% win rate) that Dutch courts pushed into **Box 1** (progressive rates to ~49.5% on actual profit) rather than Box 3, so "Box 3 is cheaper" is only true if Box 3 actually holds. **Move to a dedicated (ring-fenced) BV when either (a) your advisor judges Box 1 reclassification risk high enough that corporate-rate certainty is worth the overhead, or (b) profits are large and consistently retained/reinvested so Vpb-only compounding (19%) beats paying Box 3 tax on capital every year.** Move to a fund structure (de-minimis AIFM + FGR) only when external capital is a concrete, near-term plan — not before.
|
||||
|
||||
## 2. The accounting system (the part you can build)
|
||||
|
||||
**Recommendation: build the tax ledger inside your existing Postgres/Timescale warehouse, not in a plaintext tool (beancount/hledger).** You already run Postgres + Dagster with per-strategy tagging; a second, disconnected file-based ledger just creates a reconciliation surface between the warehouse and the books. beancount's one real advantage — a native FIFO/AVERAGE cost-basis engine — is a bounded amount of SQL to replicate, and neither tool has a Bybit importer (you build that either way). Keep beancount only as a *possible export target* if your accountant wants to inspect plain text.
|
||||
|
||||
### Schema (append-only double-entry)
|
||||
|
||||
- **`journal_entries`** (header): `entry_id, entry_date, description, source_venue, strategy_tag, source_ref` — immutable, corrections via reversing entries only (never UPDATE/DELETE).
|
||||
- **`journal_postings`** (lines): `entry_id, account_id, side ('debit'|'credit'), amount, currency, fx_rate_to_eur, eur_amount` — with a DB-level CHECK/trigger enforcing that each entry's postings sum to zero **in EUR-converted terms**.
|
||||
- **`accounts`**: per-venue sub-ledgers that each balance standalone — `IBKR-EUR-cash`, `IBKR-securities`, `Bybit-USD-cash`, `Bybit-crypto`, `Bybit-perp-margin` — plus **two distinct FX accounts per foreign balance**: `unrealized-FX-PnL` (hit at each period-end retranslation) and `realized-FX-PnL` (hit only on actual conversion/withdrawal). This split is what lets you resolve the accounting-vs-tax realization question later without re-deriving history.
|
||||
- **`cost_basis_lots`** (per venue/asset): matched lots so realized gains come from actual lot-matching (FIFO — the NL de-facto default — or whatever your advisor confirms, applied *consistently* across both venues), not cash-flow deltas.
|
||||
- **`fx_rates`**: daily ECB reference rates, keyed by value_date.
|
||||
- **`account_balances`**: a materialized view for fast reads.
|
||||
|
||||
The **strategy_tag** on every posting makes P&L-by-strategy a `GROUP BY`, reusing your existing Dagster attribution — a filter, not a rebuild.
|
||||
|
||||
### Ingestion pipeline (Dagster assets, mirroring your nightly ETL pattern)
|
||||
|
||||
1. **IBKR Flex Web Service API** (not PDF scraping): a per-query token + query ID, 2-step SendRequest→GetStatement poll, ~1 req/sec limit, 2–3 pulls/day is safe. Pull trades, cash transactions, dividends, corporate actions, positions, NAV. Archive the annual Activity Statement as an audit doc even though you compute P&L independently. Note: as a W-8BEN non-US person you get a **1042-S**, never a Consolidated 1099, and there is no IBKR "NL tax statement."
|
||||
2. **Bybit** transaction-history / Tax-API REST endpoints for fills, **funding fees**, and deposits/withdrawals. **Explicitly verify the Unified Trading Account API surfaces perp funding and mark-to-market events** — they are easy to miss and material to P&L.
|
||||
3. Land raw payloads in an append-only `raw` schema (mirrors your DB-anchor-SSOT pattern), then a transform layer normalizes both venues into the canonical `journal_postings`.
|
||||
|
||||
### FX conversion & reconciliation
|
||||
|
||||
- **Standard rate: ECB daily reference rate** (`eurofxref-daily.xml`, ~16:00 CET; `eurofxref-hist.xml` for one-time backfill), applied at each transaction's value_date (IAS 21 transaction-date convention).
|
||||
- **The ECB publishes no crypto crosses.** Two-step for Bybit: (1) treat USDT as a USD-proxy ~1:1 **and document that assumption explicitly** (or use Bybit's own USD-index for USDT legs), then (2) USD→EUR at the ECB rate for that value_date.
|
||||
- **Box 3 peildatum (1 January)** crypto valuation: use the **exchange's own displayed value at 00:00 on 1 Jan** (crypto is venue-priced), and snapshot it via API.
|
||||
- **Reconciliation target: DAC8/CARF.** From FY2026, in-scope CASPs report your crypto data straight to the Belastingdienst (first reporting Jan 2027). Your books must reconcile to what the venue reports. Given Bybit's current global entity is non-EU pre-migration, **assume no third-party feed and keep fully independent records** — this raises, not lowers, your self-reporting burden. (See §5 on the Bybit EU migration deadline.)
|
||||
|
||||
## 3. Tax-efficient profit extraction (ranked)
|
||||
|
||||
| Rank | Route | Box / rate implication | When it wins |
|
||||
|---|---|---|---|
|
||||
| **1** | **Stay personal, Box 3** | 36% on a *deemed* return (~6% "overige bezittingen" forfait for 2026 → effective ~2.2% of asset value/yr); tegenbewijsregeling lets you report actual return if lower | Pilot / own-capital, **if Box 3 classification holds** and actual returns exceed the forfait. Dramatically cheaper than any actual-profit regime in a strong year. |
|
||||
| **2** | **BV, retain earnings** | Vpb only: 19% ≤€200k, 25.8% above — no Box 2 layer until you distribute | Once profits are large and **reinvested/compounded** inside the BV; deferral of the Box 2 layer is the compounding advantage. Also removes the Box 1/3 question entirely. |
|
||||
| **3** | **BV, distribute as dividend** | Vpb (19/25.8%) **then** Box 2 (24.5% ≤~€68.8k / 31% above); combined effective ~38–44% on distributed profit | Only when you actually need the cash personally *and* Box 1 reclassification would otherwise have applied (~49.5%) — you pay for certainty. |
|
||||
| **4** | **DGA salary from a BV** | gebruikelijk loon ~€58k (2026), taxed as Box 1 wage income, progressive to ~49.5% | Not a "choice" — it's a *mandatory* cost of the BV route (an active trading BV is unlikely to qualify for the ~€5k limited-activity carve-out, and from 2026 needs an advisor-cosigned salary justification). Factor it into the BV break-even, don't treat it as extraction. |
|
||||
|
||||
**Key numbers to hold in mind:** Box 3 taxes *deemed* return on *capital*; the BV route taxes *actual* profit. So Box 3 is cheaper the more your real return exceeds ~6%, and the BV is cheaper mainly when Box 1 (~49.5% on actual) is the realistic alternative or when earnings are retained. Under the Box 3 **tegenbewijsregeling**, "actual return" includes **unrealized** mark-to-market gains and does **not** allow deduction of trading fees/funding (only actual Box-3 debt interest) — this is *not* how a trading P&L system naturally computes returns, so your ledger needs a parallel tax-basis calculation.
|
||||
|
||||
## 4. Fund-structure decision map
|
||||
|
||||
```
|
||||
STAGE 0 — Personal, Box 3 own capital only, any size
|
||||
trigger to advance ↓ external capital contemplated, OR Box 1 risk judged high, OR large retained profits
|
||||
STAGE 1 — Dedicated BV (ring-fenced) own capital; new sister/subsidiary BV, NOT commingled with bizworx
|
||||
trigger to advance ↓ first external euro is concrete & near-term
|
||||
STAGE 2 — BV(manager) + FGR + AIFMD "light" registration
|
||||
trigger to advance ↓ AUM approaches ~€20–30m (dedicated-fund fixed costs become proportionate)
|
||||
STAGE 3 — Licensed fund / RAIF (LU) or QIAIF (IE) institutional external capital, tens of €m+
|
||||
```
|
||||
|
||||
**Regulatory gates:**
|
||||
|
||||
- **"Own money only" is a genuine AIFMD safe harbor** (Art. 4(1)(a); Recital 7 family-office carve-out). No license, no registration while you trade solely your own capital — *this covers a solo BV too*. Also outside MiFID II via the Art. 2(1)(d) own-account exemption (subject to a technical check that your order/message rate doesn't hit the "high-frequency algorithmic trading" threshold), and outside MiCA CASP authorization for own-account crypto.
|
||||
- **The moment you accept one external euro**, AIF/AIFMD scope begins. Below **€100m AUM** (leveraged / redemption within 5yr) or **€500m** (unleveraged closed-end), you need only **AFM registration** (the "light"/de-minimis regime), *not* a full AIFM license — but that registration must be done **before** the first external euro, plus Wwft (AML) and Sanctiewet compliance and AFM/DNB reporting. Marketing to retail adds top-up rules (Wft 4:37p) and a <150-investor / €100k-minimum-ticket cap.
|
||||
- **Ring-fence, don't commingle.** If/when you do incorporate, put trading in a **new dedicated BV**, not inside bizworx.nl — standard NL practice isolates a volatile, potentially loss-making book from an operating business's balance sheet, credit, and any regulatory status.
|
||||
- **FGR is in flux and not fit-for-purpose at pilot scale.** The Jan-2025 reform abolished the open/closed test; a new fund must qualify directly under Wft-anchored rules and a further consultation (opened Dec 2025) is open. VBI is effectively closed to solo operators post-reform; FBI structurally disfavors active high-turnover trading; RAIF/QIAIF both *require* an external authorized AIFM and are uneconomical below ~€20–30m. So Stage 2 is a *later* decision, timed to land after the FGR rules settle.
|
||||
|
||||
## 5. What to take to a Dutch tax advisor / notaris
|
||||
|
||||
**Classification (highest stakes):**
|
||||
1. Given a fully agentic, continuously-running systematic strategy across IBKR UCITS ETFs + Bybit perps — is this **Box 3 or Box 1** (resultaat uit overige werkzaamheden / winst uit onderneming)? Reconcile the taxpayer-favorable *Gelderland/Noord-Holland* daytrading line against the *GHARL:2024:4613* bot line that landed in Box 1. **Is a vooroverleg / advance ruling advisable before scaling capital**, given the retroactive-assessment downside if reclassified after the fact?
|
||||
|
||||
**Entity & existing structure:**
|
||||
2. Is **bizworx.nl a BV or an eenmanszaak?** This determines whether housing trading is a fresh incorporation or a restructuring (eenmanszaak→BV has geruisloze inbreng considerations; a fiscale eenheid question if a second BV is created).
|
||||
3. If moving personally-held IBKR/Bybit positions into a BV — does that trigger a **deemed-disposal (afrekening)** for Box 3, and is any rollover/deferral available for a trading portfolio?
|
||||
4. **gebruikelijk loon**: does running an autonomous/agentic system (software does most of the "work") qualify for the ~€5k limited-activity carve-out, or does the algorithm-design labor count against it?
|
||||
|
||||
**Accounting method & FX:**
|
||||
5. **Cost-basis method** — FIFO or weighted-average, and must it be identical across IBKR ETF lots and Bybit crypto lots (goedkoopmansgebruik consistency)?
|
||||
6. **Unrealized FX/crypto revaluation** — book to P&L each period (RJ122/IAS21) for internal reporting while the tax return recognizes only realized (goedkoopmansgebruik; the NLFiscaal unrealized-FX-loss-denial precedent)? Do I need two parallel FX-recognition tracks (book vs tax)?
|
||||
7. **Functional-currency election** (EUR vs USD) — worth the 10-year lock-in, or is "EUR presentation + translate every Bybit leg at transaction-date ECB rate" sufficient without electing?
|
||||
8. **USDT** — treated as USD-equivalent, or its own crypto-asset (own EUR cross) for both ongoing bookkeeping and the Box 3 peildatum?
|
||||
|
||||
**Instrument-specific (a genuine gap in the research):**
|
||||
9. How are **perpetual-futures funding payments** characterized for Box 3 werkelijk-rendement — as income each period, or folded into position P&L? No authoritative Belastingdienst guidance was found.
|
||||
10. Under werkelijk rendement, are Bybit **trading fees / funding paid / margin costs non-deductible** (as the tegenbewijsregeling mechanics suggest, unlike normal trading P&L)?
|
||||
|
||||
**Reform & reporting:**
|
||||
11. Confirm the **finalized 2026 figures** at filing time (heffingsvrij vermogen — sources conflicted between ~€51,396 and ~€59,357 — forfait %, Box 2 thresholds).
|
||||
12. Does the fund trigger a formal **boekhoudplicht** (BW 2:10 / 3:15i — 7-year retention, audit-trail format), and does the Postgres pipeline satisfy it?
|
||||
13. **BTW/VAT** — confirm own-account securities/crypto trading is exempt, and re-check if a management fee is later introduced.
|
||||
|
||||
**Regulatory (Wft lawyer, not just belastingadviseur):**
|
||||
14. Does the IBKR order/message rate meet MiFID II's **HFT** technical threshold (removes the own-account exemption)? Needs a real order-flow review.
|
||||
15. **Bybit EU migration — operationally urgent:** the global platform closes EEA/NL access ~1 July 2026; you migrate to the Austria-MiCA "Bybit EU" entity (fresh KYC, Travel Rule proof-of-wallet for transfers >€1,000). **Bybit EU reportedly cannot offer USDT** (Tether has no MiCA EMT authorization), so USDT balances must convert to EUR or a MiCA stablecoin — **is that forced conversion a taxable disposal, and under which box?** Verify directly on Bybit's compliance pages before the deadline.
|
||||
|
||||
## 6. Immediate next build step
|
||||
|
||||
**Build the append-only double-entry ledger core + the IBKR Flex ingestion asset first — one Dagster asset landing IBKR Flex trades/cash/NAV into a `raw` schema, normalizing into `journal_entries`/`journal_postings` with the balanced-entry DB trigger, and joining an ECB `fx_rates` table by value_date to produce EUR-normalized postings.**
|
||||
|
||||
Why this one component first, over everything else:
|
||||
- IBKR is your EUR-native, better-documented venue with a stable Flex Web Service API and an existing importer pattern to model on — lowest integration risk, so you validate the schema and the balanced-entry invariant against real data before adding Bybit's messier surface (funding fees, USDT proxy, migration churn).
|
||||
- It exercises **every** load-bearing design decision at once: append-only immutability, the EUR-sum-to-zero constraint, the strategy_tag dimension, and the ECB-rate join — the FX-revaluation split and cost-basis lot tables extend cleanly on top once this spine is proven.
|
||||
- It's a natural Dagster asset alongside your existing nightly ETL, reusing your monitoring/DB-anchor-SSOT patterns — no new infrastructure, and it turns tax season into a query rather than an annual export scramble regardless of which box you end up in.
|
||||
|
||||
Defer the Bybit importer, the cost-basis lot engine, and any beancount export until this spine is landing correct, reconciled EUR postings from IBKR.
|
||||
29
docs/superpowers/VRP-REMOVAL-TODO.md
Normal file
29
docs/superpowers/VRP-REMOVAL-TODO.md
Normal file
@@ -0,0 +1,29 @@
|
||||
# VRP full removal — DONE (2026-07-20/21)
|
||||
|
||||
**Status: COMPLETE.** VRP code fully removed on branch `chore/remove-vrp-code` (9 commits over base a263b67),
|
||||
full suite green (2018 passed, 2 skipped), whole-branch review clean (no Critical/Important). OPRA data KEPT.
|
||||
|
||||
## What was removed
|
||||
- 7 VRP application modules (`vrp_book/eval/defined_risk_eval/exec_record/marking/pricing/research`) + `black_scholes.py`
|
||||
- The DVOL pipeline (`deribit_dvol.py`, `dvol_ingest.py`, the `VolIndexClient` port) — DVOL was the IV leg of VRP
|
||||
- The `vrp_nav` Dagster asset + its ~14 OPRA-freeze/backfill helper chain
|
||||
- The `XspOptionBarsRepo` **class** + the `_build_vrp` migration builder + the `backfill-xsp-opra`/`ingest-dvol`/
|
||||
`execute-vrp`/`vrp-eval`/`vrp-defined-risk-eval` CLI commands + the `fxhnt-opra-backfill` CronJob manifest
|
||||
- The `vrp`/`vrp_exec` registry entries + `_REF_ALIAS`/`_IBKR_ACCOUNT_SIDS`/`SIM_BOOKS` literals
|
||||
- 13 VRP test files; comment/docstring scrubs across 16 files; test updates (assert vrp-ABSENT / synthetic entries)
|
||||
|
||||
## What was KEPT (Constraint 1 — verified intact)
|
||||
- `xsp_option_bars` **table** + `XspOptionBarRow` model (`cockpit_models.py:436`) + the 13-year OPRA `.dbn` data
|
||||
- `ForwardNavRepo.xsp_freeze_max_date()` (`forward_nav.py:145`) — still reads the kept table
|
||||
- The generic freeze-staleness health axis (`_FREEZE_SOURCES = {"opra-pit"}`) stays
|
||||
- `deribit_funding` (carry edge) untouched
|
||||
|
||||
## CronJob cluster-resource — VERIFIED not present (no action needed)
|
||||
Checked 2026-07-21 on `bizworx-prod-kapsule`: `kubectl get cronjob fxhnt-opra-backfill -n foxhunt` → NotFound,
|
||||
0 hits cluster-wide. The manifest was never applied live (or already gone), so removing it from git was
|
||||
sufficient. The 5 surviving foxhunt CronJobs (ucits/bybit/ibkr rebalancers, ucits-volume, factory) are all
|
||||
non-VRP.
|
||||
|
||||
## Merged + pushed
|
||||
Branch `chore/remove-vrp-code` merged to master (`2f957bb`) and pushed to origin. Live on prod after the next
|
||||
normal git-sync + rollout-restart.
|
||||
480
docs/superpowers/plans/2026-07-13-phase0b-venue-consolidation.md
Normal file
480
docs/superpowers/plans/2026-07-13-phase0b-venue-consolidation.md
Normal file
@@ -0,0 +1,480 @@
|
||||
# Phase 0b — Venue Consolidation to Bybit + IBKR Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development to implement this
|
||||
> plan task-by-task. Steps use checkbox (`- [ ]`) syntax.
|
||||
|
||||
**Goal:** Make Bybit (crypto) + IBKR (tradfi) the only venues for execution AND data — migrate the crypto
|
||||
edges that work on Bybit (xsfunding, unlock) onto Bybit data, retire the ones that don't (crypto_tstrend
|
||||
standalone, stablecoin), and remove Binance + Alpaca entirely.
|
||||
|
||||
**Architecture:** Migrate first (xsfunding/unlock → Bybit standalone tracks, template = `positioning`),
|
||||
then remove (Alpaca exec, Binance exec, Binance data + `features` table, vestigial `state_file`). The
|
||||
`bybit_4edge` deploy book is unchanged (it already runs on Bybit).
|
||||
|
||||
**Tech Stack:** Python 3.12, Dagster assets, SQLAlchemy 2.0, Timescale/Postgres, pytest.
|
||||
|
||||
**Base:** branch off `feat/unified-db-backtest-architecture` (0a, unmerged) as
|
||||
`feat/phase0b-venue-consolidation`. Spec:
|
||||
`docs/superpowers/specs/2026-07-13-phase0b-venue-consolidation-bybit-ibkr.md`.
|
||||
|
||||
## Global Constraints
|
||||
|
||||
- **A crypto strat survives only if it works on Bybit** (instruments listed + edge holds). Decided per edge
|
||||
in the spec — do not re-litigate.
|
||||
- **Real money errs false-WAIT, never false-FUND.** Migrated tracks re-inception on cutover (new Bybit
|
||||
basis) — a VISIBLE deterministic re-inception, never a silent reset. Their Bybit refs must exist before
|
||||
the gate can PASS. The 0a reconciliation invariant test MUST stay green.
|
||||
- **Single source of truth**; no dead code / stubs / TODO; fix every review Minor in-loop.
|
||||
- **Keep the Broker port** with `IBKRBroker` as the sole implementation (do not delete the abstraction).
|
||||
- **`bybit_4edge` deploy book is UNCHANGED** — crypto_tstrend stays a sleeve in the book; 0b retires only its
|
||||
redundant *standalone* track. Do not touch `_DEFAULT_BYBIT_SLEEVES`.
|
||||
- **Binance stays gone**: after migration, nothing (data or exec) may read Binance or the `features` table.
|
||||
- Verify the cockpit LOCALLY in a real browser before finishing (`./scripts/dev-cockpit-local.sh` or a
|
||||
seeded sqlite DB).
|
||||
|
||||
---
|
||||
|
||||
## Task 1: Generic Bybit single-sleeve track builder + migrate `xsfunding` → Bybit
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/adapters/orchestration/assets.py` (add `build_bybit_sleeve_forward_nav`; refactor
|
||||
`build_positioning_forward_nav` to use it; rewrite `xsfunding_nav` asset)
|
||||
- Modify: `src/fxhnt/application/bybit_forward_track.py` (`bybit_report_sid_kwargs`: add `xsfunding`)
|
||||
- Modify: `src/fxhnt/cli.py` (`_BYBIT_BOOK_SIDS`: add `"xsfunding"`)
|
||||
- Modify: `src/fxhnt/registry.py` (`xsfunding` entry: venue + backtest.kind + params)
|
||||
- Test: `tests/unit/test_bybit_sleeve_forward_builder.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `BybitFourEdgeStrategy(store, sleeves=[...], unlock_events=…, positioning_capacity_capital=…,
|
||||
positioning_tail_cap_k=…)`, `forward_engine.run_track(repo, sid, strategy, today, at)`,
|
||||
`TimescaleFeatureStore(dsn, table="bybit_features")`, `liquid_universe(store)`.
|
||||
- Produces: `build_bybit_sleeve_forward_nav(repo, store, sleeve, *, today, at, universe=None, cost_bps=5.5,
|
||||
unlock_events=None, positioning_capacity_capital=None, positioning_tail_cap_k=None) -> dict`.
|
||||
|
||||
- [ ] **Step 1: Write the failing test** — the generic builder runs a single-sleeve Bybit strategy through
|
||||
the deterministic engine and returns a track dict, and `build_positioning_forward_nav` still works via it.
|
||||
|
||||
```python
|
||||
# tests/unit/test_bybit_sleeve_forward_builder.py
|
||||
import datetime as dt
|
||||
from fxhnt.adapters.orchestration.assets import build_bybit_sleeve_forward_nav, build_positioning_forward_nav
|
||||
from fxhnt.adapters.persistence.forward_nav import ForwardNavRepo
|
||||
from fxhnt.adapters.warehouse.timescale_feature_store import TimescaleFeatureStore
|
||||
|
||||
|
||||
def _seed(store, days=40):
|
||||
import math
|
||||
for i in range(6):
|
||||
px, rows, sign = 100.0, [], (1.0 if i % 2 == 0 else -1.0)
|
||||
for d in range(days):
|
||||
px *= 1 + sign * 0.01 + 0.003 * math.cos(0.4 * d + i)
|
||||
rows.append((d * 86400, {"close": px, "funding": 0.001 * sign, "spot_close": px,
|
||||
"long_ratio": 0.5 + 0.1 * sign}))
|
||||
store.write_features(f"SYM{i:02d}USDT", rows)
|
||||
|
||||
|
||||
def test_generic_builder_books_a_single_sleeve_track(tmp_path):
|
||||
repo = ForwardNavRepo(f"sqlite:///{tmp_path/'c.db'}"); repo.migrate()
|
||||
store = TimescaleFeatureStore("sqlite://", table="bybit_features"); _seed(store)
|
||||
out = build_bybit_sleeve_forward_nav(repo, store, "xsfunding",
|
||||
today="1970-02-09", at=dt.datetime(2026, 1, 1), cost_bps=0.0)
|
||||
assert out["days"] >= 1 and "t0" in out
|
||||
store.close()
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run to verify it fails** — `.venv/bin/python -m pytest tests/unit/test_bybit_sleeve_forward_builder.py -q` → FAIL (import error).
|
||||
|
||||
- [ ] **Step 3: Add the generic builder + refactor positioning to use it** (`assets.py`). Insert before
|
||||
`build_positioning_forward_nav`:
|
||||
|
||||
```python
|
||||
def build_bybit_sleeve_forward_nav(repo, store, sleeve, *, today, at, universe=None, cost_bps=5.5,
|
||||
unlock_events=None, positioning_capacity_capital=None,
|
||||
positioning_tail_cap_k=None): # type: ignore[no-untyped-def]
|
||||
"""Plain (Dagster-free, unit-testable) driver for a SINGLE Bybit sleeve as its own forward track: build
|
||||
the one-sleeve `BybitFourEdgeStrategy` over `store` (bybit_features) and run it through the deterministic
|
||||
engine (DB anchor + recompute). The DB (forward_nav + anchor) is the SSOT — no JSON state. Shared by the
|
||||
per-edge Bybit tracks (xsfunding / unlock / positioning)."""
|
||||
from fxhnt.application.bybit_forward_track import BybitFourEdgeStrategy
|
||||
from fxhnt.application.forward_engine import run_track
|
||||
strategy = BybitFourEdgeStrategy(
|
||||
store, universe=universe, cost_bps=cost_bps, sleeves=[sleeve], unlock_events=unlock_events,
|
||||
positioning_capacity_capital=positioning_capacity_capital, positioning_tail_cap_k=positioning_tail_cap_k)
|
||||
return run_track(repo, sleeve, strategy, today=today, at=at)
|
||||
```
|
||||
Then replace `build_positioning_forward_nav`'s body with a thin delegation:
|
||||
```python
|
||||
def build_positioning_forward_nav(repo, store, *, today, at, universe=None, cost_bps=5.5,
|
||||
positioning_capacity_capital=None, positioning_tail_cap_k=None):
|
||||
"""POSITIONING single-sleeve Bybit track — thin wrapper on `build_bybit_sleeve_forward_nav` (keeps the
|
||||
capturability haircuts). See that function."""
|
||||
return build_bybit_sleeve_forward_nav(
|
||||
repo, store, "positioning", today=today, at=at, universe=universe, cost_bps=cost_bps,
|
||||
positioning_capacity_capital=positioning_capacity_capital, positioning_tail_cap_k=positioning_tail_cap_k)
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run — the builder test + the existing positioning tests pass** —
|
||||
`.venv/bin/python -m pytest tests/unit/test_bybit_sleeve_forward_builder.py tests/integration/test_positioning_forward_track.py -q` → PASS.
|
||||
|
||||
- [ ] **Step 5: Rewrite the `xsfunding_nav` asset** (Bybit, mirroring `positioning_nav`). Replace the current
|
||||
Binance body (`XsFundingForward(BinanceXsFundingLive())` + `persist_backtest_ref=True`) with:
|
||||
|
||||
```python
|
||||
@asset(deps=[bybit_warehouse]) # match positioning_nav's freshness dep (use the same bybit warehouse asset dep)
|
||||
def xsfunding_nav(context: AssetExecutionContext) -> dict: # type: ignore[type-arg]
|
||||
"""Paper-forward cross-sectional funding-dispersion — the SINGLE xsfunding sleeve as its own edge, on
|
||||
BYBIT (`bybit_features`), the venue it would trade. Deterministic engine (DB anchor + recompute); its
|
||||
reconciliation ref is written by the weekly Bybit book persist (`_persist_bybit_book`), like positioning."""
|
||||
import datetime as _dt
|
||||
import urllib.error
|
||||
from fxhnt.adapters.persistence.forward_nav import ForwardNavRepo
|
||||
from fxhnt.adapters.warehouse.timescale_feature_store import TimescaleFeatureStore
|
||||
from fxhnt.application.bybit_liquidity import liquid_universe
|
||||
from fxhnt.config import get_settings
|
||||
s = get_settings()
|
||||
store = TimescaleFeatureStore(s.operational_dsn, table="bybit_features")
|
||||
try:
|
||||
universe = liquid_universe(store) or None
|
||||
repo = ForwardNavRepo(s.operational_dsn); repo.migrate()
|
||||
today = _dt.datetime.now(_dt.UTC).strftime("%Y-%m-%d")
|
||||
at = _dt.datetime.now(_dt.UTC).replace(tzinfo=None)
|
||||
out = build_bybit_sleeve_forward_nav(repo, store, "xsfunding", today=today, at=at,
|
||||
universe=universe, cost_bps=s.cost_bps_per_turnover)
|
||||
context.log.info(f"xsfunding_nav: T0={out['t0']}, {out['days']}d through {out['last_date']} "
|
||||
f"(reinception={out['reinception']})")
|
||||
return {"inception": out["t0"], "last_date": out["last_date"], "forward_days": out["days"]}
|
||||
except (urllib.error.URLError, TimeoutError, ConnectionError, OSError) as e:
|
||||
context.log.warning(f"xsfunding_nav step failed (record unchanged): {e}")
|
||||
return {"inception": None, "last_date": None, "forward_days": 0}
|
||||
finally:
|
||||
store.close()
|
||||
```
|
||||
*(Use the SAME `deps=[...]` and the same bybit warehouse-refresh asset that `positioning_nav` depends on —
|
||||
read positioning_nav's decorator and match it exactly; if positioning_nav has no explicit dep, match that.)*
|
||||
|
||||
- [ ] **Step 6: Wire xsfunding's Bybit ref.** (a) `bybit_forward_track.py` `bybit_report_sid_kwargs`: add
|
||||
`if strategy_id == "xsfunding": return {"sleeves": ["xsfunding"], "strategy_id": "xsfunding"}`. (b)
|
||||
`cli.py`: `_BYBIT_BOOK_SIDS = ("bybit_4edge", "bybit_4edge_levered", "positioning", "xsfunding")`. (c)
|
||||
`registry.py` `xsfunding`: set `"venue": "bybit-perp"`, and `definition.backtest.kind` from `"sleeve"` →
|
||||
`"report"` (its ref now comes from the Bybit book persist, like positioning); update
|
||||
`definition.params` to the single-sleeve Bybit construction (mirror `positioning`'s params shape).
|
||||
|
||||
- [ ] **Step 7: Update the invariant test expectation** — `tests/unit/test_backtest_invariant.py` already
|
||||
enumerates the registry; confirm it still passes (xsfunding is now `report`-kind, non-`none`, and has a
|
||||
provider via `_BYBIT_BOOK_SIDS`). Run `.venv/bin/python -m pytest tests/unit/test_backtest_invariant.py -q`.
|
||||
|
||||
- [ ] **Step 8: Run the affected suites + commit** —
|
||||
`.venv/bin/python -m pytest tests/unit/test_bybit_sleeve_forward_builder.py tests/unit/test_backtest_invariant.py tests/integration/test_bybit_persist_sleeve_ret_cli.py -q` → PASS, then:
|
||||
```bash
|
||||
git add src/fxhnt/adapters/orchestration/assets.py src/fxhnt/application/bybit_forward_track.py src/fxhnt/cli.py src/fxhnt/registry.py tests/unit/test_bybit_sleeve_forward_builder.py
|
||||
git commit -m "feat(0b): migrate xsfunding to a Bybit single-sleeve forward track (venue-consistent ref)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 2: Migrate `unlock` → Bybit (reuse the builder; carry the unlock calendar)
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/adapters/orchestration/assets.py` (rewrite `unlock_nav`)
|
||||
- Modify: `src/fxhnt/application/bybit_forward_track.py` (`bybit_report_sid_kwargs`: add `unlock`)
|
||||
- Modify: `src/fxhnt/cli.py` (`_BYBIT_BOOK_SIDS`: add `"unlock"`)
|
||||
- Modify: `src/fxhnt/registry.py` (`unlock` entry)
|
||||
- Test: extend `tests/unit/test_bybit_sleeve_forward_builder.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `build_bybit_sleeve_forward_nav` (Task 1), `load_unlock_events(operational_unlock_repo(s), path)`,
|
||||
`_data_dir()`.
|
||||
|
||||
- [ ] **Step 1: Add a failing test** that the builder threads `unlock_events` into the unlock sleeve
|
||||
(monkeypatch `BybitFourEdgeStrategy` to capture kwargs, assert `sleeves==["unlock"]` and
|
||||
`unlock_events` passed through). Run → FAIL.
|
||||
|
||||
- [ ] **Step 2: Rewrite the `unlock_nav` asset** — Bybit single-sleeve via `build_bybit_sleeve_forward_nav`
|
||||
with `sleeve="unlock"`, loading the calendar the SAME way the Bybit book does
|
||||
(`unlock_events = load_unlock_events(operational_unlock_repo(s), f"{_data_dir()}/unlock_calendar.json")`,
|
||||
`[]` on failure — never fabricate), store = `TimescaleFeatureStore(dsn, table="bybit_features")`, NO
|
||||
`persist_backtest_ref` (ref comes from `_persist_bybit_book`). Mirror the `xsfunding_nav` shape from Task 1.
|
||||
|
||||
- [ ] **Step 3: Wire unlock's Bybit ref** — `bybit_report_sid_kwargs`: add
|
||||
`if strategy_id == "unlock": return {"sleeves": ["unlock"], "strategy_id": "unlock"}`;
|
||||
`_BYBIT_BOOK_SIDS += ("unlock",)`; `registry.py` `unlock`: `venue` → `bybit-perp`, `backtest.kind`
|
||||
`recompute-replay` → `report`, params → single-sleeve Bybit construction.
|
||||
|
||||
- [ ] **Step 4: Confirm `_persist_bybit_book` writes both new refs** — it loops `_BYBIT_BOOK_SIDS`, which now
|
||||
includes xsfunding + unlock; `bybit_4edge_backtest_summary(sleeves=["unlock"], strategy_id="unlock",
|
||||
sleeve_returns=sleeve_rets)` reuses the already-computed sleeve series (unlock is in `_DEFAULT_BYBIT_SLEEVES`,
|
||||
so `sleeve_rets["unlock"]` exists — verify). No extra compute.
|
||||
|
||||
- [ ] **Step 5: Run + commit** —
|
||||
`.venv/bin/python -m pytest tests/unit/test_bybit_sleeve_forward_builder.py tests/unit/test_backtest_invariant.py -q` → PASS, then commit:
|
||||
```bash
|
||||
git commit -am "feat(0b): migrate unlock to a Bybit single-sleeve forward track"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 3: Align the Bybit reconciliation cost basis (single-source 5.5)
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/config.py` (add `bybit_cost_bps: float = 5.5`)
|
||||
- Modify: `src/fxhnt/adapters/orchestration/assets.py` (`positioning_nav`, `xsfunding_nav`, `unlock_nav`: pass `cost_bps=s.bybit_cost_bps`, NOT `s.cost_bps_per_turnover`)
|
||||
- Modify: `src/fxhnt/adapters/orchestration/migration_builders.py` (the Bybit-sleeve recompute builders: `cost_bps=settings.bybit_cost_bps`)
|
||||
- Modify: `src/fxhnt/cli.py` (`_persist_bybit_book` + `bybit-persist-sleeve-ret`/`backtest-refs` `--cost-bps`: default from `settings.bybit_cost_bps`)
|
||||
- Modify: `src/fxhnt/registry.py` (the Bybit-sleeve tracks' `params.cost_bps` 10.0→5.5, update the "mirrors Settings.cost_bps_per_turnover" comment → `bybit_cost_bps`, BUMP `definition_version` 1→2 for a clean visible re-inception)
|
||||
- Test: `tests/unit/test_bybit_cost_basis_consistent.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: `Settings.bybit_cost_bps` (= 5.5) — the SINGLE Bybit-perp per-turnover reconciliation cost, read by both the forward tracks AND their `_persist_bybit_book` refs.
|
||||
|
||||
**Why:** the standalone Bybit sleeve tracks charged `cost_bps_per_turnover=10.0` while their reconciliation ref (`_persist_bybit_book`) + the deploy book use 5.5 (the MEASURED liquid-book cost — the "mirage" finding: Sharpe 2.11 at 5.5bp drag). Forward-charged-more-than-ref biases the gate toward WAIT (safe, but a cost artifact could permanently WAIT a real edge). One correct Bybit cost, single-sourced, enforced by an invariant test. `cost_bps_per_turnover` (10) stays for the equity/tradfi tracks (different venue) — do NOT change multistrat.
|
||||
|
||||
- [ ] **Step 1: Write the failing invariant test**
|
||||
|
||||
```python
|
||||
# tests/unit/test_bybit_cost_basis_consistent.py
|
||||
from fxhnt.config import Settings
|
||||
from fxhnt.registry import STRATEGY_REGISTRY
|
||||
|
||||
_BYBIT_SLEEVE_SIDS = ("positioning", "xsfunding", "unlock") # standalone Bybit sleeve tracks
|
||||
|
||||
|
||||
def test_bybit_sleeve_tracks_use_the_single_bybit_cost():
|
||||
c = Settings().bybit_cost_bps
|
||||
assert c == 5.5
|
||||
for sid in _BYBIT_SLEEVE_SIDS:
|
||||
params = STRATEGY_REGISTRY[sid]["definition"]["params"]
|
||||
assert params["cost_bps"] == c, (
|
||||
f"{sid} registry cost_bps={params['cost_bps']} != bybit_cost_bps={c} — "
|
||||
f"track and reconciliation ref would use different costs")
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run to verify it fails** — `.venv/bin/python -m pytest tests/unit/test_bybit_cost_basis_consistent.py -q` → FAIL.
|
||||
|
||||
- [ ] **Step 3: Add `bybit_cost_bps = 5.5` to `Settings`** (config.py), documented as the measured Bybit-perp liquid-book per-turnover reconciliation cost (single source for tracks + refs).
|
||||
|
||||
- [ ] **Step 4: Repoint the tracks + refs + builders to `bybit_cost_bps`.** In `assets.py`, the three Bybit sleeve assets pass `cost_bps=s.bybit_cost_bps`. In `migration_builders.py`, the Bybit-sleeve builders use `settings.bybit_cost_bps`. In `cli.py`, `_persist_bybit_book`'s `cost_bps` default + the `--cost-bps` option defaults resolve from `get_settings().bybit_cost_bps`. VERIFY `bybit_4edge`/`bybit_4edge_levered` forward tracks: if their asset already uses 5.5 they are consistent (leave); if any Bybit forward track passes `cost_bps_per_turnover`, repoint it too — the invariant is "every Bybit forward track and its ref use `bybit_cost_bps`".
|
||||
|
||||
- [ ] **Step 5: Registry — set `cost_bps: 5.5` + bump `version` 1→2** on the Bybit sleeve tracks (positioning/xsfunding/unlock; and `carry` for consistency if present, though dormant), update the mirror comment. The param change re-hashes the definition → deterministic re-inception; the version bump is the documented protocol for a cost change (visible, not silent). Do NOT touch multistrat/equity cost_bps.
|
||||
|
||||
- [ ] **Step 6: Run the invariant + the migration/regression suites** —
|
||||
`.venv/bin/python -m pytest tests/unit/test_bybit_cost_basis_consistent.py tests/unit/test_backtest_invariant.py tests/integration/test_bybit_persist_sleeve_ret_cli.py tests/unit/test_bybit_sleeve_forward_builder.py -q` → PASS.
|
||||
|
||||
- [ ] **Step 7: Commit**
|
||||
|
||||
```bash
|
||||
git commit -am "fix(0b): single-source the Bybit reconciliation cost (bybit_cost_bps=5.5) — tracks + refs now reconcile on one measured cost basis"
|
||||
```
|
||||
|
||||
Post-deploy note (runbook): the cost change re-inceptions the affected Bybit tracks (version 2) — a VISIBLE deterministic re-inception; include `migrate-forward-anchors` for them.
|
||||
|
||||
## Task 4: Retire `crypto_tstrend` standalone track + `stablecoin_rotation`
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/adapters/orchestration/assets.py` (delete `crypto_tstrend_nav`, `stablecoin_rotation_nav`
|
||||
assets + remove from the Definitions asset list)
|
||||
- Modify: `src/fxhnt/registry.py` (delete `crypto_tstrend`, `stablecoin_rotation` entries)
|
||||
- Delete: `src/fxhnt/application/stablecoin_strategy.py`, `src/fxhnt/application/stablecoin_runner.py` (if no
|
||||
other consumer — grep first)
|
||||
- Test: `tests/unit/test_retired_tracks_gone.py`
|
||||
|
||||
**Interfaces:** none produced. **Consumes:** nothing new.
|
||||
|
||||
- [ ] **Step 1: Verify no other consumer** — `grep -rn "stablecoin_strategy\|stablecoin_runner\|StableRotationForward\|StableReversionRunner\|crypto_tstrend_nav" src/fxhnt/ | grep -v test`. Anything outside the assets/registry above is a blocker — report it. (The `crypto_tstrend` SLEEVE in `_DEFAULT_BYBIT_SLEEVES` / `bybit_book_eval` is the DEPLOY BOOK — it MUST remain; only the standalone `crypto_tstrend_nav` track + registry entry go.)
|
||||
|
||||
- [ ] **Step 2: Write the guard test**:
|
||||
```python
|
||||
# tests/unit/test_retired_tracks_gone.py
|
||||
from fxhnt.registry import STRATEGY_REGISTRY
|
||||
from fxhnt.application.bybit_book_eval import _DEFAULT_BYBIT_SLEEVES
|
||||
|
||||
|
||||
def test_standalone_binance_tracks_retired():
|
||||
assert "crypto_tstrend" not in STRATEGY_REGISTRY # standalone track gone
|
||||
assert "stablecoin_rotation" not in STRATEGY_REGISTRY
|
||||
|
||||
|
||||
def test_tstrend_stays_in_the_deploy_book():
|
||||
assert "crypto_tstrend" in _DEFAULT_BYBIT_SLEEVES # the BOOK sleeve is untouched
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Run to verify it fails** (entries still present) → FAIL.
|
||||
|
||||
- [ ] **Step 4: Delete** the two assets (remove their `@asset` defs + drop them from the `Definitions(assets=[...])` list), the two registry entries, and the two stablecoin modules (if Step 1 clean). Leave `_DEFAULT_BYBIT_SLEEVES` untouched.
|
||||
|
||||
- [ ] **Step 5: Run the guard + full suite slice** —
|
||||
`.venv/bin/python -m pytest tests/unit/test_retired_tracks_gone.py -q && .venv/bin/python -c "from fxhnt.adapters.orchestration.assets import defs; print('assets load')"` → PASS. Fix any now-dangling import.
|
||||
|
||||
- [ ] **Step 6: Commit** — `git commit -am "feat(0b): retire crypto_tstrend standalone track + stablecoin_rotation (not tradeable on Bybit); book sleeve unchanged"`.
|
||||
|
||||
---
|
||||
|
||||
## Task 5: Remove the Alpaca execution surface (keep Broker port, IBKR sole impl)
|
||||
|
||||
**Files:**
|
||||
- Delete: `src/fxhnt/adapters/broker/alpaca.py`, `infra/k8s/jobs/fxhnt-alpaca-rebalancer.yaml`,
|
||||
`infra/k8s/secrets/alpaca-credentials.yaml`
|
||||
- Modify: `src/fxhnt/config.py` (delete `AlpacaSettings` + the `alpaca` field), `src/fxhnt/cli.py` (remove
|
||||
`--broker` option + all `alpaca` branches from `execute-multistrat` (cli.py:257–325) and the peer command
|
||||
at cli.py:206–223 — IBKR implicit)
|
||||
- Test: `tests/unit/test_no_alpaca.py`
|
||||
|
||||
**Interfaces:** the Broker port stays; `IBKRBroker` becomes the only construction path.
|
||||
|
||||
- [ ] **Step 1: Grep the Alpaca surface** — `grep -rn "alpaca\|Alpaca\|AlpacaBroker\|AlpacaSettings" src/fxhnt/ infra/ | grep -v test`. Enumerate every site; each must be deleted or repointed to IBKR.
|
||||
|
||||
- [ ] **Step 2: Guard test**:
|
||||
```python
|
||||
# tests/unit/test_no_alpaca.py
|
||||
import subprocess, pathlib
|
||||
def test_no_alpaca_symbols_in_src():
|
||||
root = pathlib.Path(__file__).parents[1] / "src" / "fxhnt"
|
||||
hits = subprocess.run(["grep", "-rin", "alpaca", str(root)], capture_output=True, text=True).stdout
|
||||
assert not hits.strip(), f"residual Alpaca references:\n{hits}"
|
||||
def test_broker_port_ibkr_only():
|
||||
from fxhnt.adapters.broker.ibkr import IBKRBroker # the sole impl still imports
|
||||
assert IBKRBroker is not None
|
||||
```
|
||||
*(Adjust the IBKRBroker import to its real path — grep `class IBKRBroker`.)*
|
||||
|
||||
- [ ] **Step 3: Run → FAIL** (alpaca refs present).
|
||||
|
||||
- [ ] **Step 4: Delete + rewire** — remove `alpaca.py`, `AlpacaSettings` + `alpaca` field, the `--broker`
|
||||
option and its `alpaca` branches (execute-multistrat + the peer command become IBKR-only; keep the Broker
|
||||
port abstraction and `IBKRBroker`). Delete the two manifests. `kubectl delete cronjob/fxhnt-alpaca-rebalancer
|
||||
secret/alpaca-credentials -n foxhunt` + the superseded `multistrat-rebalancer` CronJob (locate its manifest
|
||||
via `grep -rn multistrat-rebalancer infra/`; delete file + in-cluster object) — record these as MANUAL
|
||||
cluster ops in the commit body (not run by CI).
|
||||
|
||||
- [ ] **Step 5: Run guard + cli load** —
|
||||
`.venv/bin/python -m pytest tests/unit/test_no_alpaca.py -q && .venv/bin/python -c "from fxhnt.cli import app; print('cli OK')"` → PASS.
|
||||
|
||||
- [ ] **Step 6: Commit** — `git commit -am "feat(0b): remove dead Alpaca execution; IBKR is the sole equity executor (Broker port kept)"`.
|
||||
|
||||
---
|
||||
|
||||
## Task 6: Remove the Binance execution surface
|
||||
|
||||
**Files:**
|
||||
- Delete: `src/fxhnt/adapters/exchange/binance_ccxt.py`
|
||||
- Modify: `src/fxhnt/cli.py` (delete `crypto-rebalance` (4746), `crypto-positions` (4768), `crypto-flatten`
|
||||
(4776) commands), `src/fxhnt/application/crypto_execution.py` (remove Binance coupling — delete the module
|
||||
if Binance was its only backend; grep first), `src/fxhnt/config.py` (remove execution-only `BinanceSettings`
|
||||
fields)
|
||||
- Test: `tests/unit/test_no_binance_execution.py`
|
||||
|
||||
- [ ] **Step 1: Grep** — `grep -rn "binance_ccxt\|BinanceCcxt\|crypto-rebalance\|crypto-positions\|crypto-flatten\|crypto_execution" src/fxhnt/ | grep -v test`. Determine whether `crypto_execution.py` has any non-Binance use (if not → delete the module).
|
||||
|
||||
- [ ] **Step 2: Guard test** asserting the three CLI commands are gone (`crypto-rebalance` / `-positions` /
|
||||
`-flatten` not in `app`'s registered commands) and `binance_ccxt` unimportable. Run → FAIL.
|
||||
|
||||
- [ ] **Step 3: Delete** `binance_ccxt.py`, the three CLI commands, the Binance coupling in
|
||||
`crypto_execution.py` (or the whole module), and the execution-only `BinanceSettings` fields. Keep any
|
||||
Binance *data* settings for now (Task 6 removes them once the ingests are gone).
|
||||
|
||||
- [ ] **Step 4: Run guard + cli load** → PASS. **Step 5: Commit** —
|
||||
`git commit -am "feat(0b): remove dead Binance execution surface (binance_ccxt + crypto-rebalance/positions/flatten CLI)"`.
|
||||
|
||||
---
|
||||
|
||||
## Task 7: Remove Binance DATA + retire the `features` table (depends on Tasks 1,2,4)
|
||||
|
||||
**Files:**
|
||||
- Delete: `src/fxhnt/adapters/exchange/binance_funding_history.py`,
|
||||
`src/fxhnt/adapters/data/binance_spot_history.py`, `src/fxhnt/adapters/data/binance_xsfunding_live.py`
|
||||
- Modify: `src/fxhnt/adapters/orchestration/assets.py` (delete `crypto_bars` (263), `crypto_funding` (322),
|
||||
`crypto_spot` (345) ingest assets + remove from the Definitions list), `src/fxhnt/application/bybit_edges_eval.py`
|
||||
(remove/repoint its Binance imports), `src/fxhnt/config.py` (remove the now-unused Binance data settings +
|
||||
`warehouse_table="features"` default if `features` is dropped — confirm bybit paths hardcode `bybit_features`)
|
||||
- Test: `tests/unit/test_no_binance_data.py`
|
||||
|
||||
**Interfaces:** none produced. This is the "Binance fully gone" guard. Depends on the crypto migration (Tasks 1,2) + retirement (Task 4).
|
||||
|
||||
- [ ] **Step 1: Prove nothing live reads `features` / Binance** — `grep -rn "BinanceFundingHistory\|BinanceSpotHistory\|BinanceXsFundingLive\|binance_funding_history\|binance_spot_history\|binance_xsfunding_live\|crypto_bars\|crypto_funding\|crypto_spot" src/fxhnt/ | grep -v test`. After Tasks 1–3, the only hits should be the definitions being deleted here + `bybit_edges_eval.py` (repoint/remove). Any live *reader* of the `features` table (`get_feature_store` with the default table on a live path) is a BLOCKER — report it. (The bybit book/tracks all pass `table="bybit_features"` explicitly.)
|
||||
|
||||
- [ ] **Step 2: Guard test**:
|
||||
```python
|
||||
# tests/unit/test_no_binance_data.py
|
||||
import subprocess, pathlib
|
||||
def test_no_binance_symbols_in_src():
|
||||
root = pathlib.Path(__file__).parents[1] / "src" / "fxhnt"
|
||||
hits = subprocess.run(["grep", "-rin", "binance", str(root)], capture_output=True, text=True).stdout
|
||||
assert not hits.strip(), f"residual Binance references:\n{hits}"
|
||||
```
|
||||
- [ ] **Step 3: Run → FAIL.** **Step 4:** Delete the three adapters + three ingest assets (+ Definitions
|
||||
entries) + `bybit_edges_eval.py` Binance imports + the Binance data settings. If the `features` table is
|
||||
now unreferenced, drop its default (`warehouse_table` becomes `bybit_features`, or the setting is removed if
|
||||
every consumer hardcodes the table) — confirm no path relied on the `features` default. Leave a one-time
|
||||
`DROP TABLE features` as a documented MANUAL cluster op in the commit body (not auto-run).
|
||||
|
||||
- [ ] **Step 5: Run guard + full suite** —
|
||||
`.venv/bin/python -m pytest tests/unit/test_no_binance_data.py -q && .venv/bin/python -m pytest tests/ -q` → PASS (full suite green — this is the venue-cutover checkpoint).
|
||||
|
||||
- [ ] **Step 6: Commit** — `git commit -am "feat(0b): remove Binance data (adapters + ingests) + retire the features table — crypto is 100% Bybit"`.
|
||||
|
||||
---
|
||||
|
||||
## Task 8: Remove the vestigial registry `state_file` field
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/registry.py` (delete `state_file` from all entries + the header comment lines 3,5)
|
||||
- Modify: `src/fxhnt/adapters/orchestration/assets.py` (`_run_paper_tracker` (401): drop the `state_file`
|
||||
param + all call sites)
|
||||
- Investigate: `src/fxhnt/cli.py:470` (forward-track CLI `(state_file, build_strategy)` map),
|
||||
`src/fxhnt/adapters/orchestration/migration_builders.py` (one-time migration seed)
|
||||
- Test: `tests/unit/test_no_registry_state_file.py`
|
||||
|
||||
- [ ] **Step 1: Determine migration completeness + CLI dependency.** `grep -rn "state_file" src/fxhnt/ | grep -v test`. Two non-registry consumers: (a) `migration_builders.py` (the ONE-TIME deterministic-forward-state seed — if every track is already on a DB anchor in prod, this is fully vestigial → delete the module); (b) `cli.py:470` forward-track CLI. Read cli.py:470 — if the CLI recomputes from the DB anchor (0a SSOT) and only needs a `ForwardTracker` PATH it constructs itself, the registry `state_file` is not needed there; if it reads `entry["state_file"]`, move that basename into the CLI locally (or derive `f"{sid}_state"`) so removing the registry field doesn't break it. Decide per finding; record it in the commit body.
|
||||
|
||||
- [ ] **Step 2: Guard test**:
|
||||
```python
|
||||
# tests/unit/test_no_registry_state_file.py
|
||||
from fxhnt.registry import STRATEGY_REGISTRY
|
||||
def test_no_state_file_field():
|
||||
assert all("state_file" not in e for e in STRATEGY_REGISTRY.values())
|
||||
```
|
||||
- [ ] **Step 3: Run → FAIL.** **Step 4:** Remove the `state_file` field from every entry + the header
|
||||
comment; drop the `state_file` param from `_run_paper_tracker` and all call sites; apply the Step-1
|
||||
decision for the CLI + `migration_builders.py` (delete the module if the migration is complete, else keep
|
||||
it self-contained). The JSON `ForwardTracker` + its `.bak` crash-recovery stay (used by `fxhnt forward-track`).
|
||||
|
||||
- [ ] **Step 5: Run guard + forward suites** —
|
||||
`.venv/bin/python -m pytest tests/unit/test_no_registry_state_file.py tests/integration/test_forward_gate_e2e.py -q && .venv/bin/python -c "from fxhnt.cli import app; print('cli OK')"` → PASS.
|
||||
|
||||
- [ ] **Step 6: Commit** — `git commit -am "feat(0b): remove vestigial registry state_file field (DB anchor is the forward SSOT)"`.
|
||||
|
||||
---
|
||||
|
||||
## Task 9: Cockpit-local verify + whole-branch review
|
||||
|
||||
**Files:** none (verification task).
|
||||
|
||||
- [ ] **Step 1: Local cockpit verify (real browser).** Seed a sqlite cockpit DB (or use
|
||||
`./scripts/dev-cockpit-local.sh` against prod, read-only) and confirm in a browser: `xsfunding` and
|
||||
`unlock` strategy pages render as **Bybit** tracks (venue `bybit-perp`) with their reconciliation gates;
|
||||
`crypto_tstrend` and `stablecoin_rotation` detail pages are GONE (404, not a 500); the `bybit_4edge`
|
||||
deploy book + `/paper` bybit view are unchanged. Read the actual rendered values — do not claim off logs.
|
||||
|
||||
- [ ] **Step 2: Full suite green** — `.venv/bin/python -m pytest tests/ -q` → all pass. Confirm the 0a
|
||||
reconciliation invariant + the four new guard tests (retired-tracks, no-alpaca, no-binance-data,
|
||||
no-state-file) are included and green.
|
||||
|
||||
- [ ] **Step 3: Whole-branch review** — dispatch the final code-reviewer over
|
||||
`git merge-base feat/unified-db-backtest-architecture HEAD`..HEAD with the spec's Global Constraints as the
|
||||
lens (crypto 100% Bybit; no Binance/Alpaca; Broker port kept; book unchanged; deterministic re-inception;
|
||||
invariant green). Fix Critical/Important in one fix wave; record Minors and fix them in-loop.
|
||||
|
||||
- [ ] **Step 4: Finish** — use superpowers:finishing-a-development-branch (verify tests → present merge/PR
|
||||
options). Note the post-deploy runbook in the PR body: the `migrate-forward-anchors` step for the
|
||||
re-inceptioned xsfunding/unlock tracks + the two MANUAL cluster ops (drop `features` table; delete the
|
||||
Alpaca/multistrat CronJobs + `alpaca-credentials` secret).
|
||||
|
||||
---
|
||||
|
||||
## Self-Review notes
|
||||
- **Spec coverage:** A→xsfunding (T1) + unlock (T2); B→retire (T3); C→Alpaca (T4); D→Binance-exec (T5);
|
||||
E→Binance-data (T6); F→state_file (T7); §7 0a-interaction folded into T1/T2 (ref store flips to
|
||||
bybit_features; xsfunding leaves the inline sleeve path); verification+cockpit (T8). All spec sections map.
|
||||
- **Ordering:** migrate (T1–T2) + retire (T3) BEFORE Binance-data removal (T6) — enforced by T6's Step-1
|
||||
block. Exec removals (T4–T5) are independent and can run any time after T1–T3.
|
||||
- **Type consistency:** `build_bybit_sleeve_forward_nav(repo, store, sleeve, *, …) -> dict` used by T1/T2 +
|
||||
positioning wrapper. `_BYBIT_BOOK_SIDS` extended in T1/T2. `bybit_report_sid_kwargs(sid)` extended in T1/T2.
|
||||
289
docs/superpowers/plans/2026-07-14-ibkr-paper-cockpit-parity.md
Normal file
289
docs/superpowers/plans/2026-07-14-ibkr-paper-cockpit-parity.md
Normal file
@@ -0,0 +1,289 @@
|
||||
# IBKR Paper Cockpit Parity — Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Show the IBKR paper strategies' backtest (on the Backtests page) and their REAL DU-paper-account trading across the cockpit — an overview "real trades" line, a consolidated **Paper books** page showing both the crypto (simulated) and IBKR (real) books, and a click-through **holdings** detail — reconciled against the idealized sim ("how closely real trades follow the plan"), in plain language, scaling to multiple executing strategies by adding a row not a redesign.
|
||||
|
||||
**Architecture:** A faithful EXTENSION of the existing exec path. `IbkrBroker` already reads real NLV + positions; we (a) generalize the Backtests curve source from Bybit-only to any `backtest.kind`, (b) capture real fills → the existing empty `exec_fills` table + a new `ibkr_account_nav` snapshot at each `execute-multistrat` run, (c) compute ONE shared "behind/ahead of plan" signal rendered in plain language in three places (overview real-trades line, holdings-detail card, per-strategy breakdown), (d) consolidate `/paper` into a two-book landing (crypto simulated + IBKR real) that clicks through to a holdings detail. Plain-language labels throughout. Paper-only; gate/promotion/allocation untouched; the crypto book's data/behavior untouched.
|
||||
|
||||
**Tech Stack:** Python 3.12, SQLAlchemy 2.0 (offline), FastAPI + server-rendered Jinja templates, TimescaleDB/Postgres, ib-async (IBKR), pytest (`uv run pytest`, xdist `-n auto`).
|
||||
|
||||
## Global Constraints
|
||||
|
||||
- **PLAIN LANGUAGE, NO DEV JARGON** in any user-facing copy. A single `display_name` map (internal id → human
|
||||
name) is the source of every rendered name. Use: "Crypto 4-edge book", "Multi-asset ETF portfolio", "Options
|
||||
income (put spreads)", "60/40 benchmark", "real trades", "0.3% behind plan"/"ahead of plan", "cost to trade
|
||||
0.04%", "building", "not trading — options access pending", "no trades recorded yet". NEVER surface: raw ids
|
||||
(`bybit_4edge`, `multistrat`, `vrp`), the DU account number, `bps` (use %), "sim"/"recompute-replay"/
|
||||
"exec-twin"/"divergence". Keep only the fund's vocabulary (Sharpe, backtest, worst drop).
|
||||
- **Badges:** `REAL PAPER` (blue, `.badge-exec.real`) marks the IBKR real-paper account; `PAPER` (grey,
|
||||
`.badge-exec.paper`) marks the simulated crypto book. Reuse the existing `.badge-exec` + the amber/green `.b`
|
||||
pill vocabulary; do NOT invent a palette.
|
||||
- **Consolidated `/paper`:** one "Paper books" page shows BOTH books (crypto simulated + IBKR real), each
|
||||
clickable → its holdings detail. The Bybit book's underlying data/behavior is UNTOUCHED — only its landing is
|
||||
now shared. (No EU testnet keys → the crypto book stays simulated.)
|
||||
- Paper only — NO live/real-capital path, NO arm step, NO gate/promotion/allocation semantic change (spec "Out").
|
||||
- Capture is BEST-EFFORT + paper-only: a fills/snapshot persistence failure LOGS, never fails the rebalance or blocks execution (mirrors the existing best-effort ref-persist).
|
||||
- No silent degradation: capture failures log loudly; a stalled exec track already surfaces via the health axis. Empty/absent data → clean empty state, NEVER a 500, NEVER a fabricated number.
|
||||
- ONE divergence computation, reused by overview + detail + account (the "one signal, three places" rule).
|
||||
- All data tagged by `strategy_id` so a second executor is a NEW ROW, not a redesign.
|
||||
- DB migrations idempotent + Postgres-safe (no heavy startup DDL — follow `forward_nav`/`strategy_health` create-if-not-exists patterns).
|
||||
- Reuse the existing badge/health palette + `exec_badge` macro; do NOT invent a new palette.
|
||||
- Verify the cockpit LOCALLY in a real browser before deploy (read the rendered numbers).
|
||||
- Commit messages end with `Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>`.
|
||||
|
||||
---
|
||||
|
||||
## File Structure
|
||||
|
||||
| File | Responsibility | Task |
|
||||
|---|---|---|
|
||||
| `src/fxhnt/adapters/web/app.py` | `_SIM_BOOKS`→generalized book list; `_sim_returns` dispatch on `backtest.kind`; consolidated `/paper` both-books route; holdings-detail context | 1,3,4,5 |
|
||||
| `src/fxhnt/application/sim_curves.py` (new) | `sim_returns_for(book, repo, store)` — dispatch a book's honest-cost curve by its registry `backtest.kind` (bybit→`bybit_sleeve_ret`; recompute-replay→precomputed IBKR curve) | 1 |
|
||||
| `src/fxhnt/adapters/persistence/ibkr_account_repo.py` (new) | `IbkrAccountRepo` — migrate + replace/read `ibkr_account_nav` (per-run account snapshot) + read `exec_fills` by strategy | 2 |
|
||||
| `src/fxhnt/adapters/persistence/cockpit_models.py` | new `IbkrAccountNavRow`; add `strategy_id` to `ExecFillRow` | 2 |
|
||||
| `src/fxhnt/application/exec_capture.py` (new) | `capture_fills(trades)→list[FillDTO]`, `persist_execution(repo, strategy_id, rebalance_id, fills, account_snapshot, at)` (best-effort) | 2 |
|
||||
| `src/fxhnt/adapters/broker/ibkr.py` | collect fills from placed trades; expose `account_snapshot()` (nlv/cash/gross/positions) | 2 |
|
||||
| `src/fxhnt/application/multistrat_exec_record.py` | call `persist_execution` in `plan_and_record` (additive, best-effort, paper-only) | 2 |
|
||||
| `src/fxhnt/application/exec_reconcile.py` (new) | `exec_vs_sim(strategy_id, repo)→ReconResult` — THE shared "behind/ahead of plan" signal (cum divergence, median cost-to-trade, fees) | 3 |
|
||||
| `src/fxhnt/application/display_names.py` (new) | `display_name(id)→str` — the SINGLE internal-id→human-name map; every view uses it (plain-language rule) | 3 |
|
||||
| `src/fxhnt/application/dashboard_service.py` | per-strat exec status + `exec_vs_sim` into fleet rows; `paper_books()`; holdings-detail context; per-strategy breakdown | 3,4,5 |
|
||||
| `src/fxhnt/adapters/web/templates/_macros.html` | `.badge-exec.real` "REAL PAPER" state; `plan_pill(recon)` macro (plain words, no "vs sim"/bps) | 3,4 |
|
||||
| `src/fxhnt/adapters/web/templates/strategy.html` | holdings detail: value-over-time (real vs plan), "what it's holding" table, recent trades, per-strategy breakdown | 3,5 |
|
||||
| `src/fxhnt/adapters/web/templates/cockpit.html` | overview "real trades" sub-row + "behind plan" pill + REAL PAPER badge | 4 |
|
||||
| `src/fxhnt/adapters/web/templates/paper.html` | consolidated `/paper` "Paper books" — BOTH books as clickable cards | 5 |
|
||||
|
||||
---
|
||||
|
||||
## Task 1: D1 — `/paper/sim` sources the IBKR backtest curve
|
||||
|
||||
**Files:**
|
||||
- Create: `src/fxhnt/application/sim_curves.py`
|
||||
- Create: `tests/unit/test_sim_curves.py`
|
||||
- Modify: `src/fxhnt/adapters/web/app.py:388` (`_SIM_BOOKS`), `:402-413` (`_sim_returns`)
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `registry_backtest_kind(sid)` from `fxhnt.application.backtest_refs`; `PaperRepo.bybit_sleeve_returns()`; the precomputed IBKR recompute-replay curve (persisted like `bybit_sleeve_ret` — see Step 3).
|
||||
- Produces: `sim_returns_for(book: str, *, bybit_returns: Callable[[], dict[str, dict[int,float]]], ibkr_curve: Callable[[str], dict[str, dict[int,float]]]) -> dict[str, dict[int, float]]` — a book's per-sleeve honest-cost return series keyed `{sleeve: {day_index: ret}}`, dispatched by `backtest.kind`.
|
||||
- Produces: `SIM_BOOKS: tuple[str,...] = ("bybit_4edge","bybit_4edge_levered","multistrat","vrp")`.
|
||||
|
||||
- [ ] **Step 1: Write the failing test** (`tests/unit/test_sim_curves.py`)
|
||||
|
||||
```python
|
||||
from fxhnt.application.sim_curves import sim_returns_for, SIM_BOOKS
|
||||
|
||||
def _bybit(): return {"tstrend": {0: 0.0, 1: 0.01}}
|
||||
def _ibkr(book): return {book: {0: 0.0, 1: 0.02}}
|
||||
|
||||
def test_bybit_book_reads_bybit_sleeve_returns():
|
||||
out = sim_returns_for("bybit_4edge", bybit_returns=_bybit, ibkr_curve=_ibkr)
|
||||
assert out == {"tstrend": {0: 0.0, 1: 0.01}}
|
||||
|
||||
def test_ibkr_recompute_replay_book_reads_ibkr_curve():
|
||||
out = sim_returns_for("multistrat", bybit_returns=_bybit, ibkr_curve=_ibkr)
|
||||
assert out == {"multistrat": {0: 0.0, 1: 0.02}}
|
||||
|
||||
def test_sim_books_includes_ibkr():
|
||||
assert "multistrat" in SIM_BOOKS and "bybit_4edge" in SIM_BOOKS
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run test to verify it fails** — `uv run pytest tests/unit/test_sim_curves.py -q` → FAIL (module missing).
|
||||
|
||||
- [ ] **Step 3: Implement `sim_curves.py`.** Dispatch on `registry_backtest_kind(book)`: `report`/bybit sids → `bybit_returns()`; `recompute-replay` sids → `ibkr_curve(book)`. The IBKR curve must be a PRECOMPUTED read (the sim never recomputes on the request path). Persist the recompute-replay curve where the sim reads it cheaply: extend the nightly `_persist_track_backtest_ref` / the `fxhnt-backtest-refs` Job to also write the per-day return series for `multistrat`/`vrp` into a small table (reuse the `bybit_sleeve_ret` shape or a sibling `sim_curve_ret(strategy_id, day_index, ret)`), and have `ibkr_curve(book)` read it. Wire `ibkr_curve` in `app.py` to that repo read (cached like `_sim_returns`).
|
||||
|
||||
```python
|
||||
"""Dispatch a sim book's honest-cost curve by its registry backtest.kind — bybit books read the precomputed
|
||||
bybit_sleeve_ret table; recompute-replay books (multistrat/vrp) read their precomputed IBKR curve. The sim
|
||||
NEVER recomputes an edge on the request path (both sources are nightly-precomputed)."""
|
||||
from __future__ import annotations
|
||||
from typing import Callable
|
||||
from fxhnt.application.backtest_refs import registry_backtest_kind
|
||||
|
||||
SIM_BOOKS: tuple[str, ...] = ("bybit_4edge", "bybit_4edge_levered", "multistrat", "vrp")
|
||||
|
||||
def sim_returns_for(book: str, *, bybit_returns: Callable[[], dict], ibkr_curve: Callable[[str], dict]) -> dict:
|
||||
kind = registry_backtest_kind(book)
|
||||
if kind == "recompute-replay":
|
||||
return ibkr_curve(book)
|
||||
return bybit_returns() # report-kind bybit books (the existing path)
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Wire `app.py`.** Replace `_SIM_BOOKS = (...)` with `from fxhnt.application.sim_curves import SIM_BOOKS` (use `SIM_BOOKS`). Change `_sim_returns(book)` to call `sim_returns_for(book, bybit_returns=_paper_repo().bybit_sleeve_returns, ibkr_curve=_ibkr_sim_curve)` where `_ibkr_sim_curve(book)` reads the precomputed IBKR curve (cached with the same 300s TTL). Keep the TTL cache keyed by book.
|
||||
|
||||
- [ ] **Step 5: Run tests** — `uv run pytest tests/unit/test_sim_curves.py tests/integration/test_bybit_sim_web.py tests/integration/test_paper_sim_web.py -q` → PASS (bybit sim unchanged; new dispatch covered). Then the sim web test asserting `/paper/sim?book=multistrat` returns 200 and renders a curve.
|
||||
|
||||
- [ ] **Step 6: Commit** — `git add -A && git commit -m "feat(D1): /paper/sim sources any book's curve by backtest.kind (adds IBKR multistrat/vrp)"` (+ Co-Authored-By).
|
||||
|
||||
---
|
||||
|
||||
## Task 2: D2.1 — capture the real DU account (fills + NAV snapshot)
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/adapters/persistence/cockpit_models.py` (new `IbkrAccountNavRow`; `ExecFillRow` + `strategy_id`)
|
||||
- Create: `src/fxhnt/adapters/persistence/ibkr_account_repo.py`
|
||||
- Create: `src/fxhnt/application/exec_capture.py`
|
||||
- Modify: `src/fxhnt/adapters/broker/ibkr.py` (fills + `account_snapshot`)
|
||||
- Modify: `src/fxhnt/application/multistrat_exec_record.py:89-97` (call capture)
|
||||
- Create: `tests/integration/test_exec_capture.py`, `tests/unit/test_ibkr_account_repo.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: `IbkrAccountRepo(dsn)` with `.migrate()`, `.replace_snapshot(run_date: str, nlv: float, cash: float, gross: float, positions: dict[str,float], at)`, `.nav_series() -> list[tuple[str,float]]` (date, nlv), `.latest_positions() -> dict[str,float]`, `.fills_for(strategy_id: str, limit: int) -> list[FillRow]`.
|
||||
- Produces: `capture_fills(trades) -> list[FillDTO]` (FillDTO: symbol, side, qty, price, fee, status, shortfall_bps); `persist_execution(repo, *, strategy_id, rebalance_id, fills, snapshot, at)` — best-effort (catches+logs).
|
||||
- Produces: `IbkrBroker.account_snapshot() -> AccountSnapshot(nlv, cash, gross, positions: dict[str,float])`.
|
||||
|
||||
- [ ] **Step 1: Write the failing test** (`tests/unit/test_ibkr_account_repo.py`) — migrate on sqlite, replace a snapshot, read it back:
|
||||
|
||||
```python
|
||||
import datetime as dt
|
||||
from fxhnt.adapters.persistence.ibkr_account_repo import IbkrAccountRepo
|
||||
|
||||
def test_snapshot_roundtrip(tmp_path):
|
||||
repo = IbkrAccountRepo(f"sqlite:///{tmp_path}/t.db"); repo.migrate()
|
||||
at = dt.datetime(2026,7,14,14,35)
|
||||
repo.replace_snapshot("2026-07-14", nlv=1_028_594.0, cash=500.0, gross=1_000_000.0,
|
||||
positions={"SPY": 10.0}, at=at)
|
||||
assert repo.nav_series() == [("2026-07-14", 1_028_594.0)]
|
||||
assert repo.latest_positions() == {"SPY": 10.0}
|
||||
# idempotent replace
|
||||
repo.replace_snapshot("2026-07-14", nlv=1_029_000.0, cash=1.0, gross=1.0, positions={"SPY": 11.0}, at=at)
|
||||
assert repo.nav_series() == [("2026-07-14", 1_029_000.0)]
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run it** → FAIL (module missing).
|
||||
|
||||
- [ ] **Step 3: Implement the model + repo.** In `cockpit_models.py` add:
|
||||
|
||||
```python
|
||||
class IbkrAccountNavRow(CockpitBase):
|
||||
__tablename__ = "ibkr_account_nav"
|
||||
run_date: Mapped[dt.date] = mapped_column(Date, primary_key=True)
|
||||
nlv: Mapped[float] = mapped_column(Float)
|
||||
cash: Mapped[float] = mapped_column(Float)
|
||||
gross: Mapped[float] = mapped_column(Float)
|
||||
positions_json: Mapped[str] = mapped_column(Text)
|
||||
at: Mapped[dt.datetime] = mapped_column(DateTime)
|
||||
```
|
||||
|
||||
Add `strategy_id: Mapped[str] = mapped_column(String(64), index=True, default="")` to `ExecFillRow` (tag fills per strategy; default "" is back-compat). `IbkrAccountRepo` follows the `ForwardNavRepo` pattern: `migrate()` = `CockpitBase.metadata.create_all(engine)` (checkfirst — no ALTER, no heavy DDL) + a guarded idempotent `ADD COLUMN strategy_id` on `exec_fills` for pre-existing Postgres tables (mirror the paper_repo ALTER pattern, Postgres-only). `replace_snapshot` = delete-then-insert the run_date row; positions stored as `json.dumps`.
|
||||
|
||||
- [ ] **Step 4: Run the repo test** → PASS.
|
||||
|
||||
- [ ] **Step 5: Broker + capture (failing test first)** (`tests/integration/test_exec_capture.py`): a fake trade with `.fills` (each fill: contract.symbol, execution.side/shares/price/avgPrice, commissionReport.commission) → `capture_fills` returns FillDTOs with the right fields; `persist_execution` writes them to `exec_fills` tagged `strategy_id="multistrat"` and the snapshot to `ibkr_account_nav`; a repo that RAISES on write → `persist_execution` logs and does NOT raise (best-effort). Then implement `capture_fills` (map ib-async trade fills → FillDTO, `shortfall_bps` = (fill.price − intended)/intended·1e4 when the intended price is available, else 0.0) and `persist_execution` (wrap all writes in try/except → log.warning, never raise). Add `IbkrBroker.account_snapshot()` reading `accountSummary()` + `positions()` (reuse the existing `_SUMMARY_TAGS` read at ibkr.py:39-40) and make the placed-order path return the trades so fills are collectible.
|
||||
|
||||
- [ ] **Step 6: Wire `plan_and_record`** (multistrat_exec_record.py:89-97): after `record_exec_track(...)`, when `account_is_paper` and not blocked, call `persist_execution(account_repo, strategy_id="multistrat", rebalance_id=<plan.rebalance_id or today>, fills=capture_fills(plan.trades), snapshot=broker.account_snapshot(), at=at)`. Best-effort; thread the `account_repo` + broker through (the CLI constructs them — it already builds the broker). Do NOT change the `positions_after` return basis (gate-relevant).
|
||||
|
||||
- [ ] **Step 7: Run tests** — `uv run pytest tests/integration/test_exec_capture.py tests/unit/test_ibkr_account_repo.py tests/unit/test_multistrat_exec_helpers.py -q` → PASS.
|
||||
|
||||
- [ ] **Step 8: Commit** — `feat(D2.1): capture real IBKR fills->exec_fills (strategy-tagged) + ibkr_account_nav snapshot (best-effort, paper-only)`.
|
||||
|
||||
---
|
||||
|
||||
## Task 3: D2.2/2.3 — holdings detail page + the shared "behind plan" signal + display names
|
||||
|
||||
> Also create `src/fxhnt/application/display_names.py` — the SINGLE `display_name(id) -> str` map (internal id → human name) used by EVERY view (per the plain-language global constraint). Test it covers all books/strategies.
|
||||
|
||||
**Files:**
|
||||
- Create: `src/fxhnt/application/exec_reconcile.py`, `tests/unit/test_exec_reconcile.py`
|
||||
- Modify: `src/fxhnt/application/dashboard_service.py` (feed detail), `src/fxhnt/adapters/web/app.py` (detail context), `src/fxhnt/adapters/web/templates/strategy.html`, `_macros.html`
|
||||
- Test: `tests/integration/test_ibkr_account_detail_web.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: `exec_vs_sim(strategy_id: str, *, exec_repo, forward_repo) -> ReconResult` where `ReconResult` = dataclass(has_exec: bool, cum_exec_return: float, cum_sim_return: float, divergence: float, median_slippage_bps: float, total_fees: float). THE one shared signal (reused by D3, D4).
|
||||
- Consumes: `IbkrAccountRepo` (Task 2), `ForwardNavRepo` (existing `multistrat` + `multistrat_exec` series).
|
||||
|
||||
- [ ] **Step 1: Failing test** (`tests/unit/test_exec_reconcile.py`): given a `multistrat_exec` executed cum return and a `multistrat` sim cum return + fills with known slippage, `exec_vs_sim` returns the expected `divergence` (= exec − sim) and `median_slippage_bps`; a strategy with NO exec data → `has_exec=False`, zeros. Use fakes for the two repos.
|
||||
|
||||
```python
|
||||
def test_divergence_and_slippage():
|
||||
r = exec_vs_sim("multistrat", exec_repo=FakeExec(cum=0.031, fills_bps=[3.0,5.0,4.0], fees=12.0),
|
||||
forward_repo=FakeFwd(exec_cum=0.031, sim_cum=0.038))
|
||||
assert round(r.divergence, 4) == -0.007 # exec 3.1% vs sim 3.8%
|
||||
assert r.median_slippage_bps == 4.0
|
||||
assert r.has_exec is True
|
||||
|
||||
def test_no_exec_data():
|
||||
r = exec_vs_sim("sixtyforty", exec_repo=FakeExec(cum=None, fills_bps=[], fees=0.0),
|
||||
forward_repo=FakeFwd(exec_cum=None, sim_cum=0.05))
|
||||
assert r.has_exec is False
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run it** → FAIL.
|
||||
|
||||
- [ ] **Step 3: Implement `exec_reconcile.py`.** `has_exec` = the `*_exec` forward track has ≥1 record. `divergence` = cum exec − cum sim (both from `forward_summary`/nav). `median_slippage_bps` = median of `exec_fills.shortfall_bps` for the strat. `total_fees` = sum `exec_fills.fee`. All reads via the repos; no compute on unavailable data (return zeros + `has_exec=False`).
|
||||
|
||||
- [ ] **Step 4: Run** → PASS.
|
||||
|
||||
- [ ] **Step 5: `_macros.html` — plain-language macros.** Add `plan_pill(recon)` (renders `has_exec` → the amber/green `.b` pill "{abs(divergence):.1%} behind plan" / "ahead of plan"; `has_exec=False` → nothing) and add the `.badge-exec.real` "REAL PAPER" state. Reuse the existing `.b`/`.badge-exec` classes. NO "vs sim"/"divergence"/`bps` in any rendered string — plain words only.
|
||||
|
||||
- [ ] **Step 6: holdings detail page (`strategy.html` + detail context).** `dashboard_service.detail()` attaches, for the IBKR book/strategy: `holdings` (from `IbkrAccountRepo.latest_positions()` → each asset with `display_name`, a plain "what it is" description, shares, value, %-of-book), `recent_trades` (`fills_for(sid, 20)` → time, asset, action Buy/Sell, shares, price, **cost-to-trade as a %** (not bps), fee), `account_nav` (`nav_series()`), `recon` (`exec_vs_sim(...)`). `strategy.html` renders: a "Following the plan" card + `plan_pill`, the **value-over-time** curve with the plan overlaid (dashed), a **"What it's holding right now"** holdings table, and a **"Recent trades"** table — all plain-language (see the asset→description map: SPY→"US stocks", IEF→"US Treasuries", GLD→"Gold", PDBC→"Commodities", DBMF→"Managed futures"). Empty/no-exec → "no trades recorded yet" (no 500).
|
||||
|
||||
- [ ] **Step 7: Web test** (`test_holdings_detail_web.py`): seed `ibkr_account_nav` + `exec_fills` + the two forward tracks; GET the IBKR book's detail renders the "what it's holding" table (with plain asset descriptions), a recent-trades row with cost-to-trade as a %, the value-over-time curve, and the "behind plan" pill; assert NO raw ids / `bps` / "vs sim" strings appear in the HTML. GET a strat with no exec → 200 with "no trades recorded yet", no 500.
|
||||
|
||||
- [ ] **Step 8: Commit** — `feat(D2.2/2.3): IBKR paper-account section on the strategy detail + shared exec-vs-sim divergence signal`.
|
||||
|
||||
---
|
||||
|
||||
## Task 4: D3 — overview "real trades" sub-line + "behind plan" pill
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/application/dashboard_service.py` (`fleet()`/`overview()` — per-row exec status + rollup), `src/fxhnt/adapters/web/app.py`, `src/fxhnt/adapters/web/templates/cockpit.html`
|
||||
- Test: `tests/integration/test_overview_exec_signal.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `exec_vs_sim` (Task 3), `IbkrAccountRepo` (Task 2).
|
||||
- Produces: `FleetRow` gains `exec_status: str` ("" | "EXEC") and `recon: ReconResult | None`; an `IbkrAccountRollup` (total_nav, n_executing, agg_divergence) on the overview model.
|
||||
|
||||
- [ ] **Step 1: Failing test** (`test_overview_exec_signal.py`): with a seeded executing `multistrat` (exec data present) and a pure-sim strat, `fleet()` marks `multistrat.exec_status == "EXEC"` with a non-None `recon`, and the sim strat has `exec_status == ""`. `overview()` yields an `IbkrAccountRollup` with `total_nav` = the latest `ibkr_account_nav` and `n_executing == 1`.
|
||||
|
||||
- [ ] **Step 2: Run it** → FAIL.
|
||||
|
||||
- [ ] **Step 3: Implement.** In `dashboard_service.fleet()` set `exec_status="EXEC"` + `recon=exec_vs_sim(sid,...)` when the strat has exec data (its `*_exec` track exists AND has records); else "". Build `IbkrAccountRollup` from `IbkrAccountRepo` (`nav_series()[-1]`, count of executing strats, aggregate divergence = mean of per-strat divergences). One computation via `exec_vs_sim` (no duplicate math).
|
||||
|
||||
- [ ] **Step 4: Run** → PASS.
|
||||
|
||||
- [ ] **Step 5: `cockpit.html`** — mark an executing strategy's row with the `REAL PAPER` badge, and render a nested **"real trades"** sub-row (reuse the existing `.exec-tag` + `tr.exrow` style) showing: what actually filled (real return, "N trades · M holdings →" linking to its holdings detail), the `plan_pill(recon)` ("0.3% behind plan"), and a plain "cost to trade X% · no fees" note. Only for `exec_status=="EXEC"`. No separate rollup card (the consolidated Paper-books page is the account home). Plain language only.
|
||||
|
||||
- [ ] **Step 6: Web test** — GET `/` renders the `REAL PAPER` badge + the "real trades" sub-row + the "behind plan" pill on the executing row, NOT on a sim-only row; assert plain language (no raw ids / `bps` / "vs sim"); no exec data anywhere → no sub-row, no 500.
|
||||
|
||||
- [ ] **Step 7: Commit** — `feat(D3): overview real-trades sub-line + behind-plan pill (plain language, REAL PAPER badge)`.
|
||||
|
||||
---
|
||||
|
||||
## Task 5: D4 — consolidated `/paper` "Paper books" page (both books) + per-strategy breakdown
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/adapters/web/templates/paper.html` (the existing `/paper` — make it the two-book landing), `src/fxhnt/adapters/web/app.py` (`/paper` route → both books), `src/fxhnt/application/dashboard_service.py` (`paper_books()` + the IBKR book detail's per-strategy breakdown)
|
||||
- Test: `tests/integration/test_paper_books_web.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `IbkrAccountRepo`, `exec_vs_sim` (per executing strat).
|
||||
- Produces: `dashboard_service.ibkr_account_view() -> IbkrAccountView(total_nav_series, all_positions, recent_fills, per_strategy: list[StratExecRow])` where `StratExecRow` = (strategy_id, display_name, executed_return, divergence, position_count, fill_count).
|
||||
|
||||
- [ ] **Step 1: Failing test** (`test_paper_books_web.py`): GET `/paper` renders BOTH book cards — the crypto book (`PAPER`) and the IBKR account (`REAL PAPER`) — each linking to its detail. `paper_books()` returns both books' summaries. The IBKR book DETAIL: `per_strategy` list with ONE row today; a **scale test** seeds a synthetic SECOND executing strat → `per_strategy` has TWO rows (proves "new row, not redesign"). Assert plain language throughout (no raw ids / `bps`).
|
||||
|
||||
- [ ] **Step 2: Run it** → FAIL.
|
||||
|
||||
- [ ] **Step 3: Implement `paper_books()` + per-strategy breakdown.** `paper_books()` returns both books' summaries (crypto: existing simulated stats; IBKR: `IbkrAccountRepo` account value + N strategies + `plan_pill` data). The `/paper` route renders the two-book `paper.html`. The IBKR book detail's `per_strategy` = for each strat with exec data, a row (human `display_name`, `exec_vs_sim` return + behind/ahead-of-plan, its position/trade counts from strategy-tagged `exec_fills`). All plain-language.
|
||||
|
||||
- [ ] **Step 4: Run** → PASS.
|
||||
|
||||
- [ ] **Step 5: `paper.html` — the consolidated "Paper books" landing.** Render BOTH books as clickable cards: the **Crypto 4-edge book** (`PAPER`, its existing simulated summary — value/since/Sharpe/worst-drop/status + mini curve) and the **IBKR paper account** (`REAL PAPER` — account value / N strategies trading / holdings count / `plan_pill` / mini curve with the plan overlaid). Each card links to that book's holdings detail (Task 3). The IBKR book's DETAIL page (Task 3's `strategy.html`) additionally shows the **per-strategy breakdown** table — one row per executing strategy (human name, real return, `plan_pill`, holdings/trade counts); renders a SET → 1 row today, N later, no layout change; a dimmed "not started yet" row shows the scale path. Plain language; empty account → clean empty state, no 500.
|
||||
|
||||
- [ ] **Step 6: Web test** — the scale test above passes (2-strat fixture → 2 rows); empty account → empty state, no 500.
|
||||
|
||||
- [ ] **Step 7: Commit** — `feat(D4): consolidated /paper Paper-books page (both books) + per-strategy breakdown (scales by row)`.
|
||||
|
||||
---
|
||||
|
||||
## Verify (whole feature)
|
||||
|
||||
- Full suite green FOREGROUND (`uv run pytest -q`) + `from fxhnt.cli import app`.
|
||||
- Deploy via `mode=deploy` (git-sync); the `ibkr_account_nav` table + the `exec_fills.strategy_id` column create/ALTER on startup (idempotent).
|
||||
- Browser-verify LOCALLY (port-forward prod pg → 15432, `create_app`): `/` shows the "real trades" sub-line + "behind plan" pill; `/paper` shows BOTH book cards (crypto `PAPER` + IBKR `REAL PAPER`); clicking the IBKR book shows the holdings + trades + value-over-time + per-strategy breakdown; the Backtests page shows the IBKR tracks' curves. Read the numbers; confirm NO dev jargon / raw ids / `bps` appear anywhere.
|
||||
- After the next `execute-multistrat` run, confirm `exec_fills` + `ibkr_account_nav` populate with real DU-account data.
|
||||
|
||||
## Self-Review
|
||||
|
||||
- **Spec coverage:** D1 (backtests incl. IBKR tracks)→Task1, D2.1→Task2, D2.2/2.3 (holdings detail + behind-plan signal)→Task3, D3 (overview real-trades line)→Task4, D4 (consolidated Paper books)→Task5. One "behind plan" signal (`exec_vs_sim`) used by Task3/4/5. Plain-language rule → `display_names.py` (Task3) + the "no raw ids/bps/jargon" assertions in the Task3/4/5 web tests. Multi-strat scaling → strategy-tagged `exec_fills` (Task2) + the scale test (Task5). ✓
|
||||
- **Placeholder scan:** none — each task has concrete code/tests.
|
||||
- **Type consistency:** `sim_returns_for`, `IbkrAccountRepo`, `capture_fills`/`persist_execution`, `exec_vs_sim`/`ReconResult`, `FleetRow.exec_status`/`recon`, `IbkrAccountView`/`StratExecRow` — names consistent across tasks.
|
||||
@@ -0,0 +1,225 @@
|
||||
# Capacity-Aware Capital Allocator — Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Upgrade the existing B2a allocation engine so it weights edges by inverse-vol (risk-parity, capped), caps each edge at its honest-reference capacity ceiling, and sizes from the honest-reference backtest (not the too-short forward record) — with the forward gate remaining the live-deploy switch.
|
||||
|
||||
**Architecture:** A focused change to `allocation_engine.compute_allocation` (equal-weight → inverse-vol capped) + a per-strategy capacity ceiling cap in `target_capital`, fed by a new provider that maps each deploy strategy to its honest-reference return series + capacity ceiling. `record_allocation` sources returns from that provider instead of `nav_history`. Everything else (forward-gating, cross-venue, funding-decay, killswitch, vol-target, $ conversion) is reused unchanged.
|
||||
|
||||
**Tech Stack:** Python 3.12, numpy (already used in `allocation_engine`), the merged honest-reference modules (`honest_reference`, `honest_gates`), pytest.
|
||||
|
||||
## Global Constraints
|
||||
|
||||
- **Upgrade, not a new system.** Reuse `AllocationPolicy`, `full_history_vol`, `equal_weight_marginal_prune`, `killswitch_active`, `target_capital`, `record_allocation`, `StrategyAllocationRow`, the rebalancers. Do not duplicate.
|
||||
- **Scope = paper.** No change to the `capital_ceiling` ($35k human gate) or going live; no new execution path. Forward-gating (deploy + promoted) stays the live-deploy switch.
|
||||
- **Sizing basis = the honest-reference backtest series**, NOT `nav_history` (8-day forward, fails `min_history_days`). The forward gate is the live-deploy switch only.
|
||||
- **Weighting = inverse-vol (risk-parity), capped at `max_strategy_weight`** (concentration prudence — a young, not-yet-forward-proven fund must not bet on one edge).
|
||||
- **Capacity ceiling per deploy strategy**, from the honest-reference: `bybit_4edge` = book-level (its sleeves, tstrend excluded — retired standalone), `positioning` = its sleeve curve, `multistrat` (IBKR) = a documented high **equity default** ($10M) — do NOT fabricate a crypto number for equity.
|
||||
- `capacity_min_sharpe` default **1.0**; bump `ALLOCATION_POLICY_VERSION` on any policy field change.
|
||||
- Tests run `pytest -o addopts=""` (repo `addopts` injects `-n`/xdist, absent locally). Use `python3`.
|
||||
|
||||
## File Structure
|
||||
|
||||
- `src/fxhnt/application/allocation_engine.py` — add `inverse_vol_weights`, `capacity_ceiling_from_curve`; change `compute_allocation`'s base weights; add the capacity cap to `target_capital`.
|
||||
- `src/fxhnt/application/allocation_honest_inputs.py` — NEW: `honest_allocation_inputs(store, spread_bps, unlock_events, *, deploy_sids, target_aum, ...) -> tuple[dict[str, dict[int,float]], dict[str,float]]` returning (per-strategy honest return series, per-strategy capacity ceiling). The deploy-strategy → honest-reference mapping lives here.
|
||||
- `src/fxhnt/application/allocation_policy.py` — add `capacity_min_sharpe`, `equity_default_ceiling`; bump version.
|
||||
- `src/fxhnt/application/allocation_ingest.py` — `record_allocation` sources returns + ceilings from the provider.
|
||||
- Tests: `tests/unit/test_allocation_engine.py` (extend), `tests/unit/test_allocation_honest_inputs.py`, `tests/integration/test_record_allocation_capacity.py`.
|
||||
|
||||
---
|
||||
|
||||
### Task 1: `inverse_vol_weights` — risk-parity base weights
|
||||
|
||||
**Files:** Modify `src/fxhnt/application/allocation_engine.py`; Test `tests/unit/test_allocation_engine.py`.
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `full_history_vol(returns) -> float` (already in the module).
|
||||
- Produces: `inverse_vol_weights(returns_by_strategy: dict[str, dict[str,float]]) -> dict[str,float]` (∝ 1/full_history_vol, summing to 1; zero-vol strategy → weight 0; all-degenerate → equal weight).
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
```python
|
||||
# tests/unit/test_allocation_engine.py
|
||||
from fxhnt.application.allocation_engine import inverse_vol_weights
|
||||
|
||||
def test_inverse_vol_weights_favour_low_vol_and_sum_to_one():
|
||||
rbs = {"lo": {"1": 0.001, "2": -0.001, "3": 0.001},
|
||||
"hi": {"1": 0.10, "2": -0.10, "3": 0.10}}
|
||||
w = inverse_vol_weights(rbs)
|
||||
assert abs(sum(w.values()) - 1.0) < 1e-9 and w["lo"] > w["hi"]
|
||||
```
|
||||
- [ ] **Step 2: Run** `python3 -m pytest tests/unit/test_allocation_engine.py::test_inverse_vol_weights_favour_low_vol_and_sum_to_one -v -o addopts=""` → FAIL (undefined).
|
||||
- [ ] **Step 3: Implement** (add near `full_history_vol`):
|
||||
```python
|
||||
def inverse_vol_weights(returns_by_strategy: dict[str, dict[str, float]]) -> dict[str, float]:
|
||||
"""Risk-parity base: weight each strategy ∝ 1/full_history_vol, normalized to sum 1. Zero/degenerate-vol
|
||||
strategy → 0; all-degenerate → equal weight. Reuses the ex-ante full-history vol (never trailing)."""
|
||||
inv = {s: (1.0 / v if (v := full_history_vol(r)) > 0.0 else 0.0) for s, r in returns_by_strategy.items()}
|
||||
total = sum(inv.values())
|
||||
if total <= 0.0:
|
||||
n = len(returns_by_strategy)
|
||||
return {s: 1.0 / n for s in returns_by_strategy} if n else {}
|
||||
return {s: v / total for s, v in inv.items()}
|
||||
```
|
||||
- [ ] **Step 4: Run** the test → PASS.
|
||||
- [ ] **Step 5: Commit** `git add -A && git commit -m "feat(alloc): inverse-vol (risk-parity) base weights"`
|
||||
|
||||
---
|
||||
|
||||
### Task 2: `capacity_ceiling_from_curve` — the ceiling from a capacity curve
|
||||
|
||||
**Files:** Modify `src/fxhnt/application/allocation_engine.py`; Test `tests/unit/test_allocation_engine.py`.
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: `capacity_ceiling_from_curve(curve: dict[float,float], min_sharpe: float) -> float | None` — the LARGEST aum whose honest Sharpe ≥ `min_sharpe`; `None` if the curve is empty or the first (smallest) AUM already fails (edge unviable even at the smallest size); the largest tested AUM if it never drops below (unbounded in-range).
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
```python
|
||||
def test_capacity_ceiling_is_last_aum_above_min_sharpe():
|
||||
from fxhnt.application.allocation_engine import capacity_ceiling_from_curve
|
||||
curve = {35_000: 3.5, 100_000: 2.5, 350_000: 0.3, 1_000_000: -2.5} # xsfunding-shaped
|
||||
assert capacity_ceiling_from_curve(curve, 1.0) == 100_000 # last AUM with Sh >= 1.0
|
||||
robust = {35_000: 2.9, 1_000_000: 2.6, 10_000_000: 1.6}
|
||||
assert capacity_ceiling_from_curve(robust, 1.0) == 10_000_000 # never drops -> top of range
|
||||
assert capacity_ceiling_from_curve({35_000: 0.2}, 1.0) is None # unviable even at smallest
|
||||
assert capacity_ceiling_from_curve({}, 1.0) is None
|
||||
```
|
||||
- [ ] **Step 2: Run** → FAIL.
|
||||
- [ ] **Step 3: Implement**
|
||||
```python
|
||||
def capacity_ceiling_from_curve(curve: dict[float, float], min_sharpe: float) -> float | None:
|
||||
"""Largest AUM whose honest Sharpe >= min_sharpe (the capacity ceiling). None if empty or the edge is
|
||||
unviable even at the smallest tested AUM. Report the ceiling next to the curve so the threshold is visible."""
|
||||
ok = [aum for aum in sorted(curve) if curve[aum] >= min_sharpe]
|
||||
return max(ok) if ok else None
|
||||
```
|
||||
- [ ] **Step 4: Run** → PASS.
|
||||
- [ ] **Step 5: Commit** `git add -A && git commit -m "feat(alloc): capacity ceiling from an honest capacity curve"`
|
||||
|
||||
---
|
||||
|
||||
### Task 3: `compute_allocation` uses inverse-vol (capped), not equal-weight
|
||||
|
||||
**Files:** Modify `src/fxhnt/application/allocation_engine.py`; Test `tests/unit/test_allocation_engine.py`.
|
||||
|
||||
**Interfaces:** Consumes `inverse_vol_weights` (Task 1). `compute_allocation` signature unchanged.
|
||||
|
||||
- [ ] **Step 1: Write the failing test** — the survivor base is inverse-vol capped, not 1/n:
|
||||
```python
|
||||
def test_compute_allocation_uses_inverse_vol_base_capped():
|
||||
from fxhnt.application.allocation_engine import compute_allocation
|
||||
from fxhnt.application.allocation_policy import AllocationPolicy
|
||||
# two survivors, very different vol; both pass the (disabled) marginal gate; enough history
|
||||
n = 400
|
||||
rbs = {"lo": {str(i): (0.002 if i % 2 else -0.002) for i in range(n)},
|
||||
"hi": {str(i): (0.05 if i % 2 else -0.05) for i in range(n)}}
|
||||
pol = AllocationPolicy(marginal_gate=False, min_history_days=60, max_strategy_weight=0.9)
|
||||
w = compute_allocation(rbs, pol)
|
||||
assert w["lo"] > w["hi"] # low-vol gets more (was equal under the old code)
|
||||
```
|
||||
- [ ] **Step 2: Run** → FAIL (old code gives equal base → `w["lo"] == w["hi"]`).
|
||||
- [ ] **Step 3: Implement** — in `compute_allocation`, replace the equal-weight `base` line
|
||||
```python
|
||||
base = {s: min(1.0 / n, policy.max_strategy_weight) for s in survivors}
|
||||
```
|
||||
with inverse-vol capped:
|
||||
```python
|
||||
rp = inverse_vol_weights({s: eligible[s] for s in survivors})
|
||||
base = {s: min(rp.get(s, 0.0), policy.max_strategy_weight) for s in survivors}
|
||||
```
|
||||
(Keep everything else — the NO-renormalise cap behavior, killswitch, vol-target K — identical.)
|
||||
- [ ] **Step 4: Run** the new test AND the full existing allocation suite `python3 -m pytest tests/unit/test_allocation_engine.py -v -o addopts=""`. Some existing equal-weight assertions may need updating to the inverse-vol expectation — update them to the correct new values, do NOT weaken. Expected: all pass.
|
||||
- [ ] **Step 5: Commit** `git add -A && git commit -m "feat(alloc): compute_allocation weights by inverse-vol (capped) not equal-weight"`
|
||||
|
||||
---
|
||||
|
||||
### Task 4: Honest-reference allocation inputs provider (series + ceilings per deploy strategy)
|
||||
|
||||
**Files:** Create `src/fxhnt/application/allocation_honest_inputs.py`; Test `tests/unit/test_allocation_honest_inputs.py`.
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `honest_sleeve_returns` (`honest_reference`), `book_returns`/`risk_parity_weights` (`honest_gates`), `sleeve_weights_and_contributions` (`bybit_paper_weights`), `capacity_curve` (`honest_reference`), `capacity_ceiling_from_curve` (Task 2).
|
||||
- Produces: `honest_allocation_inputs(store, spread_bps, unlock_events, *, deploy_sids, target_aum, aums, participation_cap=0.03, impact_k=1.0, min_sharpe=1.0, equity_default_ceiling=10_000_000.0) -> tuple[dict[str, dict[int,float]], dict[str,float]]` → (returns_by_strategy, capacity_ceiling_by_strategy). Pure (store reads only).
|
||||
|
||||
- [ ] **Step 1: Confirm the mapping (read, no code yet).** Read `honest_report.build_honest_report` for how sleeves → honest series + capacity curves are produced; confirm `bybit_4edge`'s sleeves in the registry (`["tstrend","unlock","xsfunding","positioning"]`) and that `tstrend` has no standalone store series (excluded). Note the mapping to write:
|
||||
- `positioning` → its own `honest_sleeve_returns` series + `capacity_ceiling_from_curve` of its capacity curve.
|
||||
- `bybit_4edge` → the risk-parity **book** of its available crypto sleeves (`xsfunding`, `unlock`, `positioning`; `tstrend` excluded) via `book_returns`; ceiling from the book's capacity curve.
|
||||
- `multistrat` (or any non-crypto deploy sid) → `equity_default_ceiling`, series left to the caller's fallback (out of v1 crypto scope; deploy set is currently crypto-only).
|
||||
- [ ] **Step 2: Write the failing test** with a fake `_FeatureStore` (model on `tests/integration/test_honest_reference.py`'s fake store) for `deploy_sids={"positioning","bybit_4edge"}`:
|
||||
```python
|
||||
def test_provider_returns_series_and_ceilings_per_deploy_strategy():
|
||||
from fxhnt.application.allocation_honest_inputs import honest_allocation_inputs
|
||||
store = _fake_store() # supports xsfunding + positioning sleeves
|
||||
aums = [35_000, 100_000, 350_000, 1_000_000, 10_000_000]
|
||||
series, ceilings = honest_allocation_inputs(store, {}, [], deploy_sids={"positioning", "bybit_4edge"},
|
||||
target_aum=35_000, aums=aums)
|
||||
assert set(series) == {"positioning", "bybit_4edge"} and all(len(s) > 1 for s in series.values())
|
||||
assert set(ceilings) == {"positioning", "bybit_4edge"} # each has a ceiling (float or None)
|
||||
```
|
||||
- [ ] **Step 3: Run** → FAIL (module missing).
|
||||
- [ ] **Step 4: Implement** the provider per the Step-1 mapping: for each crypto deploy sid, build the honest series at `target_aum` and the capacity curve across `aums` → ceiling; for a non-crypto sid, ceiling = `equity_default_ceiling` (series omitted, caller falls back). Return `(series, ceilings)`.
|
||||
- [ ] **Step 5: Run** the test → PASS. **Commit** `git add -A && git commit -m "feat(alloc): honest-reference series+ceiling provider per deploy strategy"`
|
||||
|
||||
---
|
||||
|
||||
### Task 5: `record_allocation` sizes from the provider + applies the capacity cap
|
||||
|
||||
**Files:** Modify `src/fxhnt/application/allocation_engine.py` (`target_capital` capacity cap) and `src/fxhnt/application/allocation_ingest.py`; Test `tests/integration/test_record_allocation_capacity.py`.
|
||||
|
||||
**Interfaces:** `target_capital(target_weights, equity, policy, *, capacity_ceiling=None)` — new optional per-strategy dollar ceiling.
|
||||
|
||||
- [ ] **Step 1: Write the failing test** for the capacity cap in `target_capital`:
|
||||
```python
|
||||
def test_target_capital_caps_at_per_strategy_capacity():
|
||||
from fxhnt.application.allocation_engine import target_capital
|
||||
from fxhnt.application.allocation_policy import AllocationPolicy
|
||||
pol = AllocationPolicy(capital_ceiling=1_000_000.0)
|
||||
w = {"cap_limited": 0.5, "robust": 0.5}
|
||||
d = target_capital(w, equity=1_000_000.0, policy=pol,
|
||||
capacity_ceiling={"cap_limited": 100_000.0}) # cap_limited breaks at $100k
|
||||
assert d["cap_limited"] <= 100_000.0 and d["robust"] > d["cap_limited"]
|
||||
```
|
||||
- [ ] **Step 2: Run** → FAIL (no `capacity_ceiling` param).
|
||||
- [ ] **Step 3: Implement** the cap in `target_capital` (apply `min(dollars, capacity_ceiling[s])` per strategy after the existing `capital_ceiling` scale; `None`/missing → no cap). Then wire `record_allocation`: build `returns` and `ceilings` from `honest_allocation_inputs(...)` (target_aum = the equity) instead of `nav_history`, and pass `capacity_ceiling=ceilings` to `target_capital`. Keep the deploy+promoted gating, funding-decay, and killswitch exactly as-is.
|
||||
- [ ] **Step 4: Run** the target_capital test AND an integration test of `record_allocation` against a fake store + a small in-memory DB (model on existing `allocation_ingest` tests if present; else assert the written `StrategyAllocationRow`s respect the ceilings). Expected: PASS.
|
||||
- [ ] **Step 5: Commit** `git add -A && git commit -m "feat(alloc): record_allocation sizes from honest-reference + caps at capacity (spec 4b/4c)"`
|
||||
|
||||
---
|
||||
|
||||
### Task 6: Policy knobs + capacity-ceiling visibility
|
||||
|
||||
**Files:** Modify `src/fxhnt/application/allocation_policy.py`; Modify the honest-report/cockpit display; Test `tests/unit/test_allocation_policy.py`.
|
||||
|
||||
**Interfaces:** `AllocationPolicy` gains `capacity_min_sharpe: float = 1.0` and `equity_default_ceiling: float = 10_000_000.0`; `ALLOCATION_POLICY_VERSION` bumped to 3.
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
```python
|
||||
def test_policy_has_capacity_knobs_and_version_bumped():
|
||||
from fxhnt.application.allocation_policy import AllocationPolicy, ALLOCATION_POLICY_VERSION
|
||||
p = AllocationPolicy()
|
||||
assert p.capacity_min_sharpe == 1.0 and p.equity_default_ceiling == 10_000_000.0
|
||||
assert ALLOCATION_POLICY_VERSION >= 3
|
||||
```
|
||||
- [ ] **Step 2: Run** → FAIL.
|
||||
- [ ] **Step 3: Implement** — add the two fields to `AllocationPolicy`; bump `ALLOCATION_POLICY_VERSION = 3`. Surface the per-strategy capacity ceiling in `build_honest_report`'s output (a `"capacity_ceiling"` per sleeve/book next to its curve) so the threshold choice is visible (spec §8).
|
||||
- [ ] **Step 4: Run** the test + the honest-report test (confirm the new key is present, existing shape intact). Expected: PASS.
|
||||
- [ ] **Step 5: Commit** `git add -A && git commit -m "feat(alloc): capacity policy knobs + ceiling shown next to the curve (spec 6/8)"`
|
||||
|
||||
---
|
||||
|
||||
## Self-Review
|
||||
|
||||
**Spec coverage:** §4a inverse-vol → Tasks 1,3. §4b capacity cap → Tasks 2,5. §4c sizing-from-backtest → Tasks 4,5. §5 granularity (book/sleeve/equity-default) → Task 4. §6 forward-gating → unchanged (reused; Task 5 keeps deploy+promoted). §7 scope (paper, no live) → all tasks read-only-ish (record_allocation writes only the allocation table it already owns; no new execution). §8 ceiling visibility → Task 6. §9 decisions → knobs Task 6.
|
||||
|
||||
**Placeholder scan:** Tasks 1-3,6 have complete code. Task 4 Step 1 is a deliberate READ (the deploy-strategy→honest mapping must be confirmed against the real registry/store before coding — the crux), then Steps 4 gives the mapping to implement; Task 5 Step 3 describes the wiring against the confirmed provider signature. Acceptable — these are integration tasks whose exact store fixtures depend on Task 4's confirmed API, not hidden work.
|
||||
|
||||
**Type consistency:** honest series are `dict[int,float]` (epoch-day keyed, matching `honest_sleeve_returns`); `full_history_vol`/`compute_allocation` take `dict[str,float]` (date-string keyed) — **the provider (Task 4) must return the series in the shape `compute_allocation` expects (str-keyed dates)** or convert at the seam; call this out in Task 4/5 and add an assertion. Weights/ceilings are `dict[str,float]`.
|
||||
|
||||
---
|
||||
|
||||
## Execution Handoff
|
||||
|
||||
Plan complete and saved to `docs/superpowers/plans/2026-07-15-capacity-aware-capital-allocator.md`. Two execution options:
|
||||
1. **Subagent-Driven (recommended)** — fresh subagent per task, review between tasks (this is what caught the honest-reference bugs).
|
||||
2. **Inline Execution** — execute in this session with checkpoints.
|
||||
|
||||
Which approach?
|
||||
@@ -0,0 +1,589 @@
|
||||
# Crypto Honest-Reference Pipeline — Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Build a coin-level capacity-, cost-, and leverage-honest backtest reference for the Bybit crypto sleeves, plus the falsification gates, risk-parity allocation, ex-ante risk layer, and a gated pump-detection candidate edge — as a pure research/report pipeline that does not touch live sizing or the cockpit.
|
||||
|
||||
**Architecture:** Reuse the existing `sleeve_weights_and_contributions(store, sleeve)` → `{day:{symbol:(weight, ret)}}` engine (no edge logic reimplemented). A new `honest_reference.py` applies a per-coin ADV participation cap (from `store.read_panel(symbols, "turnover")`) + square-root market-impact slippage (floored at the coin's `bybit_real_spread`) and re-aggregates to a NET daily return, parameterized by AUM. Gates, risk-parity weights, leverage-DD, and the pump candidate are pure functions on those return series. A CLI report assembles the numbers. Everything is read-only.
|
||||
|
||||
**Tech Stack:** Python 3.12, stdlib `statistics`/`math`, the existing `_FeatureStore` Protocol, `nav_stats.nav_backfill_stats`, pytest. No new dependencies.
|
||||
|
||||
## Global Constraints
|
||||
|
||||
- **Research/validation scope only.** No change to live sizing, the cockpit, `backtest_summary`, or the recon gate. No writes to prod tables.
|
||||
- **Reuse the real sleeve engines** via `sleeve_weights_and_contributions` — never reimplement edge logic.
|
||||
- **Fidelity is a hard stop:** with the cap disabled (AUM→small) and impact off (`impact_k=0`, no spread), the honest series MUST equal `sleeve_returns_from_store(cost_bps=0)` (== `reconciliation_series(...)`), within 1e-9. A mismatch blocks all downstream tasks.
|
||||
- `participation_cap` = **0.03**; AUM curve = **{35_000, 100_000, 350_000, 1_000_000, 10_000_000}**; impact-`k` sensitivity = **{1.0, 2.0, 4.0}**. Single global knobs, never per-sleeve tuned.
|
||||
- Weighting is **risk-parity / inverse-vol on honest vol** — never an in-sample Sharpe grid.
|
||||
- **G4 anti-reactive:** any de-risk rule must beat "do nothing" in an honest A/B or it is excluded. No reactive de-sizing (vol-target / drawdown-trigger) — A/B-rejected 3× historically.
|
||||
- Pure + network-free functions (inputs injected), matching the `bybit_*_eval.py` pattern.
|
||||
- Tests run with `pytest -o addopts=""` in the dev sandbox (the repo `addopts` injects `-n` / xdist which is absent locally).
|
||||
- `unlock` is **not hard-cut**; G1/G2 decide its weight (expected ≈ 0).
|
||||
|
||||
## File Structure
|
||||
|
||||
- `src/fxhnt/application/honest_reference.py` — core: `honest_sleeve_returns`, `capacity_curve`, `spread_floor_from_snapshot`. One responsibility: turn coin weights + turnover + spread into NET return series per AUM.
|
||||
- `src/fxhnt/application/honest_gates.py` — `risk_parity_weights`, `book_returns`, `gate_survives` (G1), `loo_marginal` (G2), `beats_concentration` (G3), `ab_beats_nothing` (G4).
|
||||
- `src/fxhnt/application/honest_leverage.py` — `max_drawdown_at_leverage`, `leverage_ceiling`.
|
||||
- `src/fxhnt/application/pump_detection.py` — `pump_signal`, `pump_sleeve_weights`, the candidate edge #4.
|
||||
- `src/fxhnt/adapters/persistence/bybit_spread.py` — `read_spread_snapshot(dsn) -> dict[str,float]` (reads `bybit_real_spread`, coin→spread_bps). (New tiny reader; `read_panel` covers `turnover` already.)
|
||||
- `bin/fxt`/CLI: `honest-reference-report` command → prints/writes the research table. (Wire into the existing typer app in `src/fxhnt/cli.py`.)
|
||||
- Tests: `tests/integration/test_honest_reference.py`, `tests/unit/test_honest_gates.py`, `tests/unit/test_honest_leverage.py`, `tests/unit/test_pump_detection.py`.
|
||||
|
||||
---
|
||||
|
||||
### Task 1: Honest return core + fidelity invariant (spec §5 0a/0d)
|
||||
|
||||
**Files:**
|
||||
- Create: `src/fxhnt/application/honest_reference.py`
|
||||
- Test: `tests/integration/test_honest_reference.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `sleeve_weights_and_contributions(store, sleeve, *, universe, unlock_events) -> dict[int, dict[str, tuple[float,float]]]` and `reconciliation_series(weights_by_day) -> dict[int,float]` (both `application/bybit_paper_weights.py`); `sleeve_returns_from_store(store, edge, cost_bps=0.0)` (`application/bybit_book_eval.py`).
|
||||
- Produces: `honest_sleeve_returns(weights_by_day, turnover_panel, spread_bps, *, aum, participation_cap=0.03, impact_k=1.0) -> dict[int,float]`.
|
||||
|
||||
- [ ] **Step 1: Write the failing fidelity test**
|
||||
|
||||
```python
|
||||
# tests/integration/test_honest_reference.py
|
||||
from fxhnt.application.honest_reference import honest_sleeve_returns
|
||||
|
||||
def test_no_cap_no_impact_reproduces_gross_reconciliation():
|
||||
# {day: {symbol: (weight, ret)}} — the exact shape sleeve_weights_and_contributions returns.
|
||||
wbd = {1: {"AAA": (0.5, 0.02), "BBB": (-0.5, -0.01)},
|
||||
2: {"AAA": (0.4, -0.03), "BBB": (-0.6, 0.05)}}
|
||||
turnover = {"AAA": {1: 1e9, 2: 1e9}, "BBB": {1: 1e9, 2: 1e9}} # huge ADV -> cap never binds
|
||||
# cap disabled via tiny AUM relative to ADV, impact off -> must equal Σ w·r per day
|
||||
out = honest_sleeve_returns(wbd, turnover, spread_bps={}, aum=1_000.0,
|
||||
participation_cap=0.03, impact_k=0.0)
|
||||
assert abs(out[1] - (0.5*0.02 + -0.5*-0.01)) < 1e-9
|
||||
assert abs(out[2] - (0.4*-0.03 + -0.6*0.05)) < 1e-9
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `pytest tests/integration/test_honest_reference.py::test_no_cap_no_impact_reproduces_gross_reconciliation -v -o addopts=""`
|
||||
Expected: FAIL — `ModuleNotFoundError: fxhnt.application.honest_reference`.
|
||||
|
||||
- [ ] **Step 3: Write minimal implementation**
|
||||
|
||||
```python
|
||||
# src/fxhnt/application/honest_reference.py
|
||||
"""Capacity- and cost-honest daily return series for a Bybit sleeve, from coin-level weights.
|
||||
|
||||
Pure + network-free: coin weights (sleeve_weights_and_contributions), the per-coin turnover (ADV) panel,
|
||||
and the per-coin quoted spread are INJECTED. With the cap non-binding (huge ADV vs AUM) and impact off
|
||||
(impact_k=0, no spread) this reduces to Σ_symbol weight·ret == reconciliation_series (the fidelity invariant)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
|
||||
|
||||
def honest_sleeve_returns(
|
||||
weights_by_day: dict[int, dict[str, tuple[float, float]]],
|
||||
turnover_panel: dict[str, dict[int, float]],
|
||||
spread_bps: dict[str, float],
|
||||
*, aum: float, participation_cap: float = 0.03, impact_k: float = 1.0,
|
||||
) -> dict[int, float]:
|
||||
out: dict[int, float] = {}
|
||||
for day, row in weights_by_day.items():
|
||||
gross = 0.0
|
||||
cost = 0.0
|
||||
for sym, (w, r) in row.items():
|
||||
adv = turnover_panel.get(sym, {}).get(day, 0.0)
|
||||
target_notional = abs(w) * aum
|
||||
# capacity: never hold more than participation_cap * ADV; scale the weight down if we would.
|
||||
if adv > 0.0 and target_notional > participation_cap * adv:
|
||||
scale = (participation_cap * adv) / target_notional
|
||||
else:
|
||||
scale = 1.0
|
||||
w_capped = w * scale
|
||||
gross += w_capped * r
|
||||
# slippage on the traded notional (approx daily full rebalance == |w_capped|·aum).
|
||||
traded_notional = abs(w_capped) * aum
|
||||
if traded_notional > 0.0 and adv > 0.0:
|
||||
impact = impact_k * math.sqrt(traded_notional / adv) * 1e4 # bps
|
||||
else:
|
||||
impact = 0.0
|
||||
slip_bps = max(impact, spread_bps.get(sym, 0.0))
|
||||
cost += abs(w_capped) * (slip_bps / 1e4)
|
||||
out[day] = gross - cost
|
||||
return out
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run test to verify it passes**
|
||||
|
||||
Run: `pytest tests/integration/test_honest_reference.py::test_no_cap_no_impact_reproduces_gross_reconciliation -v -o addopts=""`
|
||||
Expected: PASS.
|
||||
|
||||
- [ ] **Step 5: Add the REAL-engine fidelity test (the hard stop) + commit**
|
||||
|
||||
Add a test that builds a small fake `_FeatureStore` (model on `tests/integration/test_bybit_book_reconciliation.py`), runs `sleeve_weights_and_contributions(store, "xsfunding")`, and asserts `honest_sleeve_returns(..., aum=1.0, impact_k=0.0, spread_bps={})` equals `sleeve_returns_from_store(store, "xsfunding", cost_bps=0.0)` day-by-day within 1e-9. Then:
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/application/honest_reference.py tests/integration/test_honest_reference.py
|
||||
git commit -m "feat(honest-ref): coin-level honest return core + fidelity invariant (spec 0a/0d)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 2: Capacity cap parameterized by AUM (spec §5 0b)
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/application/honest_reference.py` (already caps in Task 1 — this task adds `capacity_curve`)
|
||||
- Test: `tests/integration/test_honest_reference.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: `capacity_curve(weights_by_day, turnover_panel, spread_bps, *, aums, participation_cap=0.03, impact_k=1.0) -> dict[float, dict[int,float]]` (AUM → net series).
|
||||
|
||||
- [ ] **Step 1: Write the failing test — the cap bites at high AUM, not low**
|
||||
|
||||
```python
|
||||
def test_capacity_cap_bites_only_at_high_aum():
|
||||
from fxhnt.application.honest_reference import honest_sleeve_returns
|
||||
wbd = {1: {"THIN": (1.0, 0.10)}} # +10% day on a thin coin
|
||||
turnover = {"THIN": {1: 1_000_000.0}} # $1M/day ADV
|
||||
lo = honest_sleeve_returns(wbd, turnover, {}, aum=35_000.0, participation_cap=0.03, impact_k=0.0)
|
||||
hi = honest_sleeve_returns(wbd, turnover, {}, aum=10_000_000.0, participation_cap=0.03, impact_k=0.0)
|
||||
# at $35k: target $35k > 3%*$1M=$30k -> scaled to 30/35; at $10M: target $10M -> scaled to 30k/10M (tiny)
|
||||
assert abs(lo[1] - 0.10 * (30_000/35_000)) < 1e-9
|
||||
assert hi[1] < lo[1] # capacity destroys the return at scale
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run** `pytest tests/integration/test_honest_reference.py::test_capacity_cap_bites_only_at_high_aum -v -o addopts=""` — Expected: PASS (the cap logic is already in Task 1; this locks its behavior).
|
||||
|
||||
- [ ] **Step 3: Implement `capacity_curve`**
|
||||
|
||||
```python
|
||||
def capacity_curve(weights_by_day, turnover_panel, spread_bps, *, aums,
|
||||
participation_cap=0.03, impact_k=1.0):
|
||||
return {aum: honest_sleeve_returns(weights_by_day, turnover_panel, spread_bps,
|
||||
aum=aum, participation_cap=participation_cap, impact_k=impact_k)
|
||||
for aum in aums}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Write + run a test that `capacity_curve` returns one series per AUM and is monotone-non-increasing in Sharpe** (compute `nav_backfill_stats` on the cumulative nav of each). Expected: PASS.
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/application/honest_reference.py tests/integration/test_honest_reference.py
|
||||
git commit -m "feat(honest-ref): AUM-parameterized capacity curve (spec 0b)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 3: Market-impact slippage + spread floor + k-sensitivity (spec §5 0c)
|
||||
|
||||
**Files:**
|
||||
- Create: `src/fxhnt/adapters/persistence/bybit_spread.py`
|
||||
- Modify: `src/fxhnt/application/honest_reference.py` (add `spread_floor_from_snapshot`)
|
||||
- Test: `tests/integration/test_honest_reference.py`, `tests/unit/test_bybit_spread.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: `read_spread_snapshot(dsn) -> dict[str,float]` (coin→spread_bps); `honest_sleeve_returns(..., impact_k)` already charges impact; this task locks its behavior + the spread floor.
|
||||
|
||||
- [ ] **Step 1: Write the failing test — impact scales with √(size/ADV) and floors at spread**
|
||||
|
||||
```python
|
||||
def test_impact_scales_and_floors_at_spread():
|
||||
from fxhnt.application.honest_reference import honest_sleeve_returns
|
||||
wbd = {1: {"C": (1.0, 0.0)}} # flat return -> net = -cost
|
||||
turnover = {"C": {1: 1_000_000.0}}
|
||||
# size $10k of $1M ADV -> impact = k*sqrt(0.01)*1e4 = k*1000 bps; k=1 -> 1000 bps? scale by traded frac.
|
||||
r = honest_sleeve_returns(wbd, turnover, spread_bps={}, aum=10_000.0, participation_cap=1.0, impact_k=1.0)
|
||||
# cost = |w|*(impact/1e4); impact = 1*sqrt(10000/1e6)*1e4 = 1000 bps -> cost = 1*0.10 = 0.10
|
||||
assert abs(r[1] - (-0.10)) < 1e-6
|
||||
# spread floor: with a wide spread and impact_k=0, cost == spread fraction
|
||||
r2 = honest_sleeve_returns(wbd, turnover, spread_bps={"C": 50.0}, aum=10_000.0,
|
||||
participation_cap=1.0, impact_k=0.0)
|
||||
assert abs(r2[1] - (-50.0/1e4)) < 1e-9
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run** the test — Expected: PASS (impact + floor already implemented in Task 1; this locks it).
|
||||
|
||||
- [ ] **Step 3: Implement the spread reader**
|
||||
|
||||
```python
|
||||
# src/fxhnt/adapters/persistence/bybit_spread.py
|
||||
"""Read the current bybit_real_spread snapshot (coin -> spread_bps). Read-only; a point-in-time cross-section
|
||||
(all rows share one fetched_at), used as the per-coin slippage FLOOR in the honest-reference model."""
|
||||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import create_engine, text
|
||||
|
||||
|
||||
def read_spread_snapshot(dsn: str) -> dict[str, float]:
|
||||
eng = create_engine(dsn)
|
||||
with eng.connect() as c:
|
||||
rows = c.execute(text("select coin, spread_bps from bybit_real_spread")).all()
|
||||
return {coin: float(bps) for coin, bps in rows if bps is not None}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Write `tests/unit/test_bybit_spread.py`** using an in-memory SQLite with a `bybit_real_spread(coin, spread_bps)` table + two rows; assert `read_spread_snapshot` returns the dict. Run — Expected: PASS.
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/adapters/persistence/bybit_spread.py tests/unit/test_bybit_spread.py tests/integration/test_honest_reference.py
|
||||
git commit -m "feat(honest-ref): sqrt-impact slippage + real-spread floor reader (spec 0c)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 4: Risk-parity weighting + book returns (spec §7 Phase 2)
|
||||
|
||||
**Files:**
|
||||
- Create: `src/fxhnt/application/honest_gates.py`
|
||||
- Test: `tests/unit/test_honest_gates.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: honest per-sleeve series `dict[int,float]` (Task 1).
|
||||
- Produces: `risk_parity_weights(returns_by_sleeve) -> dict[str,float]` (∝ 1/vol, sum to 1); `book_returns(returns_by_sleeve, weights) -> dict[int,float]` (weighted, over common days).
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
```python
|
||||
# tests/unit/test_honest_gates.py
|
||||
from fxhnt.application.honest_gates import risk_parity_weights, book_returns
|
||||
|
||||
def test_risk_parity_weights_inverse_vol_sum_to_one():
|
||||
rbs = {"lowvol": {1: 0.001, 2: -0.001, 3: 0.001}, # tiny vol -> big weight
|
||||
"hivol": {1: 0.10, 2: -0.10, 3: 0.10}} # big vol -> small weight
|
||||
w = risk_parity_weights(rbs)
|
||||
assert abs(sum(w.values()) - 1.0) < 1e-9
|
||||
assert w["lowvol"] > w["hivol"]
|
||||
|
||||
def test_book_returns_weighted_over_common_days():
|
||||
rbs = {"a": {1: 0.02, 2: 0.04}, "b": {2: 0.10, 3: 0.20}} # common day = 2 only
|
||||
br = book_returns(rbs, {"a": 0.5, "b": 0.5})
|
||||
assert set(br) == {2} and abs(br[2] - (0.5*0.04 + 0.5*0.10)) < 1e-9
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run** `pytest tests/unit/test_honest_gates.py -v -o addopts=""` — Expected: FAIL (module missing).
|
||||
|
||||
- [ ] **Step 3: Implement**
|
||||
|
||||
```python
|
||||
# src/fxhnt/application/honest_gates.py
|
||||
"""Falsification gates + risk-parity allocation on the honest per-sleeve return series. Pure functions."""
|
||||
from __future__ import annotations
|
||||
|
||||
import statistics as st
|
||||
|
||||
|
||||
def risk_parity_weights(returns_by_sleeve: dict[str, dict[int, float]]) -> dict[str, float]:
|
||||
inv = {}
|
||||
for s, series in returns_by_sleeve.items():
|
||||
vals = list(series.values())
|
||||
vol = st.pstdev(vals) if len(vals) > 1 else 0.0
|
||||
inv[s] = (1.0 / vol) if vol > 0.0 else 0.0
|
||||
total = sum(inv.values())
|
||||
if total <= 0.0:
|
||||
n = len(returns_by_sleeve)
|
||||
return {s: 1.0 / n for s in returns_by_sleeve} if n else {}
|
||||
return {s: v / total for s, v in inv.items()}
|
||||
|
||||
|
||||
def book_returns(returns_by_sleeve: dict[str, dict[int, float]],
|
||||
weights: dict[str, float]) -> dict[int, float]:
|
||||
days = None
|
||||
for series in returns_by_sleeve.values():
|
||||
days = set(series) if days is None else (days & set(series))
|
||||
days = sorted(days or set())
|
||||
return {d: sum(weights.get(s, 0.0) * returns_by_sleeve[s][d] for s in returns_by_sleeve) for d in days}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run** `pytest tests/unit/test_honest_gates.py -v -o addopts=""` — Expected: PASS.
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/application/honest_gates.py tests/unit/test_honest_gates.py
|
||||
git commit -m "feat(honest-ref): risk-parity weights + book aggregation (spec Phase 2)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 5: Gates G1 survives, G2 LOO, G3 beats-concentration (spec §6)
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/application/honest_gates.py`
|
||||
- Test: `tests/unit/test_honest_gates.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `risk_parity_weights`, `book_returns` (Task 4); `nav_backfill_stats(nav: list[float]) -> BackfillStats(.sharpe)` (`application/nav_stats.py`).
|
||||
- Produces: `sharpe_of(series) -> float`; `gate_survives(series, *, min_sharpe=1.0) -> bool`; `loo_marginal(returns_by_sleeve) -> dict[str,float]` (full_book_sharpe − drop-one_sharpe per sleeve); `beats_concentration(returns_by_sleeve) -> bool` (rp-book Sharpe > best single sleeve).
|
||||
|
||||
- [ ] **Step 1: Write the failing tests**
|
||||
|
||||
```python
|
||||
def test_gate_survives_threshold():
|
||||
from fxhnt.application.honest_gates import gate_survives
|
||||
strong = {i: (0.01 if i % 2 else -0.002) for i in range(400)} # positive drift, modest vol
|
||||
assert gate_survives(strong, min_sharpe=1.0) is True
|
||||
flat = {i: (0.001 if i % 2 else -0.001) for i in range(400)}
|
||||
assert gate_survives(flat, min_sharpe=1.0) is False
|
||||
|
||||
def test_loo_marginal_flags_dead_weight():
|
||||
from fxhnt.application.honest_gates import loo_marginal
|
||||
# 'dead' is pure noise; dropping it should NOT hurt (marginal ~ <= 0)
|
||||
good = {i: (0.01 if i % 2 else -0.003) for i in range(400)}
|
||||
dead = {i: (0.002 if i % 3 else -0.002) for i in range(400)}
|
||||
m = loo_marginal({"good": good, "dead": dead})
|
||||
assert m["good"] > m["dead"]
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run** — Expected: FAIL (functions missing).
|
||||
|
||||
- [ ] **Step 3: Implement**
|
||||
|
||||
```python
|
||||
# add to honest_gates.py
|
||||
import math
|
||||
from fxhnt.application.nav_stats import nav_backfill_stats
|
||||
|
||||
|
||||
def _nav(series: dict[int, float]) -> list[float]:
|
||||
nav, v = [], 1.0
|
||||
for d in sorted(series):
|
||||
v *= (1.0 + series[d])
|
||||
nav.append(v)
|
||||
return [1.0] + nav
|
||||
|
||||
|
||||
def sharpe_of(series: dict[int, float]) -> float:
|
||||
return nav_backfill_stats(_nav(series)).sharpe
|
||||
|
||||
|
||||
def gate_survives(series: dict[int, float], *, min_sharpe: float = 1.0) -> bool:
|
||||
return sharpe_of(series) > min_sharpe
|
||||
|
||||
|
||||
def loo_marginal(returns_by_sleeve: dict[str, dict[int, float]]) -> dict[str, float]:
|
||||
full = sharpe_of(book_returns(returns_by_sleeve, risk_parity_weights(returns_by_sleeve)))
|
||||
out = {}
|
||||
for drop in returns_by_sleeve:
|
||||
rest = {s: v for s, v in returns_by_sleeve.items() if s != drop}
|
||||
rest_sh = sharpe_of(book_returns(rest, risk_parity_weights(rest))) if rest else 0.0
|
||||
out[drop] = full - rest_sh
|
||||
return out
|
||||
|
||||
|
||||
def beats_concentration(returns_by_sleeve: dict[str, dict[int, float]]) -> bool:
|
||||
book_sh = sharpe_of(book_returns(returns_by_sleeve, risk_parity_weights(returns_by_sleeve)))
|
||||
best_single = max((sharpe_of(v) for v in returns_by_sleeve.values()), default=0.0)
|
||||
return book_sh > best_single
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run** `pytest tests/unit/test_honest_gates.py -v -o addopts=""` — Expected: PASS.
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/application/honest_gates.py tests/unit/test_honest_gates.py
|
||||
git commit -m "feat(honest-ref): gates G1 survives / G2 LOO / G3 beats-concentration (spec 6)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 6: Leverage ceiling vs real drawdown + G4 anti-reactive (spec §7 book layer, §6 G4)
|
||||
|
||||
**Files:**
|
||||
- Create: `src/fxhnt/application/honest_leverage.py`
|
||||
- Modify: `src/fxhnt/application/honest_gates.py` (add `ab_beats_nothing`)
|
||||
- Test: `tests/unit/test_honest_leverage.py`, `tests/unit/test_honest_gates.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: `max_drawdown_at_leverage(book_series, leverage) -> float` (negative); `leverage_ceiling(book_series, *, dd_limit=-0.35, stress=2.0, grid=(1.0,1.25,1.5,1.75,2.0)) -> float`; `ab_beats_nothing(base_series, derisked_series) -> bool`.
|
||||
|
||||
- [ ] **Step 1: Write the failing tests**
|
||||
|
||||
```python
|
||||
# tests/unit/test_honest_leverage.py
|
||||
from fxhnt.application.honest_leverage import max_drawdown_at_leverage, leverage_ceiling
|
||||
|
||||
def test_drawdown_scales_with_leverage():
|
||||
s = {1: 0.05, 2: -0.10, 3: 0.04}
|
||||
dd1 = max_drawdown_at_leverage(s, 1.0)
|
||||
dd2 = max_drawdown_at_leverage(s, 2.0)
|
||||
assert dd1 < 0.0 and dd2 < dd1 # 2x is a deeper (more negative) drawdown
|
||||
|
||||
def test_leverage_ceiling_respects_stress_buffer():
|
||||
s = {1: 0.02, 2: -0.05, 3: 0.03, 4: -0.02} # mild book
|
||||
# with a 2x stress buffer under -35%, the ceiling picks the largest grid L whose 2*maxDD >= -35%
|
||||
lev = leverage_ceiling(s, dd_limit=-0.35, stress=2.0)
|
||||
assert 1.0 <= lev <= 2.0
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run** — Expected: FAIL (module missing).
|
||||
|
||||
- [ ] **Step 3: Implement**
|
||||
|
||||
```python
|
||||
# src/fxhnt/application/honest_leverage.py
|
||||
"""Real fat-tail drawdown of the honest book at constant leverage, and the ex-ante leverage ceiling
|
||||
(K(D*)) — anchored to a REAL drawdown with a stress buffer, never to a smooth modeled curve."""
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
|
||||
|
||||
def max_drawdown_at_leverage(book_series: dict[int, float], leverage: float) -> float:
|
||||
peak, trough, v = 1.0, 0.0, 1.0
|
||||
for d in sorted(book_series):
|
||||
step = 1.0 + leverage * book_series[d]
|
||||
if step <= 0.0: # wipeout at this leverage
|
||||
return -1.0
|
||||
v *= step
|
||||
peak = max(peak, v)
|
||||
trough = min(trough, v / peak - 1.0)
|
||||
return trough
|
||||
|
||||
|
||||
def leverage_ceiling(book_series, *, dd_limit=-0.35, stress=2.0,
|
||||
grid=(1.0, 1.25, 1.5, 1.75, 2.0)) -> float:
|
||||
ok = [L for L in grid if stress * max_drawdown_at_leverage(book_series, L) >= dd_limit]
|
||||
return max(ok) if ok else 0.0
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Add `ab_beats_nothing` to `honest_gates.py` + test** (G4): a de-risk variant series only "wins" if its Sharpe exceeds the base by a margin AND it is not merely lower-vol; keep it simple — `sharpe_of(derisked) > sharpe_of(base)`. Test that an identical series does NOT beat nothing.
|
||||
|
||||
```python
|
||||
def ab_beats_nothing(base_series: dict[int, float], derisked_series: dict[int, float]) -> bool:
|
||||
return sharpe_of(derisked_series) > sharpe_of(base_series)
|
||||
```
|
||||
|
||||
- [ ] **Step 5: Run both test files + commit**
|
||||
|
||||
Run: `pytest tests/unit/test_honest_leverage.py tests/unit/test_honest_gates.py -v -o addopts=""` — Expected: PASS.
|
||||
```bash
|
||||
git add src/fxhnt/application/honest_leverage.py src/fxhnt/application/honest_gates.py tests/unit/test_honest_leverage.py tests/unit/test_honest_gates.py
|
||||
git commit -m "feat(honest-ref): leverage ceiling vs real DD + G4 anti-reactive gate (spec 7)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 7: Candidate edge #4 — pump/anomaly detection (spec §8)
|
||||
|
||||
**Files:**
|
||||
- Create: `src/fxhnt/application/pump_detection.py`
|
||||
- Test: `tests/unit/test_pump_detection.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `store.read_panel(symbols, "turnover")` and `crypto_close_panel()`; `honest_sleeve_returns` (Task 1) for the execution gate.
|
||||
- Produces: `pump_signal(close_panel, turnover_panel, *, z=3.0) -> dict[int, dict[str, float]]` (day→coin→signed target weight, ONLY on same-day-observable anomalies, no look-ahead); `evaluate_pump_edge(weights_by_day, turnover_panel, spread_bps, *, aum, impact_k) -> dict` (returns net Sharpe + the 3-gate verdict).
|
||||
|
||||
- [ ] **Step 1: Write the failing test — signal only fires on an anomaly, uses only past data**
|
||||
|
||||
```python
|
||||
# tests/unit/test_pump_detection.py
|
||||
from fxhnt.application.pump_detection import pump_signal
|
||||
|
||||
def test_signal_fires_on_volume_zspike_and_is_causal():
|
||||
# day 10 has a turnover z-spike vs the trailing window -> a signal; earlier calm days -> none.
|
||||
turnover = {"X": {d: 1_000_000.0 for d in range(1, 10)}}
|
||||
turnover["X"][10] = 50_000_000.0 # 50x spike
|
||||
close = {"X": {d: 100.0 for d in range(1, 11)}}
|
||||
sig = pump_signal(close, turnover, z=3.0)
|
||||
assert 10 in sig and "X" in sig[10] # fires on the spike day
|
||||
assert all("X" not in sig.get(d, {}) for d in range(1, 10)) # not before (causal)
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run** — Expected: FAIL (module missing).
|
||||
|
||||
- [ ] **Step 3: Implement** a causal detector: for each coin-day, z-score today's turnover vs the trailing N-day window (strictly `< day`); emit a signed target (e.g. short the pump, `-1`, capped) when `z > z`. No future data touched.
|
||||
|
||||
```python
|
||||
# src/fxhnt/application/pump_detection.py
|
||||
"""Candidate edge #4: causal pump/anomaly detection. A per-coin, same-day-observable turnover z-spike ->
|
||||
a (short-the-pump) target weight. Must be run through honest_sleeve_returns to see if it survives the real
|
||||
pump-day slippage (the killer). Pure; no look-ahead."""
|
||||
from __future__ import annotations
|
||||
|
||||
import statistics as st
|
||||
|
||||
_WINDOW = 20
|
||||
|
||||
|
||||
def pump_signal(close_panel, turnover_panel, *, z: float = 3.0) -> dict[int, dict[str, float]]:
|
||||
out: dict[int, dict[str, float]] = {}
|
||||
for sym, series in turnover_panel.items():
|
||||
days = sorted(series)
|
||||
for i, d in enumerate(days):
|
||||
hist = [series[days[j]] for j in range(max(0, i - _WINDOW), i)] # strictly past
|
||||
if len(hist) < _WINDOW:
|
||||
continue
|
||||
mu, sd = st.mean(hist), st.pstdev(hist)
|
||||
if sd > 0.0 and (series[d] - mu) / sd > z:
|
||||
out.setdefault(d, {})[sym] = -1.0 # short the pump (defined-risk sizing added downstream)
|
||||
return out
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Add `evaluate_pump_edge` + a test** that (a) builds `{day:{sym:(weight, ret)}}` from the signal × a synthetic reversal return, (b) runs `honest_sleeve_returns(..., impact_k=1.0)`, and asserts the returned dict has keys `sharpe`, `survives_execution` (net Sharpe > 0 after slippage), `marginal_vs_positioning`. Run — Expected: PASS.
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/application/pump_detection.py tests/unit/test_pump_detection.py
|
||||
git commit -m "feat(honest-ref): candidate edge #4 causal pump detector + 3-gate eval (spec 8)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 8: Report CLI — the research deliverable (spec §9)
|
||||
|
||||
**Files:**
|
||||
- Create: `src/fxhnt/application/honest_report.py`
|
||||
- Modify: `src/fxhnt/cli.py` (add `honest-reference-report` command)
|
||||
- Test: `tests/integration/test_honest_report.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: everything above + a real `_FeatureStore` (the analytical store), `read_spread_snapshot`, `read_panel(symbols, "turnover")`, `unlock_events` source.
|
||||
- Produces: `build_honest_report(store, spread_bps, unlock_events, *, sleeves, aums, ks) -> dict` (per-sleeve honest stats per (AUM,k), the risk-parity book, gate verdicts G1-G4, leverage ceiling, and the edge-#4 verdict) — pure; the CLI just prints it.
|
||||
|
||||
- [ ] **Step 1: Write the failing test** — `build_honest_report` on a fake store returns a dict with keys `sleeves`, `capacity_curve`, `gates`, `leverage_ceiling`, `edge4`; the `k`-sensitivity has 3 entries; the AUM curve has 5. Run — Expected: FAIL.
|
||||
|
||||
- [ ] **Step 2: Implement `build_honest_report`** — loop sleeves × aums × ks calling `honest_sleeve_returns` + `nav_backfill_stats`; assemble the risk-parity book at the base (aum=100k, k=2); call `gate_survives`/`loo_marginal`/`beats_concentration`/`leverage_ceiling`/`evaluate_pump_edge`. **Survivorship guard:** assert the coin universe count from `read_panel` matches the ingestion's survivorship-free count (log a warning if a survivorship flag is unavailable — do not silently assume).
|
||||
|
||||
- [ ] **Step 3: Wire the CLI command** in `src/fxhnt/cli.py`:
|
||||
|
||||
```python
|
||||
@app.command("honest-reference-report")
|
||||
def honest_reference_report() -> None:
|
||||
"""Research-only: print the capacity/cost/leverage-honest reference table + gate verdicts. Read-only."""
|
||||
from fxhnt.application.honest_report import build_honest_report
|
||||
# ... construct the analytical store + spread snapshot + unlock events, then:
|
||||
import json, typer
|
||||
typer.echo(json.dumps(build_honest_report(...), indent=2, default=str))
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run** `pytest tests/integration/test_honest_report.py -v -o addopts=""` — Expected: PASS. Then run the CLI once against the real store in-cluster (`kubectl exec <dagster-pod> -c daemon -- python -m fxhnt.cli honest-reference-report`) and eyeball the numbers vs the SQL research estimates (xsfunding Sh ≈ 3, unlock collapses, book Sh ≈ 1.7 at P98-equivalent). Discrepancy > ~0.3 Sharpe ⇒ investigate before trusting.
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/application/honest_report.py src/fxhnt/cli.py tests/integration/test_honest_report.py
|
||||
git commit -m "feat(honest-ref): honest-reference research report CLI (spec 9)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Self-Review
|
||||
|
||||
**Spec coverage:** §3 data → Tasks 1/3 (turnover via `read_panel`, spread via `bybit_spread.py`). §5 0a → Task 1 (fidelity invariant). §5 0b → Task 2 (capacity curve, AUM series). §5 0c → Task 3 (sqrt-impact + spread floor + k-sensitivity). §5 0d → Tasks 1/8 (`nav_backfill_stats`). §6 G1/G2/G3 → Task 5. §6 G4 → Task 6. §7 allocation → Task 4; leverage ceiling → Task 6; position cap → Tasks 1-2; tail-drop → G1/G2 (Task 5) surfaced in the report (Task 8). §8 edge #4 → Task 7. §9 report/scope → Task 8. §10 survivorship guard → Task 8 Step 2; slippage-sensitivity → Task 8 `ks`. No live-cockpit change anywhere — confirmed (all read-only, one CLI print). **Gap intentionally deferred:** the "hard ADV cap vs slippage-penalty" belt-and-suspenders — Task 1 implements BOTH (cap scales the weight, impact charges the residual), satisfying §11.
|
||||
|
||||
**Placeholder scan:** no TBD/TODO; every code step shows real code against the real APIs (`sleeve_weights_and_contributions`, `read_panel`, `nav_backfill_stats`, `reconciliation_series`). Task 8 Steps 1-2 describe the loop shape with exact keys rather than 60 lines of assembly — acceptable as the report is pure glue over tested units; the test pins its output keys.
|
||||
|
||||
**Type consistency:** series are `dict[int,float]` throughout; coin weights `dict[int,dict[str,tuple[float,float]]]`; turnover/close panels `dict[str,dict[int,float]]`; spread `dict[str,float]`. `sharpe_of`/`gate_survives`/`book_returns` signatures match across Tasks 4-8.
|
||||
|
||||
---
|
||||
|
||||
## Execution Handoff
|
||||
|
||||
Plan complete and saved to `docs/superpowers/plans/2026-07-15-crypto-honest-reference-pipeline.md`. Two execution options:
|
||||
|
||||
1. **Subagent-Driven (recommended)** — fresh subagent per task, review between tasks, fast iteration.
|
||||
2. **Inline Execution** — execute tasks in this session with checkpoints.
|
||||
|
||||
Which approach?
|
||||
@@ -0,0 +1,262 @@
|
||||
# Book Representation + Venue-Cap Knob Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development. Steps use checkbox (`- [ ]`) tracking.
|
||||
|
||||
**Goal:** Switch the crypto deploy target from per-sleeve (collapses to positioning, ~18% deployed) to the diversified `bybit_4edge` BOOK (measured 75% deployed at vol 5.2% / maxDD −3.2% — 4× more at lower risk), and add a `max_venue_weight` fund-config knob so per-venue concentration (75% on Bybit) is cappable at live.
|
||||
|
||||
**Architecture:** Registry re-points the deploy tier to `bybit_4edge` (positioning/xsfunding/unlock become tracked constituents); the honest-inputs provider re-adds the book series dispatch (keeping the sleeve generalization). A new `max_venue_weight` on `AllocationPolicy` + `apply_venue_caps` engine function caps each venue's total gross; it is a fund_config column set via `fxhnt fund-config`.
|
||||
|
||||
**Tech Stack:** Python 3.12, SQLAlchemy 2.0, typer, pytest. In-cluster measurement already done (book = 75% deploy; funding-stress check PASSED).
|
||||
|
||||
## Global Constraints
|
||||
|
||||
- Basis: the 2026-07-16 measurements — book (`_book_series` risk-parity of {xsfunding,unlock,positioning}) @ $100k: vol 0.0523, maxDD −0.0324, Sharpe 0.161/pd, deploys **75%** at target_vol 0.25 (K hits max_leverage 1.5). Funding-stress check PASSED (crises in-sample, smoothness legitimate).
|
||||
- `max_venue_weight` DEFAULT = **1.0** (no cap → byte-identical to today's deployment). It caps the fraction of `equity` deployable as gross to any single venue.
|
||||
- `record_allocation(policy=None)` and `apply_venue_caps(..., max_venue_weight=1.0)` MUST be byte-identical to today (no-op at the default).
|
||||
- `fund_config` read fallback + partial-upsert semantics unchanged; a missing `max_venue_weight` column/value → default 1.0 (never crash the nightly). Column added to the EXISTING prod table via an idempotent ALTER (mirror `forward_nav._ensure_summary_policy_columns`: Postgres `ADD COLUMN IF NOT EXISTS`, SQLite PRAGMA-check).
|
||||
- Bump `ALLOCATION_POLICY_VERSION` 3 → 4 (policy shape changed).
|
||||
- Marginal LOO gate, ex-ante full-history-vol K, anti-reactive invariant: UNTOUCHED.
|
||||
- Commit trailer: `Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>`.
|
||||
|
||||
---
|
||||
|
||||
### Task A: switch the crypto deploy target to the `bybit_4edge` book
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/registry.py`
|
||||
- Modify: `src/fxhnt/application/allocation_honest_inputs.py`
|
||||
- Modify: `src/fxhnt/adapters/web/templates/sim.html`
|
||||
- Test: `tests/unit/test_allocation_honest_inputs.py`, `tests/unit/test_registry_tiers.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: deploy set = `{bybit_4edge}`; provider sizes `bybit_4edge`→`_book_series`, `{positioning,xsfunding,unlock}`→`_sleeve_series`.
|
||||
|
||||
- [ ] **Step 1: Update the tier tests to the target state (write the expectation first)**
|
||||
|
||||
In `tests/unit/test_registry_tiers.py`: `_DEPLOY` back to `{"bybit_4edge"}` (positioning is now a research constituent). Update the test name/docstring accordingly (deploy tier = exactly the bybit_4edge book).
|
||||
|
||||
- [ ] **Step 2: Run — expect FAIL** (registry still has positioning deploy / bybit_4edge research)
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && .venv/bin/python3 -m pytest tests/unit/test_registry_tiers.py -q`
|
||||
Expected: FAIL.
|
||||
|
||||
- [ ] **Step 3: Registry edits** (`registry.py`):
|
||||
- `bybit_4edge`: `"tier": "research"` → **`"tier": "deploy"`**. Rewrite its leading comment: it is the fund's DEPLOYED crypto book (risk-parity of its sleeves; measured 75% deploy @ 0.25 vol, funding-stress-checked); constituents tracked separately as reference.
|
||||
- `positioning`: `"tier": "deploy"` → **`"tier": "research"`** (constituent of the book; still forward-tracked as reference).
|
||||
- `xsfunding`: **REMOVE** `"promote_on_pass": True` (it is a book constituent, not a standalone deploy edge — must not auto-promote and double-count the book). Keep `tier: research`.
|
||||
- `unlock`: **REMOVE** `"promote_on_pass": True`. Keep `tier: research`.
|
||||
|
||||
- [ ] **Step 4: Provider edits** (`allocation_honest_inputs.py`):
|
||||
- `_CRYPTO_DEPLOY_SIDS = frozenset({"positioning", "xsfunding", "unlock", "bybit_4edge"})`.
|
||||
- `_deploy_series(store, sid, ...)`:
|
||||
```python
|
||||
if sid == "bybit_4edge":
|
||||
return _book_series(store, spread_bps, unlock_events, aum=aum,
|
||||
participation_cap=participation_cap, impact_k=impact_k)
|
||||
if sid in _CRYPTO_DEPLOY_SIDS:
|
||||
return _sleeve_series(store, sid, spread_bps, unlock_events, aum=aum,
|
||||
participation_cap=participation_cap, impact_k=impact_k)
|
||||
return {}
|
||||
```
|
||||
- Update the module docstring's deploy→honest mapping: `bybit_4edge`→the risk-parity BOOK (the deploy target), the three sleeves→their own sleeve series (available if ever deployed standalone).
|
||||
|
||||
- [ ] **Step 5: sim.html** — revert the label: `bybit_4edge` span `· reference` → `· deploy`; and the two comments (lines ~9, ~20) `REFERENCE book`/`(reference, default)` → `DEPLOY book`/`(deploy, default)` (it is the deploy book again).
|
||||
|
||||
- [ ] **Step 6: Update `test_allocation_honest_inputs.py`** — the pre-existing tests from the prior task asserted `_CRYPTO_DEPLOY_SIDS == {positioning,xsfunding,unlock}` and `bybit_4edge` not a crypto deploy sid. Update to the new set (includes `bybit_4edge`) and assert `bybit_4edge` now gets the BOOK series (non-empty, ISO-keyed) via the provider. Keep the sleeve-series assertions.
|
||||
|
||||
- [ ] **Step 7: Run — expect PASS**
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && .venv/bin/python3 -m pytest tests/unit/test_allocation_honest_inputs.py tests/unit/test_registry_tiers.py -q`
|
||||
Expected: PASS.
|
||||
|
||||
- [ ] **Step 8: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/registry.py src/fxhnt/application/allocation_honest_inputs.py src/fxhnt/adapters/web/templates/sim.html tests/unit/test_allocation_honest_inputs.py tests/unit/test_registry_tiers.py
|
||||
git commit -m "feat(alloc): switch crypto deploy target to the bybit_4edge book (75% deploy vs 18% per-sleeve)
|
||||
|
||||
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task B: venue-cap in the allocation engine + policy
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/application/allocation_policy.py` (`max_venue_weight` field, version bump)
|
||||
- Modify: `src/fxhnt/application/allocation_engine.py` (`apply_venue_caps`)
|
||||
- Modify: `src/fxhnt/application/allocation_ingest.py` (wire it into `record_allocation`)
|
||||
- Test: `tests/unit/test_venue_caps.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: `AllocationPolicy.max_venue_weight: float = 1.0`; `apply_venue_caps(dollars, venue_by_sid, max_venue_weight, equity) -> dict[str,float]` (per-venue gross cap, sign-preserving, no-op at ≥1.0).
|
||||
|
||||
- [ ] **Step 1: Write failing tests**
|
||||
|
||||
```python
|
||||
# tests/unit/test_venue_caps.py
|
||||
from fxhnt.application.allocation_engine import apply_venue_caps
|
||||
|
||||
def test_no_cap_at_default_is_identity():
|
||||
d = {"a": 60000.0, "b": -30000.0}
|
||||
v = {"a": "bybit-perp", "b": "ibkr-equity"}
|
||||
assert apply_venue_caps(dict(d), v, 1.0, 100000.0) == d
|
||||
|
||||
def test_single_venue_over_cap_scales_down_sign_preserving():
|
||||
# both on bybit-perp, gross 75k, cap 0.6*100k=60k -> scale 0.8
|
||||
d = {"a": 50000.0, "b": -25000.0}
|
||||
v = {"a": "bybit-perp", "b": "bybit-perp"}
|
||||
out = apply_venue_caps(dict(d), v, 0.6, 100000.0)
|
||||
assert round(out["a"], 2) == 40000.0 and round(out["b"], 2) == -20000.0
|
||||
|
||||
def test_under_cap_venue_untouched():
|
||||
d = {"a": 40000.0, "b": 40000.0}
|
||||
v = {"a": "bybit-perp", "b": "ibkr-equity"} # each venue 40k < 60k cap
|
||||
assert apply_venue_caps(dict(d), v, 0.6, 100000.0) == d
|
||||
|
||||
def test_missing_venue_treated_as_own_bucket():
|
||||
d = {"a": 80000.0}
|
||||
v = {} # sid with no venue -> its own ("") bucket
|
||||
out = apply_venue_caps(dict(d), v, 0.6, 100000.0)
|
||||
assert round(out["a"], 2) == 60000.0
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run — expect FAIL**
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && .venv/bin/python3 -m pytest tests/unit/test_venue_caps.py -q`
|
||||
Expected: FAIL.
|
||||
|
||||
- [ ] **Step 3: Implement `apply_venue_caps`** (`allocation_engine.py`):
|
||||
|
||||
```python
|
||||
def apply_venue_caps(dollars: dict[str, float], venue_by_sid: dict[str, str],
|
||||
max_venue_weight: float, equity: float) -> dict[str, float]:
|
||||
"""Cap each venue's total deployed gross at `max_venue_weight * equity` (sign-preserving, proportional
|
||||
scale-down within the venue). `max_venue_weight >= 1.0` (or non-finite equity) -> no-op. A sid missing from
|
||||
`venue_by_sid` falls in the "" bucket (its own venue)."""
|
||||
if not dollars or max_venue_weight >= 1.0 or not math.isfinite(equity) or equity <= 0.0:
|
||||
return dollars
|
||||
cap = max_venue_weight * equity
|
||||
by_venue: dict[str, list[str]] = {}
|
||||
for sid in dollars:
|
||||
by_venue.setdefault(venue_by_sid.get(sid, ""), []).append(sid)
|
||||
for sids in by_venue.values():
|
||||
gross = sum(abs(dollars[s]) for s in sids)
|
||||
if gross > cap and gross > 0.0:
|
||||
scale = cap / gross
|
||||
for s in sids:
|
||||
dollars[s] *= scale
|
||||
return dollars
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Policy** (`allocation_policy.py`): add `max_venue_weight: float = 1.0 # cap on the fraction of equity deployed as gross to any single venue (1.0 = no cap)` after `max_strategy_weight`; bump `ALLOCATION_POLICY_VERSION = 3` → `4`.
|
||||
|
||||
- [ ] **Step 5: Wire into `record_allocation`** (`allocation_ingest.py`, after `dollars = target_capital(...)`):
|
||||
|
||||
```python
|
||||
venue_by_sid = {sid: STRATEGY_REGISTRY.get(sid, {}).get("venue", "") for sid in dollars}
|
||||
dollars = apply_venue_caps(dollars, venue_by_sid, getattr(pol, "max_venue_weight", 1.0), equity)
|
||||
```
|
||||
|
||||
(import `apply_venue_caps` from allocation_engine; `STRATEGY_REGISTRY` is already imported.)
|
||||
|
||||
- [ ] **Step 6: Run — expect PASS** (venue tests + existing allocation tests still green)
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && .venv/bin/python3 -m pytest tests/unit/test_venue_caps.py tests/unit/test_compute_allocation.py tests/unit/test_record_allocation_policy.py -q`
|
||||
Expected: PASS.
|
||||
|
||||
- [ ] **Step 7: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/application/allocation_policy.py src/fxhnt/application/allocation_engine.py src/fxhnt/application/allocation_ingest.py tests/unit/test_venue_caps.py
|
||||
git commit -m "feat(alloc): max_venue_weight per-venue gross cap (default 1.0 = no-op)
|
||||
|
||||
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task C: `max_venue_weight` as a fund_config knob (end-to-end)
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/adapters/persistence/cockpit_models.py` (`FundConfigRow.max_venue_weight`)
|
||||
- Modify: `src/fxhnt/application/fund_config.py` (field + validation + idempotent ALTER + read/write/seed)
|
||||
- Modify: `src/fxhnt/cli.py` (`--max-venue-weight` flag on `fund-config set`; show it)
|
||||
- Modify: `src/fxhnt/adapters/orchestration/assets.py` (`cockpit_forward` replace adds `max_venue_weight`)
|
||||
- Test: `tests/unit/test_fund_config.py`, `tests/unit/test_fund_config_cli.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `apply_venue_caps` (Task B), `AllocationPolicy.max_venue_weight` (Task B).
|
||||
- Produces: `FundConfig.max_venue_weight` (default 1.0); `fxhnt fund-config set --max-venue-weight`.
|
||||
|
||||
- [ ] **Step 1: Write failing tests** (extend `test_fund_config.py`):
|
||||
|
||||
```python
|
||||
def test_max_venue_weight_default_and_roundtrip():
|
||||
import os
|
||||
p = "./.pytest_fc_venue.db"
|
||||
if os.path.exists(p): os.remove(p)
|
||||
from fxhnt.application.fund_config import read_fund_config, write_fund_config, seed_fund_config
|
||||
seed_fund_config(f"sqlite:///{p}")
|
||||
assert read_fund_config(f"sqlite:///{p}").max_venue_weight == 1.0
|
||||
write_fund_config(f"sqlite:///{p}", max_venue_weight=0.6)
|
||||
cfg = read_fund_config(f"sqlite:///{p}")
|
||||
assert cfg.max_venue_weight == 0.6 and cfg.target_vol == 0.20 # partial-override: vol unchanged
|
||||
os.remove(p)
|
||||
|
||||
def test_max_venue_weight_validation():
|
||||
import pytest, os
|
||||
p = "./.pytest_fc_venue_val.db"
|
||||
if os.path.exists(p): os.remove(p)
|
||||
from fxhnt.application.fund_config import write_fund_config
|
||||
for bad in (0.0, -0.1, 1.5):
|
||||
with pytest.raises(ValueError):
|
||||
write_fund_config(f"sqlite:///{p}", max_venue_weight=bad)
|
||||
if os.path.exists(p): os.remove(p)
|
||||
```
|
||||
|
||||
Also extend the fallback test to assert `read_fund_config(absent).max_venue_weight == 1.0`.
|
||||
|
||||
- [ ] **Step 2: Run — expect FAIL**
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && .venv/bin/python3 -m pytest tests/unit/test_fund_config.py -q`
|
||||
Expected: FAIL.
|
||||
|
||||
- [ ] **Step 3: Model + module** :
|
||||
- `cockpit_models.FundConfigRow`: add `max_venue_weight: Mapped[float] = mapped_column(Float)`.
|
||||
- `fund_config.py`: add `max_venue_weight: float` to the `FundConfig` dataclass; `_defaults()` → `max_venue_weight=1.0`; `_validate` accepts `max_venue_weight` and enforces `0 < max_venue_weight <= 1.0`; `write_fund_config` gains the `max_venue_weight=None` kwarg (partial-overlay like the others — including the "at least one field" guard); `seed_fund_config` inserts `max_venue_weight=1.0`.
|
||||
- **Idempotent ALTER** for the existing prod table — add an `_ensure_fund_config_columns(engine)` helper (mirror `forward_nav._ensure_summary_policy_columns`) that adds `max_venue_weight DOUBLE PRECISION` (Postgres `ADD COLUMN IF NOT EXISTS`; SQLite PRAGMA-check) and BACKFILLS existing row(s) to 1.0 where NULL; call it in `write_fund_config`/`seed_fund_config` right after `create_all` (so an existing seeded table gains the column without losing the operator's target_vol). A fresh table gets the column from `create_all`, so the ALTER is a no-op there.
|
||||
|
||||
- [ ] **Step 4: CLI** (`cli.py`): `fund-config set` gains `max_venue_weight: float = typer.Option(None, "--max-venue-weight", help="Max fraction of equity deployed to any single venue (0,1]. 1.0 = no cap.")`; pass it to `write_fund_config`; include it in both `show` and `set` echo lines.
|
||||
|
||||
- [ ] **Step 5: Wire the nightly** (`assets.py` `cockpit_forward`): the `dataclasses.replace(...)` gains `max_venue_weight=_cfg.max_venue_weight`.
|
||||
|
||||
- [ ] **Step 6: Run — expect PASS**
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && .venv/bin/python3 -m pytest tests/unit/test_fund_config.py tests/unit/test_fund_config_cli.py -q`
|
||||
Expected: PASS.
|
||||
|
||||
- [ ] **Step 7: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/adapters/persistence/cockpit_models.py src/fxhnt/application/fund_config.py src/fxhnt/cli.py src/fxhnt/adapters/orchestration/assets.py tests/unit/test_fund_config.py tests/unit/test_fund_config_cli.py
|
||||
git commit -m "feat(fund-config): max_venue_weight knob (fxhnt fund-config set --max-venue-weight)
|
||||
|
||||
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task D: full-suite green
|
||||
|
||||
- [ ] **Step 1:** `cd /home/jgrusewski/Work/fxhnt && .venv/bin/python3 -m pytest tests/unit -q` → all pass.
|
||||
- [ ] **Step 2:** `.venv/bin/python3 -m pytest tests/integration -k "allocation or honest or funding" -q` → all pass.
|
||||
- [ ] **Step 3:** commit any fixes (else skip).
|
||||
|
||||
## Post-implementation (controller-run)
|
||||
Deploy (git-sync), `fxhnt migrate-cockpit` (adds the column, backfills 1.0), re-measure in-cluster: confirm the deploy set is `{bybit_4edge}` and it deploys ~75% at target_vol 0.25 with `max_venue_weight=1.0`; then demonstrate the knob (`fxhnt fund-config set --max-venue-weight 0.6` → book caps to 60%). Cockpit local verify.
|
||||
|
||||
## Self-Review
|
||||
- Coverage: book switch → Task A; venue-cap engine → Task B; venue-cap knob end-to-end → Task C; green → Task D.
|
||||
- Byte-identity guards: `apply_venue_caps` no-op at 1.0; policy=None unchanged; fund_config fallback/ALTER default 1.0.
|
||||
- Type consistency: `max_venue_weight`, `apply_venue_caps`, `FundConfig` fields used identically across tasks.
|
||||
538
docs/superpowers/plans/2026-07-16-deploy-fund-config-knobs.md
Normal file
538
docs/superpowers/plans/2026-07-16-deploy-fund-config-knobs.md
Normal file
@@ -0,0 +1,538 @@
|
||||
# Deploy-the-Fund Knobs Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Make the fund's capital + risk appetite CLI-configurable via a single persisted SSOT, and switch the crypto allocation from the double-counted `bybit_4edge` book to its individual capacity-bounded sleeves.
|
||||
|
||||
**Architecture:** A single-row `fund_config` ORM table (rides the existing `CockpitBase.metadata.create_all`), a `application/fund_config.py` with read (settings-fallback) / write (partial-upsert + validation) / seed, a `fxhnt fund-config show|set` typer CLI, the nightly `cockpit_forward` reads it for equity + an `AllocationPolicy` override, and the registry+provider re-point the crypto allocation to sleeves.
|
||||
|
||||
**Tech Stack:** Python 3.12, SQLAlchemy 2.0 (Mapped/mapped_column, DeclarativeBase), typer CLI, pytest. In-memory SQLite for tests (StaticPool), Postgres/Timescale in prod.
|
||||
|
||||
## Global Constraints
|
||||
|
||||
- Spec: `docs/superpowers/specs/2026-07-16-deploy-fund-config-knobs-design.md`. Copy exact values from it.
|
||||
- Initial fund-config values: **capital 100_000, target_vol 0.20, kelly_fraction 0.5**.
|
||||
- Validation (fail loud at the boundary, never silently clamp): `capital > 0`, `0 < target_vol <= 1.0`, `0 < kelly_fraction <= 1.0`.
|
||||
- **Graceful fallback**: when the `fund_config` table or row id=1 is absent, `read_fund_config` returns `FundConfig(capital=settings.paper_capital, target_vol=AllocationPolicy().target_vol, kelly_fraction=AllocationPolicy().kelly_fraction)` — byte-identical to today's nightly behaviour. A fresh DB or a failed read must NEVER change the allocation.
|
||||
- **Partial-override upsert**: `write_fund_config(capital=X)` changes only `capital`, leaving `target_vol`/`kelly_fraction` at their current effective values (classic upsert bug to avoid).
|
||||
- **Seed is insert-if-absent** (`id=1` PK collision → no-op); a re-run must never clobber an operator-set value.
|
||||
- Marginal LOO gate is KEPT (do not weaken/remove it to force diversification). The ex-ante `K` stays computed from full-history vol (never trailing) — the anti-reactive invariant.
|
||||
- No heavy DDL on the startup hot path (one tiny `create_all` table is fine; it is IF-NOT-EXISTS and a no-op after first run).
|
||||
- `record_allocation(dsn, equity, store=None, ...)` with `store=None` and `policy=None` must remain byte-identical to today.
|
||||
- Commit messages end with: `Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>`.
|
||||
|
||||
---
|
||||
|
||||
### Task 1: `fund_config` model + read/write/fallback/validation
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/adapters/persistence/cockpit_models.py` (add `FundConfigRow`)
|
||||
- Create: `src/fxhnt/application/fund_config.py`
|
||||
- Test: `tests/unit/test_fund_config.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `CockpitBase` (cockpit_models), `get_settings()` (config), `AllocationPolicy` (allocation_policy).
|
||||
- Produces:
|
||||
- `FundConfigRow` — ORM model, `__tablename__ = "fund_config"`, single-row (PK `id` fixed 1).
|
||||
- `@dataclass(frozen=True) FundConfig(capital: float, target_vol: float, kelly_fraction: float)`
|
||||
- `read_fund_config(dsn: str) -> FundConfig` (settings-fallback on absent table/row)
|
||||
- `write_fund_config(dsn: str, *, capital: float | None = None, target_vol: float | None = None, kelly_fraction: float | None = None) -> FundConfig` (partial-upsert + validation; returns new effective config)
|
||||
- `seed_fund_config(dsn: str) -> None` (insert-if-absent: 100_000 / 0.20 / 0.5)
|
||||
|
||||
- [ ] **Step 1: Write failing tests**
|
||||
|
||||
```python
|
||||
# tests/unit/test_fund_config.py
|
||||
import pytest
|
||||
from fxhnt.application.fund_config import (
|
||||
FundConfig, read_fund_config, write_fund_config, seed_fund_config,
|
||||
)
|
||||
from fxhnt.adapters.persistence.cockpit_models import CockpitBase, FundConfigRow
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.pool import StaticPool
|
||||
|
||||
def _dsn():
|
||||
return "sqlite://"
|
||||
|
||||
def _fresh_engine_dsn():
|
||||
# a dsn whose table has NOT been created yet (fallback path)
|
||||
return "sqlite://"
|
||||
|
||||
def test_read_fallback_when_table_absent(monkeypatch):
|
||||
# No migrate() called → table absent → settings/policy defaults
|
||||
from fxhnt.application.allocation_policy import AllocationPolicy
|
||||
from fxhnt.config import get_settings
|
||||
cfg = read_fund_config("sqlite://")
|
||||
assert cfg.capital == get_settings().paper_capital
|
||||
assert cfg.target_vol == AllocationPolicy().target_vol
|
||||
assert cfg.kelly_fraction == AllocationPolicy().kelly_fraction
|
||||
|
||||
def test_write_then_read_roundtrip():
|
||||
dsn = "sqlite:///./.pytest_fund_cfg_rt.db"
|
||||
import os
|
||||
if os.path.exists("./.pytest_fund_cfg_rt.db"): os.remove("./.pytest_fund_cfg_rt.db")
|
||||
write_fund_config(dsn, capital=250_000.0, target_vol=0.20, kelly_fraction=0.5)
|
||||
cfg = read_fund_config(dsn)
|
||||
assert (cfg.capital, cfg.target_vol, cfg.kelly_fraction) == (250_000.0, 0.20, 0.5)
|
||||
os.remove("./.pytest_fund_cfg_rt.db")
|
||||
|
||||
def test_partial_override_leaves_other_fields():
|
||||
dsn = "sqlite:///./.pytest_fund_cfg_po.db"
|
||||
import os
|
||||
if os.path.exists("./.pytest_fund_cfg_po.db"): os.remove("./.pytest_fund_cfg_po.db")
|
||||
write_fund_config(dsn, capital=100_000.0, target_vol=0.20, kelly_fraction=0.5)
|
||||
write_fund_config(dsn, capital=300_000.0) # only capital
|
||||
cfg = read_fund_config(dsn)
|
||||
assert cfg.capital == 300_000.0
|
||||
assert cfg.target_vol == 0.20 # unchanged
|
||||
assert cfg.kelly_fraction == 0.5 # unchanged
|
||||
os.remove("./.pytest_fund_cfg_po.db")
|
||||
|
||||
@pytest.mark.parametrize("kw", [
|
||||
{"capital": 0.0}, {"capital": -5.0},
|
||||
{"target_vol": 0.0}, {"target_vol": 1.5},
|
||||
{"kelly_fraction": 0.0}, {"kelly_fraction": 2.0},
|
||||
])
|
||||
def test_validation_rejects(kw):
|
||||
dsn = "sqlite:///./.pytest_fund_cfg_val.db"
|
||||
import os
|
||||
if os.path.exists("./.pytest_fund_cfg_val.db"): os.remove("./.pytest_fund_cfg_val.db")
|
||||
with pytest.raises(ValueError):
|
||||
write_fund_config(dsn, **kw)
|
||||
if os.path.exists("./.pytest_fund_cfg_val.db"): os.remove("./.pytest_fund_cfg_val.db")
|
||||
|
||||
def test_seed_is_insert_if_absent():
|
||||
dsn = "sqlite:///./.pytest_fund_cfg_seed.db"
|
||||
import os
|
||||
if os.path.exists("./.pytest_fund_cfg_seed.db"): os.remove("./.pytest_fund_cfg_seed.db")
|
||||
seed_fund_config(dsn)
|
||||
assert read_fund_config(dsn) == FundConfig(100_000.0, 0.20, 0.5)
|
||||
write_fund_config(dsn, capital=500_000.0)
|
||||
seed_fund_config(dsn) # must NOT clobber
|
||||
assert read_fund_config(dsn).capital == 500_000.0
|
||||
os.remove("./.pytest_fund_cfg_seed.db")
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run tests — expect ImportError/failures**
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && python3 -m pytest tests/unit/test_fund_config.py -q`
|
||||
Expected: FAIL (module/model not defined).
|
||||
|
||||
- [ ] **Step 3: Add the ORM model**
|
||||
|
||||
In `cockpit_models.py`, after the existing simple rows (match the `Mapped`/`mapped_column` style; import `Integer`, `Float`, `DateTime` are already imported — verify and add any missing):
|
||||
|
||||
```python
|
||||
class FundConfigRow(CockpitBase):
|
||||
"""Single-row (id=1) fund-level knobs — the SSOT for capital + risk appetite, set ONLY via
|
||||
`fxhnt fund-config` (never hand-edited), read by the nightly allocation. Absent row/table → the nightly
|
||||
falls back to settings/policy defaults (see application/fund_config.read_fund_config)."""
|
||||
__tablename__ = "fund_config"
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True) # always 1
|
||||
capital: Mapped[float] = mapped_column(Float)
|
||||
target_vol: Mapped[float] = mapped_column(Float)
|
||||
kelly_fraction: Mapped[float] = mapped_column(Float)
|
||||
updated_at: Mapped[dt.datetime] = mapped_column(DateTime)
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Write `application/fund_config.py`**
|
||||
|
||||
```python
|
||||
"""The fund-level knobs SSOT: capital + risk appetite (target_vol, kelly_fraction) as a single-row DB record,
|
||||
set ONLY via the `fxhnt fund-config` CLI, read by the nightly allocation. Absent table/row → settings/policy
|
||||
defaults (byte-identical to the pre-fund_config nightly). No hand-editing — the CLI is the sole writer."""
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import create_engine, select
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy.pool import StaticPool
|
||||
|
||||
from fxhnt.adapters.persistence.cockpit_models import CockpitBase, FundConfigRow
|
||||
from fxhnt.application.allocation_policy import AllocationPolicy
|
||||
from fxhnt.config import get_settings
|
||||
|
||||
_ROW_ID = 1
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FundConfig:
|
||||
capital: float
|
||||
target_vol: float
|
||||
kelly_fraction: float
|
||||
|
||||
|
||||
def _defaults() -> FundConfig:
|
||||
p = AllocationPolicy()
|
||||
return FundConfig(capital=get_settings().paper_capital,
|
||||
target_vol=p.target_vol, kelly_fraction=p.kelly_fraction)
|
||||
|
||||
|
||||
def _engine(dsn: str):
|
||||
kw: dict[str, Any] = {"future": True}
|
||||
if dsn == "sqlite://" or (dsn.startswith("sqlite://") and ":memory:" in dsn):
|
||||
kw |= {"connect_args": {"check_same_thread": False}, "poolclass": StaticPool}
|
||||
return create_engine(dsn, **kw)
|
||||
|
||||
|
||||
def _validate(capital: float | None, target_vol: float | None, kelly_fraction: float | None) -> None:
|
||||
if capital is not None and not (capital > 0):
|
||||
raise ValueError(f"capital must be > 0, got {capital}")
|
||||
if target_vol is not None and not (0 < target_vol <= 1.0):
|
||||
raise ValueError(f"target_vol must be in (0, 1.0], got {target_vol}")
|
||||
if kelly_fraction is not None and not (0 < kelly_fraction <= 1.0):
|
||||
raise ValueError(f"kelly_fraction must be in (0, 1.0], got {kelly_fraction}")
|
||||
|
||||
|
||||
def read_fund_config(dsn: str) -> FundConfig:
|
||||
"""Row id=1 as a FundConfig; settings/policy defaults if the table or row is absent (never raises on a
|
||||
fresh DB — the nightly must not break)."""
|
||||
try:
|
||||
eng = _engine(dsn)
|
||||
with Session(eng) as s:
|
||||
row = s.get(FundConfigRow, _ROW_ID)
|
||||
except Exception: # noqa: BLE001 — absent table on a fresh DB → defaults
|
||||
return _defaults()
|
||||
if row is None:
|
||||
return _defaults()
|
||||
return FundConfig(capital=row.capital, target_vol=row.target_vol, kelly_fraction=row.kelly_fraction)
|
||||
|
||||
|
||||
def write_fund_config(dsn: str, *, capital: float | None = None, target_vol: float | None = None,
|
||||
kelly_fraction: float | None = None) -> FundConfig:
|
||||
"""Partial upsert of row id=1: overlay only the provided fields on the current effective config, validate,
|
||||
write. Returns the new effective FundConfig. Creates the table if absent."""
|
||||
if capital is None and target_vol is None and kelly_fraction is None:
|
||||
raise ValueError("write_fund_config: provide at least one of capital/target_vol/kelly_fraction")
|
||||
_validate(capital, target_vol, kelly_fraction)
|
||||
eng = _engine(dsn)
|
||||
CockpitBase.metadata.create_all(eng, tables=[FundConfigRow.__table__])
|
||||
cur = read_fund_config(dsn)
|
||||
new = FundConfig(
|
||||
capital=cur.capital if capital is None else float(capital),
|
||||
target_vol=cur.target_vol if target_vol is None else float(target_vol),
|
||||
kelly_fraction=cur.kelly_fraction if kelly_fraction is None else float(kelly_fraction),
|
||||
)
|
||||
with Session(eng) as s:
|
||||
s.merge(FundConfigRow(id=_ROW_ID, capital=new.capital, target_vol=new.target_vol,
|
||||
kelly_fraction=new.kelly_fraction,
|
||||
updated_at=dt.datetime.now(dt.UTC).replace(tzinfo=None)))
|
||||
s.commit()
|
||||
return new
|
||||
|
||||
|
||||
def seed_fund_config(dsn: str) -> None:
|
||||
"""Insert row id=1 with the initial knobs (100k / 0.20 / 0.5) IF ABSENT. Never clobbers an operator-set
|
||||
value (idempotent)."""
|
||||
eng = _engine(dsn)
|
||||
CockpitBase.metadata.create_all(eng, tables=[FundConfigRow.__table__])
|
||||
with Session(eng) as s:
|
||||
if s.get(FundConfigRow, _ROW_ID) is not None:
|
||||
return
|
||||
s.add(FundConfigRow(id=_ROW_ID, capital=100_000.0, target_vol=0.20, kelly_fraction=0.5,
|
||||
updated_at=dt.datetime.now(dt.UTC).replace(tzinfo=None)))
|
||||
s.commit()
|
||||
```
|
||||
|
||||
- [ ] **Step 5: Run tests — expect PASS**
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && python3 -m pytest tests/unit/test_fund_config.py -q`
|
||||
Expected: PASS (all).
|
||||
|
||||
- [ ] **Step 6: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/adapters/persistence/cockpit_models.py src/fxhnt/application/fund_config.py tests/unit/test_fund_config.py
|
||||
git commit -m "feat(fund-config): single-row fund_config SSOT (read/write/fallback/validate/seed)
|
||||
|
||||
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 2: `fxhnt fund-config` CLI + wire seed into `migrate-cockpit`
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/cli.py` (add `fund-config` typer sub-app or two commands; wire `seed_fund_config` into `migrate_cockpit`)
|
||||
- Test: `tests/unit/test_fund_config_cli.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `read_fund_config`, `write_fund_config`, `seed_fund_config` (Task 1), `typer`, `get_settings().operational_dsn`.
|
||||
- Produces: CLI `fxhnt fund-config show` and `fxhnt fund-config set [--capital] [--target-vol] [--kelly]`.
|
||||
|
||||
- [ ] **Step 1: Write failing tests** (use typer's `CliRunner`)
|
||||
|
||||
```python
|
||||
# tests/unit/test_fund_config_cli.py
|
||||
import os
|
||||
from typer.testing import CliRunner
|
||||
from fxhnt.cli import app
|
||||
from fxhnt.application.fund_config import read_fund_config
|
||||
|
||||
runner = CliRunner()
|
||||
|
||||
def _isolate(monkeypatch, path):
|
||||
if os.path.exists(path): os.remove(path)
|
||||
monkeypatch.setattr("fxhnt.cli.get_settings", lambda: _S(f"sqlite:///{path}"))
|
||||
|
||||
class _S:
|
||||
def __init__(self, dsn): self.operational_dsn = dsn; self.paper_capital = 100_000.0
|
||||
|
||||
def test_set_then_show(monkeypatch):
|
||||
path = "./.pytest_fund_cli.db"; _isolate(monkeypatch, path)
|
||||
r = runner.invoke(app, ["fund-config", "set", "--capital", "250000", "--target-vol", "0.2", "--kelly", "0.5"])
|
||||
assert r.exit_code == 0, r.output
|
||||
assert read_fund_config(f"sqlite:///{path}").capital == 250_000.0
|
||||
r2 = runner.invoke(app, ["fund-config", "show"])
|
||||
assert r2.exit_code == 0
|
||||
assert "250000" in r2.output.replace(",", "") or "250000.0" in r2.output.replace(",", "")
|
||||
os.remove(path)
|
||||
|
||||
def test_set_rejects_bad_value(monkeypatch):
|
||||
path = "./.pytest_fund_cli_bad.db"; _isolate(monkeypatch, path)
|
||||
r = runner.invoke(app, ["fund-config", "set", "--target-vol", "1.5"])
|
||||
assert r.exit_code != 0
|
||||
if os.path.exists(path): os.remove(path)
|
||||
|
||||
def test_set_no_flags_errors(monkeypatch):
|
||||
path = "./.pytest_fund_cli_noflag.db"; _isolate(monkeypatch, path)
|
||||
r = runner.invoke(app, ["fund-config", "set"])
|
||||
assert r.exit_code != 0
|
||||
if os.path.exists(path): os.remove(path)
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run — expect FAIL** (`fund-config` command absent)
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && python3 -m pytest tests/unit/test_fund_config_cli.py -q`
|
||||
Expected: FAIL.
|
||||
|
||||
- [ ] **Step 3: Implement the CLI** (typer sub-app `fund_config_app` registered on `app`)
|
||||
|
||||
```python
|
||||
# in cli.py, near the other command groups
|
||||
fund_config_app = typer.Typer(help="Fund-level knobs (capital + risk appetite) — the allocation SSOT.")
|
||||
app.add_typer(fund_config_app, name="fund-config")
|
||||
|
||||
@fund_config_app.command("show")
|
||||
def fund_config_show() -> None:
|
||||
"""Print the effective fund knobs (capital, target_vol, kelly_fraction)."""
|
||||
from fxhnt.application.fund_config import read_fund_config
|
||||
cfg = read_fund_config(get_settings().operational_dsn)
|
||||
typer.echo(f"capital={cfg.capital:.2f} target_vol={cfg.target_vol:.4f} kelly_fraction={cfg.kelly_fraction:.4f}")
|
||||
|
||||
@fund_config_app.command("set")
|
||||
def fund_config_set(
|
||||
capital: float = typer.Option(None, help="Fund capital (equity the allocation sizes against). > 0."),
|
||||
target_vol: float = typer.Option(None, "--target-vol", help="Annualised vol target (0, 1.0]."),
|
||||
kelly: float = typer.Option(None, "--kelly", help="Kelly fraction (0, 1.0]."),
|
||||
) -> None:
|
||||
"""Set one or more fund knobs (partial override; unspecified fields unchanged)."""
|
||||
from fxhnt.application.fund_config import write_fund_config
|
||||
try:
|
||||
cfg = write_fund_config(get_settings().operational_dsn, capital=capital,
|
||||
target_vol=target_vol, kelly_fraction=kelly)
|
||||
except ValueError as e:
|
||||
typer.echo(f"error: {e}"); raise typer.Exit(code=1) from e
|
||||
typer.echo(f"set: capital={cfg.capital:.2f} target_vol={cfg.target_vol:.4f} kelly_fraction={cfg.kelly_fraction:.4f}")
|
||||
```
|
||||
|
||||
Then in `migrate_cockpit()` (after `ForwardNavRepo(dsn).migrate()`), add:
|
||||
|
||||
```python
|
||||
from fxhnt.application.fund_config import seed_fund_config
|
||||
seed_fund_config(dsn)
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run — expect PASS**
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && python3 -m pytest tests/unit/test_fund_config_cli.py -q`
|
||||
Expected: PASS.
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/cli.py tests/unit/test_fund_config_cli.py
|
||||
git commit -m "feat(fund-config): fxhnt fund-config show|set CLI + seed on migrate-cockpit
|
||||
|
||||
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 3: nightly reads `fund_config` → equity + `AllocationPolicy`
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/application/allocation_ingest.py` (optional `policy` param on `record_allocation`)
|
||||
- Modify: `src/fxhnt/adapters/orchestration/assets.py` (`cockpit_forward`: read fund_config → equity + policy)
|
||||
- Test: `tests/unit/test_record_allocation_policy.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `read_fund_config` (Task 1), `resolve_allocation_policy` / `AllocationPolicy` (allocation_policy), `dataclasses.replace`.
|
||||
- Produces: `record_allocation(dsn, equity, *, store=None, spread_bps=None, unlock_events=None, policy=None)` — `policy=None` keeps today's resolution (byte-identical); a provided `AllocationPolicy` is used verbatim for both the honest and nav paths.
|
||||
|
||||
- [ ] **Step 1: Inspect** how `record_allocation` currently builds its policy (`resolve_allocation_policy`) so the `policy=None` default is provably identical. Read `allocation_ingest.py` lines around the policy build and `allocation_policy.resolve_allocation_policy`.
|
||||
|
||||
- [ ] **Step 2: Write failing tests**
|
||||
|
||||
```python
|
||||
# tests/unit/test_record_allocation_policy.py
|
||||
import dataclasses
|
||||
from fxhnt.application.allocation_policy import AllocationPolicy, resolve_allocation_policy
|
||||
|
||||
def test_policy_none_matches_resolved_default(monkeypatch):
|
||||
# record_allocation(policy=None) must build the SAME policy resolve_allocation_policy() returns today.
|
||||
# Assert via the helper the code will use: a None arg resolves to resolve_allocation_policy().
|
||||
from fxhnt.application import allocation_ingest as ai
|
||||
resolved = ai._effective_policy(None) # helper introduced in Step 3
|
||||
assert resolved == resolve_allocation_policy()
|
||||
|
||||
def test_policy_passthrough():
|
||||
from fxhnt.application import allocation_ingest as ai
|
||||
p = dataclasses.replace(resolve_allocation_policy(), target_vol=0.20, kelly_fraction=0.5)
|
||||
assert ai._effective_policy(p) is p
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Implement.** In `allocation_ingest.py` add a tiny helper and thread it:
|
||||
|
||||
```python
|
||||
def _effective_policy(policy):
|
||||
from fxhnt.application.allocation_policy import resolve_allocation_policy
|
||||
return resolve_allocation_policy() if policy is None else policy
|
||||
```
|
||||
|
||||
Add `policy=None` to the `record_allocation` signature and replace its internal `resolve_allocation_policy()` call with `pol = _effective_policy(policy)`, using `pol` wherever the policy is consumed (compute_allocation / target_capital / hashing). Verify every prior use of the resolved policy now reads `pol`.
|
||||
|
||||
- [ ] **Step 4: Wire `cockpit_forward`** (`assets.py` ~line 1073). Replace `equity = get_settings().paper_capital` with:
|
||||
|
||||
```python
|
||||
from fxhnt.application.fund_config import read_fund_config
|
||||
import dataclasses as _dc
|
||||
from fxhnt.application.allocation_policy import resolve_allocation_policy
|
||||
_cfg = read_fund_config(dsn)
|
||||
equity = _cfg.capital
|
||||
_policy = _dc.replace(resolve_allocation_policy(), target_vol=_cfg.target_vol,
|
||||
kelly_fraction=_cfg.kelly_fraction)
|
||||
```
|
||||
|
||||
and pass `policy=_policy` to BOTH `record_allocation(...)` calls (the honest-store path and the nav-fallback path).
|
||||
|
||||
- [ ] **Step 5: Run tests — expect PASS**
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && python3 -m pytest tests/unit/test_record_allocation_policy.py tests/unit/test_compute_allocation.py -q`
|
||||
Expected: PASS (new + existing allocation tests still green).
|
||||
|
||||
- [ ] **Step 6: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/application/allocation_ingest.py src/fxhnt/adapters/orchestration/assets.py tests/unit/test_record_allocation_policy.py
|
||||
git commit -m "feat(fund-config): nightly allocation reads fund_config for equity + policy (target_vol/kelly)
|
||||
|
||||
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 4: per-sleeve allocation — registry + provider
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/registry.py` (`xsfunding`/`unlock` → `promote_on_pass: True`; `bybit_4edge` → `tier: research`)
|
||||
- Modify: `src/fxhnt/application/allocation_honest_inputs.py` (crypto-sleeve generalization)
|
||||
- Test: `tests/unit/test_allocation_honest_inputs.py` (extend existing if present, else create)
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `_sleeve_series` (existing, works for any sleeve), `honest_allocation_inputs` (existing signature unchanged).
|
||||
- Produces: `_CRYPTO_DEPLOY_SIDS = frozenset({"positioning", "xsfunding", "unlock"})`; `_deploy_series` returns the per-sleeve honest series for each of those three (no `bybit_4edge` branch in the allocation path).
|
||||
|
||||
- [ ] **Step 1: Write failing tests** (a fake store returning deterministic per-sleeve weights/turnover; assert xsfunding/unlock are sized as sleeves, and bybit_4edge yields no series in the allocation path)
|
||||
|
||||
```python
|
||||
# tests/unit/test_allocation_honest_inputs.py (extend / create)
|
||||
from fxhnt.application.allocation_honest_inputs import (
|
||||
honest_allocation_inputs, _CRYPTO_DEPLOY_SIDS,
|
||||
)
|
||||
|
||||
def test_crypto_deploy_sids_are_the_three_sleeves():
|
||||
assert _CRYPTO_DEPLOY_SIDS == frozenset({"positioning", "xsfunding", "unlock"})
|
||||
|
||||
def test_xsfunding_and_unlock_get_a_series_not_equity_default(fake_store, spread_bps, unlock_events):
|
||||
# fake_store yields non-empty weights for xsfunding & unlock
|
||||
returns, ceilings = honest_allocation_inputs(
|
||||
fake_store, spread_bps, unlock_events,
|
||||
deploy_sids={"xsfunding", "unlock"}, target_aum=35_000.0, aums=[35_000.0, 350_000.0])
|
||||
assert "xsfunding" in returns and returns["xsfunding"] # sleeve-sized, ISO-keyed
|
||||
assert "unlock" in returns and returns["unlock"]
|
||||
# ceilings present (not the equity default 10M sentinel for these crypto sleeves)
|
||||
assert ceilings["xsfunding"] != 10_000_000.0
|
||||
|
||||
def test_bybit_4edge_is_not_a_crypto_deploy_sid():
|
||||
assert "bybit_4edge" not in _CRYPTO_DEPLOY_SIDS
|
||||
```
|
||||
|
||||
(Reuse the existing test's `fake_store`/fixtures if the test file already exists; otherwise build a minimal fake exposing `read_panel` + the `sleeve_weights_and_contributions` path — mirror the pattern already used in the honest-reference tests.)
|
||||
|
||||
- [ ] **Step 2: Run — expect FAIL**
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && python3 -m pytest tests/unit/test_allocation_honest_inputs.py -q`
|
||||
Expected: FAIL.
|
||||
|
||||
- [ ] **Step 3: Registry edits** (`registry.py`):
|
||||
- `xsfunding` entry: add `"promote_on_pass": True,` (top-level key, sibling of `tier`).
|
||||
- `unlock` entry: add `"promote_on_pass": True,`.
|
||||
- `bybit_4edge` entry: change `"tier": "deploy"` → `"tier": "research"`. Leave everything else (gate_spec, definition, inception_t0) unchanged. Update the entry's leading comment to note it is now a tracked REFERENCE aggregate (allocation runs per-sleeve; see the 2026-07-16 spec).
|
||||
|
||||
- [ ] **Step 4: Provider edits** (`allocation_honest_inputs.py`):
|
||||
- `_CRYPTO_DEPLOY_SIDS = frozenset({"positioning", "xsfunding", "unlock"})`.
|
||||
- `_deploy_series`: replace the positioning/bybit_4edge branches with: `if sid in _CRYPTO_DEPLOY_SIDS: return _sleeve_series(store, sid, spread_bps, unlock_events, aum=aum, participation_cap=participation_cap, impact_k=impact_k)` then `return {}`.
|
||||
- Keep `_book_series` / `_BOOK_SLEEVES` (still used by the honest-reference REPORT), but remove the `bybit_4edge` dispatch from `_deploy_series`. Update the module docstring's deploy→honest mapping to the three sleeves + note the book is no longer an allocation target.
|
||||
|
||||
- [ ] **Step 5: Run — expect PASS** (and re-run the honest-reference tests to confirm `_book_series` still works)
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && python3 -m pytest tests/unit/test_allocation_honest_inputs.py tests/unit/ -k "honest or allocation" -q`
|
||||
Expected: PASS.
|
||||
|
||||
- [ ] **Step 6: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/registry.py src/fxhnt/application/allocation_honest_inputs.py tests/unit/test_allocation_honest_inputs.py
|
||||
git commit -m "feat(alloc): per-sleeve crypto allocation (promote xsfunding/unlock; bybit_4edge->reference)
|
||||
|
||||
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 5: full-suite green + book-family sanity
|
||||
|
||||
**Files:** none (verification task)
|
||||
|
||||
- [ ] **Step 1: Run the allocation + fund-config + honest-reference unit suites**
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && python3 -m pytest tests/unit -k "fund_config or allocation or honest or compute_allocation" -q`
|
||||
Expected: all PASS.
|
||||
|
||||
- [ ] **Step 2: Grep for any remaining hard reference to `bybit_4edge` as a deploy-tier allocation target** (e.g. code that assumed the book is allocated). Fix or note any found.
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && grep -rn "bybit_4edge" src/fxhnt/application/ src/fxhnt/adapters/ | grep -iv "exec\|levered\|ref\|comment"`
|
||||
Expected: no code that *allocates* to `bybit_4edge` (execution wiring is out of scope / suspended).
|
||||
|
||||
- [ ] **Step 3: Commit any fixes** from Step 2 (if none, skip).
|
||||
|
||||
---
|
||||
|
||||
## Post-implementation (controller-run, NOT a subagent task — in-cluster + local)
|
||||
|
||||
These are the spec's §5 falsification gate + cockpit verification. The controller runs them after the branch is green, per the user's "run queries in the cluster" and "verify cockpit locally" directives:
|
||||
|
||||
1. **Deploy** (git-sync nightly picks up master) + `fxhnt migrate-cockpit` (creates `fund_config`, seeds 100k/0.20/0.5) in-cluster.
|
||||
2. **G1/G2/G3** — in-cluster diagnostic: dump `survivors_after_marginal_prune`, the per-sleeve weights, deployed gross $ and %, and confirm `K` uses full-history vol. Report the numbers (do NOT override the gate).
|
||||
3. **Cockpit local verify** — `bybit_4edge`-as-reference renders sanely (deploy section shows the sleeves, no broken headline).
|
||||
4. Report the honest verdict (per-sleeve deploys more, or it concentrates to positioning + the vol-target is the lever).
|
||||
|
||||
## Self-Review
|
||||
|
||||
- **Spec coverage:** §4a→T1, §4b→T2, §4c→T3, §4d→T4, §4e→T1(seed)+T2(wire), §5→post-impl gate, §7 risks→Global Constraints + T4 + post-impl. Covered.
|
||||
- **Placeholders:** none — every step has concrete code/commands.
|
||||
- **Type consistency:** `FundConfig`, `read_fund_config`, `write_fund_config`, `seed_fund_config`, `_effective_policy`, `_CRYPTO_DEPLOY_SIDS` used identically across tasks.
|
||||
298
docs/superpowers/plans/2026-07-16-equity-leg-deployable.md
Normal file
298
docs/superpowers/plans/2026-07-16-equity-leg-deployable.md
Normal file
@@ -0,0 +1,298 @@
|
||||
# Equity Leg Deployable Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: superpowers:subagent-driven-development. Checkbox (`- [ ]`) steps.
|
||||
|
||||
**Goal:** Make the IBKR US-ETF `multistrat` book honest-sized + capacity-bounded (real ADV from ingested ETF volume, real trading costs), so it deploys as the growth vehicle above the crypto book's ~$350k break.
|
||||
|
||||
**Architecture:** An equity analog of the crypto honest-reference. Ingest real Yahoo ETF volume (PIT); a real cost model (sqrt-impact on dollar-ADV + per-ETF spread + IBKR commission) net of `multistrat`'s turnover; a cost-netted `book_series` sizing series + ADV capacity curve; an `equity_allocation_inputs` provider merged into `record_allocation`.
|
||||
|
||||
**Tech Stack:** Python 3.12, SQLAlchemy 2.0, typer, pytest. Spec: `docs/superpowers/specs/2026-07-16-equity-leg-deployable-design.md`.
|
||||
|
||||
## Global Constraints
|
||||
|
||||
- `FUND_INSTRUMENTS = ["SPY","IEF","GLD","PDBC","DBMF"]` (from `multistrat`).
|
||||
- **Per-ETF half-spread constants (bps), documented — the one estimated input** (no ETF bid/ask feed): `{"SPY":0.3,"IEF":1.0,"GLD":1.0,"PDBC":4.0,"DBMF":8.0}`. Default for an unlisted symbol: 5.0 bps.
|
||||
- **IBKR commission**: `commission_per_share = 0.0035` USD; `commission_bps = 100 * commission_per_share / price` (tiny for high-priced ETFs — dominant cost is spread).
|
||||
- **Dollar-ADV**: `adv_usd[sym][day] = pit_close[sym][day] * volume[sym][day]`, trailing **63-day MEDIAN** (outlier-robust; a volume spike must not inflate capacity). A day with missing volume is excluded (no fabricated ADV).
|
||||
- **PIT discipline**: volume snapshot is first-seen-frozen (a re-fetch must NOT overwrite a stored (symbol,date) — mirror `snapshot_yahoo_closes`).
|
||||
- **Byte-identity**: `record_allocation` with `store=None`, or with no equity sid in the deploy set, must be byte-identical to today. The equity provider only adds series/ceilings for `ibkr-equity` deploy sids.
|
||||
- **Cost invariant**: costs must REDUCE the equity Sharpe (net ≤ gross); the turnover charged is the ACTUAL traded weight change (two-way), from `multistrat.target_weights`, not the raw signal.
|
||||
- Reuse the crypto templates (`honest_sleeve_returns` shape, `capacity_ceiling_from_curve`, load-once/reprice). Do NOT reintroduce a flat 15bp cost.
|
||||
- Commit trailer: `Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>`.
|
||||
|
||||
---
|
||||
|
||||
### Task 1: Ingest ETF volume (fetch + PIT store + snapshot + backfill CLI)
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/adapters/data/yahoo_daily.py` (`.volumes()`)
|
||||
- Modify: `src/fxhnt/adapters/persistence/cockpit_models.py` (`YahooPitVolumeRow`)
|
||||
- Modify: `src/fxhnt/adapters/persistence/forward_nav.py` (`snapshot_yahoo_volumes`, `yahoo_pit_volumes`)
|
||||
- Modify: `src/fxhnt/adapters/orchestration/assets.py` (nightly snapshot of volumes)
|
||||
- Modify: `src/fxhnt/cli.py` (`backfill-etf-volume`)
|
||||
- Test: `tests/unit/test_yahoo_volume_pit.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: `YahooDailyClient.volumes(symbol) -> dict[str,float]` (date→raw share volume); `ForwardNavRepo.snapshot_yahoo_volumes(symbol, series, at)` / `.yahoo_pit_volumes(symbol) -> dict[str,float]`.
|
||||
|
||||
- [ ] **Step 1: Write failing tests** — a fake Yahoo response dict → `volumes()` extracts `quote[0]["volume"]` mapped to dates (skipping None); a repo roundtrip: `snapshot_yahoo_volumes` then `yahoo_pit_volumes` returns the series; first-seen-frozen (a second snapshot with a changed value for an existing date does NOT overwrite).
|
||||
|
||||
```python
|
||||
# tests/unit/test_yahoo_volume_pit.py
|
||||
import datetime as dt
|
||||
from fxhnt.adapters.persistence.forward_nav import ForwardNavRepo
|
||||
|
||||
def test_snapshot_and_read_volumes_first_seen_frozen():
|
||||
r = ForwardNavRepo("sqlite://"); r.migrate()
|
||||
at = dt.datetime(2026, 7, 16)
|
||||
r.snapshot_yahoo_volumes("SPY", {"2026-07-01": 1000.0, "2026-07-02": 2000.0}, at)
|
||||
r.snapshot_yahoo_volumes("SPY", {"2026-07-01": 9999.0, "2026-07-03": 3000.0}, at) # 07-01 must NOT change
|
||||
assert r.yahoo_pit_volumes("SPY") == {"2026-07-01": 1000.0, "2026-07-02": 2000.0, "2026-07-03": 3000.0}
|
||||
|
||||
def test_volumes_extracts_quote_volume(monkeypatch):
|
||||
from fxhnt.adapters.data.yahoo_daily import YahooDailyClient
|
||||
c = YahooDailyClient()
|
||||
fake = {"chart": {"result": [{"timestamp": [1719792000, 1719878400],
|
||||
"indicators": {"quote": [{"volume": [111.0, None]}], "adjclose": [{"adjclose": [1.0, 2.0]}]}}]}}
|
||||
monkeypatch.setattr(c, "_get", lambda url: fake)
|
||||
out = c.volumes("SPY")
|
||||
assert list(out.values()) == [111.0] # None volume day skipped
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run — expect FAIL**
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && .venv/bin/python3 -m pytest tests/unit/test_yahoo_volume_pit.py -q`
|
||||
Expected: FAIL.
|
||||
|
||||
- [ ] **Step 3: Implement.**
|
||||
- `yahoo_daily.py` — `volumes(symbol)` mirrors `adj_closes` but reads `ind["quote"][0]["volume"]`:
|
||||
```python
|
||||
def volumes(self, symbol: str) -> dict[str, float]:
|
||||
res = self._get(_BASE.format(sym=symbol, rng=self._rng))["chart"]["result"][0]
|
||||
ts = res["timestamp"]
|
||||
vol = res["indicators"]["quote"][0].get("volume") or []
|
||||
out: dict[str, float] = {}
|
||||
for t, v in zip(ts, vol, strict=False):
|
||||
if v is not None:
|
||||
out[dt.datetime.fromtimestamp(t, dt.timezone.utc).strftime("%Y-%m-%d")] = float(v)
|
||||
return out
|
||||
```
|
||||
- `cockpit_models.py` — `YahooPitVolumeRow` mirroring `YahooPitCloseRow` (`__tablename__="yahoo_pit_volume"`, PK (symbol,date), `volume: Mapped[float]`, `first_seen_at`).
|
||||
- `forward_nav.py` — `snapshot_yahoo_volumes` / `yahoo_pit_volumes` mirroring the close methods exactly (first-seen-frozen).
|
||||
- `assets.py` — in the block that calls `repo.snapshot_yahoo_closes(sym, live.adj_closes(sym), at)` (~line 421), add `repo.snapshot_yahoo_volumes(sym, live.volumes(sym), at)` in the same loop (wrap the volume fetch in try/except so a volume-fetch failure never breaks the close snapshot — WARN + continue).
|
||||
- `cli.py` — `@app.command("backfill-etf-volume")`: for each `FUND_INSTRUMENTS` sym, `YahooDailyClient(rng="25y").volumes(sym)` → `repo.snapshot_yahoo_volumes(...)`; echo counts. (Import `FUND_INSTRUMENTS` from `fxhnt.domain.strategies.multistrat`.)
|
||||
|
||||
- [ ] **Step 4: Run — expect PASS**
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && .venv/bin/python3 -m pytest tests/unit/test_yahoo_volume_pit.py -q`
|
||||
Expected: PASS.
|
||||
|
||||
- [ ] **Step 5: Commit** (`git add` the 5 files + test; message `feat(equity): ingest ETF volume (PIT) + backfill-etf-volume CLI`).
|
||||
|
||||
---
|
||||
|
||||
### Task 2: `equity_honest.py` — real cost model + capacity curve
|
||||
|
||||
**Files:**
|
||||
- Create: `src/fxhnt/application/equity_honest.py`
|
||||
- Test: `tests/unit/test_equity_honest.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `IMPACT_COEFF_BPS` (honest_reference), `sharpe_of`/`per_period_sharpe`, `capacity_ceiling_from_curve` (allocation_engine).
|
||||
- Produces:
|
||||
- `ETF_HALF_SPREAD_BPS: dict[str,float]` (the documented constants) + `_spread_bps(sym) -> float`.
|
||||
- `dollar_adv(pit_closes, pit_volumes, window=63) -> dict[str, dict[str,float]]` (per-sym trailing-median dollar volume).
|
||||
- `equity_honest_returns(book_ret_by_day, weights_by_day, adv_usd, *, aum, participation_cap=0.03, impact_k=1.0, commission_per_share=0.0035) -> dict[str,float]`
|
||||
- `equity_capacity_curve(book_ret_by_day, weights_by_day, adv_usd, *, aums, ...) -> dict[float, dict[str,float]]`
|
||||
|
||||
- [ ] **Step 1: Write failing tests.**
|
||||
- `dollar_adv`: given closes + volumes over N days, the ADV is the trailing 63d (or shorter early) median of `close*volume`; a missing-volume day is excluded.
|
||||
- `equity_honest_returns`: (a) with ZERO turnover (constant weights) → net == gross book_ret (no cost). (b) with turnover + tiny AUM (adv >> notional) → cost ≈ spread-only (impact ~0), net < gross by `Σ traded_w * spread/1e4`. (c) net Sharpe ≤ gross Sharpe. (d) a position exceeding `participation_cap * adv` is scaled down (capacity), reducing that day's gross contribution.
|
||||
- `equity_capacity_curve`: monotonic-ish — higher AUM → lower or equal net Sharpe (impact grows); returns one series per aum.
|
||||
|
||||
```python
|
||||
# tests/unit/test_equity_honest.py (sketch — fill real numbers)
|
||||
from fxhnt.application.equity_honest import dollar_adv, equity_honest_returns, equity_capacity_curve, ETF_HALF_SPREAD_BPS
|
||||
|
||||
def test_zero_turnover_no_cost():
|
||||
days = ["2026-01-01","2026-01-02","2026-01-03"]
|
||||
book = {d: 0.001 for d in days}
|
||||
w = {d: {"SPY": 0.5} for d in days} # constant weights -> no turnover after day 1
|
||||
adv = {"SPY": {d: 1e12 for d in days}} # infinite liquidity -> no impact
|
||||
net = equity_honest_returns(book, w, adv, aum=100_000.0)
|
||||
# day 1 enters the position (turnover from 0->0.5) -> small spread cost; days 2-3 no turnover -> net==book
|
||||
assert abs(net["2026-01-02"] - book["2026-01-02"]) < 1e-12
|
||||
assert abs(net["2026-01-03"] - book["2026-01-03"]) < 1e-12
|
||||
|
||||
def test_costs_reduce_sharpe():
|
||||
... # net sharpe <= gross sharpe over a turnover-y series
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run — expect FAIL**
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && .venv/bin/python3 -m pytest tests/unit/test_equity_honest.py -q`
|
||||
Expected: FAIL.
|
||||
|
||||
- [ ] **Step 3: Implement `equity_honest.py`.** Mirror `honest_sleeve_returns`'s structure (turnover-basis cost, participation cap), but with equity inputs. Reference implementation:
|
||||
|
||||
```python
|
||||
"""Equity honest-reference: the IBKR US-ETF book's cost-netted return series + capacity, the analog of the
|
||||
crypto honest_reference. Real costs: sqrt market-impact on REAL dollar-ADV (ingested Yahoo volume) + a
|
||||
documented per-ETF spread + IBKR commission, charged on the book's ACTUAL traded weight (two-way turnover).
|
||||
Pure; no I/O (callers pass preloaded closes/volumes/weights)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import statistics
|
||||
|
||||
from fxhnt.application.honest_reference import IMPACT_COEFF_BPS
|
||||
from fxhnt.domain.gauntlet import per_period_sharpe
|
||||
|
||||
ETF_HALF_SPREAD_BPS: dict[str, float] = {"SPY": 0.3, "IEF": 1.0, "GLD": 1.0, "PDBC": 4.0, "DBMF": 8.0}
|
||||
_DEFAULT_SPREAD_BPS = 5.0
|
||||
|
||||
|
||||
def _spread_bps(sym: str) -> float:
|
||||
return ETF_HALF_SPREAD_BPS.get(sym, _DEFAULT_SPREAD_BPS)
|
||||
|
||||
|
||||
def dollar_adv(pit_closes: dict[str, dict[str, float]], pit_volumes: dict[str, dict[str, float]],
|
||||
*, window: int = 63) -> dict[str, dict[str, float]]:
|
||||
"""Per-symbol trailing-`window` MEDIAN dollar volume (close*volume). Missing-volume days excluded."""
|
||||
out: dict[str, dict[str, float]] = {}
|
||||
for sym, vols in pit_volumes.items():
|
||||
closes = pit_closes.get(sym, {})
|
||||
dv = {d: closes[d] * vols[d] for d in sorted(vols) if d in closes and vols[d] is not None}
|
||||
dates = sorted(dv)
|
||||
cur: dict[str, float] = {}
|
||||
for i, d in enumerate(dates):
|
||||
lo = max(0, i - window + 1)
|
||||
cur[d] = statistics.median(dv[x] for x in dates[lo:i + 1])
|
||||
out[sym] = cur
|
||||
return out
|
||||
|
||||
|
||||
def equity_honest_returns(book_ret_by_day: dict[str, float], weights_by_day: dict[str, dict[str, float]],
|
||||
adv_usd: dict[str, dict[str, float]], *, aum: float,
|
||||
participation_cap: float = 0.03, impact_k: float = 1.0,
|
||||
commission_per_share: float = 0.0035,
|
||||
price_by_day: dict[str, dict[str, float]] | None = None) -> dict[str, float]:
|
||||
"""Cost-netted daily book return. Cost = Σ_sym traded_w * (max(impact, spread) + commission)/1e4, on the
|
||||
two-way traded weight; positions exceeding participation_cap*ADV are scaled down (capacity)."""
|
||||
out: dict[str, float] = {}
|
||||
prev: dict[str, float] = {}
|
||||
for day in sorted(book_ret_by_day):
|
||||
row = weights_by_day.get(day, {})
|
||||
cur: dict[str, float] = {}
|
||||
scale_hit = 0.0
|
||||
for sym, w in row.items():
|
||||
adv = adv_usd.get(sym, {}).get(day, 0.0)
|
||||
notional = abs(w) * aum
|
||||
if adv <= 0.0:
|
||||
cur[sym] = 0.0 # no ADV -> zero capacity (honest, not infinite)
|
||||
scale_hit += abs(w) # this weight can't be held -> its return is forgone
|
||||
elif notional > participation_cap * adv:
|
||||
cur[sym] = w * (participation_cap * adv) / notional
|
||||
scale_hit += abs(w) - abs(cur[sym])
|
||||
else:
|
||||
cur[sym] = w
|
||||
cost = 0.0
|
||||
for sym in set(cur) | set(prev):
|
||||
traded_w = abs(cur.get(sym, 0.0) - prev.get(sym, 0.0))
|
||||
if traded_w <= 0.0:
|
||||
continue
|
||||
adv = adv_usd.get(sym, {}).get(day, 0.0)
|
||||
traded_notional = traded_w * aum
|
||||
impact = impact_k * IMPACT_COEFF_BPS * math.sqrt(traded_notional / adv) if adv > 0.0 and traded_notional > 0.0 else 0.0
|
||||
price = (price_by_day or {}).get(sym, {}).get(day, 0.0)
|
||||
commission_bps = (100.0 * commission_per_share / price) if price > 0.0 else 0.0
|
||||
slip_bps = max(impact, _spread_bps(sym)) + commission_bps
|
||||
cost += traded_w * (slip_bps / 1e4)
|
||||
# capacity-forgone return: reduce the day's gross by the scaled-out fraction of the book's own return
|
||||
out[day] = book_ret_by_day[day] * (1.0 - scale_hit) - cost if scale_hit < 1.0 else -cost
|
||||
prev = cur
|
||||
return out
|
||||
|
||||
|
||||
def equity_capacity_curve(book_ret_by_day, weights_by_day, adv_usd, *, aums,
|
||||
participation_cap: float = 0.03, impact_k: float = 1.0,
|
||||
commission_per_share: float = 0.0035, price_by_day=None):
|
||||
"""{aum: equity_honest_returns(...at aum)} — load-once inputs, reprice per AUM."""
|
||||
return {aum: equity_honest_returns(book_ret_by_day, weights_by_day, adv_usd, aum=aum,
|
||||
participation_cap=participation_cap, impact_k=impact_k,
|
||||
commission_per_share=commission_per_share, price_by_day=price_by_day)
|
||||
for aum in aums}
|
||||
```
|
||||
|
||||
**Implementer note:** the `scale_hit` capacity-forgone term is an approximation (scales the whole book's return by the un-held fraction). Confirm it is monotone (higher AUM → more scale_hit → lower net) and document it. If a cleaner per-symbol attribution is available from `book_series`'s internals, prefer it — but do NOT block on it; the book-level approximation is acceptable for v1 and must be stated.
|
||||
|
||||
- [ ] **Step 4: Run — expect PASS.** Run the test file; fix until green.
|
||||
|
||||
- [ ] **Step 5: Commit** (`feat(equity): real ETF cost model + ADV capacity curve (equity_honest.py)`).
|
||||
|
||||
---
|
||||
|
||||
### Task 3: `equity_allocation_inputs` provider + `record_allocation` merge
|
||||
|
||||
**Files:**
|
||||
- Create: `src/fxhnt/application/equity_allocation_inputs.py`
|
||||
- Modify: `src/fxhnt/application/allocation_ingest.py` (merge equity series/ceilings)
|
||||
- Test: `tests/unit/test_equity_allocation_inputs.py`, `tests/integration/test_record_allocation_equity.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `MultiStratStrategy` + `book_series`/`target_weights` (multistrat), `YahooPitClient` + `yahoo_pit_volumes` (repo), `equity_honest.*`, `capacity_ceiling_from_curve`.
|
||||
- Produces: `equity_allocation_inputs(repo, deploy_sids, *, target_aum, aums, participation_cap=0.03, impact_k=1.0, min_sharpe=1.0) -> tuple[dict[str,dict[str,float]], dict[str,float|None]]`.
|
||||
|
||||
- [ ] **Step 1: Confirm the multistrat shapes.** Read `multistrat.book_series` and `multistrat.target_weights` signatures/returns and `MultiStratStrategy.advance` — determine the exact type of the daily book-return series and the weights vector (keys = ISO date? epoch? sym→weight?). The provider must feed `equity_honest_returns` `book_ret_by_day: {isodate: ret}` and `weights_by_day: {isodate: {sym: weight}}`. Adapt keys as needed (ISO strings, matching the nav record for `deploy_book_returns` alignment).
|
||||
|
||||
- [ ] **Step 2: Write failing tests.** A fake repo returning seeded `yahoo_pit_closes`/`yahoo_pit_volumes` for FUND_INSTRUMENTS over enough days; `equity_allocation_inputs(repo, {"multistrat"}, target_aum=100_000, aums=[...])` returns a non-empty ISO-keyed series for `multistrat` (len ≥ 60) + a finite ceiling. And in `record_allocation`: with an equity deploy sid present + store injected, `multistrat` gets a series (is eligible, not dropped); with `store=None` OR no equity sid, the result is byte-identical to before (add a regression assertion).
|
||||
|
||||
- [ ] **Step 3: Run — expect FAIL.**
|
||||
|
||||
- [ ] **Step 4: Implement `equity_allocation_inputs.py`:**
|
||||
- For each equity deploy sid (registry `venue == "ibkr-equity"`; today only `multistrat`):
|
||||
- `closes = {sym: repo.yahoo_pit_closes(sym) for sym in FUND_INSTRUMENTS}`; `volumes = {sym: repo.yahoo_pit_volumes(sym) for sym in FUND_INSTRUMENTS}`.
|
||||
- `book_ret = book_series(closes, ...)` → `{isodate: ret}`; `weights = target_weights(closes, ...)` → `{isodate: {sym: w}}` (adapt from Step 1).
|
||||
- `adv = dollar_adv(closes, volumes)`.
|
||||
- base series = `equity_honest_returns(book_ret, weights, adv, aum=target_aum, price_by_day=closes)`; store ISO-keyed under the sid.
|
||||
- ceiling = `capacity_ceiling_from_curve({aum: sharpe_of(equity_honest_returns(...at aum)) for aum in aums}, min_sharpe)` via `equity_capacity_curve` (load-once).
|
||||
- Absent/short PIT data → no series (caller falls back), ceiling `None`.
|
||||
- [ ] **Step 4b: Wire `record_allocation`** (`allocation_ingest.py`, in the `store is not None` branch after `honest_allocation_inputs`):
|
||||
|
||||
```python
|
||||
from fxhnt.application.equity_allocation_inputs import equity_allocation_inputs
|
||||
eq_sids = {sid for sid in funded_sids
|
||||
if STRATEGY_REGISTRY.get(sid, {}).get("venue") == "ibkr-equity"}
|
||||
if eq_sids:
|
||||
eq_returns, eq_ceilings = equity_allocation_inputs(
|
||||
repo, eq_sids, target_aum=equity,
|
||||
aums=[100_000, 1_000_000, 10_000_000, 50_000_000],
|
||||
min_sharpe=getattr(pol, "capacity_min_sharpe", 1.0))
|
||||
honest_returns = {**honest_returns, **eq_returns}
|
||||
ceilings = {**ceilings, **eq_ceilings}
|
||||
```
|
||||
|
||||
(`repo` is the `ForwardNavRepo` already constructed at the top of `record_allocation`.)
|
||||
|
||||
- [ ] **Step 5: Run — expect PASS** (unit + integration; existing allocation tests still green — byte-identity for no-equity/store=None).
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && .venv/bin/python3 -m pytest tests/unit/test_equity_allocation_inputs.py tests/integration/test_record_allocation_equity.py tests/unit -k "allocation or record" -q`
|
||||
|
||||
- [ ] **Step 6: Commit** (`feat(equity): equity_allocation_inputs provider merged into record_allocation`).
|
||||
|
||||
---
|
||||
|
||||
### Task 4: full-suite green
|
||||
|
||||
- [ ] **Step 1:** `cd /home/jgrusewski/Work/fxhnt && .venv/bin/python3 -m pytest tests/unit -q` → all pass.
|
||||
- [ ] **Step 2:** `.venv/bin/python3 -m pytest tests/integration -k "allocation or honest or equity or multistrat or yahoo" -q` → all pass.
|
||||
- [ ] **Step 3:** commit any fixes (else skip).
|
||||
|
||||
## Post-implementation (controller-run, in-cluster)
|
||||
Deploy (git-sync) + `fxhnt migrate-cockpit` (creates `yahoo_pit_volume`) + `fxhnt backfill-etf-volume`. Then the §5 gates: G1 multistrat sizes (cost-netted Sharpe < gross), G2 real ADV ceiling (finite, DBMF/PDBC-bound), G3 cost realism (SPY/IEF/GLD effective bps << 15), G4 combined marginal gate {crypto book, multistrat} — report the honest verdict + combined deployment.
|
||||
|
||||
## Self-Review
|
||||
- Coverage: §4a→T1, §4b→T2, §4c→T2, §4d→T3, gates→post-impl.
|
||||
- Byte-identity: equity merge only fires for ibkr-equity deploy sids with store injected; no-equity/store=None unchanged (T3 regression test).
|
||||
- Reuse: IMPACT_COEFF_BPS, capacity_ceiling_from_curve, load-once/reprice, first-seen PIT.
|
||||
- The estimated spread constants + the `scale_hit` capacity approximation are documented, not hidden.
|
||||
87
docs/superpowers/plans/2026-07-16-equity-ucits-rebase.md
Normal file
87
docs/superpowers/plans/2026-07-16-equity-ucits-rebase.md
Normal file
@@ -0,0 +1,87 @@
|
||||
# Equity Capacity/Cost UCITS Re-base Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: superpowers:subagent-driven-development. Checkbox (`- [ ]`) steps.
|
||||
|
||||
**Goal:** Re-base the equity leg's capacity ceiling + cost model onto the UCITS lines the EU account actually trades (CSPX/IDTM/IGLN/ICOM/DBMF-LU) instead of the US ETFs — with a regime flag so it switches back to the deep US-ETF liquidity once the fund elects MiFID-II professional status (~€500k). Returns (`book_series` on US underlyings) are unchanged — only ADV/spread inputs are re-sourced.
|
||||
|
||||
**Architecture:** The equity honest-reference already exists (`equity_honest.py`, `equity_allocation_inputs.py`) sourcing ADV from US-ETF volume + US spreads. This adds: UCITS volume ingestion (Yahoo LSE tickers), per-line UCITS spreads, and a `equity_execution_regime` config (`ucits` default | `us`). `equity_allocation_inputs` sources each US sleeve's ADV + spread from the regime's instrument (US ticker's own data, or its UCITS-mapped ticker's) while `book_series`/`target_weights` stay US.
|
||||
|
||||
**Tech Stack:** Python 3.12, SQLAlchemy 2.0, typer, pytest. Spec context: `docs/superpowers/specs/2026-07-16-equity-leg-deployable-design.md` (this refines §4a/§4b for EU/UCITS reality).
|
||||
|
||||
## Global Constraints
|
||||
|
||||
- The US→UCITS map is `settings.ucits.map` (`config.py`): `SPY→CSPX (IE00B5BMR087)`, `IEF→IDTM (IE00B1FZS798)`, `GLD→IGLN (IE00B4ND3602)`, `PDBC→ICOM (IE00BDFL4P12)`, `DBMF→DBMF (LU2951555585)`.
|
||||
- **Yahoo LSE tickers** for UCITS volume: `<ticker>.L` (CSPX.L, IDTM.L, IGLN.L, ICOM.L, DBMF.L). Per-symbol fetch failures are isolated (Task-1-of-the-prior-feature pattern) — a missing/thin line (esp. DBMF-UCITS) must NOT break ingestion.
|
||||
- **UCITS per-line half-spread (bps), documented estimates** (wider than US): `CSPX 1.5, IDTM 2.0, IGLN 2.0, ICOM 8.0, DBMF 30.0` (DBMF-UCITS is new/small/thin). Default 10.0.
|
||||
- **Regime flag** `equity_execution_regime: "ucits" | "us"`, DEFAULT `"ucits"` (the fund is retail EU today). Switching to `"us"` is a deliberate operator action AFTER electing MiFID-II professional status (~€500k portfolio + trading frequency) — NEVER auto-switched on AUM.
|
||||
- **Returns unchanged**: `book_series`/`target_weights` always use US closes (the edge proxy holds in both regimes; UCITS track the US underlyings). Only `adv_usd` + `spread_bps` fed to `equity_honest_returns` change by regime.
|
||||
- **Byte-identity of the `us` regime**: with `equity_execution_regime="us"`, the equity model is byte-identical to today (US ADV + `ETF_HALF_SPREAD_BPS`). The re-base only changes behavior under the (new default) `ucits` regime.
|
||||
- **ADV-coverage guard (from the prior fix)** still applies — if the UCITS volume is absent/thin (< coverage threshold), the sid emits NO series → nav fallback (a thin DBMF-UCITS must not fabricate capacity).
|
||||
- Commit trailer: `Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>`.
|
||||
|
||||
---
|
||||
|
||||
### Task 1: UCITS volume ingestion + config (spreads, yahoo ticker, regime flag)
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/config.py` (`UcitsListing.yahoo_ticker` + `.half_spread_bps`; `equity_execution_regime`)
|
||||
- Modify: `src/fxhnt/cli.py` (`backfill-etf-volume` also ingests the UCITS LSE tickers)
|
||||
- Modify: `src/fxhnt/adapters/orchestration/assets.py` (nightly snapshots UCITS volume too)
|
||||
- Test: `tests/unit/test_ucits_volume_config.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: `UcitsListing.yahoo_ticker` (default `f"{ticker}.L"`) + `half_spread_bps`; `settings.<...>.equity_execution_regime`.
|
||||
|
||||
- [ ] **Step 1: Failing tests** — the UCITS map exposes a `yahoo_ticker` (`CSPX.L`, `DBMF.L`, ...) and a `half_spread_bps` per line; `equity_execution_regime` defaults to `"ucits"`; an env override sets it to `"us"`.
|
||||
- [ ] **Step 2: Run — expect FAIL.**
|
||||
- [ ] **Step 3: Implement.**
|
||||
- `config.py` `UcitsListing`: add `yahoo_ticker: str = ""` (post-init/validator: default to `f"{ticker}.L"` when empty) and `half_spread_bps: float`. Populate the map's 5 entries with the Global-Constraints spread values (CSPX 1.5, IDTM 2.0, IGLN 2.0, ICOM 8.0, DBMF 30.0). DBMF's yahoo_ticker may need to be `"DBMF.L"` — set it explicitly; if Yahoo lacks it the ingestion skips gracefully.
|
||||
- Add `equity_execution_regime: str = "ucits"` to a settings group (recommend `AllocationSettings`, env `FXHNT_ALLOCATION_EQUITY_EXECUTION_REGIME`) with a comment: `ucits` (retail EU, default) | `us` (after electing MiFID-II professional, ~€500k) — sets which instruments' ADV/spread the equity capacity+cost model uses; returns stay US.
|
||||
- `cli.py` `backfill-etf-volume`: after the US `FUND_INSTRUMENTS` loop, ALSO iterate `settings.ucits.map.values()` and `YahooDailyClient(rng="25y").volumes(listing.yahoo_ticker)` → `repo.snapshot_yahoo_volumes(listing.yahoo_ticker, ...)`. Per-symbol try/except (a thin UCITS line that Yahoo lacks is logged + skipped, not fatal). Echo counts.
|
||||
- `assets.py` nightly: in the same volume-snapshot loop, also snapshot each UCITS `yahoo_ticker` (guarded like the US volume snapshot).
|
||||
- [ ] **Step 4: Run — expect PASS.**
|
||||
- [ ] **Step 5: Commit** (`feat(equity): ingest UCITS LSE volume + regime flag + UCITS spread config`).
|
||||
|
||||
---
|
||||
|
||||
### Task 2: regime-aware equity ADV/spread sourcing
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/application/equity_allocation_inputs.py`
|
||||
- Test: `tests/unit/test_equity_allocation_inputs.py` (extend)
|
||||
|
||||
**Interfaces:**
|
||||
- `equity_allocation_inputs(repo, deploy_sids, *, target_aum, aums, min_sharpe=1.0, regime="ucits", ucits_map=None, us_spread_bps=None, ucits_spread_bps=None)` — new `regime` + injectable maps (default from settings). Returns unchanged shape.
|
||||
|
||||
- [ ] **Step 1: Failing tests.**
|
||||
- regime="us": `adv_usd[sym]` sourced from `yahoo_pit_volumes(sym)` (US) + spread `ETF_HALF_SPREAD_BPS[sym]` — byte-identical to current behavior (regression assertion against the pre-regime output).
|
||||
- regime="ucits": `adv_usd[sym]` sourced from the UCITS-mapped ticker's `yahoo_pit_volumes(ucits_ticker)` (remapped to the US sleeve key so `equity_honest_returns`'s US-keyed weights line up) + spread = the per-line UCITS spread. With a seeded LOW UCITS volume and a HIGH US volume, the `ucits` ceiling must be MATERIALLY LOWER than the `us` ceiling (proves the re-base bites).
|
||||
- regime="ucits" with a thin/absent UCITS line (e.g. DBMF.L empty) → that sleeve's ADV is absent → coverage guard trips or that sleeve is capacity-scaled — verify it does NOT fabricate capacity (no series if coverage < threshold).
|
||||
- [ ] **Step 2: Run — expect FAIL.**
|
||||
- [ ] **Step 3: Implement.**
|
||||
- In `equity_allocation_inputs`, build the per-sleeve ADV + spread from the regime:
|
||||
- `us`: `adv_source_ticker[sym] = sym`; `spread[sym] = (us_spread_bps or ETF_HALF_SPREAD_BPS).get(sym, default)`.
|
||||
- `ucits`: `adv_source_ticker[sym] = ucits_map[sym].yahoo_ticker`; `spread[sym] = ucits_map[sym].half_spread_bps`.
|
||||
- Load `volumes_by_source = {sym: repo.yahoo_pit_volumes(adv_source_ticker[sym])}`; `adv = dollar_adv({sym: us_closes[sym]}, {sym: volumes_by_source[sym]})` — **KEY**: `dollar_adv` needs price×volume; use the US close as the price proxy for the USD-denominated UCITS line (the USD UCITS lines track the US underlying's USD price closely; document this — a small basis, acceptable for a capacity estimate; a follow-on could use the real UCITS price). So `adv_usd` is keyed by the US sleeve `sym` but its VOLUME comes from the UCITS line. Spread passed as the regime spread dict.
|
||||
- Pass `spread_bps=<regime spread dict>` into `equity_honest_returns`/`equity_capacity_curve` (add a `spread_bps` param there if it currently hardcodes `ETF_HALF_SPREAD_BPS` — check Task-2-of-the-prior-feature `equity_honest.py`; if `_spread_bps(sym)` reads the module constant, thread an optional `spread_bps: dict|None` override so the regime dict wins). Do NOT mutate the module constant.
|
||||
- Defaults: `regime = settings...equity_execution_regime`, `ucits_map = settings.ucits.map`, spreads from config — resolved at the call in `record_allocation` (pass them in) OR defaulted inside via `get_settings()`; keep it injectable for tests.
|
||||
- [ ] **Step 4: `record_allocation`** passes `regime=getattr(pol/settings, ...)` (thread the regime from settings; the equity block already exists). Confirm the `us`-regime path stays byte-identical.
|
||||
- [ ] **Step 5: Run — expect PASS** (unit + the byte-identity regression + full allocation tests).
|
||||
- [ ] **Step 6: Commit** (`feat(equity): regime-aware ADV/spread (ucits default | us) for the capacity+cost model`).
|
||||
|
||||
---
|
||||
|
||||
### Task 3: full-suite green
|
||||
|
||||
- [ ] **Step 1:** `.venv/bin/python3 -m pytest tests/unit -q` → all pass.
|
||||
- [ ] **Step 2:** `.venv/bin/python3 -m pytest tests/integration -k "allocation or equity or yahoo or multistrat" -q` → all pass.
|
||||
- [ ] **Step 3:** commit any fixes.
|
||||
|
||||
## Post-implementation (controller-run, in-cluster)
|
||||
Deploy (git-sync) + `fxhnt migrate-cockpit` + `fxhnt backfill-etf-volume` (now also fetches the UCITS LSE lines). Then re-run the equity gates UNDER `equity_execution_regime=ucits`: report the UCITS-real ceiling (expect MATERIALLY below the US ≥$10M — likely $0.5–3M, DBMF/ICOM-bound) + which UCITS line binds + the net Sharpe under UCITS spreads (expect below the 1.58 US estimate). Confirm DBMF.L coverage (if Yahoo lacks it, the sleeve falls to the coverage guard). Contrast with `regime=us` to show the professional-status upside. NOTE: run LIGHT (equity provider only — do NOT re-run the 2× crypto-book path that OOM-killed the daemon; keep in-cluster diagnostics to one honest path at a time).
|
||||
|
||||
## Self-Review
|
||||
- Byte-identity: `regime="us"` reproduces today's equity model (regression test).
|
||||
- Returns unchanged: only adv/spread inputs re-sourced; `book_series`/`target_weights` untouched.
|
||||
- Thin-line safety: ADV-coverage guard + per-symbol ingestion isolation protect against a missing/thin DBMF-UCITS.
|
||||
- Documented approximation: US close as the price proxy for the USD UCITS line's dollar-ADV (small basis; follow-on = real UCITS price).
|
||||
@@ -0,0 +1,173 @@
|
||||
# Honest-Inputs Load-Once/Reprice Refactor Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: superpowers:subagent-driven-development. Checkbox (`- [ ]`) steps.
|
||||
|
||||
**Goal:** Refactor `allocation_honest_inputs` so each crypto sleeve's AUM-INDEPENDENT panels (weights + turnover) are loaded ONCE and repriced across all capacity-curve AUMs, instead of a full panel reload per AUM — so the nightly can use a wider AUM grid without OOM-killing on the book deploy set.
|
||||
|
||||
**Architecture:** Split the current load+reprice `_sleeve_series`/`_book_series`/`_deploy_series` into a load step (`_sleeve_inputs` → (wbd, turnover), heavy, once) and a reprice step (reuse `honest_reference.honest_sleeve_returns` / `capacity_curve`, cheap, per AUM). `honest_allocation_inputs` loads once per sid then reprices for base + the whole curve. Byte-identical honest series/ceilings; only the compute schedule changes. Then widen the nightly AUM grid.
|
||||
|
||||
**Tech Stack:** Python 3.12, pytest. The honest-reference REPORT path (`honest_report.py`) already does load-once/reprice — this brings the allocation provider to the same pattern (and reuses `capacity_curve`).
|
||||
|
||||
## Global Constraints
|
||||
|
||||
- **Byte-identity:** for the same store/spread/unlock_events/aums, the NEW `honest_allocation_inputs` must return the SAME `returns_by_strategy` (ISO-keyed) and the SAME `ceilings` as the OLD one. The series math is unchanged — same `wbd`, same `turnover`, same `honest_sleeve_returns(...)` calls; only WHEN the panel is loaded changes.
|
||||
- **Load-once invariant:** `store.read_panel(...)` must be called ONCE per distinct sleeve (1 for a sleeve sid, 3 for the `bybit_4edge` book) regardless of how many AUMs are in the grid — NOT once per (sleeve × AUM). Enforce with a test that counts `read_panel` calls.
|
||||
- `_BOOK_SLEEVES = ("xsfunding", "unlock", "positioning")`, `_CRYPTO_DEPLOY_SIDS = {positioning, xsfunding, unlock, bybit_4edge}` — unchanged. tstrend still excluded (no standalone store series).
|
||||
- The old `_sleeve_series`/`_book_series`/`_deploy_series` become dead after the refactor → DELETE them (wire-or-delete; no dead code). They are used ONLY inside `allocation_honest_inputs.py` (verified — the report path has its own impl).
|
||||
- Reuse `honest_reference.capacity_curve` for the sleeve reprice (do not reimplement the per-AUM loop).
|
||||
- Commit trailer: `Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>`.
|
||||
|
||||
---
|
||||
|
||||
### Task 1: load-once/reprice refactor of the provider
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/application/allocation_honest_inputs.py`
|
||||
- Test: `tests/unit/test_allocation_honest_inputs.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `sleeve_weights_and_contributions` (bybit_paper_weights), `honest_sleeve_returns` + `capacity_curve` (honest_reference), `book_returns` + `risk_parity_weights` + `sharpe_of` (honest_gates), `capacity_ceiling_from_curve` (allocation_engine).
|
||||
- Produces (new internal helpers, replacing the old ones):
|
||||
- `_sleeve_inputs(store, sleeve, unlock_events) -> tuple[dict, dict] | None`
|
||||
- `_book_curve(store, spread_bps, unlock_events, aums, *, participation_cap, impact_k) -> dict[float, dict[int,float]]`
|
||||
- `_deploy_curve(store, sid, spread_bps, unlock_events, aums, *, participation_cap, impact_k) -> dict[float, dict[int,float]]`
|
||||
- `honest_allocation_inputs(...)` — SAME public signature and return type.
|
||||
|
||||
- [ ] **Step 1: Write the load-once-count test first** (extend `test_allocation_honest_inputs.py`). Wrap the fake store so `read_panel` increments a counter; call `honest_allocation_inputs(deploy_sids={"bybit_4edge"}, target_aum=100_000, aums=[35_000,100_000,350_000,1_000_000,10_000_000])` (5 AUMs) and assert `read_panel` was called exactly **3 times** (one per book sleeve), NOT 3×(1+5). Also add a sleeve case: `deploy_sids={"positioning"}` with the same 5 AUMs → `read_panel` called exactly **1 time**.
|
||||
|
||||
Look at the existing test file's fake store / `_seed(...)` pattern and mirror it — wrap or subclass the store used there to count `read_panel`.
|
||||
|
||||
- [ ] **Step 2: Run — expect FAIL** (current code reloads per AUM → 18 and 6 calls)
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && .venv/bin/python3 -m pytest tests/unit/test_allocation_honest_inputs.py -q`
|
||||
Expected: the new count tests FAIL (too many read_panel calls); the existing series/ceiling tests still PASS.
|
||||
|
||||
- [ ] **Step 3: Implement the refactor.** Replace `_sleeve_series`, `_book_series`, `_deploy_series` with:
|
||||
|
||||
```python
|
||||
def _sleeve_inputs(store: Any, sleeve: str,
|
||||
unlock_events: list[dict[str, Any]] | None) -> tuple[dict, dict] | None:
|
||||
"""AUM-INDEPENDENT heavy inputs for a sleeve: (weights_by_day, turnover_panel). Load ONCE, reprice across
|
||||
AUMs via honest_sleeve_returns/capacity_curve — the turnover panel read is the memory cost, so it must NOT
|
||||
be reloaded per AUM. None when the sleeve is uncomputable (e.g. unlock with no events). READ-ONLY."""
|
||||
wbd = sleeve_weights_and_contributions(store, sleeve, unlock_events=unlock_events)
|
||||
if not wbd:
|
||||
return None
|
||||
symbols = sorted({s for row in wbd.values() for s in row})
|
||||
turnover = store.read_panel(symbols, "turnover")
|
||||
return wbd, turnover
|
||||
|
||||
|
||||
def _book_curve(store: Any, spread_bps: dict[str, float], unlock_events: list[dict[str, Any]] | None,
|
||||
aums, *, participation_cap: float, impact_k: float) -> dict[float, dict[int, float]]:
|
||||
"""{aum: risk-parity BOOK series} across `aums`, loading each `_BOOK_SLEEVES` sleeve's inputs ONCE then
|
||||
repricing per AUM. Empty sleeves (uncomputable) are dropped; an AUM with no computable sleeve -> {}."""
|
||||
loaded = {sleeve: inp for sleeve in _BOOK_SLEEVES
|
||||
if (inp := _sleeve_inputs(store, sleeve, unlock_events)) is not None}
|
||||
out: dict[float, dict[int, float]] = {}
|
||||
for aum in aums:
|
||||
per: dict[str, dict[int, float]] = {}
|
||||
for sleeve, (wbd, turnover) in loaded.items():
|
||||
series = honest_sleeve_returns(wbd, turnover, spread_bps, aum=aum,
|
||||
participation_cap=participation_cap, impact_k=impact_k)
|
||||
if series:
|
||||
per[sleeve] = series
|
||||
out[aum] = book_returns(per, risk_parity_weights(per)) if per else {}
|
||||
return out
|
||||
|
||||
|
||||
def _deploy_curve(store: Any, sid: str, spread_bps: dict[str, float],
|
||||
unlock_events: list[dict[str, Any]] | None, aums, *,
|
||||
participation_cap: float, impact_k: float) -> dict[float, dict[int, float]]:
|
||||
"""{aum: honest series} for a crypto deploy `sid` across `aums`, loading AUM-independent inputs ONCE.
|
||||
`bybit_4edge` -> the risk-parity book; a sleeve sid -> its own series (reuses `capacity_curve`); other -> {}."""
|
||||
if sid == "bybit_4edge":
|
||||
return _book_curve(store, spread_bps, unlock_events, aums,
|
||||
participation_cap=participation_cap, impact_k=impact_k)
|
||||
if sid in _CRYPTO_DEPLOY_SIDS:
|
||||
inputs = _sleeve_inputs(store, sid, unlock_events)
|
||||
if inputs is None:
|
||||
return {aum: {} for aum in aums}
|
||||
wbd, turnover = inputs
|
||||
return capacity_curve(wbd, turnover, spread_bps, aums=aums,
|
||||
participation_cap=participation_cap, impact_k=impact_k)
|
||||
return {aum: {} for aum in aums}
|
||||
```
|
||||
|
||||
Rewrite the loop body of `honest_allocation_inputs` (keep the signature + the non-crypto branch) so it loads/reprices once:
|
||||
|
||||
```python
|
||||
for sid in deploy_sids:
|
||||
if sid not in _CRYPTO_DEPLOY_SIDS:
|
||||
ceilings[sid] = equity_default_ceiling # equity / non-crypto — caller falls back; no series
|
||||
continue
|
||||
# AUM-independent panels loaded ONCE inside _deploy_curve; base + curve are repriced from them.
|
||||
all_aums = sorted({float(target_aum), *(float(a) for a in aums)})
|
||||
curve_series = _deploy_curve(store, sid, spread_bps, unlock_events, all_aums,
|
||||
participation_cap=participation_cap, impact_k=impact_k)
|
||||
base = curve_series.get(float(target_aum), {})
|
||||
if base:
|
||||
returns_by_strategy[sid] = {_iso(day): ret for day, ret in base.items()}
|
||||
curve = {aum: sharpe_of(curve_series.get(float(aum), {})) for aum in aums}
|
||||
ceilings[sid] = capacity_ceiling_from_curve(curve, min_sharpe)
|
||||
```
|
||||
|
||||
Update imports (add `capacity_curve` from honest_reference) and the module docstring (deploy→honest mapping now describes the load-once/reprice; the report path already does this).
|
||||
|
||||
**Note on float keying:** the curve dict is keyed by the AUM values passed to `_deploy_curve`. Build `all_aums` and look up with the SAME float() cast on both sides (as shown) so `target_aum` and grid entries match by key even if one is int and one is float.
|
||||
|
||||
- [ ] **Step 4: Run — expect PASS** (count tests pass; series/ceiling tests unchanged and still pass = byte-identity held)
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && .venv/bin/python3 -m pytest tests/unit/test_allocation_honest_inputs.py -q`
|
||||
Expected: PASS (all).
|
||||
|
||||
- [ ] **Step 5: Confirm no other consumer of the deleted helpers**
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && grep -rn "_sleeve_series\|_book_series\|_deploy_series" src/ tests/ | grep -v "docstring\|# "`
|
||||
Expected: no CODE references remain (only prose/docstring mentions, which you should also update if they name the removed functions).
|
||||
|
||||
- [ ] **Step 6: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/application/allocation_honest_inputs.py tests/unit/test_allocation_honest_inputs.py
|
||||
git commit -m "refactor(alloc): load-once/reprice honest inputs (panels loaded once, repriced per AUM)
|
||||
|
||||
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 2: widen the nightly AUM grid + full suite green
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/application/allocation_ingest.py`
|
||||
|
||||
- [ ] **Step 1:** In `record_allocation`, widen the capacity-curve grid back to characterize the book's break with headroom (now cheap — one load, many reprices): `aums=[35_000, 100_000, 350_000]` → `aums=[35_000, 100_000, 350_000, 1_000_000, 10_000_000]`. Update the leading comment: the load-once/reprice provider (Task 1) makes grid length cheap (repricing, not reloading), so the wider grid no longer OOMs; it restores capacity resolution above the book's ~$350k break so growth past it is detectable.
|
||||
|
||||
- [ ] **Step 2: Full unit suite**
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && .venv/bin/python3 -m pytest tests/unit -q`
|
||||
Expected: all pass.
|
||||
|
||||
- [ ] **Step 3: Allocation/honest integration**
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && .venv/bin/python3 -m pytest tests/integration -k "allocation or honest or funding" -q`
|
||||
Expected: all pass (2 skipped ok).
|
||||
|
||||
- [ ] **Step 4: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/application/allocation_ingest.py
|
||||
git commit -m "feat(alloc): widen nightly capacity-curve AUM grid (load-once makes it cheap)
|
||||
|
||||
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
## Post-implementation (controller-run, in-cluster)
|
||||
Deploy (git-sync), re-run `record_allocation` with the honest store: confirm (1) NO OOM with the widened 5-AUM grid on the book deploy set, (2) `read_panel` load-once holds (the whole point), (3) the book ceiling is resolved over the wider grid (still ~$350k or refined), (4) deployment still ~$75k (75%) — byte-identical sizing, just a wider capacity probe.
|
||||
|
||||
## Self-Review
|
||||
- Byte-identity: same wbd/turnover/honest_sleeve_returns → identical series+ceilings (only load schedule changed); existing series/ceiling tests are the guard.
|
||||
- Load-once: enforced by the read_panel-count test (invariant, not a comment).
|
||||
- Dead code: old three helpers deleted; grep gate in Task 1 Step 5.
|
||||
- Reuse: `capacity_curve` reused for the sleeve path (DRY).
|
||||
@@ -0,0 +1,73 @@
|
||||
# IBKR UCITS Volume Source Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: superpowers:subagent-driven-development. Checkbox (`- [ ]`) steps.
|
||||
|
||||
**Goal:** Source the UCITS regime's ADV from **IBKR real daily volume** (`reqHistoricalData TRADES` — the authoritative on-exchange volume of the exact line we execute) instead of Yahoo's coarse LSE feed. Fetch runs in the IBKR-network-reachable rebalancer context → DB; the daemon/nightly reads the DB.
|
||||
|
||||
**Architecture:** A parallel `ibkr_pit_volume` PIT table (first-seen-frozen, mirrors `yahoo_pit_volume`); an `IbkrBroker.historical_daily_volume` method (reqHistoricalData TRADES); a `backfill-ucits-volume-ibkr` CLI (rebalancer context); a `ucits_volume_source` config flag (`yahoo` default | `ibkr`) that `equity_allocation_inputs` honors; a fetch CronJob. Spread stays the estimated per-line constant (real IBKR bid/ask needs a market-data subscription — Error 354, a documented follow-on).
|
||||
|
||||
**Tech Stack:** Python 3.12, SQLAlchemy 2.0, `ib_async` (runtime-installed in the rebalancer image), typer, pytest, k8s CronJob.
|
||||
|
||||
## Global Constraints
|
||||
|
||||
- **Probe-verified (2026-07-16):** IBKR `reqHistoricalData(contract, "", "N D", "1 day", "TRADES", useRTH=True, formatDate=1)` returns real daily volume for the UCITS lines (CSPX/CBU0/ICOM/DBMF all resolved via LSEETF+ISIN; DBMF ~16 shares/day confirms genuine illiquidity). `reqMktData` bid/ask = NaN / Error 354 (no market-data subscription) → **spread stays estimated; do NOT attempt real-time spread here.**
|
||||
- **Network reality:** only pods labeled `app: multistrat-rebalancer` (or trading-service/broker-gateway) can reach `ib-gateway:4004` (NetworkPolicy). The daemon CANNOT. So the IBKR fetch MUST run in the rebalancer context (a CronJob), write to the DB; the daemon reads the DB. `ib_async` is a runtime `pip install ib_async==2.1.0` (extra `ibkr = ["ib-async>=2.0"]`), NOT in the base image.
|
||||
- **PIT discipline:** `ibkr_pit_volume` is first-seen-frozen (mirror `snapshot_yahoo_volumes` — insert only (symbol,date) not present).
|
||||
- **Byte-identity of `ucits_volume_source="yahoo"` (the default):** `equity_allocation_inputs` must be byte-identical to today when the flag is `yahoo`. The IBKR read path only activates on `ibkr`.
|
||||
- Store IBKR volume keyed by the UCITS `yahoo_ticker` string (e.g. `CSPX.L`) so the equity provider's existing per-sleeve key mapping is reused unchanged — the source flag only switches WHICH table is read.
|
||||
- Commit trailer: `Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>`.
|
||||
|
||||
---
|
||||
|
||||
### Task 1: IBKR volume storage + broker fetch method + config flag
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/adapters/persistence/cockpit_models.py` (`IbkrPitVolumeRow`)
|
||||
- Modify: `src/fxhnt/adapters/persistence/forward_nav.py` (`snapshot_ibkr_volumes` / `ibkr_pit_volumes`)
|
||||
- Modify: `src/fxhnt/adapters/broker/ibkr.py` (`historical_daily_volume`)
|
||||
- Modify: `src/fxhnt/config.py` (`AllocationSettings.ucits_volume_source`)
|
||||
- Test: `tests/unit/test_ibkr_volume_pit.py`
|
||||
|
||||
**Interfaces:**
|
||||
- `IbkrPitVolumeRow` (`ibkr_pit_volume`, PK (symbol,date), volume Float, first_seen_at) — rides `create_all`.
|
||||
- `ForwardNavRepo.snapshot_ibkr_volumes(symbol, series, at)` / `.ibkr_pit_volumes(symbol) -> dict[str,float]` (mirror the yahoo methods exactly, first-seen-frozen).
|
||||
- `IbkrBroker.historical_daily_volume(*, ticker, isin, exchange="LSEETF", currency="USD", days=90) -> dict[str,float]`: resolve via `_resolve_contract`, `reqHistoricalData(contract, "", f"{days} D", "1 day", "TRADES", useRTH=True, formatDate=1)`, return `{bar.date.isoformat() or str(bar.date): float(bar.volume)}` for bars with `volume not in (None, -1)`. Empty dict on resolve-fail/no-data (log + return {}, never raise into the caller's loop).
|
||||
- `AllocationSettings.ucits_volume_source: str = "yahoo"` (yahoo | ibkr), env `FXHNT_ALLOCATION_UCITS_VOLUME_SOURCE`, comment: which store the UCITS-regime ADV reads from; `ibkr` = the real IBKR on-exchange volume (fetched by the `ucits-volume-ibkr` CronJob), `yahoo` = the Yahoo LSE feed (default until the CronJob is running).
|
||||
|
||||
- [ ] **Step 1: Failing tests** — `ibkr_pit_volume` roundtrip + first-seen-frozen (mirror the yahoo volume test); `ucits_volume_source` default "yahoo" + env override "ibkr". (The broker method needs a live gateway — do NOT unit-test the fetch; test the storage + config only. Note in the test file why the broker fetch is integration-only.)
|
||||
- [ ] **Step 2: Run — expect FAIL.**
|
||||
- [ ] **Step 3: Implement** all four (mirror the yahoo volume + the existing `market_price` reqHistoricalData pattern for the broker method; `bar.date` from ib_async is a `datetime.date` — format to ISO `YYYY-MM-DD`).
|
||||
- [ ] **Step 4: Run — expect PASS.**
|
||||
- [ ] **Step 5: Commit** (`feat(equity): IBKR pit-volume store + historical_daily_volume + ucits_volume_source flag`).
|
||||
|
||||
---
|
||||
|
||||
### Task 2: `backfill-ucits-volume-ibkr` CLI + source-flag read
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/cli.py` (`backfill-ucits-volume-ibkr`)
|
||||
- Modify: `src/fxhnt/application/equity_allocation_inputs.py` (read ibkr vs yahoo by flag)
|
||||
- Test: `tests/unit/test_equity_allocation_inputs.py` (extend)
|
||||
|
||||
- [ ] **Step 1: Failing tests** — `equity_allocation_inputs(..., regime="ucits", volume_source="ibkr")` reads `ibkr_pit_volumes(ucits_ticker)`; `volume_source="yahoo"` (default) reads `yahoo_pit_volumes` (byte-identical to today, regression). Seed one repo with DIFFERENT ibkr vs yahoo volume for a UCITS ticker → the ceiling differs by source (proves the switch reads the right table).
|
||||
- [ ] **Step 2: Run — expect FAIL.**
|
||||
- [ ] **Step 3: Implement:**
|
||||
- `equity_allocation_inputs` gains `volume_source: str = "yahoo"` (default from `settings.allocation.ucits_volume_source` at the `record_allocation` call). In the ucits branch, `vol_read = repo.ibkr_pit_volumes if volume_source == "ibkr" else repo.yahoo_pit_volumes`; `ucits_volumes = {sym: vol_read(resolved_map[sym].yahoo_ticker) ...}`. The us branch is unchanged (always yahoo US volume — Databento is the future US source, out of scope). `record_allocation` threads `volume_source=get_settings().allocation.ucits_volume_source`.
|
||||
- `cli.py` `backfill-ucits-volume-ibkr`: `pip`-installed `ib_async` is assumed present (rebalancer context). For each `settings.ucits.map` listing, `IbkrBroker(host, port, client_id).historical_daily_volume(ticker=listing.ticker, isin=listing.isin, exchange=listing.exchange, currency=listing.currency, days=<opt, default 90>)` → `repo.snapshot_ibkr_volumes(listing.yahoo_ticker, series, at)`. Per-line try/except (a thin/unresolvable line logs + continues). Echo per-line day counts. Use `get_settings().ibkr` for host/port/client_id.
|
||||
- [ ] **Step 4: Run — expect PASS** (+ full unit suite).
|
||||
- [ ] **Step 5: Commit** (`feat(equity): backfill-ucits-volume-ibkr CLI + source-flag read in equity provider`).
|
||||
|
||||
---
|
||||
|
||||
### Task 3: fetch CronJob (infra) + full green — CONTROLLER-run (not a subagent task)
|
||||
- [ ] Add `infra/k8s/.../ucits-volume-ibkr-cronjob.yaml`: label `app: multistrat-rebalancer`, image `fxhnt-cockpit:latest`, `imagePullSecrets: scw-registry`, env (FXHNT_OPERATIONAL_DSN + PGPASSWORD from the same secret the rebalancer uses, FXHNT_IBKR_HOST=ib-gateway, PORT=4004, CLIENT_ID=27), command `pip install --quiet ib_async==2.1.0 && fxhnt backfill-ucits-volume-ibkr`. Schedule daily before the ucits-rebalancer (e.g. `30 8 * * 1-5`). Clone the ucits-rebalancer's spec for the DSN/secret wiring.
|
||||
- [ ] `.venv/bin/python3 -m pytest tests/unit -q` + `tests/integration -k "equity or allocation or ibkr or yahoo"` → green.
|
||||
|
||||
## Post-implementation (controller-run, in-cluster)
|
||||
Deploy (git-sync) + migrate-cockpit (creates `ibkr_pit_volume`). Run the fetch as a one-off Job (rebalancer context) → confirm `ibkr_pit_volume` populates with real volume (DBMF ~16 shares/day). Flip `FXHNT_ALLOCATION_UCITS_VOLUME_SOURCE=ibkr` and re-run the equity gate → the book sizes on the real IBKR ADV (partial: liquid lines + DBMF scaled to ~0). Contrast the IBKR-sourced ceiling vs the Yahoo-sourced one. NOTE: the fetch MUST run labeled `app: multistrat-rebalancer` (netpol); the daemon cannot reach the gateway.
|
||||
|
||||
## Self-Review
|
||||
- Byte-identity: `ucits_volume_source="yahoo"` (default) unchanged; ibkr path additive.
|
||||
- Network-correct: fetch in rebalancer context → DB → daemon reads DB (no IBKR in the daemon).
|
||||
- Spread deferred honestly (subscription follow-on; Error 354 documented).
|
||||
- PIT first-seen-frozen for ibkr_pit_volume.
|
||||
@@ -0,0 +1,221 @@
|
||||
# Foxhunt → Bizworx Consolidation — Implementation Plan
|
||||
|
||||
> **For the operator/agent:** execute **inline, with a checkpoint after every task** (this is live infra +
|
||||
> real-money fund + secrets — NOT a subagent-driven code plan). Steps use `- [ ]` for tracking. Spec:
|
||||
> `docs/superpowers/specs/2026-07-18-foxhunt-to-bizworx-consolidation-design.md`.
|
||||
|
||||
**Goal:** Move the fxhnt fund onto the bizworx Kapsule + consolidate to one gitea, verify it fully, THEN
|
||||
retire the foxhunt Kapsule (cost saving).
|
||||
|
||||
**Architecture:** Build the fund on bizworx **alongside** the still-live foxhunt cluster, migrate the one
|
||||
stateful thing (Postgres/Timescale), prove it works in parallel, cut over DNS, observe, and only then tear
|
||||
down. Reuse bizworx's existing argo + observability + gitea.
|
||||
|
||||
**Tech stack:** Scaleway Kapsule (2 clusters, 2 projects, same org), kubectl (contexts `bizworx-prod-…` +
|
||||
`foxhunt-…`), `scw` CLI, Terragrunt/Terraform (`infra/live/production`), argo (git-sync deploy), gitea,
|
||||
TimescaleDB/Postgres, IBKR ib-gateway, tailscale.
|
||||
|
||||
## Global Constraints (apply to EVERY task)
|
||||
|
||||
- **VERIFY-FIRST, NOTHING DESTRUCTIVE UNTIL PROVEN.** No delete / teardown / `terraform destroy` / PVC
|
||||
removal / foxhunt scale-down happens before Phase 1 is fully verified AND the observation window (Task 10)
|
||||
passes. Tasks 1–9 are strictly additive; foxhunt stays fully live through Task 8, suspended-but-intact
|
||||
through teardown.
|
||||
- **EVALUATE PER STEP.** Every task ends with an explicit `EVAL:` gate. If it fails → STOP, do not proceed.
|
||||
- **ROLLBACK is always available** until Task 12: re-point DNS + un-suspend foxhunt CronJobs (foxhunt's
|
||||
Postgres still holds pre-cutover state).
|
||||
- **Secrets NEVER through the transcript.** Create them with `! kubectl create secret …` / openbao /
|
||||
external-secrets. Never echo/commit secret values.
|
||||
- **Real-money fund:** before arming any rebalancer on bizworx, a dry-run + recon-gate check must pass.
|
||||
- **Two contexts, always explicit:** `FOX=foxhunt-34a1e3c4-ac35-48c8-ab49-5f6ec4df32c1`,
|
||||
`BIZ=bizworx-prod-kapsule-05e1dae3-8f35-4ad4-8fce-0f9a7691a760`. Never rely on current-context.
|
||||
- Cockpit changes verified LOCALLY in a real browser + on prod, reading the numbers.
|
||||
- Per-item operator sign-off before deleting any Phase-2 data.
|
||||
|
||||
---
|
||||
|
||||
## PHASE 1 — build + verify on bizworx (foxhunt stays live) — additive only
|
||||
|
||||
### Task 1: Prerequisites + target namespace
|
||||
|
||||
**Files:** none (cluster ops).
|
||||
- [ ] Confirm both kube-contexts respond: `kubectl --context $BIZ get ns` and `kubectl --context $FOX get ns`.
|
||||
- [ ] Pick + confirm a StorageClass on bizworx for the fund (foxhunt used `sbs-default`/`scw-bssd`; bizworx has
|
||||
`sbs-5k`,`sbs-default`): `kubectl --context $BIZ get storageclass`.
|
||||
- [ ] Create the target namespace: `kubectl --context $BIZ create ns foxhunt`.
|
||||
- [ ] Record the foxhunt Postgres + TimescaleDB versions to match on bizworx:
|
||||
`kubectl --context $FOX exec -n foxhunt deploy/postgres -- psql -U postgres -tAc "select version(); select extversion from pg_extension where extname='timescaledb';"`
|
||||
|
||||
**EVAL:** both contexts work, ns `foxhunt` exists on bizworx, TimescaleDB version noted. **No foxhunt change.**
|
||||
|
||||
### Task 2: Secrets on bizworx
|
||||
|
||||
**Files:** none (secrets via `!`/openbao — never committed).
|
||||
- [ ] List the secrets the fund workloads consume on foxhunt (names only, NOT values):
|
||||
`kubectl --context $FOX get secrets -n foxhunt` → expect `db-credentials`, `databento-credentials`, an
|
||||
IBKR/ib-gateway secret, `scw-registry`, `argo-git-ssh-key`/`git-ssh-key`, tailscale, cockpit TLS.
|
||||
- [ ] For each: recreate on bizworx sourced from openbao/external-secrets, or by hand with
|
||||
`! kubectl --context $BIZ create secret … -n foxhunt` (operator runs; values never in transcript).
|
||||
- [ ] Verify presence: `kubectl --context $BIZ get secrets -n foxhunt` shows every required name.
|
||||
|
||||
**EVAL:** every fund secret exists on bizworx (by name). A missing secret wedges a workload later — do not
|
||||
proceed until complete. **No foxhunt change.**
|
||||
|
||||
### Task 3: Consolidate gitea (mirror repos; DO NOT re-point DNS yet)
|
||||
|
||||
**Files:** none (gitea admin ops).
|
||||
- [ ] Inventory both giteas' repos (names/sizes) to confirm scope:
|
||||
foxhunt gitea (via `git.fxhnt.ai` today) has `gitadmin/fxhnt`, `gitadmin/foxhunt` (+ any others — list them).
|
||||
- [ ] On the **bizworx gitea**, create the `gitadmin` org/user + empty `fxhnt` and `foxhunt` repos.
|
||||
- [ ] Mirror all refs/tags/LFS from foxhunt gitea → bizworx gitea (gitea migration UI/API, or
|
||||
`git clone --mirror` from `git.fxhnt.ai` then `git push --mirror` to the bizworx gitea endpoint via a
|
||||
temporary port-forward / tailnet address).
|
||||
- [ ] Verify parity: for `fxhnt` and `foxhunt`, `git ls-remote` ref count + latest master/main sha match
|
||||
between the two giteas.
|
||||
|
||||
**EVAL:** bizworx gitea holds both repos with matching refs/tags. `git.fxhnt.ai` STILL points at foxhunt
|
||||
(unchanged) — this task is a non-destructive copy. **No foxhunt change; both giteas intact.**
|
||||
|
||||
### Task 4: Re-target the k8s manifests (branch, dry-run only)
|
||||
|
||||
**Files:** `infra/k8s/**` (on this branch `chore/foxhunt-bizworx-consolidation`).
|
||||
- [ ] Identify manifests that hardcode the foxhunt cluster/endpoints: gitea `gitea-sshd.foxhunt.svc…`
|
||||
(still valid — same ns name), image registry `rg.fr-par.scw.cloud/bizworx/…` (already bizworx — good),
|
||||
StorageClass names, any nodeSelector/affinity, the `git.fxhnt.ai` git-sync URL, ingress hosts.
|
||||
- [ ] Change only what must change for bizworx (StorageClass if different; git-sync stays on `git.fxhnt.ai`
|
||||
since DNS will re-point; ns stays `foxhunt`). Commit to the branch.
|
||||
- [ ] Dry-run against bizworx: `kubectl --context $BIZ apply -n foxhunt --dry-run=server -f <manifest>` for
|
||||
each (server-side dry-run catches admission/StorageClass errors).
|
||||
|
||||
**EVAL:** every fund manifest passes server-side dry-run on bizworx. **Nothing applied for real yet.**
|
||||
|
||||
### Task 5: Stand up Postgres/Timescale on bizworx (empty)
|
||||
|
||||
**Files:** `infra/k8s/databases/**`.
|
||||
- [ ] Apply the Postgres/Timescale manifest to bizworx ns `foxhunt` (new PVC on the chosen StorageClass).
|
||||
- [ ] Verify it comes up + the TimescaleDB extension version MATCHES foxhunt (from Task 1):
|
||||
`kubectl --context $BIZ exec -n foxhunt deploy/postgres -- psql -U postgres -tAc "select extversion from pg_extension where extname='timescaledb';"`
|
||||
|
||||
**EVAL:** bizworx Postgres is Running and its TimescaleDB version == foxhunt's. If mismatch → pin the image to
|
||||
match BEFORE any restore (a version skew breaks hypertable restore).
|
||||
|
||||
### Task 6: DB migration REHEARSAL (non-destructive dump/restore)
|
||||
|
||||
**Files:** none (a rehearsal; foxhunt keeps running/serving).
|
||||
- [ ] `pg_dump` the foxhunt `fxhnt` DB using the **hypertable-aware** path (call `timescaledb_pre_restore()`
|
||||
on the target before restore and `timescaledb_post_restore()` after; use `--no-owner` as needed). Stream
|
||||
foxhunt→bizworx (e.g. `kubectl exec … pg_dump … | kubectl exec -i … pg_restore …`) — read-only on foxhunt.
|
||||
- [ ] Verify row counts match on the load-bearing tables:
|
||||
`forward_nav`, `backtest_summary`, `bybit_sleeve_ret`, `xsp_option_bars`, the feature/`*_features` tables,
|
||||
`exec_fills`, `ibkr_account_nav`. Compare `SELECT count(*)` foxhunt vs bizworx for each.
|
||||
|
||||
**EVAL:** every checked table's row count matches. This proves the migration procedure works BEFORE cutover.
|
||||
foxhunt untouched (dump is read-only). (The real cutover in Task 8 re-runs this on the frozen DB.)
|
||||
|
||||
### Task 7: Stand up the fund on bizworx IN PARALLEL (no DNS yet) — the "alles werkt eerst" gate
|
||||
|
||||
**Files:** `infra/k8s/{jobs,gitea,monitoring→skip,...}`.
|
||||
- [ ] Apply the fund workloads to bizworx ns `foxhunt`: dagster, ib-gateway, fxhnt-dashboard (cockpit),
|
||||
pointed at the bizworx Postgres (restored in Task 6) + git-sync from the **bizworx gitea** (use its
|
||||
tailnet/ClusterIP endpoint directly for now, since `git.fxhnt.ai` still points at foxhunt).
|
||||
- [ ] Apply the CronJobs **all SUSPENDED** (`suspend: true`) initially.
|
||||
- [ ] **ib-gateway:** verify it connects to IBKR paper on bizworx (no 2FA/IP-limit) + a `whatIf` order is
|
||||
accepted: exec the gateway health / a dry-run probe.
|
||||
- [ ] **Cockpit:** port-forward the bizworx cockpit and verify in a **real browser** it renders the migrated
|
||||
data (NAVs, gates, holdings) — read the numbers; compare against the live foxhunt cockpit.
|
||||
- [ ] **Rebalancer dry-run:** run `fxhnt-ucits-rebalancer` once manually as a dry-run Job on bizworx (EU hours)
|
||||
→ it should plan/price the UCITS lines without error (or fill, on paper).
|
||||
- [ ] **Dagster:** trigger the nightly asset graph once on bizworx → green.
|
||||
|
||||
**EVAL (the big one):** the ENTIRE fund works on bizworx in parallel with foxhunt still live — cockpit correct,
|
||||
ib-gateway connected, rebalancer dry-run clean, dagster green, recon gate reconciles the migrated refs. **Only
|
||||
if ALL pass** do we proceed to cutover. Nothing on foxhunt has changed.
|
||||
|
||||
---
|
||||
|
||||
## PHASE 1 — cutover (only after Task 7 passes)
|
||||
|
||||
### Task 8: Cutover window (downtime OK, flexible)
|
||||
|
||||
**Files:** DNS/tailnet + ingress config.
|
||||
- [ ] Announce/pick a quiet window (fund is ~1×/day; any convenient time).
|
||||
- [ ] Freeze foxhunt: `kubectl --context $FOX patch cronjob <each fund cronjob> -n foxhunt -p '{"spec":{"suspend":true}}'`
|
||||
and scale down foxhunt dagster/cockpit writers so its Postgres is quiescent.
|
||||
- [ ] Final delta dump/restore foxhunt→bizworx Postgres (re-run Task 6's procedure on the now-frozen DB) so
|
||||
bizworx has the latest state. Re-verify the key row counts.
|
||||
- [ ] Copy `unlock_calendar.json` (or rely on the DB-first loader), ib-gateway settings if not re-authed.
|
||||
- [ ] Re-point DNS: `git.fxhnt.ai` and `dashboard.fxhnt.ai` → bizworx (tailnet node / ingress; ACME TLS for
|
||||
the dashboard). Fund git-sync now resolves the bizworx gitea.
|
||||
- [ ] Un-suspend the fund CronJobs on **bizworx** (start with the UCITS rebalancer; leave US/bybit suspended
|
||||
as they already are).
|
||||
|
||||
**EVAL:** on bizworx via the real hostnames — cockpit at `dashboard.fxhnt.ai` renders correctly, `git.fxhnt.ai`
|
||||
serves the repos + git-sync works, the armed rebalancer's next run succeeds. foxhunt is now suspended (NOT
|
||||
deleted). **Rollback still trivial** (re-point DNS + un-suspend foxhunt).
|
||||
|
||||
### Task 9: Cockpit + prod verification post-cutover
|
||||
|
||||
**Files:** none.
|
||||
- [ ] Verify cockpit LOCALLY in a real browser against the bizworx DB (dev-cockpit loop) AND on
|
||||
`dashboard.fxhnt.ai`; read the numbers (`feedback_verify_cockpit_locally_not_prod`).
|
||||
- [ ] Confirm the reconciliation gate + forward tracks are intact (no spurious re-inception; deterministic
|
||||
recompute from the migrated DB anchor).
|
||||
|
||||
**EVAL:** prod cockpit + gate correct on bizworx. If anything is off → rollback (foxhunt still intact).
|
||||
|
||||
---
|
||||
|
||||
## PHASE 1 — observation (keep BOTH, delete NOTHING)
|
||||
|
||||
### Task 10: Parallel observation window
|
||||
|
||||
**Files:** none.
|
||||
- [ ] Run the fund on bizworx for an agreed window (e.g. several trading days) with foxhunt suspended-but-intact.
|
||||
- [ ] Each day, evaluate: dagster nightly green, the UCITS rebalancer run clean, recon gate healthy, cockpit
|
||||
numbers sane, no stray-holdings/venue surprises.
|
||||
|
||||
**EVAL:** N consecutive clean days on bizworx. **This is the gate that unlocks any teardown.** Until it passes,
|
||||
nothing is deleted.
|
||||
|
||||
---
|
||||
|
||||
## PHASE 2 — triage the rest (per-item operator sign-off before any delete)
|
||||
|
||||
### Task 11: Keep/drop inventory of non-fund ns `foxhunt`
|
||||
|
||||
**Files:** produce `docs/superpowers/specs/…-phase2-triage.md` (the inventory).
|
||||
- [ ] For every non-fund workload (0-replica legacy: api/backtesting-service/broker-gateway/
|
||||
data-acquisition-service/ml-training-service/questdb/loki/pushgateway; other: minio/redis/kanidm/
|
||||
mattermost/stalwart/netbird) and every orphan PVC (training-data 500Gi, gitlab 50Gi, feature-cache,
|
||||
fxhnt-backtest-data, multistrat-state, test-data, tempo, …): classify DROP / MIGRATE / KEEP-EXTERNAL.
|
||||
- [ ] Operator signs off **per item** before its data is deleted (look before deleting).
|
||||
|
||||
**EVAL:** a signed-off inventory; no ambiguous items remain.
|
||||
|
||||
---
|
||||
|
||||
## PHASE 3 — teardown (ONLY after Task 10 + Task 11)
|
||||
|
||||
### Task 12: Retire the foxhunt Kapsule
|
||||
|
||||
**Files:** `infra/live/production/kapsule/**` (Terragrunt, project `c293eb98`), `infra/k8s/**`.
|
||||
- [ ] Remove the foxhunt k8s manifests / argo apps.
|
||||
- [ ] `terraform destroy` (via Terragrunt) the foxhunt Kapsule: control plane + `platform` (3× dev1_l) +
|
||||
`ci-compile-cpu` pools.
|
||||
- [ ] Explicitly delete retain-class volumes that won't auto-release (`*-retain` PVCs), the LB, private
|
||||
network `4ac46c9e…`, and other foxhunt-project scw resources; drop the confirmed-dead PVCs from Task 11.
|
||||
- [ ] Remove the temporary foxhunt API ACL entry for this box (`scw k8s acl` — the `bizworx-devbox-v4` entry),
|
||||
unless still useful.
|
||||
- [ ] Update memory: foxhunt Kapsule retired; fund lives on bizworx; single gitea = bizworx.
|
||||
|
||||
**EVAL:** foxhunt Kapsule + nodes gone; fund still healthy on bizworx; scw monthly cost dropped by the
|
||||
foxhunt control plane + 3 dev1_l nodes + its block storage. No wanted data lost.
|
||||
|
||||
---
|
||||
|
||||
## Self-review notes
|
||||
- **Verify-first honored:** Tasks 1–7 are additive (foxhunt untouched); the only foxhunt change before teardown
|
||||
is *suspend* (Task 8), fully reversible. First delete is Task 12, gated behind Tasks 10 + 11.
|
||||
- **Per-step eval honored:** every task has an `EVAL:` gate.
|
||||
- **Secrets:** Task 2 uses `!`/openbao, never the transcript.
|
||||
- **Rollback:** available through Task 11 (foxhunt intact-and-suspended).
|
||||
@@ -0,0 +1,104 @@
|
||||
# Production Scheduling — Consolidate onto Dagster (K8sRunLauncher) + build the proper image
|
||||
|
||||
> **For the operator/agent:** execute inline, checkpoint per task. Live infra + real-money fund. Operator
|
||||
> decision (2026-07-19): go ALL-DAGSTER properly — upgrade the Dagster infra so heavy compute runs in
|
||||
> per-run K8s Jobs (own memory), fold the cron precompute into the DAG, and build the image the right way
|
||||
> (podman → bizworx registry, bypassing the dead Argo/Kaniko/GitLab pipeline).
|
||||
|
||||
**Goal:** One orchestrator (Dagster) owns the whole nightly ETL + precompute as an asset graph, with per-run
|
||||
memory isolation via `K8sRunLauncher` (heavy assets get 6-8Gi Jobs, the daemon stays 4Gi). Remove the
|
||||
`compare-measured-precompute` cron (folded into the DAG). Keep EXECUTION (rebalancers) in K8s CronJobs
|
||||
(deliberate money-exec blast-radius isolation). Delete the dead VRP. Result: consistent, documented, not-confusing.
|
||||
|
||||
## Why this is the right end-state (and why it needs an infra upgrade)
|
||||
- The split felt "buggy" because `bybit_sleeve_ret` is written by a SUSPENDED own-memory cron, cross-timed to the
|
||||
Dagster 23:30 ingest. Folding it into Dagster removes the timing coupling.
|
||||
- BUT the cron exists for a HARD reason: the sleeve-ret compute needs **6-8Gi** and OOM-killed the 4Gi daemon.
|
||||
Naively making it a daemon-run asset re-breaks the nightly. The CORRECT fix = `K8sRunLauncher`: each Dagster
|
||||
run launches as its own right-sized K8s Job; the daemon only schedules/launches (stays tiny). This is the
|
||||
standard Dagster production pattern.
|
||||
|
||||
## Current-state facts (verified 2026-07-19)
|
||||
- Dagster ETL HEALTHY: 23:30 run SUCCESS, all 14 assets materialized, forward_nav fresh 07-18. (working_directory
|
||||
/app→/code/src change already applied — harmless, more-correct.)
|
||||
- `dagster_k8s` NOT in the image (orchestration extra = dagster/webserver/postgres only). RBAC: dagster SA
|
||||
`tailscale-dagster` CANNOT create jobs. Run launcher: default in-process (4Gi daemon).
|
||||
- Image `rg.fr-par.scw.cloud/bizworx/fxhnt-cockpit:latest` (633MB) built via Dockerfile
|
||||
`infra/docker/fxhnt.Dockerfile` (`pip install .[web,pg,data,factory,orchestration,poc]`). The Argo/Kaniko
|
||||
build pipeline is DEAD (ci-compile-cpu pool gone + clones the killed GitLab).
|
||||
- Build capability: devbox has **podman 4.9.3, x86_64** = cluster amd64/linux. Registry live + reachable.
|
||||
|
||||
## Task 1: Add dagster-k8s to deps + build the image RIGHT (podman → bizworx registry)
|
||||
**Files:** `pyproject.toml`, `infra/docker/fxhnt.Dockerfile` (verify), build via podman.
|
||||
- [ ] Add `dagster-k8s>=0.28,<0.29` to the `orchestration` extra in pyproject.toml (matches dagster 1.12 / the
|
||||
0.28 dagster-postgres pin). `uv lock` to update uv.lock.
|
||||
- [ ] `podman login rg.fr-par.scw.cloud` with an scw secret-key (username `nologin`, password = SCW secret key;
|
||||
NEVER print it). Verify login OK.
|
||||
- [ ] `podman build --platform linux/amd64 -f infra/docker/fxhnt.Dockerfile -t
|
||||
rg.fr-par.scw.cloud/bizworx/fxhnt-cockpit:dagster-k8s-<sha> .` (tag with the git sha, NOT :latest yet).
|
||||
- [ ] `podman push` the sha-tag. VERIFY it lands in the registry (`scw registry image list`).
|
||||
- **EVAL:** image pushed; `podman run --rm <tag> python -c "import dagster_k8s; print(dagster_k8s.__version__)"`
|
||||
succeeds. Do NOT retag :latest until the new image is proven on-cluster (Task 4). Roll back = keep :latest.
|
||||
|
||||
## Task 2: RBAC — let the dagster SA launch Jobs + a Job SA with fund access
|
||||
**Files:** `infra/k8s/orchestration/dagster.yaml` (Role/RoleBinding + a run-job SA).
|
||||
- [ ] Add a Role granting the dagster SA: create/get/list/watch/delete on `jobs` (batch) + `pods`, get on
|
||||
`pods/log`. Bind to `tailscale-dagster` (or a dedicated `dagster-runner` SA).
|
||||
- [ ] Define the SA the LAUNCHED Jobs use — needs the fund secrets (db-credentials, databento, tiingo),
|
||||
git-sync key, and the same NP egress (postgres, gitea, apiserver, public HTTPS). Mirror the cron pods'
|
||||
access. (Heavy assets that hit ib-gateway would need that NP too — but the nightly precompute does not.)
|
||||
- [ ] **VERIFY:** `kubectl auth can-i create jobs --as=system:serviceaccount:foxhunt:<sa>` = yes.
|
||||
- **EVAL:** RBAC in place, least-privilege (namespaced Role, not cluster-admin).
|
||||
|
||||
## Task 3: Configure K8sRunLauncher + per-asset memory + make sleeve-ret/backtest-refs assets
|
||||
**Files:** `infra/k8s/orchestration/dagster.yaml` (dagster.yaml instance config), `src/fxhnt/adapters/
|
||||
orchestration/{assets.py,definitions.py}`.
|
||||
- [ ] dagster.yaml: add `run_launcher: module dagster_k8s class K8sRunLauncher` with job_namespace foxhunt,
|
||||
the new image, image_pull_secrets scw-registry, the Job SA, and default resources. Add a `run_k8s_config`
|
||||
/ tag-based per-run memory (heavy assets → 6Gi req/8Gi limit, pinned to the large pool).
|
||||
- [ ] Wrap `persist_bybit_sleeve_returns` (already an import-light helper) as a Dagster `@asset(deps=[
|
||||
bybit_warehouse_refresh])` with a k8s memory tag; add a `backtest_refs` asset (deps on the sleeve-ret +
|
||||
nav assets) wrapping the `backtest-refs` logic. Add both to combined_book_job selection + defs.
|
||||
- [ ] Keep the plain helper functions intact (unit-testable); the assets just call them (no logic duplication).
|
||||
- [ ] Add/point tests: the existing sleeve-ret + backtest-refs unit tests still pass; a new test asserts the
|
||||
assets are in the job selection.
|
||||
- **EVAL:** `uv run pytest` for the touched modules green; `python -c "import ...definitions"` lists the new
|
||||
assets in combined_book_job. Do NOT deploy yet.
|
||||
|
||||
## Task 4: Deploy the new image + config, materialize the nightly once, VERIFY end-to-end
|
||||
**Files:** none (deploy).
|
||||
- [ ] Point dagster deploy at the new sha-tagged image (edit dagster.yaml image or a kustomize overlay), apply,
|
||||
rollout restart. git-sync pulls the new src (with the new assets).
|
||||
- [ ] Trigger `combined_book_forward_job` once. WATCH: the daemon LAUNCHES a K8s Job (not in-process); the Job
|
||||
gets 6-8Gi on the large pool; it materializes ALL assets incl the new sleeve-ret + backtest-refs WITHOUT
|
||||
OOM-killing the daemon.
|
||||
- [ ] **VERIFY:** `bybit_sleeve_ret` max(run_date) = today, src_updated = now; backtest_summary fresh; forward
|
||||
tracks still fresh; cockpit reflects it; daemon did NOT restart/OOM. Check `kubectl get jobs -n foxhunt`
|
||||
shows the dagster-launched run Job completed.
|
||||
- **EVAL:** full nightly runs in an isolated Job, data fresh, daemon healthy. THIS is the money shot. If OOM or
|
||||
launcher fails → debug (RBAC/image/config), do NOT retag :latest. Roll back = old image + re-arm the cron.
|
||||
- [ ] Once proven: retag the sha image as `:latest` (or update the manifest to the sha) so it's the deployed default.
|
||||
|
||||
## Task 5: Remove the now-redundant compare-measured cron + delete VRP + arm execution + document
|
||||
**Files:** `infra/k8s/jobs/*.yaml`, memory, progress anchor.
|
||||
- [ ] DELETE `fxhnt-compare-measured-precompute` cron + its NP (its work is now the Dagster assets). `git rm` the
|
||||
manifest. (Keep `factory` cron — it's migrate-cockpit + factory-cycle, separate concern; ARM it.)
|
||||
- [ ] DELETE `fxhnt-vrp-rebalancer` cron + NP + manifest (shelved/falsified; vrp_nav already de-wired). Update
|
||||
memory (project_fxhnt_diversifier_hunt_2026_07_15, project_fxhnt_vrp_cvar_crash_research): VRP exec cron DELETED.
|
||||
- [ ] ARM the EXECUTION crons that should run: `factory` (daily), `ucits-volume-ibkr` + `ucits-rebalancer`
|
||||
(weekday, ucits already armed). Keep SUSPENDED: `bybit-rebalancer` (no creds), `ibkr-rebalancer` (PRIIPs).
|
||||
Patch live + set suspend in manifests (source of truth).
|
||||
- [ ] **VERIFY:** cronjobs = {factory, ucits-volume-ibkr, ucits-rebalancer} armed; {bybit, ibkr} suspended;
|
||||
{compare-measured, vrp} GONE. Dagster owns all ETL/precompute.
|
||||
- [ ] Document the final architecture in the progress anchor + a CLAUDE-visible note: Dagster (K8sRunLauncher,
|
||||
per-run memory) = ALL nightly ETL+precompute; K8s CronJobs = execution only. Why exec stays crons
|
||||
(blast-radius). Update reference_fxhnt_prod_deploy_mechanics with the podman build path (Argo build dead).
|
||||
- **EVAL:** consistent single-orchestrator ETL, execution isolated in crons, nothing suspended-by-accident,
|
||||
VRP gone, architecture documented. Done.
|
||||
|
||||
## Global constraints
|
||||
- VERIFY-FIRST, per-task eval. REAL MONEY: only ucits-rebalancer trades (paper). Never arm bybit/ibkr exec.
|
||||
- NEVER print the SCW secret key / registry password. podman login via stdin or env, not argv.
|
||||
- Image: tag with git sha, prove on-cluster BEFORE :latest. Old :latest stays as instant rollback.
|
||||
- Manifests are source of truth: every live change also edits the committed yaml.
|
||||
- Don't duplicate logic: assets call the existing import-light helpers; keep them unit-testable.
|
||||
@@ -0,0 +1,599 @@
|
||||
# Positioning Per-Coin Gross Cap Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Add an ex-ante per-coin gross-weight cap (0.07) to the `positioning` sleeve so no single coin dominates the book, then re-inception + backfill the forward anchor so the live gate keeps running against the capped construction.
|
||||
|
||||
**Architecture:** The cap is applied to the FORMING weights in `positioning_weights` (before returns), so it is strictly causal (no lookahead). It threads through the existing seams (`_book_breakdown` → `positioning_returns_from_store` → `_positioning_series` → `sleeve_returns_from_store`) as an optional `coin_gross_cap` param that defaults to `None` (OFF → byte-identical to today). It is switched ON at 0.07 only in `bybit_book_persist.py`. The registry `positioning` + `bybit_4edge` definition versions bump to re-inception; `inception_t0` pins the existing OOS start (2026-07-06) so the forward track backfills the window with the capped construction instead of resetting the gate to 0.
|
||||
|
||||
**Tech Stack:** Python 3.12, numpy-free pure math, pytest (`-n auto`), TimescaleFeatureStore (Postgres, table `bybit_features`).
|
||||
|
||||
## Global Constraints
|
||||
|
||||
- The cap is applied to WEIGHTS (in `positioning_weights`), NOT to returns/contributions — it must be ex-ante/causal (the cap decision cannot see the coin's realized return).
|
||||
- `coin_gross_cap` default is `None` (OFF) at EVERY seam. With it OFF, every existing test and the whole backtest must be byte-identical to today. The OFF path must not add per-term division drift.
|
||||
- After clipping `|w| ≤ coin_gross_cap`, RENORMALIZE the day's weights to unit gross (`Σ|w| = 1`), matching `positioning_weights`' existing unit-gross contract.
|
||||
- The validated cap value is **0.07** (full-history sweep 1295d: Sharpe 2.60→2.69, maxDD −32.3%→−29.9%, per-year Sharpe non-decreasing every year). Do NOT use 0.04 (falsified: Sharpe drops to 2.43).
|
||||
- Re-inception uses the EXISTING pattern: bump `version` in `definition.params.version`, and set `inception_t0: {"date": "2026-07-06", "version": <new>}` (outside `params`, NOT hashed) so the anchor backdates and the forward window is recomputed, not reset. See `forward_definition.inception_override`.
|
||||
- `positioning` is a CONSTITUENT of `bybit_4edge`; both registry entries' definition versions must bump together (the book's sleeve construction changed).
|
||||
- Never touch real money / execution. This is a construction change to the modeled+forward track only.
|
||||
- Run tests with `uv run --extra dev python -m pytest ...`. The positioning eval integration tests need no extra services (they use a fake store).
|
||||
|
||||
---
|
||||
|
||||
### Task 1: Add the ex-ante `coin_gross_cap` to `positioning_weights`
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/application/bybit_positioning_eval.py:78-104` (the `positioning_weights` function)
|
||||
- Test: `tests/integration/test_bybit_positioning_eval.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: nothing new.
|
||||
- Produces: `positioning_weights(ratio_by_day, *, direction="contrarian", coin_gross_cap: float | None = None) -> dict[int, dict[str, float]]`. When `coin_gross_cap` is a float, each day's weights are clipped to `|w| ≤ coin_gross_cap` then renormalized to unit gross. When `None`, output is byte-identical to today.
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
Add to `tests/integration/test_bybit_positioning_eval.py`:
|
||||
|
||||
```python
|
||||
def test_coin_gross_cap_clips_and_renormalizes() -> None:
|
||||
# One coin dominates (retail extreme) -> uncapped weight far exceeds 0.07; cap must clip + renormalize.
|
||||
ratio = {0: {"DOMUSDT": 0.95, "AUSDT": 0.50, "BUSDT": 0.50, "CUSDT": 0.05}}
|
||||
uncapped = positioning_weights(ratio)[0]
|
||||
assert max(abs(w) for w in uncapped.values()) > 0.07 # DOM dominates uncapped
|
||||
capped = positioning_weights(ratio, coin_gross_cap=0.07)[0]
|
||||
assert max(abs(w) for w in capped.values()) <= 0.07 + 1e-9 # no coin exceeds the cap
|
||||
assert abs(sum(abs(w) for w in capped.values()) - 1.0) < 1e-9 # still unit gross
|
||||
# sign structure preserved: DOM still short (over-long crowd), C still long (over-short)
|
||||
assert capped["DOMUSDT"] < 0.0 and capped["CUSDT"] > 0.0
|
||||
|
||||
|
||||
def test_coin_gross_cap_none_is_byte_identical() -> None:
|
||||
ratio = {0: {"AAAUSDT": 0.9, "BBBUSDT": 0.5, "CCCUSDT": 0.1}}
|
||||
assert positioning_weights(ratio, coin_gross_cap=None) == positioning_weights(ratio)
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `uv run --extra dev python -m pytest tests/integration/test_bybit_positioning_eval.py::test_coin_gross_cap_clips_and_renormalizes -v`
|
||||
Expected: FAIL with `TypeError: positioning_weights() got an unexpected keyword argument 'coin_gross_cap'`
|
||||
|
||||
- [ ] **Step 3: Implement the cap in `positioning_weights`**
|
||||
|
||||
Replace the body of `positioning_weights` (lines 91-104) so the signature gains `coin_gross_cap` and, after the existing unit-gross normalization, applies the clip+renormalize when the cap is set:
|
||||
|
||||
```python
|
||||
def positioning_weights(ratio_by_day: dict[int, dict[str, float]], *,
|
||||
direction: str = "contrarian",
|
||||
coin_gross_cap: float | None = None) -> dict[int, dict[str, float]]:
|
||||
"""`{day: {coin: w}}` market-neutral cross-sectional weights from the long/short positioning signal.
|
||||
|
||||
`direction="contrarian"` (DEFAULT) SHORTs the high-buyRatio coins (retail over-long → fade) and LONGs the
|
||||
low-buyRatio coins (retail over-short). `direction="momentum"` is the EXACT SIGN-FLIP: LONG the over-long
|
||||
/ SHORT the over-short (informed-flow continuation). Construction per day:
|
||||
1. DEMEAN the per-day buyRatio across coins (so the book is dollar/market-neutral, Σw ≈ 0);
|
||||
2. apply the DIRECTION sign — contrarian = −demeaned (short the rich/over-long), momentum = +demeaned;
|
||||
3. normalise to UNIT GROSS (Σ|w| = 1);
|
||||
4. (OPTIONAL) if `coin_gross_cap` is set, CLIP each |w| to the cap and RENORMALISE to unit gross —
|
||||
an EX-ANTE per-coin concentration limit (applied to the FORMING weight, so it is strictly causal:
|
||||
it cannot see the coin's realized return). Validated at 0.07 (full-history sweep: Sharpe 2.60→2.69,
|
||||
maxDD −32.3%→−29.9%, per-year non-decreasing) — caps the LABUSDT-class implosions (a 10.3% long leg
|
||||
that did −59%/−79.5% on 2026-07-06/07) without over-throttling the edge (0.04 falsified: Sharpe→2.43).
|
||||
|
||||
Because the gross is identical for both directions, `momentum` weights are the exact per-coin negation of
|
||||
`contrarian`. A day with <2 coins, or zero gross after demeaning, yields no weights (skipped). Pure."""
|
||||
if direction not in ("contrarian", "momentum"):
|
||||
raise ValueError(f"direction must be 'contrarian' or 'momentum', got {direction!r}")
|
||||
sign = -1.0 if direction == "contrarian" else 1.0
|
||||
out: dict[int, dict[str, float]] = {}
|
||||
for day, row in ratio_by_day.items():
|
||||
if len(row) < 2:
|
||||
continue
|
||||
mean = st.mean(row.values())
|
||||
raw = {c: sign * (v - mean) for c, v in row.items()}
|
||||
gross = sum(abs(w) for w in raw.values())
|
||||
if gross <= 0.0:
|
||||
continue
|
||||
w = {c: x / gross for c, x in raw.items()}
|
||||
if coin_gross_cap is not None:
|
||||
w = {c: max(-coin_gross_cap, min(coin_gross_cap, x)) for c, x in w.items()}
|
||||
capped_gross = sum(abs(x) for x in w.values())
|
||||
if capped_gross <= 0.0:
|
||||
continue
|
||||
w = {c: x / capped_gross for c, x in w.items()}
|
||||
out[int(day)] = w
|
||||
return out
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run test to verify it passes**
|
||||
|
||||
Run: `uv run --extra dev python -m pytest tests/integration/test_bybit_positioning_eval.py -v`
|
||||
Expected: PASS (the two new tests + all existing positioning_weights tests still green)
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/application/bybit_positioning_eval.py tests/integration/test_bybit_positioning_eval.py
|
||||
git commit -m "feat(positioning): ex-ante per-coin gross cap in positioning_weights (default off)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 2: Thread `coin_gross_cap` through `_book_breakdown` and `positioning_returns_from_store`
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/application/bybit_positioning_eval.py:140-166` (`_book_breakdown` signature + the `positioning_weights` call at line 177)
|
||||
- Modify: `src/fxhnt/application/bybit_positioning_eval.py:287-303` (`positioning_returns_from_store` signature + its `_book_breakdown` call)
|
||||
- Test: `tests/integration/test_bybit_positioning_eval.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `positioning_weights(..., coin_gross_cap=...)` from Task 1.
|
||||
- Produces:
|
||||
- `_book_breakdown(store, *, universe, cost_bps, direction, capacity_capital=None, participation=0.10, turnover_window=30, tail_cap_k=None, book_vol_window=60, coin_gross_cap=None) -> dict`
|
||||
- `positioning_returns_from_store(store, *, universe=None, cost_bps=5.5, direction="contrarian", capacity_capital=None, tail_cap_k=None, coin_gross_cap=None) -> dict[int, float]`
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
Add to `tests/integration/test_bybit_positioning_eval.py` (reuse the module's existing fake store fixture — grep the file for the store used by `test_capacity_haircut_caps_gain_only_at_scale`, e.g. a `_FakeStore`/`_ReadOnlyStore` builder, and construct one where a single coin has an extreme long_ratio + a large adverse next-day return):
|
||||
|
||||
```python
|
||||
def test_coin_gross_cap_reduces_single_coin_loss_in_book(positioning_store_with_shock):
|
||||
# positioning_store_with_shock: one coin (retail over-short) craters next day; uncapped it dominates the
|
||||
# book loss, capped its weight (and thus its loss contribution) is bounded.
|
||||
store = positioning_store_with_shock
|
||||
uncapped = positioning_returns_from_store(store, cost_bps=0.0)
|
||||
capped = positioning_returns_from_store(store, cost_bps=0.0, coin_gross_cap=0.07)
|
||||
shock_day = min(uncapped, key=lambda d: uncapped[d]) # the worst day uncapped
|
||||
assert capped[shock_day] > uncapped[shock_day] # the cap made the worst day less bad
|
||||
```
|
||||
|
||||
If the module has no reusable multi-coin shock fixture, build the store inline in the test from a `long_ratio` panel (one coin at ratio 0.95, three near 0.5) + a `close` panel where the over-short coin drops ~50% the next day, mirroring the `test_capacity_haircut_*` store construction in the same file.
|
||||
|
||||
- [ ] **Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `uv run --extra dev python -m pytest tests/integration/test_bybit_positioning_eval.py::test_coin_gross_cap_reduces_single_coin_loss_in_book -v`
|
||||
Expected: FAIL with `TypeError: positioning_returns_from_store() got an unexpected keyword argument 'coin_gross_cap'`
|
||||
|
||||
- [ ] **Step 3: Thread the param through both functions**
|
||||
|
||||
In `_book_breakdown`, add `coin_gross_cap: float | None = None` to the signature (after `book_vol_window: int = 60`), and change the weight-forming call (currently line 177):
|
||||
|
||||
```python
|
||||
weights_by_day = positioning_weights(_ratio_by_day(long_ratio), direction=direction,
|
||||
coin_gross_cap=coin_gross_cap)
|
||||
```
|
||||
|
||||
In `positioning_returns_from_store`, add `coin_gross_cap: float | None = None` to the signature (after `tail_cap_k`) and pass it through:
|
||||
|
||||
```python
|
||||
return _book_breakdown(store, universe=universe, cost_bps=cost_bps, direction=direction,
|
||||
capacity_capital=capacity_capital, tail_cap_k=tail_cap_k,
|
||||
coin_gross_cap=coin_gross_cap)["net"]
|
||||
```
|
||||
|
||||
Update the `positioning_returns_from_store` docstring's param note to add: `coin_gross_cap` (None default) applies the ex-ante per-coin concentration cap (see `positioning_weights`).
|
||||
|
||||
- [ ] **Step 4: Run test to verify it passes**
|
||||
|
||||
Run: `uv run --extra dev python -m pytest tests/integration/test_bybit_positioning_eval.py -v`
|
||||
Expected: PASS
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/application/bybit_positioning_eval.py tests/integration/test_bybit_positioning_eval.py
|
||||
git commit -m "feat(positioning): thread coin_gross_cap through book_breakdown + returns_from_store"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 3: Thread `coin_gross_cap` through the sleeve seam (`_positioning_series` + `sleeve_returns_from_store`)
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/application/bybit_book_eval.py:226-240` (`_positioning_series`)
|
||||
- Modify: `src/fxhnt/application/bybit_book_eval.py:243-278` (`sleeve_returns_from_store` signature + the positioning branch)
|
||||
- Test: `tests/integration/test_bybit_positioning_eval.py` (or `tests/integration/test_web_new_edges.py` if that is where sleeve_returns_from_store is exercised — grep first)
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `positioning_returns_from_store(..., coin_gross_cap=...)` from Task 2.
|
||||
- Produces:
|
||||
- `_positioning_series(store, *, universe, cost_bps, direction="contrarian", capacity_capital=None, tail_cap_k=None, coin_gross_cap=None) -> dict[int, float]`
|
||||
- `sleeve_returns_from_store(store, edge, *, universe=None, cost_bps=0.0, unlock_events=None, stablecoin_spot_panel=None, capacity_capital=None, tail_cap_k=None, coin_gross_cap=None) -> dict[int, float]`. The `coin_gross_cap` is forwarded ONLY to the `positioning` branch (other edges ignore it).
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
Add (same test file, reuse the store from Task 2's fixture/inline construction):
|
||||
|
||||
```python
|
||||
def test_sleeve_returns_positioning_forwards_coin_gross_cap(positioning_store_with_shock):
|
||||
from fxhnt.application.bybit_book_eval import sleeve_returns_from_store
|
||||
store = positioning_store_with_shock
|
||||
uncapped = sleeve_returns_from_store(store, "positioning", cost_bps=0.0)
|
||||
capped = sleeve_returns_from_store(store, "positioning", cost_bps=0.0, coin_gross_cap=0.07)
|
||||
shock_day = min(uncapped, key=lambda d: uncapped[d])
|
||||
assert capped[shock_day] > uncapped[shock_day] # cap forwarded -> worst day improved
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `uv run --extra dev python -m pytest tests/integration/test_bybit_positioning_eval.py::test_sleeve_returns_positioning_forwards_coin_gross_cap -v`
|
||||
Expected: FAIL with `TypeError: sleeve_returns_from_store() got an unexpected keyword argument 'coin_gross_cap'`
|
||||
|
||||
- [ ] **Step 3: Thread through both functions**
|
||||
|
||||
In `_positioning_series` (line 226), add `coin_gross_cap: float | None = None` to the signature (after `tail_cap_k`) and forward it:
|
||||
|
||||
```python
|
||||
return positioning_returns_from_store(store, universe=universe, cost_bps=cost_bps, direction=direction,
|
||||
capacity_capital=capacity_capital, tail_cap_k=tail_cap_k,
|
||||
coin_gross_cap=coin_gross_cap)
|
||||
```
|
||||
|
||||
In `sleeve_returns_from_store` (line 243), add `coin_gross_cap: float | None = None` to the signature (after `tail_cap_k`), and change ONLY the positioning branch (line 276) to forward it:
|
||||
|
||||
```python
|
||||
if edge == "positioning":
|
||||
# ... existing comment about raw store + universe unchanged ...
|
||||
return _positioning_series(store, universe=universe, cost_bps=cost_bps,
|
||||
capacity_capital=capacity_capital, tail_cap_k=tail_cap_k,
|
||||
coin_gross_cap=coin_gross_cap)
|
||||
```
|
||||
|
||||
Leave the tstrend / unlock / stablecoin / xsfunding branches unchanged (they never receive `coin_gross_cap`).
|
||||
|
||||
- [ ] **Step 4: Run test to verify it passes**
|
||||
|
||||
Run: `uv run --extra dev python -m pytest tests/integration/test_bybit_positioning_eval.py -v`
|
||||
Expected: PASS
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/application/bybit_book_eval.py tests/integration/test_bybit_positioning_eval.py
|
||||
git commit -m "feat(positioning): forward coin_gross_cap through sleeve_returns_from_store"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 4: Turn the cap ON at 0.07 in the live precompute + add the constant
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/application/bybit_forward_track.py:47` (add `POSITIONING_COIN_GROSS_CAP` next to `POSITIONING_TAIL_CAP_K`)
|
||||
- Modify: `src/fxhnt/application/bybit_book_persist.py:77-84` (pass the cap into `sleeve_returns_from_store`)
|
||||
- Test: `tests/integration/test_bybit_paper_reconciliation.py` (or the persist test — grep for a test that calls `_persist_bybit_book` / `persist_bybit_sleeve_returns`; if none exists at unit level, assert the constant is wired via a focused test in `test_bybit_positioning_eval.py`)
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `sleeve_returns_from_store(..., coin_gross_cap=...)` from Task 3.
|
||||
- Produces: `POSITIONING_COIN_GROSS_CAP = 0.07` in `bybit_forward_track.py`; the live `_persist_bybit_book` sleeve computation applies it to the positioning sleeve.
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
Add to `tests/integration/test_bybit_positioning_eval.py`:
|
||||
|
||||
```python
|
||||
def test_positioning_coin_gross_cap_constant_is_007() -> None:
|
||||
from fxhnt.application.bybit_forward_track import POSITIONING_COIN_GROSS_CAP
|
||||
assert POSITIONING_COIN_GROSS_CAP == 0.07
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `uv run --extra dev python -m pytest tests/integration/test_bybit_positioning_eval.py::test_positioning_coin_gross_cap_constant_is_007 -v`
|
||||
Expected: FAIL with `ImportError: cannot import name 'POSITIONING_COIN_GROSS_CAP'`
|
||||
|
||||
- [ ] **Step 3: Add the constant + wire it into the precompute**
|
||||
|
||||
In `src/fxhnt/application/bybit_forward_track.py`, directly after line 47 (`POSITIONING_TAIL_CAP_K = 4.0`):
|
||||
|
||||
```python
|
||||
# Ex-ante per-coin gross-weight cap for the positioning sleeve — no single coin's |weight| may exceed this
|
||||
# fraction of the book's unit gross. Validated 2026-07-20 over the full 1295-day history (Sharpe 2.60→2.69,
|
||||
# maxDD −32.3%→−29.9%, per-year Sharpe non-decreasing every year); caps LABUSDT-class single-coin implosions
|
||||
# (a 10.3% long leg that did −59%/−79.5% on 2026-07-06/07) without over-throttling (0.04 falsified → 2.43).
|
||||
POSITIONING_COIN_GROSS_CAP = 0.07
|
||||
```
|
||||
|
||||
In `src/fxhnt/application/bybit_book_persist.py`, update the import (line 78) and the sleeve computation (lines 80-83):
|
||||
|
||||
```python
|
||||
from fxhnt.application.bybit_forward_track import POSITIONING_COIN_GROSS_CAP, POSITIONING_TAIL_CAP_K
|
||||
sleeve_rets = {
|
||||
s: sleeve_returns_from_store(store, s, universe=universe, cost_bps=cost_bps,
|
||||
unlock_events=unlock_events,
|
||||
capacity_capital=settings.paper_capital,
|
||||
tail_cap_k=POSITIONING_TAIL_CAP_K,
|
||||
coin_gross_cap=POSITIONING_COIN_GROSS_CAP)
|
||||
for s in _DEFAULT_BYBIT_SLEEVES}
|
||||
```
|
||||
|
||||
(`coin_gross_cap` is ignored by every sleeve except positioning — Task 3 only forwards it in the positioning branch — so passing it for all sleeves in the dict comprehension is correct and harmless.)
|
||||
|
||||
- [ ] **Step 4: Run test to verify it passes**
|
||||
|
||||
Run: `uv run --extra dev python -m pytest tests/integration/test_bybit_positioning_eval.py -v`
|
||||
Expected: PASS
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/application/bybit_forward_track.py src/fxhnt/application/bybit_book_persist.py tests/integration/test_bybit_positioning_eval.py
|
||||
git commit -m "feat(positioning): enable coin_gross_cap=0.07 in the live bybit book precompute"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 5: Bump definition versions to re-inception positioning + bybit_4edge (backdated anchor, no gate reset)
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/registry.py` (the `positioning` entry ~line 249-259 and the `bybit_4edge` entry ~line 198-211; also `bybit_4edge_levered` ~line 229-241 which shares the sleeve construction)
|
||||
- Test: `tests/` — grep for `test_forward_definition` / a registry-consistency test; add an assertion that the bumped versions + inception_t0 are consistent
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: the capped construction from Task 4 (the version bump signals the construction changed).
|
||||
- Produces: `positioning`, `bybit_4edge`, `bybit_4edge_levered` definitions at `version: 3` with `inception_t0: {"date": "2026-07-06", "version": 3}`.
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
Add `tests/unit/test_positioning_cap_reinception.py`:
|
||||
|
||||
```python
|
||||
from fxhnt.registry import STRATEGY_REGISTRY
|
||||
|
||||
|
||||
def test_positioning_and_book_bumped_to_v3_backdated():
|
||||
for sid in ("positioning", "bybit_4edge", "bybit_4edge_levered"):
|
||||
d = STRATEGY_REGISTRY[sid]["definition"]
|
||||
assert d["params"]["version"] == 3, f"{sid} version not bumped to 3"
|
||||
ov = d.get("inception_t0")
|
||||
assert ov and ov["version"] == 3, f"{sid} inception_t0 not re-pinned to v3"
|
||||
# backdated to the existing OOS start, NOT reset to today -> the forward window is recomputed, not lost
|
||||
assert ov["date"] == "2026-07-06"
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `uv run --extra dev python -m pytest tests/unit/test_positioning_cap_reinception.py -v`
|
||||
Expected: FAIL (versions are currently 2)
|
||||
|
||||
- [ ] **Step 3: Bump the three registry entries**
|
||||
|
||||
In each of the `positioning`, `bybit_4edge`, and `bybit_4edge_levered` entries in `src/fxhnt/registry.py`, change `"version": 2` to `"version": 3` inside `definition.params`, and change the `inception_t0` to `{"date": "2026-07-06", "version": 3}`. Note: `positioning`'s existing `inception_t0` date is `2026-07-06`; `bybit_4edge` / `bybit_4edge_levered` currently use `2026-07-07` — for this backfill set ALL three to `2026-07-06` (the positioning shock's first day) so the whole book re-warms over the same window. Add a comment on each:
|
||||
|
||||
```python
|
||||
"inception_t0": {"date": "2026-07-06", "version": 3}, # v3: positioning coin_gross_cap=0.07 (2026-07-20 backfill)
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run test to verify it passes**
|
||||
|
||||
Run: `uv run --extra dev python -m pytest tests/unit/test_positioning_cap_reinception.py tests/ -k "forward_definition or registry" -v`
|
||||
Expected: PASS (the new test + any existing registry/definition consistency tests)
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/registry.py tests/unit/test_positioning_cap_reinception.py
|
||||
git commit -m "feat(positioning): re-inception positioning+bybit_4edge v3 (coin_gross_cap, backdated 2026-07-06)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 6: Full-suite verification + lint/type gate
|
||||
|
||||
**Files:** none (verification task)
|
||||
|
||||
**Interfaces:** consumes all prior tasks.
|
||||
|
||||
- [ ] **Step 1: Run the full positioning + book + forward suite**
|
||||
|
||||
Run: `uv run --extra dev python -m pytest tests/ -k "positioning or book or forward or reconcil or bybit" -q`
|
||||
Expected: all pass (no regression; the OFF-path byte-identity tests from Tasks 1-3 guarantee unchanged behavior everywhere the cap is not enabled).
|
||||
|
||||
- [ ] **Step 2: Lint + type-check the changed files**
|
||||
|
||||
Run: `uv run --extra dev ruff check src/fxhnt/application/bybit_positioning_eval.py src/fxhnt/application/bybit_book_eval.py src/fxhnt/application/bybit_book_persist.py src/fxhnt/application/bybit_forward_track.py src/fxhnt/registry.py`
|
||||
Then: `uv run --extra dev --extra orchestration mypy src/fxhnt/application/bybit_positioning_eval.py src/fxhnt/application/bybit_book_eval.py`
|
||||
Expected: no NEW errors on changed lines (pre-existing whole-file findings that are not in the diff are acceptable — confirm via `git diff --unified=0 ... | grep '^+' | awk 'length>120'` returning nothing).
|
||||
|
||||
- [ ] **Step 3: Commit any lint fixes** (only if Step 2 surfaced issues on changed lines)
|
||||
|
||||
```bash
|
||||
git add -A
|
||||
git commit -m "chore(positioning): lint/type fixes for coin_gross_cap"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Post-merge deploy + backfill (OPERATOR steps — outside the coding plan, run after merge)
|
||||
|
||||
These are runtime/prod steps, NOT code, listed so the operator has the full picture. Do them after the branch is merged to master and pushed:
|
||||
|
||||
1. **Deploy the code:** `git push origin master` (the Dagster K8sRunLauncher git-syncs master HEAD; a code-only change needs no image rebuild). Rollout-restart the dagster daemon if it caches code: `kubectl --context=$BIZ -n foxhunt rollout restart deploy/dagster-daemon`.
|
||||
2. **Backfill the anchor:** the re-inception is AUTOMATIC on the next `combined_book_daily` run (23:30 UTC) — the bumped version + backdated `inception_t0` cause `forward_engine` to reset the anchor to `t0=2026-07-06` and recompute the forward window with the capped construction. Do NOT run `migrate-forward-anchors` (re-inception is automatic — see `reference_fxhnt_prod_deploy_mechanics`).
|
||||
3. **Refresh the backtest ref:** the `bybit_book_precompute` asset re-persists the `positioning` / `bybit_4edge` / `bybit_4edge_levered` `backtest_summary` from the capped `_persist_bybit_book` in the same nightly run, so the gate reconciles the forward track against the capped (capturable) reference.
|
||||
4. **Verify next morning:** query the prod DB (port-forward svc/postgres) — `positioning` forward should re-show ~13 days (recomputed, capped), the 2026-07-06/07 daily returns should be materially less negative than −7.32%/−6.89%, and the gate should read WAIT "building" (not reset to 0) or its real verdict.
|
||||
|
||||
---
|
||||
|
||||
## Self-Review
|
||||
|
||||
**1. Spec coverage:** cap in weights (T1) → threaded through book (T2) → sleeve seam (T3) → enabled live (T4) → re-inception+backfill (T5) → verified (T6). The "backfill not reset" constraint is implemented via `inception_t0` backdating in T5 + operator step 2. Covered.
|
||||
|
||||
**2. Placeholder scan:** every code step shows the exact code; the only deferred item is the shock-store fixture in T2/T3 which is explicitly instructed to reuse the existing `test_capacity_haircut_*` store construction in the same file (a real, named pattern, not a placeholder). Acceptable — the implementer has a concrete reference.
|
||||
|
||||
**3. Type consistency:** `coin_gross_cap: float | None = None` is identical across `positioning_weights`, `_book_breakdown`, `positioning_returns_from_store`, `_positioning_series`, `sleeve_returns_from_store`. Constant `POSITIONING_COIN_GROSS_CAP = 0.07` in `bybit_forward_track.py`, imported in `bybit_book_persist.py`. Versions all → 3, `inception_t0.date` all → "2026-07-06". Consistent.
|
||||
|
||||
---
|
||||
|
||||
### Task 7: Thread `coin_gross_cap` into the LIVE forward-nav path (Critical fix from final review)
|
||||
|
||||
**Why:** The final whole-branch review found the cap threads to the reconciliation REF (via `_persist_bybit_book`) but NOT to the live FORWARD NAV (built by `BybitFourEdgeStrategy`), so the gate would compare a capped ref against an uncapped forward and read a healthy book as "diverging below backtest" (reproduced: forward −25% vs ref −12.5% on a LABUSDT shock). Fail-safe (WAIT, never false-PASS) but defeats the branch's purpose. This threads the cap the same way `positioning_tail_cap_k` is already threaded.
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/application/bybit_forward_track.py` (`BybitFourEdgeStrategy.__init__` ~135-157, `advance` ~168-172)
|
||||
- Modify: `src/fxhnt/adapters/orchestration/assets.py` (builders at 105-122, 128-143, 149-158, 164-169; nightly call sites 868-876, 916-924, 957-965)
|
||||
- Modify: `src/fxhnt/adapters/orchestration/migration_builders.py` (builders at 106-117, 127-141, 148-156)
|
||||
- Modify: `src/fxhnt/application/bybit_positioning_eval.py:280` (`positioning_weights_and_contributions` — the Important SSOT-attribution fix)
|
||||
- Test: `tests/integration/test_positioning_forward_track.py` (+ the attribution invariant in `tests/integration/test_bybit_paper_reconciliation.py`)
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `POSITIONING_COIN_GROSS_CAP = 0.07` (Task 4), `sleeve_returns_from_store(..., coin_gross_cap=...)` (Task 3), `positioning_weights(..., coin_gross_cap=...)` (Task 1).
|
||||
- Produces: `BybitFourEdgeStrategy(..., positioning_coin_gross_cap: float | None = None, ...)`; the four `assets.py` builders + three `migration_builders.py` builders gain a `positioning_coin_gross_cap` param forwarded to the strategy; the three nightly call sites pass `POSITIONING_COIN_GROSS_CAP`.
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
Add to `tests/integration/test_positioning_forward_track.py` — a shock store where the forward NAV's worst day is materially less negative when the cap is threaded (mirror the shock-store construction already used in the positioning tests):
|
||||
|
||||
```python
|
||||
def test_forward_nav_applies_coin_gross_cap() -> None:
|
||||
"""The forward NAV (BybitFourEdgeStrategy store-read path) must apply the per-coin cap when given, so it
|
||||
reconciles against the capped ref instead of diverging. Shock store: one over-short coin craters ~50%."""
|
||||
from fxhnt.adapters.warehouse.timescale_feature_store import TimescaleFeatureStore
|
||||
from fxhnt.application.bybit_forward_track import BybitFourEdgeStrategy
|
||||
|
||||
_DAY = 86_400
|
||||
store = TimescaleFeatureStore("sqlite://", table="bybit_features")
|
||||
px = {"LABUSDT": 100.0, "AAAUSDT": 100.0, "BBBUSDT": 100.0, "CCCUSDT": 100.0}
|
||||
for d in range(4):
|
||||
store.write_features("LABUSDT", [(d * _DAY, {"long_ratio": 0.05, "close": px["LABUSDT"]})])
|
||||
store.write_features("AAAUSDT", [(d * _DAY, {"long_ratio": 0.50, "close": px["AAAUSDT"]})])
|
||||
store.write_features("BBBUSDT", [(d * _DAY, {"long_ratio": 0.48, "close": px["BBBUSDT"]})])
|
||||
store.write_features("CCCUSDT", [(d * _DAY, {"long_ratio": 0.52, "close": px["CCCUSDT"]})])
|
||||
if d == 2:
|
||||
px["LABUSDT"] *= 0.50
|
||||
uncapped_rows, _ = BybitFourEdgeStrategy(store, sleeves=["positioning"]).advance(None, {})
|
||||
capped_rows, _ = BybitFourEdgeStrategy(
|
||||
store, sleeves=["positioning"], positioning_coin_gross_cap=0.07).advance(None, {})
|
||||
store.close()
|
||||
assert uncapped_rows and capped_rows, "forward strategy produced no rows — fixture broken"
|
||||
u = dict(uncapped_rows); c = dict(capped_rows)
|
||||
worst = min(u, key=lambda k: u[k])
|
||||
assert c[worst] > u[worst], f"cap must reduce the forward worst day: uncapped={u[worst]}, capped={c[worst]}"
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `uv run --extra dev python -m pytest tests/integration/test_positioning_forward_track.py::test_forward_nav_applies_coin_gross_cap -v`
|
||||
Expected: FAIL with `TypeError: BybitFourEdgeStrategy.__init__() got an unexpected keyword argument 'positioning_coin_gross_cap'`
|
||||
|
||||
- [ ] **Step 3: Add the param to `BybitFourEdgeStrategy` and thread it in `advance`**
|
||||
|
||||
In `__init__` (after `positioning_tail_cap_k: float | None = None,` on line 142) add:
|
||||
```python
|
||||
positioning_coin_gross_cap: float | None = None,
|
||||
```
|
||||
And in the body (after `self._positioning_tail_cap_k = positioning_tail_cap_k` line 154):
|
||||
```python
|
||||
self._positioning_coin_gross_cap = positioning_coin_gross_cap
|
||||
```
|
||||
In `advance`'s store-read `sleeve_returns_from_store(...)` call (lines 168-172), add the kwarg:
|
||||
```python
|
||||
series = sleeve_returns_from_store(
|
||||
self._store, sleeve, universe=self._universe, cost_bps=self._cost_bps,
|
||||
unlock_events=self._unlock_events, stablecoin_spot_panel=self._stablecoin_spot_panel,
|
||||
capacity_capital=self._positioning_capacity_capital,
|
||||
tail_cap_k=self._positioning_tail_cap_k,
|
||||
coin_gross_cap=self._positioning_coin_gross_cap)
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Thread through every builder (parallel to `positioning_tail_cap_k`)**
|
||||
|
||||
In `src/fxhnt/adapters/orchestration/assets.py`, for EACH of the four builders (`build_bybit_4edge_forward_nav`, `build_bybit_4edge_levered_forward_nav`, and the two single-sleeve builders at 149 and 164): add a `positioning_coin_gross_cap=None` param to the signature (next to `positioning_tail_cap_k=None`) and forward `positioning_coin_gross_cap=positioning_coin_gross_cap` into the `BybitFourEdgeStrategy(...)` call. For the THREE nightly call sites (868-876, 916-924, 957-965) that already pass `positioning_tail_cap_k=POSITIONING_TAIL_CAP_K`: add `positioning_coin_gross_cap=POSITIONING_COIN_GROSS_CAP` (import it alongside `POSITIONING_TAIL_CAP_K`).
|
||||
|
||||
Do the identical change in `src/fxhnt/adapters/orchestration/migration_builders.py` for its three builders that pass `positioning_tail_cap_k=POSITIONING_TAIL_CAP_K` (106-117, 127-141, 148-156): add `positioning_coin_gross_cap=POSITIONING_COIN_GROSS_CAP` to each `BybitFourEdgeStrategy(...)` call, importing the constant.
|
||||
|
||||
- [ ] **Step 5: Fix the attribution SSOT (the Important finding)**
|
||||
|
||||
In `src/fxhnt/application/bybit_positioning_eval.py`, `positioning_weights_and_contributions` (line 263): add `coin_gross_cap: float | None = None` to its signature and forward it in the `positioning_weights(...)` call at line 280:
|
||||
```python
|
||||
weights_by_day = positioning_weights(_ratio_by_day(long_ratio), direction=direction,
|
||||
coin_gross_cap=coin_gross_cap)
|
||||
```
|
||||
Then thread it through `sleeve_weights_and_contributions` in `src/fxhnt/application/bybit_paper_weights.py` (the caller at ~80-84) so the paper-book per-symbol weights match the capped sleeve return (preserving the `Σ weight·return == sleeve_returns_from_store` invariant documented at `bybit_paper_weights.py:9`). Default None everywhere.
|
||||
|
||||
- [ ] **Step 6: Run the tests + verify no regression**
|
||||
|
||||
Run: `uv run --extra dev python -m pytest tests/integration/test_positioning_forward_track.py tests/integration/test_bybit_paper_reconciliation.py tests/integration/test_bybit_positioning_eval.py -v`
|
||||
Expected: all pass (new forward-nav cap test passes; existing reconciliation invariant test still passes with default-None on both sides).
|
||||
|
||||
- [ ] **Step 7: Commit**
|
||||
|
||||
```bash
|
||||
git add -A
|
||||
git commit -m "fix(positioning): thread coin_gross_cap into the LIVE forward-nav path + attribution (final-review Critical)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 8: Cap the LIVE paper-book POSITIONS + the report-ref path (Critical + Important from Task-7 review)
|
||||
|
||||
**Why:** The Task-7 review traced two more uncapped paths:
|
||||
- **Critical:** the live paper-book position sizing (`combined_symbol_weights*` / `latest_raw_sleeve_weights` → `derive_and_persist_bybit_paper_book` → the nightly `bybit_paper_book` asset) never passes the positioning haircuts, so the fund's ACTUAL persisted positions for the positioning sleeve are sized uncapped even though the reconciled return is capped. It turns out capacity + tail_cap were ALSO never threaded here (pre-existing gap). Operator decision (2026-07-20): thread ALL THREE haircuts so executed positions match the reconciled returns (full SSOT).
|
||||
- **Important:** `bybit_4edge_backtest_summary` + its caller `report_backtest_summary` pass `positioning_tail_cap_k` but no `positioning_coin_gross_cap` — a real, independently-callable uncapped ref path (masked in the scheduled flow only by the `_BYBIT_BOOK_SIDS` special-case).
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/application/bybit_paper_book.py` — `combined_symbol_weights` (~48), `combined_symbol_weights_and_returns` (~105), `latest_raw_sleeve_weights` (~80), and `derive_and_persist_bybit_paper_book` (the persist driver): add `positioning_capacity_capital`, `positioning_tail_cap_k`, `positioning_coin_gross_cap` (all `float | None = None`) and forward into every `sleeve_weights_and_contributions(...)` call.
|
||||
- Modify: `src/fxhnt/adapters/orchestration/assets.py` — `build_bybit_paper_book` (~180): add the 3 params, forward into `derive_and_persist_bybit_paper_book`; nightly call site (~1105): pass `positioning_capacity_capital=s.paper_capital, positioning_tail_cap_k=POSITIONING_TAIL_CAP_K, positioning_coin_gross_cap=POSITIONING_COIN_GROSS_CAP` (import the two constants).
|
||||
- Modify: `src/fxhnt/application/bybit_forward_track.py` — `bybit_4edge_backtest_summary` (~210): add `positioning_coin_gross_cap: float | None = None` next to `positioning_tail_cap_k`, forward into the `BybitFourEdgeStrategy(...)` call.
|
||||
- Modify: `src/fxhnt/application/backtest_refs_report.py` — `report_backtest_summary` (~42): import `POSITIONING_COIN_GROSS_CAP`, pass `positioning_coin_gross_cap=POSITIONING_COIN_GROSS_CAP` into `bybit_4edge_backtest_summary(...)`.
|
||||
- Test: `tests/integration/test_bybit_paper_reconciliation.py` (or the paper-book test file — grep for a test that calls `combined_symbol_weights_and_returns` / `derive_and_persist_bybit_paper_book`).
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `sleeve_weights_and_contributions(..., positioning_capacity_capital=None, positioning_tail_cap_k=None, positioning_coin_gross_cap=None)` (Task 7 added coin_gross_cap; VERIFY it also has capacity/tail params — if `sleeve_weights_and_contributions` does NOT yet accept capacity_capital/tail_cap_k, add them there too, forwarding into its `positioning_weights`/underlying call, same as coin_gross_cap).
|
||||
- Produces: the 3 haircut params on `combined_symbol_weights`, `combined_symbol_weights_and_returns`, `latest_raw_sleeve_weights`, `derive_and_persist_bybit_paper_book`, `build_bybit_paper_book`; `positioning_coin_gross_cap` on `bybit_4edge_backtest_summary` + `report_backtest_summary`. All default None.
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
Add a paper-book position test: on a LABUSDT-shock store, the positioning sleeve's persisted per-symbol weight for the crashing coin is smaller when `positioning_coin_gross_cap=0.07` is threaded through `combined_symbol_weights_and_returns` than without.
|
||||
|
||||
```python
|
||||
def test_paper_book_positions_capped_by_coin_gross_cap() -> None:
|
||||
"""The live paper-book per-symbol weights must honor the per-coin cap so executed positions match the
|
||||
capped reconciled return (SSOT). On a store with one dominant positioning coin, the capped weight for that
|
||||
coin is strictly smaller than uncapped."""
|
||||
from fxhnt.adapters.warehouse.timescale_feature_store import TimescaleFeatureStore
|
||||
from fxhnt.application.bybit_paper_book import combined_symbol_weights_and_returns
|
||||
_DAY = 86_400
|
||||
store = TimescaleFeatureStore("sqlite://", table="bybit_features")
|
||||
for d in range(4):
|
||||
store.write_features("LABUSDT", [(d * _DAY, {"long_ratio": 0.02, "close": 100.0})])
|
||||
for i, sym in enumerate(("AAAUSDT", "BBBUSDT", "CCCUSDT", "DDDUSDT")):
|
||||
store.write_features(sym, [(d * _DAY, {"long_ratio": 0.45 + 0.02 * i, "close": 100.0})])
|
||||
w_unc, _r, _d = combined_symbol_weights_and_returns(store, sleeves=("positioning",))
|
||||
w_cap, _r2, _d2 = combined_symbol_weights_and_returns(
|
||||
store, sleeves=("positioning",), positioning_coin_gross_cap=0.07)
|
||||
store.close()
|
||||
assert w_unc.get("positioning") and w_cap.get("positioning"), "no positioning weights — fixture broken"
|
||||
lab_unc = abs(w_unc["positioning"].get("LABUSDT", 0.0))
|
||||
lab_cap = abs(w_cap["positioning"].get("LABUSDT", 0.0))
|
||||
assert lab_unc > 0.0
|
||||
assert lab_cap < lab_unc, f"cap must shrink the dominant coin's position: unc={lab_unc}, cap={lab_cap}"
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run to verify it fails**
|
||||
|
||||
Run: `uv run --extra dev python -m pytest tests/integration/test_bybit_paper_reconciliation.py::test_paper_book_positions_capped_by_coin_gross_cap -v`
|
||||
Expected: FAIL with `TypeError: combined_symbol_weights_and_returns() got an unexpected keyword argument 'positioning_coin_gross_cap'`
|
||||
|
||||
- [ ] **Step 3: Thread the 3 haircuts through the paper-book weight functions**
|
||||
|
||||
For `combined_symbol_weights`, `combined_symbol_weights_and_returns`, and `latest_raw_sleeve_weights` in `bybit_paper_book.py`: add `positioning_capacity_capital: float | None = None`, `positioning_tail_cap_k: float | None = None`, `positioning_coin_gross_cap: float | None = None` to each signature, and forward all three into their `sleeve_weights_and_contributions(...)` call(s) (as `capacity_capital=`/`tail_cap_k=`/`positioning_coin_gross_cap=` per that function's actual param names — grep `def sleeve_weights_and_contributions` for the exact names). If `sleeve_weights_and_contributions` lacks capacity_capital/tail_cap_k params, add them there first (forwarding into its positioning path), then here.
|
||||
|
||||
- [ ] **Step 4: Thread through the persist driver + builder**
|
||||
|
||||
`derive_and_persist_bybit_paper_book` (bybit_paper_book.py): add the 3 params, forward into its `combined_symbol_weights_and_returns(...)` call. `build_bybit_paper_book` (assets.py:180): add the 3 params, forward into `derive_and_persist_bybit_paper_book(...)`. Nightly call site (assets.py:~1105): pass `positioning_capacity_capital=s.paper_capital, positioning_tail_cap_k=POSITIONING_TAIL_CAP_K, positioning_coin_gross_cap=POSITIONING_COIN_GROSS_CAP` (import both constants from `bybit_forward_track`).
|
||||
|
||||
- [ ] **Step 5: Fix the report-ref path (Important)**
|
||||
|
||||
`bybit_4edge_backtest_summary` (bybit_forward_track.py:~210): add `positioning_coin_gross_cap: float | None = None` next to `positioning_tail_cap_k`, forward into `BybitFourEdgeStrategy(..., positioning_coin_gross_cap=positioning_coin_gross_cap)`. `report_backtest_summary` (backtest_refs_report.py:~42): import `POSITIONING_COIN_GROSS_CAP`, pass `positioning_coin_gross_cap=POSITIONING_COIN_GROSS_CAP` into the `bybit_4edge_backtest_summary(...)` call.
|
||||
|
||||
- [ ] **Step 6: Run tests + no regression**
|
||||
|
||||
Run: `uv run --extra dev python -m pytest tests/integration/test_bybit_paper_reconciliation.py tests/integration/test_positioning_forward_track.py tests/unit/test_backtest_refs_report.py tests/integration/test_bybit_positioning_eval.py -v`
|
||||
Expected: all pass. Then confirm no site was missed: `grep -c positioning_tail_cap_k` vs `grep -c positioning_coin_gross_cap` should MATCH in bybit_paper_book.py, assets.py, bybit_forward_track.py, backtest_refs_report.py.
|
||||
|
||||
- [ ] **Step 7: Commit**
|
||||
|
||||
```bash
|
||||
git add -A
|
||||
git commit -m "fix(positioning): cap live paper-book positions + report-ref path (Task-7-review Critical+Important)"
|
||||
```
|
||||
474
docs/superpowers/plans/2026-07-20-vrp-code-removal.md
Normal file
474
docs/superpowers/plans/2026-07-20-vrp-code-removal.md
Normal file
@@ -0,0 +1,474 @@
|
||||
# VRP Strategy Full Removal — Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development to implement this plan task-by-task. Steps use checkbox tracking; delete leaves before callers so imports never break mid-removal.
|
||||
|
||||
**Derived from:** a discovery workflow (24 file classifications + 3 deep-traces) — `wf_ffbdc8b5-c02`. The
|
||||
tricky shared-vs-VRP calls were resolved by trace: **DVOL pipeline = delete** (only VRP reads it),
|
||||
**black_scholes.py = delete** (VRP-only), **surfer vendor = untouched** (docstring mention only),
|
||||
**xsp_option_bars TABLE/MODEL/data = KEEP** (only the VRP-only `XspOptionBarsRepo` *class* is deleted).
|
||||
|
||||
## Goal
|
||||
|
||||
Fully delete the VRP (variance-risk-premium) strategy — the shelved/FALSIFIED equity-VRP
|
||||
sleeve and its executed twin `vrp_exec` — from the fxhnt repo, leaving **zero orphaned code,
|
||||
zero dead registry keys, and zero dangling imports**. The nightly Dagster pipeline, the cockpit
|
||||
web app, live IBKR/UCITS execution, the Bybit paper book, and all four kept crypto edges must
|
||||
remain fully functional and the pytest suite must be green at the end.
|
||||
|
||||
The removal spans four coupled clusters, all of which are VRP-exclusive per the deep-traces:
|
||||
|
||||
1. **VRP application modules**: `vrp_book`, `vrp_eval`, `vrp_defined_risk_eval`,
|
||||
`vrp_exec_record`, `vrp_marking`, `vrp_pricing`, `vrp_research`.
|
||||
2. **VRP option-math + IV plumbing**: `black_scholes.py` (VRP-only), the entire DVOL pipeline
|
||||
(`deribit_dvol.py`, `dvol_ingest.py`, the `dvol` feature panel, the unused
|
||||
`VolIndexClient.dvol()` port).
|
||||
3. **VRP persistence/orchestration**: `xsp_option_bars.py` repo class, the `vrp_nav` Dagster
|
||||
asset + its ~14 private OPRA-freeze/backfill helpers, the `backfill-xsp-opra` ingest path,
|
||||
the `VrpStrategy` builder, and the `vrp`/`vrp_exec` registry entries.
|
||||
4. **Comment/registry-string scrubs** across shared cockpit infrastructure that merely *names*
|
||||
VRP as an illustrative example.
|
||||
|
||||
## Architecture
|
||||
|
||||
VRP was ARCHIVED 2026-07-15 (fails the LOO marginal-Sharpe gate; see
|
||||
`pearl_fxhnt_diversifier_hunt_2026_07_15`). The two registry entries already carry
|
||||
`archived: True`, and `vrp_nav` is already **de-wired** from `definitions.py`'s nightly asset
|
||||
graph — so no live nightly asset materializes VRP today. What remains is dead weight plus two
|
||||
active-but-purposeless CLI/CronJob ingest paths (`ingest-dvol`, `backfill-xsp-opra`) and a set
|
||||
of stale registry keys/comment references.
|
||||
|
||||
Dependency direction (leaf → root), which dictates task order so imports never break mid-removal:
|
||||
|
||||
```
|
||||
black_scholes.py ────────────► vrp_defined_risk_eval.py ──┐
|
||||
deribit_dvol.py ─► dvol_ingest.py ─► (dvol panel) ────────┤
|
||||
xsp_option_bars.py (repo class) ──► vrp_book.py ──────────┤──► cli.py commands
|
||||
vrp_pricing.py / vrp_marking.py ──► vrp_book/eval/exec ───┤ migration_builders (_build_vrp)
|
||||
└──► assets.py vrp_nav + helpers
|
||||
registry.py vrp / vrp_exec
|
||||
forward_ingest _REF_ALIAS
|
||||
sim_curves / dashboard_service /
|
||||
app.py / definitions.py (scrub)
|
||||
```
|
||||
|
||||
Rule: delete leaf modules **and their tests together** first, then the callers that referenced
|
||||
them (asset graph, registry, CLI, migration builder), then the pure comment-scrub edits last.
|
||||
|
||||
## DVOL and black_scholes — resolved decisions (per deep-traces)
|
||||
|
||||
- **DVOL pipeline → DELETE (do not keep).** Exhaustive grep confirms the only readers of the
|
||||
`dvol` feature panel (`store.read_panel(..., "dvol")`) are `vrp_eval.py` and
|
||||
`vrp_defined_risk_eval.py`. `VolIndexClient.dvol()` (`ports/market_data.py:22`) has **zero**
|
||||
callers. DVOL is not a Dagster asset. DVOL is the Deribit IV index — a data source entirely
|
||||
distinct from the OPRA `xsp_option_bars` data being kept; there is no keep-carve-out for it.
|
||||
Delete `deribit_dvol.py`, `dvol_ingest.py`, the `ingest-dvol` CLI command, the
|
||||
`VolIndexClient.dvol()` port method, and their tests. The `dvol` warehouse rows in
|
||||
`bybit_features` are inert once nothing reads them — no migration needed; they can age out.
|
||||
|
||||
- **black_scholes.py → DELETE (VRP-only, not shared math).** The only importers are
|
||||
`vrp_defined_risk_eval.py` and its own unit test `tests/unit/test_black_scholes.py`.
|
||||
`vrp_pricing.py` defines a *separate* `bs_put` (different signature) and does **not** import
|
||||
from `black_scholes.py`, so there is no hidden cross-file sharing. Delete both files.
|
||||
|
||||
- **`xsp_option_bars` TABLE + `XspOptionBarRow` model + OPRA `.dbn` → KEEP.** Only the
|
||||
`XspOptionBarsRepo` *repo class* (`adapters/persistence/xsp_option_bars.py`) is VRP-only and
|
||||
gets deleted. The ORM row model `XspOptionBarRow` in `cockpit_models.py`, the physical table,
|
||||
the 13-year OPRA data, and `ForwardNavRepo.xsp_freeze_max_date()` (raw `SELECT max(date)`)
|
||||
all stay untouched.
|
||||
|
||||
## Global Constraints
|
||||
|
||||
1. **KEEP the OPRA data.** Do **not** drop or migrate the `xsp_option_bars` table, do **not**
|
||||
remove `XspOptionBarRow` from `cockpit_models.py`, do **not** delete OPRA `.dbn` files, and
|
||||
do **not** touch `ForwardNavRepo.xsp_freeze_max_date()`. Only the VRP *ingest/consumer*
|
||||
machinery (`XspOptionBarsRepo` class, `_backfill_month`/`_freeze_*` helpers, `backfill-xsp-opra`)
|
||||
is removed. The freeze-staleness health axis (`_FREEZE_SOURCES = {"opra-pit"}`) stays generic.
|
||||
2. **Real-money paths must err false-WAIT, never false-EXECUTE.** VRP execution
|
||||
(`execute-vrp`/`vrp_exec_record`) is being deleted, not disabled — but during the removal no
|
||||
edit may cause any *surviving* real-money path (multistrat/UCITS IBKR, Bybit paper) to
|
||||
silently promote/execute on missing state. After removing the `vrp`/`vrp_exec` registry keys,
|
||||
confirm the cockpit gate and `forward_ingest` promotion loop still default to WAIT for every
|
||||
surviving strategy and never KeyError into a false-PASS.
|
||||
3. **Full pytest suite green before finishing.** Run `uv run pytest -n auto` (or
|
||||
`python -m pytest -n auto` per `pyproject.toml addopts = "-n auto"`) from the repo root and
|
||||
confirm 0 failures / 0 errors before the final commit. Per-task commands below are the fast
|
||||
local gate; the full-suite run in the final task is the release gate.
|
||||
4. **The false VRP STALE health alarm must be gone.** After removal, verify **no `vrp` key
|
||||
remains in `STRATEGY_REGISTRY`** (`vrp` and `vrp_exec` both removed) so the forward-health
|
||||
freeze/staleness axis and `dashboard_service._active_ibkr_account_sids()` can no longer
|
||||
surface a ghost VRP STALE row. Verification command in the final task.
|
||||
|
||||
### Pre-flight note (already-broken test)
|
||||
|
||||
`tests/unit/test_vrp_rebalancer_yaml.py` asserts the existence of
|
||||
`infra/k8s/jobs/fxhnt-vrp-rebalancer.yaml`, which **no longer exists** in the tree. This test is
|
||||
already failing/erroring at HEAD; it is deleted in Task 1 alongside the other VRP tests, which
|
||||
also removes that pre-existing red.
|
||||
|
||||
---
|
||||
|
||||
## Task 1 — Delete VRP application modules + their tests (leaves first)
|
||||
|
||||
**Files to delete (whole file):**
|
||||
- `src/fxhnt/application/vrp_book.py`
|
||||
- `src/fxhnt/application/vrp_eval.py`
|
||||
- `src/fxhnt/application/vrp_defined_risk_eval.py`
|
||||
- `src/fxhnt/application/vrp_exec_record.py`
|
||||
- `src/fxhnt/application/vrp_marking.py`
|
||||
- `src/fxhnt/application/vrp_pricing.py`
|
||||
- `src/fxhnt/application/vrp_research.py`
|
||||
- `src/fxhnt/application/black_scholes.py`
|
||||
- `tests/unit/test_vrp_book.py`
|
||||
- `tests/unit/test_vrp_pricing.py`
|
||||
- `tests/unit/test_vrp_exec_helpers.py`
|
||||
- `tests/unit/test_vrp_exec_ladder_cap.py`
|
||||
- `tests/unit/test_vrp_exec_record.py`
|
||||
- `tests/unit/test_vrp_rebalancer_yaml.py` (also clears the pre-existing red — see pre-flight note)
|
||||
- `tests/unit/test_registry_vrp.py`
|
||||
- `tests/unit/test_black_scholes.py`
|
||||
- `tests/integration/test_vrp_eval.py`
|
||||
- `tests/integration/test_vrp_defined_risk_eval.py`
|
||||
- `tests/integration/test_vrp_nav_asset.py`
|
||||
- `tests/integration/test_cockpit_vrp_render.py`
|
||||
|
||||
**Rationale:** every one of these modules is imported only by other `vrp_*` modules or by the
|
||||
`cli.py`/`assets.py`/`migration_builders.py` callers handled in later tasks. Deleting the tests
|
||||
in the *same* task prevents collection-time `ImportError` on the deleted modules. Do NOT yet
|
||||
touch `assets.py`, `cli.py`, `registry.py`, or `migration_builders.py` — those still reference
|
||||
these modules only via **lazy (in-function) imports**, so the module tree still imports cleanly
|
||||
after this task (the broken imports are inside function bodies not executed at import time). They
|
||||
are removed in Tasks 3–5.
|
||||
|
||||
**Covering test command** (must import-clean and not collect the deleted files):
|
||||
```
|
||||
uv run pytest tests/unit tests/integration -q --co 2>&1 | tail -20
|
||||
```
|
||||
Expect: collection succeeds with the deleted VRP tests absent and no `ImportError`.
|
||||
|
||||
**Commit:** `refactor(vrp): delete VRP application modules + black_scholes + their tests`
|
||||
|
||||
---
|
||||
|
||||
## Task 2 — Delete the DVOL pipeline + its tests
|
||||
|
||||
**Files to delete (whole file):**
|
||||
- `src/fxhnt/adapters/data/deribit_dvol.py`
|
||||
- `src/fxhnt/application/dvol_ingest.py`
|
||||
- `tests/integration/test_deribit_dvol.py`
|
||||
- `tests/integration/test_dvol_ingest.py`
|
||||
|
||||
**Edit `src/fxhnt/ports/market_data.py`:**
|
||||
- Remove the `VolIndexClient` Protocol (the `dvol()` method block, ~lines 21–24). Confirmed
|
||||
zero implementers and zero callers repo-wide; it is dead scaffolding.
|
||||
|
||||
**Do NOT touch:** `deribit_funding.py` and the `ingest-deribit-funding` CronJob/command —
|
||||
that is the *funding* (cross-venue carry) edge, a separate kept data source that reads the
|
||||
`funding`/`deribit_funding` features, never `dvol`.
|
||||
|
||||
**Rationale:** `ingest_dvol` and `DeribitDVOL` are consumed only by the `ingest-dvol` CLI
|
||||
command (removed in Task 3) which fed only the two VRP evaluators (deleted in Task 1). The
|
||||
`dvol` feature panel now has no readers.
|
||||
|
||||
**Covering test command:**
|
||||
```
|
||||
uv run pytest tests/integration -q --co 2>&1 | tail -20
|
||||
grep -rn "VolIndexClient\|import.*deribit_dvol\|import.*dvol_ingest" src/ | grep -v "/cli.py"
|
||||
```
|
||||
Expect: no collection errors; the grep returns only the (still-present, removed next task)
|
||||
`cli.py` lazy imports.
|
||||
|
||||
**Commit:** `refactor(vrp): delete DVOL pipeline (deribit_dvol, dvol_ingest, VolIndexClient port) + tests`
|
||||
|
||||
---
|
||||
|
||||
## Task 3 — Delete VRP + DVOL commands from `cli.py`
|
||||
|
||||
**File to edit:** `src/fxhnt/cli.py`
|
||||
|
||||
**Exact edits (delete each command block in full, including its in-function lazy imports):**
|
||||
- Delete `@app.command("execute-vrp")` `execute_vrp` (lines ~439–499), including the lazy
|
||||
`from fxhnt.application.vrp_exec_record import plan_and_record_vrp`. **Keep** the module-level
|
||||
and other commands' use of `ForwardNavRepo` — only remove this command's local import line if
|
||||
it has one; the `ForwardNavRepo` module itself is reused by `execute-multistrat` and others.
|
||||
- Delete `@app.command("ingest-dvol")` `ingest_dvol_cmd` (lines ~1217–1266), including the lazy
|
||||
`from fxhnt.adapters.data.deribit_dvol import DeribitDVOL` and
|
||||
`from fxhnt.application.dvol_ingest import ingest_dvol`.
|
||||
- Delete `@app.command("vrp-eval")` `vrp_eval` (lines ~2193–2288), including the lazy
|
||||
`from fxhnt.application.vrp_eval import (verify_vrp_edge, vrp_metrics_from_store, walk_forward_vrp)`.
|
||||
- Delete `@app.command("vrp-defined-risk-eval")` `vrp_defined_risk_eval` (lines ~2291–2403),
|
||||
including the lazy `from fxhnt.application.vrp_defined_risk_eval import (...)`.
|
||||
- Delete the private helpers `_print_vrp_verify` (~2406–2446) and `_print_vrp_walk_forward`
|
||||
(~2449–2469) — called only by the two eval commands above.
|
||||
|
||||
**Decision — `backfill-xsp-opra` (lines ~578–623): DELETE this command too.** It calls
|
||||
`_backfill_month` and `XspOptionBarsRepo`, both of which are deleted in Tasks 4/5, and its sole
|
||||
purpose is freezing OPRA marks into `xsp_option_bars` **for VRP recompute**. Keeping the OPRA
|
||||
*data* (Constraint 1) does not require keeping the *ingest path* — the data already exists and
|
||||
the only consumer of freshly-frozen marks was VRP. Delete the `@app.command("backfill-xsp-opra")`
|
||||
`backfill_xsp_opra` function and its `from fxhnt.adapters.orchestration.assets import _backfill_month`
|
||||
+ `from fxhnt.adapters.persistence.xsp_option_bars import XspOptionBarsRepo` local imports. (If
|
||||
the executor finds a documented non-VRP consumer of freshly-frozen OPRA marks, stop and flag —
|
||||
none was found in the trace.)
|
||||
|
||||
**Covering test command:**
|
||||
```
|
||||
uv run python -c "import fxhnt.cli" # module must import clean
|
||||
uv run fxhnt --help >/dev/null # Typer app builds with commands gone
|
||||
grep -n "vrp\|dvol\|backfill-xsp\|VrpStrategy\|black_scholes\|XspOptionBarsRepo" src/fxhnt/cli.py
|
||||
```
|
||||
Expect: import + `--help` succeed; grep returns no hits.
|
||||
|
||||
**Commit:** `refactor(vrp): remove execute-vrp/ingest-dvol/vrp-eval/vrp-defined-risk-eval/backfill-xsp-opra CLI commands`
|
||||
|
||||
---
|
||||
|
||||
## Task 4 — Remove the `vrp_nav` asset + OPRA-freeze helpers from `assets.py`
|
||||
|
||||
**File to edit:** `src/fxhnt/adapters/orchestration/assets.py`
|
||||
|
||||
**Exact edits (delete code):**
|
||||
- Delete the `@asset`-decorated `vrp_nav` function (lines ~686–718), including its inline
|
||||
imports of `DatabentoDataProvider`, `XspOptionBarsRepo`, `VrpStrategy` (~line 694).
|
||||
- Delete the exclusive private helper chain (lines ~492–683), all VRP-only and now callerless
|
||||
after Tasks 1/3: `_iso_date`, `_forward_from_marks` (+ its inline
|
||||
`fxhnt.application.vrp_pricing.parity_forward` import ~502), `_band_contracts`,
|
||||
`_band_rows_from_marks`, `_day_marks`, `_xsp_forward_estimate`, `_freeze_xsp_slice`,
|
||||
`_freeze_day`, `_prev_trading_day`, `_DEGENERATE_FORWARD_MARKERS`,
|
||||
`_is_degenerate_forward_error`, `_freeze_latest_session` (+ its inline
|
||||
`DatabentoRangeUnavailable` import ~614), `_rough_forward`, `_backfill_month`.
|
||||
- Delete the trailing `# vrp_nav ARCHIVED 2026-07-15 — de-wired (fn kept below)` comment
|
||||
(~line 1131) — the function no longer exists below.
|
||||
- Reword the module docstring (~line 4): drop the trailing `/vrp` from the forward-track asset
|
||||
enumeration.
|
||||
|
||||
**Do NOT touch:** `deribit_funding_bars` (~825–856, unrelated Deribit perp *funding*, kept for
|
||||
cross-venue-carry), and all other assets (`futures_bars`, `bybit_*`, `multistrat_*`,
|
||||
`sixtyforty_nav`, `positioning_nav`, `xsfunding_nav`, `unlock_nav`, `cockpit_forward`, …). The
|
||||
RETIRED-track comments at ~721 and ~748 do not mention vrp — leave as-is.
|
||||
|
||||
**Covering test command:**
|
||||
```
|
||||
uv run python -c "import fxhnt.adapters.orchestration.assets"
|
||||
uv run python -c "import fxhnt.adapters.orchestration.definitions" # Dagster code-location loads
|
||||
grep -n "vrp\|Vrp\|_backfill_month\|_freeze_\|XspOptionBarsRepo\|vrp_pricing" src/fxhnt/adapters/orchestration/assets.py
|
||||
uv run pytest tests/integration/test_orchestration_definitions.py -q
|
||||
```
|
||||
Expect: both modules import clean, grep returns no hits, definitions test passes.
|
||||
|
||||
**Commit:** `refactor(vrp): delete vrp_nav Dagster asset + OPRA-freeze/backfill helper chain`
|
||||
|
||||
---
|
||||
|
||||
## Task 5 — Remove `XspOptionBarsRepo` class + the `_build_vrp` migration builder
|
||||
|
||||
**File to delete (whole file):**
|
||||
- `src/fxhnt/adapters/persistence/xsp_option_bars.py` (the `XspOptionBarsRepo` repo class).
|
||||
All callers (`vrp_book.py`, `vrp_exec_record.py`, `vrp_marking.py`, the `vrp_nav` asset, the
|
||||
`backfill-xsp-opra` command) are gone after Tasks 1/3/4.
|
||||
- **VERIFIED 2026-07-20: `tests/integration/test_xsp_option_bars_repo.py` does NOT exist in the tree** —
|
||||
do not attempt to delete it. If a repo-class test surfaces under another name during removal, delete it;
|
||||
otherwise there is no test file for this class.
|
||||
|
||||
**KEEP (Constraint 1):** `cockpit_models.XspOptionBarRow`, the `xsp_option_bars` table, and
|
||||
`ForwardNavRepo.xsp_freeze_max_date()` — none of these import or depend on the repo class.
|
||||
|
||||
**File to edit:** `src/fxhnt/adapters/orchestration/migration_builders.py`
|
||||
- Delete the `_build_vrp` closure (lines ~71–76) and its `builders["vrp"] = ("vrp_state",
|
||||
_build_vrp)` registration (~78), plus the preceding `# ---- vrp_nav: ...` comment banner (~71).
|
||||
|
||||
**Covering test command:**
|
||||
```
|
||||
uv run python -c "import fxhnt.adapters.orchestration.migration_builders"
|
||||
grep -rn "XspOptionBarsRepo\|_build_vrp\|builders\[.vrp" src/
|
||||
uv run pytest tests/ -q -k "migration or migrate" 2>&1 | tail -20
|
||||
```
|
||||
Expect: import clean, grep returns no hits (the `XspOptionBarRow` model in `cockpit_models.py`
|
||||
is a different symbol and must still be present — verify separately:
|
||||
`grep -n "class XspOptionBarRow" src/fxhnt/adapters/persistence/cockpit_models.py` returns 1 hit).
|
||||
|
||||
**Commit:** `refactor(vrp): delete XspOptionBarsRepo class + _build_vrp migration builder (keep table + XspOptionBarRow)`
|
||||
|
||||
---
|
||||
|
||||
## Task 6 — Remove `vrp`/`vrp_exec` registry keys + `forward_ingest` alias
|
||||
|
||||
**File to edit:** `src/fxhnt/registry.py`
|
||||
- Delete the `"vrp"` `STRATEGY_REGISTRY` entry (lines ~161–172) and its preceding comment
|
||||
block (~157).
|
||||
- Delete the `"vrp_exec"` `STRATEGY_REGISTRY` entry (lines ~177–186) and its preceding comment
|
||||
block (~173–176).
|
||||
- Keep the dict itself and all 9 non-VRP entries and the `FUND_INSTRUMENTS` import/re-export.
|
||||
|
||||
**File to edit:** `src/fxhnt/application/forward_ingest.py`
|
||||
- Remove the `"vrp_exec": "vrp"` pair from `_REF_ALIAS` (line ~45), leaving
|
||||
`{"multistrat_exec": "multistrat", "bybit_4edge_exec": "bybit_4edge"}`.
|
||||
|
||||
**Rationale + Constraint 4:** removing the `vrp` key is what actually kills the false VRP STALE
|
||||
health alarm — with no `vrp` in `STRATEGY_REGISTRY`, the forward-health freeze/staleness loop
|
||||
and `dashboard_service._active_ibkr_account_sids()` can no longer iterate a ghost VRP row.
|
||||
(Note: `dashboard_service._IBKR_ACCOUNT_SIDS` hard-codes `"vrp"`; that literal is fixed in
|
||||
Task 7 in the *same PR* so the window where the key is gone but the literal remains never ships.)
|
||||
|
||||
**Covering test command:**
|
||||
```
|
||||
uv run python -c "from fxhnt.registry import STRATEGY_REGISTRY; assert 'vrp' not in STRATEGY_REGISTRY and 'vrp_exec' not in STRATEGY_REGISTRY, STRATEGY_REGISTRY.keys(); print('registry clean')"
|
||||
uv run pytest tests/unit/test_forward_definition.py tests/integration/test_forward_health.py tests/integration/test_dashboard_service.py -q 2>&1 | tail -20
|
||||
```
|
||||
Expect: assertion passes; forward/dashboard tests green (these confirm surviving strategies
|
||||
still WAIT-by-default — Constraint 2).
|
||||
|
||||
**Commit:** `refactor(vrp): remove vrp/vrp_exec registry entries + forward_ingest _REF_ALIAS`
|
||||
|
||||
---
|
||||
|
||||
## Task 7 — Fix the one behavioral literal + `sim_curves` book pill (data edits)
|
||||
|
||||
These are the two non-comment edits that gate on `vrp` being a *present* registry key and would
|
||||
misbehave once it is gone (Task 6). They ship together with — or immediately after — Task 6.
|
||||
|
||||
**File to edit:** `src/fxhnt/application/dashboard_service.py`
|
||||
- Line ~30: change `_IBKR_ACCOUNT_SIDS = frozenset({"multistrat", "vrp"})` to
|
||||
`frozenset({"multistrat"})`. **Why this is behavioral, not cosmetic:** `_active_ibkr_account_sids()`
|
||||
computes `not STRATEGY_REGISTRY.get("vrp", {}).get("archived")`; once the `vrp` key is gone
|
||||
this becomes `not None` = `True`, wrongly treating `vrp` as an ACTIVE ibkr-account sid and
|
||||
rendering a ghost/broken VRP row on the IBKR account page. Removing the literal eliminates it.
|
||||
|
||||
**File to edit:** `src/fxhnt/application/sim_curves.py`
|
||||
- Line ~15: remove the `"vrp"` string from the `SIM_BOOKS` tuple, leaving
|
||||
`("bybit_4edge", "bybit_4edge_levered", "multistrat", "multistrat_levered")`. **Why:** once
|
||||
the `vrp` registry entry is gone, a `vrp` Backtest-page pill would `KeyError`/500 at request
|
||||
time in `app.py` (`registry_backtest_kind("vrp")`). The dispatch logic in `sim_returns_for`
|
||||
is generic and unchanged.
|
||||
|
||||
**Covering test command:**
|
||||
```
|
||||
uv run pytest tests/unit/test_sim_curves.py tests/integration/test_dashboard_service.py tests/integration/test_ibkr_account_detail_web.py tests/integration/test_dashboard_ibkr_holdings.py tests/integration/test_paper_books_web.py tests/integration/test_ibkr_sim_web.py -q 2>&1 | tail -20
|
||||
```
|
||||
Expect: green; no VRP row/pill; surviving books render.
|
||||
|
||||
**Commit:** `fix(vrp): drop vrp from _IBKR_ACCOUNT_SIDS and SIM_BOOKS so ghost VRP row/pill can't render`
|
||||
|
||||
---
|
||||
|
||||
## Task 8 — Delete the now-purposeless VRP/OPRA CronJob manifest
|
||||
|
||||
**File to delete:**
|
||||
- `infra/k8s/jobs/fxhnt-opra-backfill.yaml` — invokes `fxhnt backfill-xsp-opra` (deleted in
|
||||
Task 3). With the command gone this CronJob would crash-loop; its only job was freezing OPRA
|
||||
marks for VRP recompute. (The `fxhnt-vrp-rebalancer.yaml` manifest is already absent from the
|
||||
tree — nothing to delete there; its dangling test is removed in Task 1.)
|
||||
|
||||
**Do NOT touch:** `fxhnt-deribit-carry-job.yaml` (funding/carry, not DVOL),
|
||||
`fxhnt-bybit-*`, `fxhnt-ucits-rebalancer.yaml`, `fxhnt-ibkr-rebalancer.yaml`, and all other
|
||||
manifests.
|
||||
|
||||
**Covering check:**
|
||||
```
|
||||
grep -rln "vrp\|ingest-dvol\|execute-vrp\|backfill-xsp-opra" infra/ # expect: no hits
|
||||
```
|
||||
|
||||
**Commit:** `chore(vrp): remove fxhnt-opra-backfill CronJob (backfill-xsp-opra command deleted)`
|
||||
|
||||
---
|
||||
|
||||
## Task 9 — Comment/docstring scrubs (no logic change) — LAST
|
||||
|
||||
Pure prose edits that dangling-reference the deleted `vrp` strategy id. No imports, logic,
|
||||
registry entries, or types change. Batched last because none affect importability.
|
||||
|
||||
**Files + edits (reword to drop the `vrp` example, keep meaning):**
|
||||
- `src/fxhnt/adapters/orchestration/definitions.py` — lines ~10–12: drop the
|
||||
"vrp_nav ARCHIVED … KEPT in assets.py but DE-WIRED" paragraph (the "KEPT" claim is now false);
|
||||
replace with a one-line "VRP removed entirely 2026-07-20" note or delete outright.
|
||||
- `src/fxhnt/adapters/persistence/cockpit_models.py` — line ~46 (drop the `(… that would have
|
||||
caught vrp)` parenthetical or generalize to "a dead strategy"); line ~228 (drop `/vrp` from
|
||||
`(multistrat/vrp …)`).
|
||||
- `src/fxhnt/adapters/persistence/forward_nav.py` — lines ~146–147 (generalize "the vrp freeze
|
||||
table" / "a vrp recompute is bounded by this max date" to opra-pit-anchored/strategy-generic
|
||||
wording; the `xsp_freeze_max_date` query stays); line ~199 (drop the "shelved/FALSIFIED
|
||||
vrp/vrp_exec" example from the `archived: True` registry note).
|
||||
- `src/fxhnt/adapters/persistence/paper_repo.py` — line ~519 (drop `vrp` from the
|
||||
`(multistrat/vrp …)` recompute-replay parenthetical).
|
||||
- `src/fxhnt/adapters/web/app.py` — lines ~449, 453, 473, 482, 551, 567, 728, 752, 810: drop the
|
||||
`/vrp` example from each `multistrat/vrp` phrase. No code change (the archived-filter
|
||||
comprehension already yields one fewer element once `vrp` left `SIM_BOOKS`/registry).
|
||||
- `src/fxhnt/adapters/web/templates/_macros.html` — line ~61 ("2-week vrp rot"): generalize.
|
||||
- `src/fxhnt/adapters/web/templates/cockpit.html` — line ~78: drop `/vrp` from
|
||||
"two IBKR real-account books (multistrat/vrp)".
|
||||
- `src/fxhnt/application/dashboard_service.py` — comment-only lines ~27, 55, 89, 390, 416, 417:
|
||||
drop/generalize the `vrp` examples (the code literal was already fixed in Task 7).
|
||||
- `src/fxhnt/application/sim_curves.py` — lines ~2, 12, 22: drop `vrp` from the
|
||||
`multistrat/vrp` recompute-replay examples (the tuple was already fixed in Task 7).
|
||||
- `src/fxhnt/application/forward_health.py` — docstring lines ~3, 11, 13, 14, 15, 24, 31 and the
|
||||
inline comment at ~147 ("for the vrp freeze axis"): generalize away from naming `vrp` as the
|
||||
mechanism's reason for existing; the `_FREEZE_SOURCES = {"opra-pit"}` logic and
|
||||
`xsp_freeze_max_date()` call are unchanged.
|
||||
- `src/fxhnt/application/display_names.py` — lines ~2 and ~51: drop the `vrp` example id /
|
||||
option-contract example from the illustrative prose.
|
||||
- `src/fxhnt/application/em_asia_marginal_eval.py` — lines ~6, 19, 71: replace "VRP" in the
|
||||
Stage-1 kill-gate precedent comments with a VRP-agnostic phrase ("prior failed diversifier
|
||||
candidates"). No logic change.
|
||||
- `src/fxhnt/config.py` — line ~215: drop `(… that would have caught vrp)` from the
|
||||
`stale_after_business_days` comment; keep the setting + default.
|
||||
- `src/fxhnt/ports/dashboard.py` — lines ~72 and ~190: drop the `vrp` example from the
|
||||
`multistrat/vrp` IBKR-real-account docstrings. (Line ~34 is unrelated carry — leave.)
|
||||
- `src/fxhnt/vendor/surfer/__init__.py` — lines ~11–13: compress the `surfer_poc`/
|
||||
`MomentumVrpStrategy` retirement note to a plain historical line without the VRP signpost.
|
||||
|
||||
**Do NOT edit** as part of scrubbing (already covered / must not change): the `xsp_option_bars`
|
||||
query, `_FREEZE_SOURCES`, `XspOptionBarRow`, any settings field, or any Protocol field.
|
||||
|
||||
**Covering test command** (broad — scrubs must not have touched logic):
|
||||
```
|
||||
uv run python -c "import fxhnt.adapters.web.app, fxhnt.application.dashboard_service, fxhnt.application.forward_health, fxhnt.config, fxhnt.ports.dashboard, fxhnt.application.display_names, fxhnt.application.em_asia_marginal_eval, fxhnt.application.sim_curves"
|
||||
uv run pytest tests/unit/test_display_names.py tests/integration/test_dashboard_service.py -q 2>&1 | tail -20
|
||||
```
|
||||
Expect: all import clean; tests green.
|
||||
|
||||
**Commit:** `docs(vrp): scrub stale VRP comment/docstring references across cockpit + shared infra`
|
||||
|
||||
---
|
||||
|
||||
## Task 10 — Full-suite green gate + removal verification (release gate)
|
||||
|
||||
**No file edits.** Final verification against all Global Constraints.
|
||||
|
||||
**Commands (all must pass):**
|
||||
```
|
||||
# (3) full suite green
|
||||
uv run pytest -n auto 2>&1 | tail -30 # 0 failed, 0 errors
|
||||
|
||||
# (4) no vrp in registry — false STALE alarm gone
|
||||
uv run python -c "from fxhnt.registry import STRATEGY_REGISTRY; assert not any(k.startswith('vrp') for k in STRATEGY_REGISTRY), list(STRATEGY_REGISTRY); print('no vrp in registry')"
|
||||
|
||||
# no orphaned VRP code / imports anywhere in src
|
||||
grep -rn "import.*vrp_\|VrpStrategy\|deribit_dvol\|dvol_ingest\|black_scholes\|XspOptionBarsRepo\|VolIndexClient\|_backfill_month\|plan_and_record_vrp" src/ # expect: no hits
|
||||
ls src/fxhnt/application/vrp_*.py 2>&1 # expect: No such file
|
||||
ls src/fxhnt/application/black_scholes.py src/fxhnt/application/dvol_ingest.py src/fxhnt/adapters/data/deribit_dvol.py src/fxhnt/adapters/persistence/xsp_option_bars.py 2>&1 # expect: No such file (all four)
|
||||
|
||||
# (1) OPRA data kept — table model + freeze query still present
|
||||
grep -n "class XspOptionBarRow" src/fxhnt/adapters/persistence/cockpit_models.py # expect: 1 hit
|
||||
grep -n "def xsp_freeze_max_date" src/fxhnt/adapters/persistence/forward_nav.py # expect: 1 hit
|
||||
|
||||
# (2) Dagster code-location + cockpit app both import clean (no false-execute / no outage)
|
||||
uv run python -c "import fxhnt.adapters.orchestration.definitions; import fxhnt.adapters.web.app; print('definitions + app import clean')"
|
||||
|
||||
# no residual vrp mentions in infra
|
||||
grep -rln "vrp\|ingest-dvol\|execute-vrp\|backfill-xsp-opra" infra/ # expect: no hits
|
||||
```
|
||||
|
||||
**Constraint 2 spot-check:** confirm `forward_ingest`'s promotion loop and the cockpit gate
|
||||
default to WAIT for every surviving strategy (the recon-gate never false-PASSes) — covered by
|
||||
`test_forward_health.py` / `test_dashboard_service.py` in the full run; if either regresses, the
|
||||
removal touched a shared gate path and must be reverted to the last green task.
|
||||
|
||||
**Commit (only after everything above is green):** `chore(vrp): verify full VRP removal — suite green, registry clean, OPRA data intact`
|
||||
|
||||
---
|
||||
|
||||
## Post-removal note
|
||||
|
||||
After merge, update `docs/superpowers/VRP-REMOVAL-TODO.md` and project memory
|
||||
(`pearl_fxhnt_diversifier_hunt_2026_07_15.md` already says "VRP now fully DELETED (shelved)") to
|
||||
reflect that the code deletion — not just shelving — is complete, and that the OPRA
|
||||
`xsp_option_bars` data was retained per Constraint 1.
|
||||
269
docs/superpowers/plans/2026-07-21-fxhnt-cicd-pipeline.md
Normal file
269
docs/superpowers/plans/2026-07-21-fxhnt-cicd-pipeline.md
Normal file
@@ -0,0 +1,269 @@
|
||||
# fxhnt CI/CD Pipeline — Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development to implement this
|
||||
> plan task-by-task. Steps use checkbox tracking. This plan writes/edits YAML manifests and applies them to a
|
||||
> LIVE cluster — every apply is gated behind a dry-run + explicit verify; a broken sensor/workflow must never
|
||||
> silently swallow a push or false-deploy.
|
||||
|
||||
**Goal:** A push to `master` on `gitadmin/fxhnt` auto-builds (only when deps/Dockerfile change) and auto-deploys
|
||||
the fxhnt cockpit+dagster daemons — no manual `kubectl rollout restart`.
|
||||
|
||||
**Architecture:** git-sync-first, image-on-demand. A minimal Sensor in `argo-events` hears the shared Gitea
|
||||
webhook, path-filters the push to pick `mode=build` (Kaniko rebuild) vs `mode=deploy` (rollout only), and submits
|
||||
the fxhnt-owned `fxhnt-cockpit` WorkflowTemplate into `foxhunt`.
|
||||
|
||||
**Tech Stack:** Argo Events v1.9.10 (Sensor), Argo Workflows (Kaniko build + kubectl deploy), Gitea webhook,
|
||||
Scaleway registry, K8s (bizworx Kapsule).
|
||||
|
||||
## Global Constraints
|
||||
|
||||
- **Project boundary:** fxhnt-owned objects (WorkflowTemplate, build/deploy pods, secrets, RBAC) live in
|
||||
`foxhunt`. ONLY the Sensor lives in `argo-events` (unavoidable). No fxhnt secrets in `argo`/`argo-events`; no
|
||||
cross-namespace RBAC.
|
||||
- **Node pool:** all build/deploy pods target `nodeSelector: k8s.scaleway.com/pool-name: large` (the
|
||||
`ci-compile-cpu` pool is DECOMMISSIONED — only `large`/`sfs` exist). A pod pinned to `ci-compile-cpu` never
|
||||
schedules.
|
||||
- **Registry:** `rg.fr-par.scw.cloud/bizworx/fxhnt-cockpit`, tagged `:<sha7>` AND `:latest`. Secret
|
||||
`scw-registry` (already in `foxhunt`).
|
||||
- **Real-money safety:** the pipeline only builds/deploys the cockpit+dagster DAEMONS. It must NEVER touch the
|
||||
execution CronJobs (ucits/bybit/ibkr rebalancers) or arm any executor.
|
||||
- **No silent failures:** a Sensor that can't match, or a Workflow that errors, must surface loudly (a failed
|
||||
push = a visible failed Workflow, never a swallowed no-op). Every manifest is `kubectl apply --dry-run=server`
|
||||
validated before a real apply.
|
||||
- **Reproducibility:** the deploy step annotates deployed Deployments with `fxhnt.io/deployed-sha=<sha7>`.
|
||||
|
||||
## File Structure
|
||||
|
||||
- `infra/argo/fxhnt-cockpit-workflowtemplate.yaml` — the reanimated WorkflowTemplate (ns foxhunt). Supersedes
|
||||
the old `infra/argo/cockpit-build-deploy.yaml` (which is renamed/removed in the final task).
|
||||
- `infra/argo/fxhnt-build-sensor.yaml` — the new Sensor (ns argo-events).
|
||||
- Gitea webhook on `gitadmin/fxhnt` — created via Gitea API/UI (not a repo file; documented in the plan).
|
||||
|
||||
---
|
||||
|
||||
## Task 1 — Reanimate + fix the WorkflowTemplate (build side)
|
||||
|
||||
**Files:**
|
||||
- Create: `infra/argo/fxhnt-cockpit-workflowtemplate.yaml` (start from `infra/argo/cockpit-build-deploy.yaml`)
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: a `WorkflowTemplate` named `fxhnt-cockpit` in ns `foxhunt` with params `commit-sha` (default HEAD)
|
||||
and `mode` (default `build`; values `build`|`deploy`). The Sensor (Task 3) submits Workflows referencing it.
|
||||
|
||||
- [ ] **Step 1: Copy the old template as the new file, fix the node pool**
|
||||
|
||||
Copy `infra/argo/cockpit-build-deploy.yaml` → `infra/argo/fxhnt-cockpit-workflowtemplate.yaml`. In BOTH the
|
||||
`kaniko-build` and `kubectl-deploy` templates, change every
|
||||
`nodeSelector: { k8s.scaleway.com/pool-name: ci-compile-cpu, ... }` to
|
||||
`nodeSelector: { k8s.scaleway.com/pool-name: large, topology.kubernetes.io/zone: fr-par-2 }`.
|
||||
Grep to confirm no `ci-compile-cpu` remains: `grep -n ci-compile-cpu infra/argo/fxhnt-cockpit-workflowtemplate.yaml` → no hits.
|
||||
|
||||
- [ ] **Step 2: Add the immutable `:<sha7>` tag to the Kaniko destination**
|
||||
|
||||
In the `kaniko-build` container args, alongside the existing
|
||||
`--destination=rg.fr-par.scw.cloud/bizworx/fxhnt-cockpit:latest`, ADD a second line:
|
||||
`--destination=rg.fr-par.scw.cloud/bizworx/fxhnt-cockpit:{{=sprig.trunc(7, workflow.parameters.commit-sha)}}`
|
||||
(the `sprig.trunc` expression matches how `build-image` truncates the sha). Keep `--cache=true` and the
|
||||
`--cache-repo`.
|
||||
|
||||
- [ ] **Step 3: Server-side dry-run validate the template**
|
||||
|
||||
Run: `kubectl apply -f infra/argo/fxhnt-cockpit-workflowtemplate.yaml --dry-run=server -n foxhunt`
|
||||
Expected: `workflowtemplate.argoproj.io/fxhnt-cockpit created (server dry run)` with no schema errors.
|
||||
(If it errors on an unknown field, the Argo CRD version differs — STOP and report; do not force-apply.)
|
||||
|
||||
- [ ] **Step 4: Commit**
|
||||
|
||||
```bash
|
||||
git add infra/argo/fxhnt-cockpit-workflowtemplate.yaml
|
||||
git commit -m "ci(fxhnt): reanimate cockpit WorkflowTemplate (node pool large, sha7 tag)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 2 — Fix the deploy step (verify the dashboard external route first)
|
||||
|
||||
**Files:**
|
||||
- Modify: `infra/argo/fxhnt-cockpit-workflowtemplate.yaml` (the `kubectl-deploy` template's script)
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: the WorkflowTemplate from Task 1.
|
||||
- Produces: a deploy step that refreshes the daemons correctly for whichever external route is live.
|
||||
|
||||
- [ ] **Step 1: Determine the live dashboard external route (decides restart vs scale-0→1)**
|
||||
|
||||
Run these and read the result:
|
||||
```bash
|
||||
kubectl get deploy fxhnt-dashboard -n foxhunt -o jsonpath='{range .spec.template.spec.containers[*]}{.name}{"\n"}{end}'
|
||||
kubectl get ingress -n foxhunt 2>/dev/null
|
||||
kubectl -n foxhunt describe svc fxhnt-ingress 2>/dev/null | grep -iE 'endpoints|selector'
|
||||
```
|
||||
Decide:
|
||||
- If `dashboard.fxhnt.ai` is served THROUGH the tailscale sidecar (sidecar present AND no ingress route to the
|
||||
dashboard) → KEEP the old `scale 0→1` dance (the tailscale session race is real).
|
||||
- If it is served through `fxhnt-ingress` (an ingress/route points at the dashboard svc; the tailscale sidecar
|
||||
is vestigial) → REPLACE the scale-dance with a plain `kubectl -n foxhunt rollout restart deploy/fxhnt-dashboard`.
|
||||
|
||||
Record the decision in the plan's progress ledger with the evidence. If ambiguous, KEEP `scale 0→1` (the
|
||||
conservative choice that cannot cause the stale-proxy "down").
|
||||
|
||||
- [ ] **Step 2: Apply the deploy-step decision + add the deployed-sha annotation**
|
||||
|
||||
Edit the `kubectl-deploy` script. Regardless of the restart mechanism chosen, add BEFORE the rollout:
|
||||
```
|
||||
kubectl -n foxhunt annotate deploy/fxhnt-dashboard deploy/dagster \
|
||||
fxhnt.io/deployed-sha="$(cd /workspace/src && git rev-parse --short HEAD)" --overwrite
|
||||
```
|
||||
Keep the existing `kubectl apply -f` of the three manifests (fxhnt-dashboard.yaml, dagster.yaml,
|
||||
network-policies/fxhnt-cockpit.yaml) and the `rollout status` waits. Keep `rollout restart deploy/dagster`
|
||||
(dagster has no proxy race).
|
||||
|
||||
- [ ] **Step 3: Dry-run validate again**
|
||||
|
||||
Run: `kubectl apply -f infra/argo/fxhnt-cockpit-workflowtemplate.yaml --dry-run=server -n foxhunt`
|
||||
Expected: server dry-run OK.
|
||||
|
||||
- [ ] **Step 4: Commit**
|
||||
|
||||
```bash
|
||||
git add infra/argo/fxhnt-cockpit-workflowtemplate.yaml
|
||||
git commit -m "ci(fxhnt): deploy step — <restart-choice> + deployed-sha annotation"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 3 — The Sensor (simple, single-trigger) — mode decided IN the workflow
|
||||
|
||||
**Design note (verified 2026-07-21):** the Argo Events Sensor CRD here has an OPEN schema (no strict property
|
||||
validation — `kubectl explain sensor.spec.triggers.template.conditions` errors, and the only `conditions:` in
|
||||
live sensors are in `status`, not trigger-level). Rather than gamble on a fragile sensor-side path-negation
|
||||
filter to pick build-vs-deploy, we keep the SENSOR SIMPLE (one trigger, exactly like `build-msp-portal`) and
|
||||
move the build-vs-deploy decision INTO the workflow, where a robust `git diff --name-only` between the previous
|
||||
and pushed commit is trivial and testable. This is the more robust design, not a fallback.
|
||||
|
||||
**Files:**
|
||||
- Create: `infra/argo/fxhnt-build-sensor.yaml` (ns argo-events)
|
||||
- Modify: `infra/argo/fxhnt-cockpit-workflowtemplate.yaml` (add a `decide-mode` step that git-diffs)
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: the `gitea-webhook` eventsource (shared), the `fxhnt-cockpit` WorkflowTemplate (Task 1/2).
|
||||
- Produces: a `Sensor` `build-fxhnt` (argo-events) that submits ONE Workflow per master push; the workflow
|
||||
self-selects build vs deploy.
|
||||
|
||||
- [ ] **Step 1: Write the single-trigger Sensor (copy build-msp-portal structure exactly)**
|
||||
|
||||
`kubectl get sensor build-msp-portal -n argo-events -o yaml` and mirror it. ONE dependency `push-to-master`:
|
||||
- `body.ref` == `refs/heads/master`
|
||||
- `body.pusher.username` != `argocd-image-updater`
|
||||
(NO `body.commits.#.modified` filter — the mode decision moved to the workflow.)
|
||||
ONE trigger `submit`: submits a Workflow `generateName: build-fxhnt-`, `namespace: foxhunt`,
|
||||
`workflowTemplateRef: fxhnt-cockpit`, injecting `commit-sha`=`body.after`. Do NOT set `mode` (the workflow
|
||||
computes it). This is structurally identical to the proven msp sensors → lowest risk.
|
||||
|
||||
- [ ] **Step 2: Add a `decide-mode` step to the WorkflowTemplate**
|
||||
|
||||
In `fxhnt-cockpit-workflowtemplate.yaml`, change the DAG so the FIRST step (after git-clone) is `decide-mode`:
|
||||
a small `alpine/git` script that runs, in the cloned repo at `commit-sha`:
|
||||
```
|
||||
CHANGED=$(git diff --name-only "${SHA}~1" "${SHA}" 2>/dev/null || echo "ALL")
|
||||
if echo "$CHANGED" | grep -qE '^pyproject\.toml$|^infra/docker/fxhnt\.Dockerfile$'; then
|
||||
echo build; else echo deploy; fi
|
||||
```
|
||||
Output the result (`build`/`deploy`) as an Argo output parameter `mode`. Then gate the `kaniko-build` task on
|
||||
`{{tasks.decide-mode.outputs.parameters.mode}} == build`, and `kubectl-deploy` depends on
|
||||
`build.Succeeded || build.Skipped` (unchanged). Edge case: if `${SHA}~1` doesn't exist (first commit / shallow),
|
||||
default to `build` (safe — a rebuild is never wrong, only slower). The explicit `mode` PARAM stays on the
|
||||
template (default `auto`) so a human can still force `mode=build`/`mode=deploy` on a manual `argo submit`;
|
||||
`decide-mode` only runs when `mode=auto`.
|
||||
|
||||
- [ ] **Step 3: Dry-run validate both**
|
||||
|
||||
```bash
|
||||
kubectl apply -f infra/argo/fxhnt-build-sensor.yaml --dry-run=server -n argo-events
|
||||
kubectl apply -f infra/argo/fxhnt-cockpit-workflowtemplate.yaml --dry-run=server -n foxhunt
|
||||
```
|
||||
Expected: both server-dry-run OK.
|
||||
|
||||
- [ ] **Step 4: Commit**
|
||||
|
||||
```bash
|
||||
git add infra/argo/fxhnt-build-sensor.yaml infra/argo/fxhnt-cockpit-workflowtemplate.yaml
|
||||
git commit -m "ci(fxhnt): single-trigger sensor + in-workflow git-diff mode decision"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 4 — Apply to the cluster + wire the Gitea webhook
|
||||
|
||||
**Files:** none new — this task applies the manifests and creates the webhook.
|
||||
|
||||
- [ ] **Step 1: Apply the WorkflowTemplate + Sensor for real**
|
||||
|
||||
```bash
|
||||
kubectl apply -f infra/argo/fxhnt-cockpit-workflowtemplate.yaml -n foxhunt
|
||||
kubectl apply -f infra/argo/fxhnt-build-sensor.yaml -n argo-events
|
||||
kubectl get workflowtemplate fxhnt-cockpit -n foxhunt
|
||||
kubectl get sensor build-fxhnt -n argo-events
|
||||
```
|
||||
Expected: both present. The sensor pod (`kubectl get pods -n argo-events | grep build-fxhnt`) reaches Running.
|
||||
|
||||
- [ ] **Step 2: Create the Gitea webhook on `gitadmin/fxhnt`**
|
||||
|
||||
The `gitea-webhook` eventsource endpoint is the same target the `bizworx-ops/infrastructure` repo already POSTs
|
||||
to. Find it: `kubectl get eventsource gitea-webhook -n argo-events -o yaml` (the webhook `port`/`endpoint`) and
|
||||
how the existing infra-repo webhook is configured (inspect via Gitea admin UI or API on an existing repo).
|
||||
Create the equivalent webhook on `gitadmin/fxhnt`: push events, active, POST JSON to the eventsource URL. Record
|
||||
the exact URL used in the ledger. Do NOT paste any webhook secret into the transcript.
|
||||
|
||||
- [ ] **Step 3: Verify the webhook reaches the sensor (dry, safe probe)**
|
||||
|
||||
Trigger a harmless test push (or use Gitea's "Test Delivery" webhook button) and confirm the sensor received it:
|
||||
```bash
|
||||
kubectl logs -n argo-events -l sensor-name=build-fxhnt --tail=50 | grep -iE 'triggered|push-to-master|dispatch'
|
||||
```
|
||||
Expected: the sensor logs the received push. If Gitea's test-delivery uses a non-master ref, the sensor
|
||||
correctly does NOT fire — confirm it's reachable via the eventsource logs instead.
|
||||
|
||||
---
|
||||
|
||||
## Task 5 — End-to-end verification (both modes) + retire the old file
|
||||
|
||||
**Files:**
|
||||
- Remove: `infra/argo/cockpit-build-deploy.yaml` (superseded)
|
||||
|
||||
- [ ] **Step 1: Verify mode=deploy end-to-end (code-only push)**
|
||||
|
||||
Make a trivial code-only commit on `master` (e.g. touch a comment in `src/`), push, and watch:
|
||||
```bash
|
||||
kubectl get wf -n foxhunt --sort-by=.metadata.creationTimestamp | tail -3
|
||||
```
|
||||
Expected: a `build-fxhnt-*` Workflow runs with `mode=deploy`, SKIPS the kaniko-build step, deploys, and the
|
||||
dashboard stays UP (check `dashboard.fxhnt.ai` reachable post-deploy, and
|
||||
`kubectl get deploy fxhnt-dashboard -n foxhunt -o jsonpath='{.metadata.annotations.fxhnt\.io/deployed-sha}'`
|
||||
shows the new sha). Total time ~1-2 min, no Kaniko build.
|
||||
|
||||
- [ ] **Step 2: Verify mode=build end-to-end (dep push)**
|
||||
|
||||
Make a no-op-but-real change to `pyproject.toml` (e.g. reorder a dependency, or bump a harmless comment), push,
|
||||
and watch: the Workflow runs `mode=build`, Kaniko produces `fxhnt-cockpit:<sha7>`+`:latest`
|
||||
(`kubectl run` a crane/skopeo check OR verify the workflow's kaniko step succeeded), then deploys. Confirm the
|
||||
`:<sha7>` tag exists in the registry.
|
||||
|
||||
- [ ] **Step 3: Confirm no false trigger + real-money safety**
|
||||
|
||||
- Push a change touching ONLY `docs/` → confirm a `mode=deploy` Workflow runs (harmless) OR, if docs are
|
||||
excluded, no Workflow — either is fine, but it must NOT trigger a build.
|
||||
- Confirm NO execution CronJob was modified/triggered by any pipeline run:
|
||||
`kubectl get cronjob -n foxhunt` (ucits/bybit/ibkr rebalancers unchanged, bybit/ibkr still Suspend=True).
|
||||
|
||||
- [ ] **Step 4: Retire the old file + commit**
|
||||
|
||||
```bash
|
||||
git rm infra/argo/cockpit-build-deploy.yaml
|
||||
git commit -m "ci(fxhnt): retire old cockpit-build-deploy (superseded by workflowtemplate + sensor)"
|
||||
```
|
||||
|
||||
- [ ] **Step 5: Final verification summary**
|
||||
|
||||
Confirm all Global Constraints hold: sensor in argo-events only, workflow+pods in foxhunt, node pool `large`,
|
||||
no cross-ns RBAC, no fxhnt secrets in platform ns, real-money CronJobs untouched. Record in the ledger.
|
||||
382
docs/superpowers/plans/CONSOLIDATION-PROGRESS.md
Normal file
382
docs/superpowers/plans/CONSOLIDATION-PROGRESS.md
Normal file
@@ -0,0 +1,382 @@
|
||||
# Foxhunt→Bizworx Consolidation — Live Progress (resume anchor)
|
||||
|
||||
## 🚦 T8 CUTOVER EXECUTED (2026-07-18 ~20:15 UTC) — fxhnt.ai NOW SERVES FROM BIZWORX ✅
|
||||
**The fund has cut over. foxhunt is FROZEN but INTACT (rollback available). DELETE NOTHING until T10 observation passes.**
|
||||
- **C1 (freeze foxhunt):** suspended fund cronjobs (ucits-rebalancer, ucits-volume-ibkr, compare-measured-precompute)
|
||||
+ scaled foxhunt dagster/fxhnt-dashboard/ib-gateway → 0/0. Infra cronjobs (postgres-backup/pvc-autoscaler/
|
||||
billing-scraper) LEFT RUNNING (they die at teardown, not now). PASS.
|
||||
- **C2 (delta DB migrate):** pg_dump -Fc fxhnt (47.8M) from frozen foxhunt → restore into bizworx. GOTCHA:
|
||||
`pg_restore --clean --if-exists` over the existing rehearsal copy FAILED (dropped _timescaledb_internal schema
|
||||
mid-restore → 1136 errors, DB stuck restoring:on). FIX (rehearsal-proven path): DROP+CREATE empty fxhnt DB,
|
||||
then pre_restore → pg_restore (NO --clean) → post_restore = exit 0, 0 errors. EVAL C2 PASS: all counts match
|
||||
frozen source (forward_nav 106, backtest_summary 6, bybit_sleeve_ret 7023, xsp_option_bars 814373, exec_fills 9,
|
||||
ibkr_account_nav 4, forward_anchor 20; max_anchor_t0 2026-07-14, max_fwd_nav 2026-07-17; 37 tables/3 hypertables).
|
||||
NOTE: counts were IDENTICAL to the rehearsal (fund already suspended during parallel work → zero real delta).
|
||||
LESSON: NEVER pg_restore --clean over a live timescaledb DB; always restore into a fresh empty DB.
|
||||
- **C3 (bizworx fund live):** scaled dagster/cockpit/ib-gateway→1 + rollout-restart cockpit&dagster (fresh DB
|
||||
conns). EVAL C3 PASS: cockpit serves real migrated data over the proxy (/ + /strategy/multistrat + /xsfunding
|
||||
= 200, shows PASS/WAIT/Sharpe/multistrat/xsfunding); ib-gateway 1/1 STABLE (~12min, foxhunt released IBKR
|
||||
session); dagster 3/3, ingress 3/3, postgres 1/1.
|
||||
- **C4 (DNS flip):** Scaleway fxhnt.ai records dashboard/cockpit/dagster/git A → **100.93.141.11** (bizworx
|
||||
fxhnt-ingress), TTL 60 (was 100.95.225.27 foxhunt proxy). SYNTAX: `scw dns record set dns-zone=fxhnt.ai
|
||||
name=<n> type=A ttl=60 values.0=100.93.141.11` (values.0=, NOT data=). EVAL C4 PASS over REAL public DNS:
|
||||
dashboard/git/dagster.fxhnt.ai → 100.93.141.11, HTTP 200, tls_verify=0. Content confirmed: git=bizworx Gitea,
|
||||
dashboard=cockpit "Overview · fxhnt" w/ real data, SSH `git ls-remote ssh://git@git.fxhnt.ai/gitadmin/fxhnt.git`
|
||||
returns real refs (HEAD d66d1132) via socat path. (SSH hostkey-changed warning = correct, it's bizworx now.)
|
||||
- **C5 (arm fund):** un-suspended ONLY fxhnt-ucits-rebalancer on bizworx (schedule 0 9 * * 1-5 = 09:00 UTC wkdays;
|
||||
next run tomorrow AM). US(ibkr)/bybit/vrp/factory/ucits-volume all stay SUSPENDED. EVAL C5 (first armed run
|
||||
green) is OBSERVABLE at next 09:00 UTC — deps verified (DB+ib-gateway+git-sync all live).
|
||||
- **ROLLBACK (still trivial until T11):** `scw dns record set ... values.0=100.95.225.27` (back to foxhunt proxy)
|
||||
+ `kubectl --context $FOX -n foxhunt scale deploy/dagster deploy/fxhnt-dashboard deploy/ib-gateway --replicas=1`
|
||||
+ un-suspend foxhunt fund cronjobs. foxhunt Postgres still holds pre-cutover state.
|
||||
- **NEXT:** T9 verify (cockpit LOCAL browser + prod, gate/forward-track intact) → T10 observation window (watch
|
||||
the first ucits run + a day) → T11 Phase-2 triage → T12 teardown. **ROTATE the pasted tailscale key.** DELETE NOTHING.
|
||||
|
||||
|
||||
Branch `chore/foxhunt-bizworx-consolidation`. Real date 2026-07-18. Plan:
|
||||
`docs/superpowers/plans/2026-07-18-foxhunt-to-bizworx-consolidation.md`. Contexts:
|
||||
`FOX=foxhunt-34a1e3c4-…`, `BIZ=bizworx-prod-kapsule-05e1dae3-…` (both in ~/.kube/config).
|
||||
|
||||
## Done + verified
|
||||
- **Access restored:** appended this box's egress `51.158.111.225/32` to the foxhunt Kapsule API ACL
|
||||
(`scw k8s acl`, entry `bizworx-devbox-v4`) + `scw k8s kubeconfig install` → foxhunt context works.
|
||||
- **T1** ns `foxhunt` created on bizworx; StorageClass `sbs-default` matches; source = PG16.13/tsdb2.25.1.
|
||||
- **T2** 6 fund secrets copied FOX→BIZ ns foxhunt (db-credentials, databento, tiingo-api, ibkr-credentials,
|
||||
argo-git-ssh-key, scw-registry). NOT copied: bybit-testnet (absent/suspended), tailscale-auth (deferred).
|
||||
- **T3** gitea consolidated → bizworx gitea (ns gitea): `gitadmin/fxhnt` (8 heads/0 tags, master d66d113)
|
||||
+ `gitadmin/foxhunt` (13 heads/105 tags, main 8213f29a5), refs verified identical. foxhunt (625MB) done
|
||||
via direct pack-copy (HTTP push + server-side index-pack both failed on size/OOM). `git.fxhnt.ai` STILL
|
||||
points at foxhunt (DNS re-point is T8). Caveat: foxhunt archive's gitea DB metadata may lag (disk-correct).
|
||||
- **T4** manifests re-targeted (committed): git-sync `gitea-sshd.foxhunt`→`gitea-ssh.gitea.svc.cluster.local:22`;
|
||||
nodeSelector→`large`; ib-gateway pull-secret removed (public ghcr image). Server-side dry-run clean.
|
||||
- **T5/T6** Postgres on bizworx PINNED to `timescale/timescaledb:2.25.1-pg16` (latest=2.28.3 broke catalog
|
||||
restore — REHEARSAL caught it). Rehearsal restore of `fxhnt` DB: exit 0, 0 errors, all row counts match
|
||||
(forward_nav 106, backtest_summary 6, bybit_sleeve_ret 7023, xsp_option_bars 814373, exec_fills 9,
|
||||
ibkr_account_nav 4), 37/37 tables, 3/3 hypertables. bizworx fxhnt DB currently holds this rehearsal copy.
|
||||
|
||||
## T7 IN PROGRESS (parallel verify — foxhunt still fully live)
|
||||
- **7.1 DONE** — argo-git-ssh-key deploy-key registered read-only on bizworx gitadmin/fxhnt (id 2);
|
||||
verified `git ls-remote` via the key = 8 refs.
|
||||
- **cockpit DONE** — `fxhnt-dashboard` deployed on bizworx; git-sync OK (cloned d66d113 from bizworx gitea);
|
||||
cockpit container Running+ready, renders migrated data (overview + /strategy/xsfunding + /strategy/multistrat
|
||||
all HTTP OK with real WAIT/PASS gates). Verified from INSIDE the container (port-forward blocked — see below).
|
||||
- **BLOCKER / decision pending — tailscale:** dagster + cockpit pods have a tailscale sidecar needing
|
||||
`tailscale-auth` (not copied) → sidecar CreateContainerConfigError → pod phase Pending → port-forward
|
||||
refuses. Cockpit itself is fine. DECIDE: **A** fresh reusable tailscale auth-key (bizworx pods join as
|
||||
extra nodes alongside live foxhunt ones; removed at cutover) OR **B** disable sidecar for the parallel
|
||||
phase, verify via port-forward, enable tailscale at T8 cutover with real hostnames.
|
||||
|
||||
## T7 REMAINING
|
||||
- ib-gateway → apply + verify IBKR paper connect (no 2FA/IP-limit per operator) + whatIf.
|
||||
- dagster → apply (check its NetworkPolicy allows egress to ns gitea for git-sync — cross-ns) + trigger
|
||||
nightly asset graph green.
|
||||
- CronJobs → apply all SUSPENDED; then one `fxhnt-ucits-rebalancer` dry-run (EU hours).
|
||||
- NP note: fund NPs (dagster, cronjobs) allow egress to gitea in-ns; on bizworx gitea is ns `gitea` →
|
||||
may need namespaceSelector. (cockpit had no NP → git-sync worked.)
|
||||
|
||||
## HARD RULE (operator): verify-first, per-step eval, DELETE NOTHING until Phase-1 verified + observation
|
||||
window (T10) passes. foxhunt stays live/intact through T8, suspended-but-intact until T12 teardown.
|
||||
Rollback until T11 = re-point DNS + un-suspend foxhunt.
|
||||
|
||||
## T7 update (2026-07-18, later)
|
||||
- tailscale-auth secret created on bizworx (operator gave a fresh reusable key — OPTION A; key was pasted in
|
||||
transcript, ROTATE after cutover). cockpit pod now 2/2 Running (cockpit + tailscale sidecar joined).
|
||||
- ib-gateway deployed: IBC logs in to DU9600528 (Simulated Trading) — IBKR paper connect WORKS (no 2FA/IP).
|
||||
⚠️ pod had 2 restarts during IBC startup churn — WATCH it stabilises to 1/1 ready; verify API port 4003/4004
|
||||
+ a whatIf before arming any rebalancer.
|
||||
- dagster was Pending on missing PVC `fxhnt-surfer-data` (dropped from migration; manifest still mounts it) →
|
||||
created empty 5Gi PVC (sbs-default); dagster now scheduling (git-sync init). unlock_calendar.json rebuilds.
|
||||
- REMAINING T7: confirm ib-gateway 1/1 + whatIf; confirm dagster git-sync (cross-ns NP to gitea) + trigger
|
||||
nightly asset graph green; apply CronJobs SUSPENDED; one fxhnt-ucits-rebalancer dry-run (EU hours).
|
||||
- Then T8 cutover (freeze foxhunt, delta restore, DNS re-point git.fxhnt.ai + dashboard.fxhnt.ai) → T9 verify
|
||||
→ T10 observe → T11 triage → T12 teardown. foxhunt still fully live; delete NOTHING yet.
|
||||
|
||||
## T7 update 2 (2026-07-18)
|
||||
- dagster FIXED + 3/3 Running: (1) created `dagster` DB on bizworx postgres (fresh dagster, not migrated);
|
||||
(2) dagster NP gitea-egress needed a cross-ns `namespaceSelector {kubernetes.io/metadata.name: gitea}`
|
||||
(committed in dagster.yaml) → git-sync from bizworx gitea now OK. webserver+daemon+tailscale all ready.
|
||||
- ib-gateway: connects to DU9600528 (IBC login OK) but exits 0 after ~6min = IBKR single-session conflict with
|
||||
the STILL-LIVE foxhunt ib-gateway. Connect verified; stable run only after foxhunt-suspend (T8). NOT a bug.
|
||||
- T7 CORE VERIFIED: postgres+cockpit+dagster all up on bizworx against migrated data.
|
||||
- REMAINING T7: (a) apply the SAME cross-ns gitea-NP fix to every cronjob NP (jobs/fxhnt-*.yaml) BEFORE they
|
||||
run (else git-sync blocked like dagster was); (b) apply CronJobs SUSPENDED; (c) one ucits dry-run (needs
|
||||
ib-gateway sole-session → do at/after cutover, or briefly scale foxhunt ib-gateway to 0); (d) optional dagster
|
||||
asset-graph run. Then T8 cutover.
|
||||
- ALSO for cutover: the tailscale bizworx pods joined as extra nodes (fresh key); at T8 suspend foxhunt so the
|
||||
real hostnames + IBKR session move cleanly. ROTATE the pasted tailscale key post-cutover.
|
||||
|
||||
## P1 EXECUTION (2026-07-18) — wildcard cert + the cross-project DNS-01 IAM fix
|
||||
- Applied `infra/k8s/services/fxhnt-wildcard-cert.yaml` (Certificate fxhnt.ai + *.fxhnt.ai, ClusterIssuer
|
||||
letsencrypt-prod, secret fxhnt-wildcard-tls, ns foxhunt) on BIZ.
|
||||
- **DNS-01 FAILED: `403 Forbidden: domain not found`.** Root cause: `fxhnt.ai` DNS zone lives in the **foxhunt
|
||||
project c293eb98**; the bizworx cert-manager key (`SCWW9261…`, app `bizworx-prod-runtime` 48418a71) has
|
||||
`DomainsDNSFullAccess` but SCOPED to the **bizworx project 96d0717d** only → can't touch fxhnt.ai's zone.
|
||||
- **DEAD END (operator's first pick "move the domain to bizworx project"): Scaleway BLOCKS it.** fxhnt.ai is an
|
||||
EXTERNAL domain (`scw domain domain list` → IS EXTERNAL=true, REGISTRAR=EXTERNAL). `scw dns zone update
|
||||
… project-id=…` on the apex → **`Root zone can't be updated`**; on a scratch sub-zone → **`Subdomain not
|
||||
allowed`**; `scw domain domain` has NO move-project subcommand. Verified via a throwaway `_cutovertest.fxhnt.ai`
|
||||
sub-zone (created+move-attempt+deleted). So the domain CANNOT be reassigned to the bizworx project via API.
|
||||
- **ACTUAL FIX (operator's 2nd pick, applied): add an IAM rule.** `scw iam rule create
|
||||
policy-id=8573ce25-… permission-set-names.0=DomainsDNSFullAccess project-ids.0=c293eb98-…` → policy
|
||||
`bizworx-prod-runtime-scoped` now has 2 rules (orig 6-perms@96d0717d untouched + new DNS@c293eb98). This lets
|
||||
the existing bizworx cert-manager key write the DNS-01 TXT on fxhnt.ai. Domain STAYS external in c293eb98;
|
||||
only IAM changed. NOTE for teardown: this rule references the foxhunt project — harmless to keep, or tighten
|
||||
later; if the foxhunt PROJECT (not just cluster) is ever deleted, the fxhnt.ai domain/zone must first move
|
||||
registrar-side or certs break. (Cluster teardown ≠ project deletion; the DNS zone + domain survive T12.)
|
||||
- Forced cert-manager retry (deleted order+CR) after the IAM change → 403 GONE, challenges progressing normally
|
||||
(apex fxhnt.ai challenge = `valid`/"Successfully authorized domain"; wildcard `*.fxhnt.ai` challenge finishing
|
||||
its DNS-01 propagation). TXT is being written to the fxhnt.ai zone by cert-manager (verified via
|
||||
`scw dns record list dns-zone=fxhnt.ai | grep acme`). P1 is SUCCEEDING — cert will go READY shortly.
|
||||
If resuming and cert not yet READY: `kubectl --context $BIZ -n foxhunt get certificate fxhnt-wildcard -o wide`;
|
||||
if stuck >15min, `kubectl delete order,challenge,certificaterequest --all -n foxhunt` to force a clean retry
|
||||
(IAM rule is permanent so it'll succeed).
|
||||
|
||||
## P2 DRAFTED (2026-07-18) — fxhnt-ingress proxy (NOT applied yet; waiting on P1 cert READY)
|
||||
- `infra/k8s/services/fxhnt-ingress-proxy.yaml` (committed ce1c978, draft): nginx (wildcard TLS via
|
||||
fxhnt-wildcard-tls) + socat (git SSH→gitea-ssh.gitea:22) + tailscale sidecar (TS_HOSTNAME=fxhnt-ingress).
|
||||
Fronts 3 fund vhosts BY CLUSTERIP: git→gitea-http.gitea:3000, dashboard/cockpit→fxhnt-dashboard.foxhunt:80,
|
||||
dagster→dagster.foxhunt:80. Has its own NP (egress to those svcs + apiserver 172.16.0.11:6443 for TS + DERP).
|
||||
Deps satisfied: tailscale-auth secret present; dagster NP ingress already allows fxhnt-ingress label.
|
||||
## P1+P2 COMPLETE (2026-07-18) — bizworx fxhnt.ai front verified, NO DNS change yet ✅
|
||||
- **P1 DONE:** fxhnt-wildcard cert READY (LE, SAN=*.fxhnt.ai + fxhnt.ai). IAM cross-project fix held.
|
||||
- **P2 DONE:** fxhnt-ingress proxy applied + 3/3 Running. Tailnet identity = **`fxhnt-ingress` @ `100.93.141.11`**
|
||||
(clean, no -1 suffix). THIS IS THE C4 DNS TARGET.
|
||||
- **EVAL P2 PASS (from devbox over the real tailnet → 100.93.141.11, the exact browser path post-flip):**
|
||||
dashboard / cockpit / dagster / git .fxhnt.ai ALL → **HTTP 200, tls=0 (cert chain valid)**. foxhunt still live.
|
||||
- Fixes needed to get P2 green (all committed):
|
||||
1. **RBAC:** TS sidecar needs SA+Role+RoleBinding for get/create/update/patch on its ts-state Secret (added
|
||||
`tailscale-fxhnt-ingress` SA, mirrors tailscale-dagster). Without it: "missing get/update permission".
|
||||
2. **dagster NP ingress was stale on-cluster:** the 9396f53 edit (foxhunt→fxhnt-ingress label) was committed
|
||||
but the cluster still had `tailscale-gitlab-proxy` → dagster vhost hung. Re-applied dagster.yaml → fixed.
|
||||
LESSON: `git commit` ≠ `kubectl apply`; always re-apply after editing a live manifest.
|
||||
3. **gitea headless-service / CGNAT trap:** gitea-http & gitea-ssh are HEADLESS (ClusterIP:None) → DNS returns
|
||||
the gitea POD IP in 100.64.0.0/15, which the proxy's KERNEL-MODE tailscale sidecar (TS_USERSPACE=false)
|
||||
CANNOT reach (CGNAT overlap). FIX: added a non-headless **`gitea-clusterip`** Service (ns gitea, 10.32.x,
|
||||
selector app.kubernetes.io/instance=gitea+name=gitea, ports 3000+22); nginx+socat now target it. cockpit &
|
||||
dagster were fine because they have real 10.32.x ClusterIPs. (Kernel-mode TS reaches 10.32.x ClusterIPs but
|
||||
NOT 100.64.x pod IPs — verified both ways.)
|
||||
- **Verify artifact:** `curl --resolve <host>:443:100.93.141.11 https://<host>/` from any tailnet node.
|
||||
- PHASE-1 PREP COMPLETE. Ready to schedule the C1–C5 cutover window (see T8-CUTOVER-RUNBOOK.md). C4 DNS target
|
||||
= 100.93.141.11 for records dashboard/cockpit/dagster/git. foxhunt fully live; delete NOTHING.
|
||||
|
||||
## T8 PREP (2026-07-18) — DNS/ingress map + fixes (foxhunt still fully live)
|
||||
|
||||
### Gotchas hit this session
|
||||
- **dagster tailscale sidecar CrashLoop (context deadline exceeded reaching kubernetes.default.svc).** TWO-STAGE
|
||||
root cause — the FIRST fix (06d12fd, `10.32.0.1/32:443`) was WRONG and did not hold (regressed to 2/3):
|
||||
* `kubernetes.default.svc` → ClusterIP `10.32.0.1:443`, BUT **kube-proxy DNATs it to the REAL apiserver
|
||||
endpoint `172.16.0.11:6443` BEFORE the NetworkPolicy is evaluated.** The NP sees dest `172.16.0.11:6443`,
|
||||
which is inside the NP's OTHER except entry `172.16.0.0/16` → blocked. So allowing the ClusterIP never matched.
|
||||
* CORRECT FIX: `ipBlock 172.16.0.11/32` port **6443** (the real endpoint, post-DNAT). apiserver endpoint is
|
||||
`kubectl -n default get endpoints kubernetes` = 172.16.0.11 (Kapsule fixed control-plane).
|
||||
* SIDE EFFECT this exposed: when the tailscale sidecar is NotReady, the whole dagster pod is NotReady →
|
||||
**dagster Service has NO endpoints → dagster.fxhnt.ai vhost would 502**. So this MUST be green before C4 DNS flip.
|
||||
* daemon+webserver themselves were always healthy. Cockpit's sidecar works only because cockpit has NO NP.
|
||||
* GENERAL LESSON for any pod with a tailscale sidecar + a restrictive egress NP on bizworx: must allow
|
||||
egress to 172.16.0.11:6443 (apiserver). The cronjob/rebalancer NPs do NOT have tailscale sidecars so they're
|
||||
unaffected — but if any future fund pod gets a TS sidecar under an NP, add this rule.
|
||||
- **foxhunt API ACL lapsed AGAIN**: devbox egress IP drifted `51.158.111.225`→`51.158.125.1`. Foxhunt Kapsule
|
||||
admin API (`*.api.k8s.fr-par.scw.cloud:6443`) is IP-allowlisted; bizworx keeps working (tailscale-cgnat range).
|
||||
FIX: `scw k8s acl add cluster-id=34a1e3c4… acls.0.ip=<newIP>/32 acls.0.description=bizworx-devbox-vN` (APPENDS;
|
||||
verified all entries survive). Re-check `curl ifconfig.me` vs `scw k8s acl list` whenever foxhunt 6443 resets.
|
||||
|
||||
### THE fxhnt.ai DNS/ingress architecture (drives T8 DNS cutover)
|
||||
- DNS zone = **Scaleway `fxhnt.ai`** (project c293eb98=foxhunt). ALL 13 A-records → **`100.95.225.27`** = tailnet
|
||||
node **`foxhunt-gitlab`** = pod **`tailscale-gitlab-proxy` (ns foxhunt, 4/4)** = nginx(vhosts)+2×socat+tailscale.
|
||||
**This pod DIES at foxhunt teardown (T12) → DNS MUST move off 100.95.225.27 regardless.**
|
||||
- nginx vhost map (what each fxhnt.ai host fronts):
|
||||
- `git.fxhnt.ai` → gitea-web.foxhunt:3000 ✅FUND → **bizworx gitea (ns gitea), already mirrored**
|
||||
- `dashboard/cockpit.fxhnt.ai` → 100.81.150.18:80 (fxhnt-dashboard node→cockpit:8080) ✅FUND → **bizworx cockpit**
|
||||
- `dagster.fxhnt.ai` → dagster.foxhunt:80 ✅FUND → **bizworx dagster**
|
||||
- grafana ❌ (operator: "grafana can go too" — drop, bizworx has own obs) · api(50051 legacy Rust gRPC) ❌
|
||||
· mail(stalwart)/chat(mattermost)/minio ❌legacy → all Phase-2 drop. ONLY git+dashboard/cockpit+dagster move.
|
||||
- **CRITICAL SAFETY FACT:** the FUND PIPELINE never uses fxhnt.ai — git-sync=`gitea-ssh.gitea.svc.cluster.local:22`
|
||||
(in-cluster), cockpit/dagster serve over ClusterIP+own tailnet IP. fxhnt.ai is BROWSER/human access only.
|
||||
→ the fund keeps running through the DNS flip; DNS cutover only affects operator browser access. Low-risk.
|
||||
- Tailnet identities: live foxhunt `fxhnt-dashboard`=100.81.150.18, `fxhnt-dagster`=100.116.123.29. New bizworx
|
||||
cockpit joined as **`fxhnt-dashboard-1`=100.107.99.35** (`-1` because live one holds the name). Each bizworx pod
|
||||
serves TLS DIRECTLY via `tailscale serve` on its own MagicDNS cert (verified: cockpit `tailscale serve status`
|
||||
shows https://fxhnt-dashboard-1.tailff20e.ts.net → 127.0.0.1:8080). No bizworx proxy pod needed.
|
||||
|
||||
### DNS cutover DECISION (operator): "RENAME bizworx pods to take over the identity"
|
||||
- At cutover: suspend/scale foxhunt fund pods → they RELEASE `fxhnt-dashboard`/`fxhnt-dagster` tailnet names →
|
||||
restart bizworx pods so they re-register WITHOUT the `-1` suffix (reclaim the names + get their IPs) → then
|
||||
repoint the Scaleway `fxhnt.ai` A-records (dashboard,cockpit,dagster,git) from the dying 100.95.225.27 to the
|
||||
reclaimed bizworx pod tailnet IPs. Records to change: `scw dns record` set on zone fxhnt.ai.
|
||||
- TLS RESOLVED: old proxy served a cert-manager wildcard `*.fxhnt.ai` (ClusterIssuer letsencrypt-prod, Scaleway
|
||||
DNS-01 webhook). **bizworx already has the SAME stack** (cert-manager + scaleway-certmanager-webhook 1/1 +
|
||||
scaleway-api creds + letsencrypt-prod ClusterIssuer, already issuing LE certs in ~10 ns incl grafana-tls).
|
||||
→ issuing `*.fxhnt.ai` on bizworx = just a Certificate resource. NO blocker.
|
||||
- CHOSEN CUTOVER SHAPE (supersedes the naive pod-IP flip): build a **stripped fund-only nginx+tailscale proxy on
|
||||
bizworx** (`fxhnt-ingress`) fronting the 3 fund vhosts via ClusterIP (gitea-http.gitea:3000, fxhnt-dashboard
|
||||
.foxhunt:80, dagster.foxhunt:80) + socat SSH gitea-ssh.gitea:22, terminating the wildcard `*.fxhnt.ai` cert.
|
||||
Point the 4 fxhnt.ai A-records at THIS proxy's tailnet IP. Because it uses ClusterIP not pod tailnet names,
|
||||
the "reclaim fxhnt-dashboard identity" rename is NO LONGER on the critical path (optional cosmetic).
|
||||
- **FULL EXECUTABLE RUNBOOK written: `docs/superpowers/plans/T8-CUTOVER-RUNBOOK.md`** (P1 wildcard cert →
|
||||
P2 build+verify proxy pre-flip → C1 freeze foxhunt → C2 delta dump/restore → C3 bizworx live+ib-gw sole session
|
||||
→ C4 DNS flip → C5 arm ucits. Every step has VERIFY + ROLLBACK. Nothing destructive; foxhunt suspended-not-deleted.)
|
||||
- Confirmed svc names for the proxy: gitea-http.gitea:3000 (headless), gitea-ssh.gitea:22 (headless),
|
||||
fxhnt-dashboard.foxhunt:80, dagster.foxhunt:80, ib-gateway.foxhunt:4002/4004.
|
||||
|
||||
## T7 update 3 (2026-07-18) — cronjobs
|
||||
- Cross-ns gitea-NP fix applied to ALL 15 jobs/fxhnt-*.yaml (namespaceSelector kubernetes.io/metadata.name=gitea).
|
||||
- 7 CronJobs applied to bizworx ns foxhunt, ALL suspend=true (verified): bybit-rebalancer,
|
||||
compare-measured-precompute, factory, ibkr-rebalancer, ucits-rebalancer, ucits-volume-ibkr, vrp-rebalancer.
|
||||
(real-money safety: none run parallel to the live foxhunt ucits-rebalancer.)
|
||||
- The 9 one-time Jobs (opra-backfill/warehouse-migrate/paper-backfill/bybit-*-job/deribit) NOT applied (they
|
||||
run immediately); apply on demand post-cutover with the NP fix already in place.
|
||||
- REMAINING before T8: one ucits dry-run (needs SOLE IBKR session — briefly scale foxhunt ib-gateway to 0,
|
||||
or do it as the first act of cutover) + optional dagster asset-graph run. Then T8 cutover.
|
||||
|
||||
## T9 VERIFY (2026-07-18) — cockpit + gate intact against MIGRATED bizworx DB ✅
|
||||
- **Local cockpit** (`./scripts/dev-cockpit-local.sh`, current-context=bizworx → serves the MIGRATED DB read-only
|
||||
on http://127.0.0.1:8801). ENV GOTCHA fixed: the script's `uv run` needed `--extra web --extra pg` (uvicorn in
|
||||
`web`, psycopg in `pg`); a bare `uv run` fails ModuleNotFoundError. Patched dev-cockpit-local.sh.
|
||||
- **NUMBERS CROSS-CHECKED (read the numbers, per feedback_verify_cockpit_locally_not_prod):**
|
||||
- /paper IBKR card = **$1,029,015** == ibkr_account_nav.nlv latest (2026-07-17 = 1029014.63). "not connected"
|
||||
ABSENT (the card that silently degraded in a prior session renders the REAL DU account here).
|
||||
- bybit Sh 2.11 == backtest_summary bybit_4edge 2.1067; unlock Sh 1.03 == 1.0258. All real, DB-backed.
|
||||
- /, /strategy/multistrat, /strategy/xsfunding, /paper all HTTP 200 with real content.
|
||||
- CAVEAT: Playwright MCP browser not connected this session → verified by rendering real HTML + reading
|
||||
numbers (not pixels). Local cockpit left running on :8801 for the operator to click through.
|
||||
- **T9b forward tracks INTACT (no spurious re-inception from migration):** forward_anchor t0 dates are ORIGINAL
|
||||
historical inceptions (sixtyforty 2026-06-15, unlock 2026-06-20, positioning 2026-07-06, bybit_4edge/multistrat/
|
||||
xsfunding 2026-07-07, vrp/multistrat_exec 2026-07-14) — NOT today (2026-07-18). definition_hash/version chains +
|
||||
archived_at + record_mode all preserved. DB-anchor-SSOT survived byte-for-byte; recon gate will recompute correctly.
|
||||
- **STILL PENDING:** T9 real-money confirm = watch the first armed fxhnt-ucits-rebalancer run at 09:00 UTC (tomorrow).
|
||||
Then T10 observe → T11 triage → T12 teardown. ROTATE the pasted tailscale key. DELETE NOTHING.
|
||||
|
||||
## T9 VISUAL BROWSER PASS (2026-07-18) — real Chrome via Playwright MCP ✅
|
||||
- Playwright MCP now available (plugins reloaded); installed the `chrome` channel (Google Chrome 150 at
|
||||
/opt/google/chrome/chrome — the MCP hardcodes that path, bundled chromium alone isn't enough).
|
||||
- Drove the LOCAL cockpit (http://127.0.0.1:8801, bizworx migrated DB) in a REAL browser — full render, all data:
|
||||
- Overview: Bybit 4-edge $832,534 +738% Sh 1.74 maxDD -13%, deploy $75k/75%, gate 11/21 WAIT; 4 edges
|
||||
(trend Sh0.72 / unlock Sh1.03 / xsfunding Sh3.62 / positioning Sh2.55); forward-tracks table w/ correct
|
||||
gates: sixtyforty 23/20 GO, unlock 26/14 PASS, multistrat 9/20 WAIT + real IBKR subtrack "real -0.77% ·
|
||||
9 trades · 5 holdings".
|
||||
- /paper: IBKR paper account **$1,029,015 · 1 strategy trading · 5 holdings · 0.9% behind plan** (the real
|
||||
DU account — NOT "not connected"). Bybit book $832,534.
|
||||
- Full-page screenshot saved (.playwright-mcp/cockpit-overview-bizworx.png). 1 non-material console error.
|
||||
- Matches DB cross-checks exactly. T9 cockpit verification COMPLETE (visual + numeric).
|
||||
|
||||
## FOXHUNT SHUTDOWN (2026-07-18 ~20:55 UTC) — compute stopped, cluster+PVCs INTACT (not destroyed)
|
||||
Operator decision after verifying migration complete: "shut down everything now, we can spin up if we need to
|
||||
reboot." Verified first: (1) bizworx has ZERO runtime dep on foxhunt (its postgres/gitea/git-sync/registry are
|
||||
all bizworx-local — the `postgres.foxhunt.svc` refs are bizworx's OWN ns foxhunt, different cluster); (2) the
|
||||
pre-cutover state is durably backed up OFF-cluster: `~/Work/fxhnt-cutover-backup/fxhnt_precutover_2026-07-18.dump`
|
||||
(48M, 805 objects, all tables verified via pg_restore --list in the bizworx pod); (3) fund live+serving on
|
||||
bizworx (cockpit 2/2, dagster 3/3, ingress 3/3, ib-gateway 1/1; dashboard.fxhnt.ai HTTP 200).
|
||||
- Scaled ALL foxhunt-ns deploys + statefulsets to 0 (incl postgres/gitea/tailscale-gitlab-proxy = the
|
||||
instant-rollback path — operator accepted dropping it). Also scaled cert-manager + tailscale-subnet-router to 0.
|
||||
- Suspended ALL foxhunt cronjobs (fund + infra: postgres-backup/pvc-autoscaler/billing-scraper). Deleted leftover
|
||||
Completed/Error Job pods so they don't pin nodes.
|
||||
- Pool `platform` (8b520ebb…): autoscaling=true, **min-size=0** (was 1). Can't force size=0 while autoscaling on;
|
||||
the Kapsule autoscaler drains nodes to 0 once only DaemonSets+kube-system remain (~10-15min scale-down delay).
|
||||
Monitoring the 3 dev1_l nodes → 0. `ci-compile-cpu` pool already 0.
|
||||
- **ROLLBACK now = restore the .dump into a fresh pg (few min), NOT an instant DNS flip.** foxhunt cluster + ALL
|
||||
PVCs still EXIST (training-data 500Gi, minio 150Gi, postgres 100Gi, gitlab-gitaly 50Gi, gitea valkey, etc.) —
|
||||
scaling back up reschedules pods on data intact. terraform destroy (T12, irreversible) still DEFERRED.
|
||||
- REVERSIBILITY: `scw k8s pool update 8b520ebb… min-size=1 size=1` + scale deploys back up → foxhunt returns.
|
||||
- STILL PENDING: tomorrow 09:00 UTC bizworx ucits-rebalancer run (real-money confirm); then T12 terraform destroy
|
||||
(with operator confirm). ROTATE the pasted tailscale key.
|
||||
|
||||
## DATABENTO / .dbn DATA MIGRATION (2026-07-18 ~21:10 UTC) — operator caught this before teardown
|
||||
Found databento .dbn data on foxhunt PVCs NOT covered by the Postgres migration. Split cleanly:
|
||||
- **Fund-relevant (migrated):** `/data/surfer/*.dbn` — 22 futures .dbn (ES/NQ/GC/CL/NG/6E/ZC/ZS... universe,
|
||||
updated Jul 12-13) + *_state.json. READ BY the fund's tsmom/carry RESEARCH dagster assets
|
||||
(tsmom_research.py, carry_research.py, assets.py:247 read_front_arrays `$FXHNT_SURFER_DATA_DIR/<sym>.dbn`).
|
||||
bizworx had only an EMPTY fxhnt-surfer-data PVC → research would've failed. **MIGRATED**: tar (33M gz) foxhunt
|
||||
inspector → devbox → bizworx writer pod → extracted into fxhnt-surfer-data. Verified from the dagster pod's
|
||||
OWN view: 22 .dbn at /data/surfer/*.dbn + state JSONs. (Layout note: PVC mounted at /data on dagster,
|
||||
FXHNT_SURFER_DATA_DIR=/data/surfer → files live at PVC-root/surfer/.)
|
||||
- **Archived Rust-ML (NOT fund-used): ~171GB .dbn** on training-data-pvc (59 files, 151GB: ES.FUT MBP10/trades
|
||||
quarterly) + test-data-pvc (21 files, 20GB). Operator decision (option 3): push to PRIVATE Scaleway Object
|
||||
Storage, NOT a bizworx PVC, and "cannot move 175GB over a tunnel" → **upload INTRA-SCALEWAY** (foxhunt pod →
|
||||
s3.fr-par.scw.cloud, both fr-par, no tunnel/devbox hop).
|
||||
- Created private bucket **`fxhnt-dbn-archive`** (fr-par, bizworx project 96d0717d, versioning off, ACL empty).
|
||||
- S3 creds → foxhunt secret `dbn-archive-s3` (from scw config, never printed).
|
||||
- `dbn-uploader` pod (rclone/rclone image) mounts training+test PVCs RO; `rclone copy --include "*.dbn*"
|
||||
--transfers 8` → scw:fxhnt-dbn-archive/{training,test}/. Connectivity tested (rclone lsd rc=0). 171.4GB / 80 files.
|
||||
- Upload running in background in-pod (logs /tmp/rclone-*.log, /tmp/rclone-done sentinel). Monitored.
|
||||
- To restore archived data later: `rclone copy scw:fxhnt-dbn-archive/... <dest>` (creds in bizworx-prod-runtime
|
||||
key or scw config). Kept for the archived Rust codebase per [[project_rust_infra_decommissioned_phase1]].
|
||||
- NOTE: I bumped platform pool min-size back to 1 (from 0) so the inspector/uploader pods can schedule; after the
|
||||
upload completes + surfer verified, set min-size=0 again to resume the drain. Cleanup pods: data-inspector, dbn-uploader.
|
||||
|
||||
## .dbn ARCHIVE — PIVOTED to block snapshots (rclone upload was a dead end) ✅
|
||||
- The rclone S3 upload from the STARVED foxhunt cluster STALLED: dev1_l node egress + slow scw-bssd PVC read
|
||||
(20MB/s) → 76KiB/s, ETA 4 DAYS, byte count frozen. Killed it.
|
||||
- **RIGHT SOLUTION: Scaleway block-volume SNAPSHOTS** (instant, storage-layer, no tunnel, restorable, cheap).
|
||||
The .dbn PVCs are scw block volumes → snapshotted directly:
|
||||
- `fxhnt-training-dbn-archive-2026-07-18` (snap 564a4ccb-ee69-4514-8a64-9932222a0334, 537GB vol = the 151GB
|
||||
training .dbn) — ParentVol cb327a05 (training-data-pvc).
|
||||
- `fxhnt-test-dbn-archive-2026-07-18` (snap 5ba0049f-799a-4694-b7a0-0192cf0cda11, 50GB = the 20GB test .dbn)
|
||||
— ParentVol dc144e62 (test-data-pvc).
|
||||
- BOTH `available` in ~20s, in the **bizworx project 96d0717d** (survive foxhunt cluster teardown), zone fr-par-2.
|
||||
- **RESTORE:** `scw block volume create from-snapshot-id=<snap> zone=fr-par-2` → attach to a pod → read .dbn.
|
||||
- Cleaned up: dbn-uploader + data-inspector pods deleted, dbn-archive-s3 secret deleted.
|
||||
- CORRECTION (2026-07-18 later): there is NO stray junk. The earlier "132MB / 23 objects" were the FIRST partial
|
||||
`.dbn` transfers under `training/` — the resumed upload continues them (rclone skips already-present). Verified:
|
||||
bucket holds ONLY legit `training/*.dbn*` (singletest/ + speedtest/ test files were deleted during testing = 0
|
||||
objects). **DO NOT wipe/delete `fxhnt-dbn-archive` — it IS the active archive we're building.**
|
||||
- Node drain RESUMED (platform pool min-size 1->0; no blocking pods). Autoscaler drops the last node to 0.
|
||||
|
||||
## STATE SUMMARY (2026-07-18 ~21:15 UTC)
|
||||
- fxhnt.ai fund LIVE on bizworx (T8+T9 done, cockpit verified in real Chrome). foxhunt compute → 0 (draining).
|
||||
- Data fully preserved: pg dump (~/Work/fxhnt-cutover-backup), surfer .dbn (migrated to bizworx PVC + verified
|
||||
from dagster pod), archived 171GB .dbn (2 block snapshots in bizworx project). foxhunt PVCs still exist too.
|
||||
- PENDING: tomorrow 09:00 UTC ucits run (real-money confirm); then T12 terraform destroy (operator confirm) —
|
||||
which also removes the source PVCs, but snapshots + dump + bizworx-live cover everything. ROTATE tailscale key.
|
||||
Cleanup the stray fxhnt-dbn-archive bucket.
|
||||
|
||||
## .dbn S3 COPY (2026-07-18 ~21:53 UTC) — operator: also copy compressed .dbn.zst to S3, KEEP snapshots
|
||||
The .dbn.zst are already compressed → S3 copy is a good browsable archive alongside the snapshots (belt+braces).
|
||||
- Measured real S3 throughput: ~10-17 MB/s from these small VMs (dev1_l/gp1_xs) — NOT compression-limited, it's
|
||||
node network egress. Single-file test: 537MiB in 52s = 10MB/s, PVC read 217MB/s (fine). The earlier "0 B/s
|
||||
stall" was rclone stats not updating until a big multipart completed + too many concurrent 64M-chunk streams
|
||||
choking the node. FIX: --transfers 2 --s3-chunk-size 32M --s3-upload-concurrency 2 (node-friendly) → moving.
|
||||
- Restore-snapshot-to-volume path abandoned (Kapsule CSI import of a raw scw volume = rabbit hole). Instead
|
||||
uploading directly from a foxhunt pod (dbn-uploader) that mounts training+test PVCs RO — data still live there.
|
||||
- Upload RUNNING in background: rclone copy .dbn* → scw:fxhnt-dbn-archive/{training,test}/. 137GiB, ETA ~6-7h at
|
||||
~6MB/s. Persistent Monitor watching for completion. Sentinel /tmp/rclone-done in the pod.
|
||||
- KEEP THE SNAPSHOTS (operator) — validated safety net; only after S3 upload verified (object count + sizes) do
|
||||
we consider anything. Both archives coexist.
|
||||
- foxhunt platform pool held at min-size=1 for the uploader; set min-size=0 to resume drain AFTER upload done.
|
||||
- RESUME/CHECK: `kubectl --context $FOX -n foxhunt exec dbn-uploader -- sh -c 'cat /tmp/rclone-done; tail -1 /tmp/rclone-training.log'`
|
||||
and `... rclone size scw:fxhnt-dbn-archive`. If pod died: re-create it (manifest in git history commit 2438294-ish) — rclone copy is resumable/idempotent.
|
||||
|
||||
## .dbn S3 COPY COMPLETE + VERIFIED (2026-07-19 ~02:41 UTC) ✅
|
||||
- Upload finished: **80 objects / 171.393 GiB** in `fxhnt-dbn-archive` (private, fr-par, bizworx project).
|
||||
- VERIFIED byte-for-byte vs source PVCs: `rclone check --size-only --one-way`:
|
||||
- training: **0 differences, 59 matching files** ✅
|
||||
- test: **0 differences, 21 matching files** ✅
|
||||
- source .dbn* count = 80 = bucket object count = exact.
|
||||
- **TWO independent verified archives now exist** (operator: keep snapshots): (1) S3 object storage
|
||||
`s3://fxhnt-dbn-archive/{training,test}/*.dbn.zst` (browsable, per-file restore), (2) block snapshots
|
||||
fxhnt-training/test-dbn-archive-2026-07-18 (564a4ccb / 5ba0049f, bizworx project). Both safe post-teardown.
|
||||
- Cleaned up: dbn-uploader + s3-speedtest pods, dbn-archive-s3 secrets (both clusters). Node drain RESUMED
|
||||
(platform pool min-size 1->0; no blocking pods) → autoscaler drops the last dev1_l to 0. foxhunt compute → ~$0.
|
||||
- REMAINING (next session / operator): tomorrow 09:00 UTC first armed ucits run (real-money confirm); T12
|
||||
terraform destroy foxhunt Kapsule (removes source PVCs — both .dbn archives + pg dump + bizworx-live cover it);
|
||||
rotate the pasted tailscale key. Restore .dbn: `rclone copy scw:fxhnt-dbn-archive/... <dest>` OR
|
||||
`scw block volume create from-snapshot.snapshot-id=<snap> perf-iops=5000 zone=fr-par-2`.
|
||||
|
||||
## SCHEDULING CONSISTENCY — ALL-DAGSTER K8sRunLauncher UPGRADE DONE (2026-07-19) ✅
|
||||
Operator: "go all-Dagster properly + build the image right." Executed the 5-task plan
|
||||
(docs/superpowers/plans/2026-07-19-production-scheduling-consistency.md). Result = consistent single-orchestrator
|
||||
ETL, execution isolated in crons, nothing suspended-by-accident, VRP gone, documented.
|
||||
- DIAGNOSIS (corrected): Dagster ETL was NEVER broken (23:30 ran SUCCESS, forward_nav fresh). The stale
|
||||
bybit_sleeve_ret was the SUSPENDED compare-measured cron (exists because the compute needs 6-8Gi, OOM-killed
|
||||
the 4Gi daemon). Fix = K8sRunLauncher, not naive daemon assets.
|
||||
- BUILT: dagster-k8s dep added; image built via in-cluster **Kaniko** (devbox podman blocked by sandbox mount
|
||||
perms) → `fxhnt-cockpit:dagster-k8s-25e3d7c`, verified + retagged `:latest` (skopeo Job).
|
||||
- RBAC: tailscale-dagster SA can create Jobs/pods/events. K8sRunLauncher configured (launched Jobs git-sync
|
||||
master, no PVC, fund secrets). New asset `bybit_book_precompute` (op_tags 6-8Gi) wraps `_persist_bybit_book`
|
||||
(moved to application/bybit_book_persist.py). MERGED branch→master (git-sync pulls master).
|
||||
- VERIFIED (the money shot): triggered combined_book_forward_job → launched Job dagster-run-* → materialized all
|
||||
14 assets incl the heavy precompute → **daemon 0 restarts** → bybit_sleeve_ret + backtest_summary + forward_nav
|
||||
ALL fresh 2026-07-19. Run SUCCESS 6m54s.
|
||||
- CRONS (final): DELETED compare-measured (→ asset) + vrp (shelved). ARMED factory + ucits-rebalancer +
|
||||
ucits-volume-ibkr. SUSPENDED bybit-rebalancer (no creds) + ibkr-rebalancer (PRIIPs). Manifests updated.
|
||||
- Memory: [[reference_fxhnt_dagster_k8srunlauncher_scheduling]] created; VRP files + MEMORY.md updated.
|
||||
- NOTE: the fund now runs off MASTER (28-commit consolidation branch merged 06156bd; +cron changes 1ecf398).
|
||||
127
docs/superpowers/plans/T8-CUTOVER-RUNBOOK.md
Normal file
127
docs/superpowers/plans/T8-CUTOVER-RUNBOOK.md
Normal file
@@ -0,0 +1,127 @@
|
||||
# T8 — Cutover Runbook (foxhunt → bizworx)
|
||||
|
||||
> Executable, verify-first, reversible. Expands Task 8 of the consolidation plan with the exact values
|
||||
> discovered in T8-prep. **foxhunt stays intact (suspended, NOT deleted) — rollback = re-point DNS +
|
||||
> un-suspend foxhunt.** Fund is ~1×/day; downtime OK. Run in a quiet window.
|
||||
|
||||
## Contexts / constants (verified this session)
|
||||
```
|
||||
FOX=foxhunt-34a1e3c4-ac35-48c8-ab49-5f6ec4df32c1 # foxhunt Kapsule (retiring)
|
||||
BIZ=bizworx-prod-kapsule-05e1dae3-8f35-4ad4-8fce-0f9a7691a760 # survivor
|
||||
```
|
||||
- **foxhunt API is IP-allowlisted.** If `6443` resets: `curl -s ifconfig.me` → compare to
|
||||
`scw k8s acl list cluster-id=34a1e3c4-…`; if drifted, `scw k8s acl add cluster-id=34a1e3c4-…
|
||||
acls.0.ip=<newIP>/32 acls.0.description=bizworx-devbox-vN` (APPENDS — verify all entries survive).
|
||||
bizworx never hits this (tailscale-cgnat range).
|
||||
- **DNS zone:** Scaleway `fxhnt.ai` (project c293eb98). Records set with `scw dns record set dns-zone=fxhnt.ai ...`.
|
||||
|
||||
## The single most important safety fact
|
||||
The **fund pipeline never uses `fxhnt.ai`** — git-sync = `gitea-ssh.gitea.svc.cluster.local:22` (in-cluster),
|
||||
cockpit/dagster serve over ClusterIP + their own tailnet IP. `fxhnt.ai` is **browser/human access only**.
|
||||
→ The fund keeps running through the DNS flip; the DNS cutover only affects the operator's browser access.
|
||||
|
||||
## Current fxhnt.ai ingress (all → 100.95.225.27 = `tailscale-gitlab-proxy` pod, ns foxhunt, DIES at teardown)
|
||||
| host | fronts | cutover |
|
||||
|---|---|---|
|
||||
| git.fxhnt.ai | gitea-web.foxhunt:3000 | → **bizworx gitea** (ns gitea, mirrored) |
|
||||
| dashboard/cockpit.fxhnt.ai | fxhnt-dashboard node → cockpit:8080 | → **bizworx cockpit** |
|
||||
| dagster.fxhnt.ai | dagster.foxhunt:80 | → **bizworx dagster** |
|
||||
| grafana / api / mail / chat / minio | legacy | **DROP** (operator confirmed grafana can go too) |
|
||||
|
||||
Only **git + dashboard/cockpit + dagster** move. Everything else is dropped at Phase-2/teardown.
|
||||
|
||||
## TLS: solved-path (no blocker)
|
||||
Old proxy served a cert-manager wildcard `*.fxhnt.ai` (ClusterIssuer `letsencrypt-prod`, Scaleway DNS-01).
|
||||
**bizworx already has the identical stack** — cert-manager + `scaleway-certmanager-webhook` (1/1) + `scaleway-api`
|
||||
creds + `letsencrypt-prod` ClusterIssuer, already issuing real LE certs in ~10 ns. → Issuing `*.fxhnt.ai` on
|
||||
bizworx is just a `Certificate` resource.
|
||||
|
||||
---
|
||||
|
||||
## PRE-CUTOVER (do these BEFORE the window; all non-destructive, foxhunt untouched)
|
||||
|
||||
### P1. Issue the `*.fxhnt.ai` wildcard cert on bizworx (ns foxhunt) — needs the DNS-01 solver, so do it early
|
||||
- [ ] Create `infra/k8s/services/fxhnt-wildcard-cert.yaml`: a cert-manager `Certificate` (dnsNames
|
||||
`fxhnt.ai`,`*.fxhnt.ai`; issuerRef ClusterIssuer `letsencrypt-prod`; secretName `fxhnt-wildcard-tls`;
|
||||
namespace `foxhunt`). Apply on `$BIZ`.
|
||||
- [ ] **VERIFY:** `kubectl --context $BIZ -n foxhunt get certificate fxhnt-wildcard -o wide` → READY=True.
|
||||
(DNS-01 provisions a TXT on the fxhnt.ai zone; the same zone we control. Wait until True — can take minutes.)
|
||||
- **EVAL P1:** wildcard cert READY on bizworx. foxhunt untouched. If it fails → debug the webhook, do NOT proceed.
|
||||
|
||||
### P2. Decide + build the fxhnt.ai front on bizworx (the thing DNS will point at)
|
||||
Two acceptable shapes — **default = a stripped fund-only nginx proxy** mirroring the foxhunt one (faithful,
|
||||
single tailnet node for DNS, wildcard TLS termination):
|
||||
- [ ] Create `infra/k8s/services/fxhnt-ingress-proxy.yaml` on `$BIZ` ns foxhunt: nginx + tailscale sidecar
|
||||
(`TS_HOSTNAME=fxhnt-ingress`), wildcard `fxhnt-wildcard-tls` mounted at `/etc/nginx/certs`, vhosts:
|
||||
- `git.fxhnt.ai` → `http://gitea-http.gitea.svc.cluster.local:3000` (CONFIRMED svc; headless/ClusterIP:None
|
||||
→ use nginx `resolver` + variable upstream, or proxy_pass by name works via cluster DNS round-robin)
|
||||
- `dashboard.fxhnt.ai cockpit.fxhnt.ai` → `http://fxhnt-dashboard.foxhunt.svc.cluster.local:80` (CONFIRMED)
|
||||
- `dagster.fxhnt.ai` → `http://dagster.foxhunt.svc.cluster.local:80` (CONFIRMED)
|
||||
+ socat sidecar for `git.fxhnt.ai:22` SSH → `gitea-ssh.gitea.svc.cluster.local:22` (CONFIRMED; human git over SSH).
|
||||
- [ ] **VERIFY** (still pre-flip, over tailnet directly to the proxy's tailnet IP, Host header spoofed):
|
||||
`curl -k --resolve dashboard.fxhnt.ai:443:<proxy-tailnet-ip> https://dashboard.fxhnt.ai/ -I` → 200,
|
||||
and cert CN/SAN covers `*.fxhnt.ai`. Repeat for git + dagster.
|
||||
- **EVAL P2:** all 3 fund vhosts serve 200 with a valid `*.fxhnt.ai` cert via the bizworx proxy — WITHOUT any
|
||||
DNS change yet. Record the proxy's tailnet IP for step C4. If any vhost fails → fix before the window.
|
||||
- NOTE: this proxy points at the in-cluster **Services** (`fxhnt-dashboard.foxhunt`, `dagster.foxhunt`), so the
|
||||
"reclaim tailnet identity" rename of the pods is **not required for browser access** — the proxy uses
|
||||
ClusterIP, not the pod's tailnet name. (Identity rename becomes optional cosmetic; keep it out of the
|
||||
critical path.)
|
||||
|
||||
---
|
||||
|
||||
## CUTOVER WINDOW (the only steps that touch live foxhunt — each is reversible)
|
||||
|
||||
### C1. Freeze foxhunt writers (make its Postgres quiescent)
|
||||
- [ ] Suspend every foxhunt fund CronJob:
|
||||
`for cj in $(kubectl --context $FOX -n foxhunt get cronjobs -o name); do kubectl --context $FOX -n foxhunt patch $cj -p '{"spec":{"suspend":true}}'; done`
|
||||
- [ ] Scale down foxhunt writers so nothing writes to its DB during the delta dump:
|
||||
`kubectl --context $FOX -n foxhunt scale deploy/dagster deploy/fxhnt-dashboard --replicas=0`
|
||||
(cockpit is read-only but scale it too to release the `fxhnt-dashboard` tailnet name; dagster daemon writes.)
|
||||
- [ ] Scale foxhunt `ib-gateway` to 0 → **releases the sole IBKR session** so bizworx ib-gateway can hold it:
|
||||
`kubectl --context $FOX -n foxhunt scale deploy/ib-gateway --replicas=0`
|
||||
- **VERIFY C1:** `kubectl --context $FOX -n foxhunt get deploy` all fund writers at 0/0; cronjobs SUSPEND=true.
|
||||
- **ROLLBACK:** scale back up + un-suspend — foxhunt fully restored.
|
||||
|
||||
### C2. Final delta Postgres dump/restore (frozen source → bizworx)
|
||||
Re-run the proven Task-6 procedure on the now-frozen `fxhnt` DB (version-pinned tsdb 2.25.1 both sides).
|
||||
- [ ] `pg_dump -Fc` the `fxhnt` DB from foxhunt postgres (port-forward or exec), `pg_restore --no-owner
|
||||
--no-acl --clean --if-exists` into the bizworx `fxhnt` DB (which currently holds the rehearsal copy →
|
||||
`--clean` refreshes it), wrapped in `timescaledb_pre_restore()` / `post_restore()`.
|
||||
- [ ] **VERIFY C2:** re-check the canonical row counts on bizworx match the (now-frozen) foxhunt source:
|
||||
`forward_nav, backtest_summary, bybit_sleeve_ret, xsp_option_bars, exec_fills, ibkr_account_nav`
|
||||
+ `SELECT max(t0) FROM forward_anchor` matches. 37 tables / 3 hypertables.
|
||||
- **EVAL C2:** every checked count matches the frozen source. If not → STOP, do NOT flip DNS (rollback = C1 up).
|
||||
|
||||
### C3. Bring the bizworx fund fully live (still no DNS)
|
||||
- [ ] `kubectl --context $BIZ -n foxhunt scale deploy/dagster deploy/fxhnt-dashboard deploy/ib-gateway --replicas=1`
|
||||
(they were already deployed; ensure 1 replica). ib-gateway should now stay 1/1 (sole IBKR session).
|
||||
- [ ] **VERIFY C3:** on bizworx — postgres 1/1, cockpit 2/2, dagster 3/3, **ib-gateway 1/1 stable** (no more
|
||||
crashloop now that foxhunt released the session). Confirm ib-gateway API port + one **whatIf** dry-run.
|
||||
- **EVAL C3:** all bizworx fund pods healthy incl. ib-gateway stable. If ib-gateway still crashes → confirm
|
||||
foxhunt ib-gateway is really at 0; do NOT arm any rebalancer.
|
||||
|
||||
### C4. Flip DNS (Scaleway fxhnt.ai zone → bizworx proxy tailnet IP)
|
||||
**TARGET IP (verified in EVAL P2): `100.93.141.11` (tailnet node `fxhnt-ingress`).**
|
||||
- [ ] `scw dns record set dns-zone=fxhnt.ai name=dashboard type=A data=100.93.141.11 ttl=60`
|
||||
- [ ] same for `name=cockpit`, `name=dagster`, `name=git`. (TTL was 300; lower to 60 during cutover.)
|
||||
(Records currently point at 100.95.225.27 = the dying foxhunt tailscale-gitlab-proxy.)
|
||||
- [ ] Leave grafana/api/mail/chat/minio records as-is for now (dropped at Phase-2; harmless dangling until then).
|
||||
- **VERIFY C4:** from an external network (phone off-wifi is fine): `dig +short dashboard.fxhnt.ai` = new IP;
|
||||
open `https://dashboard.fxhnt.ai` in a browser → cockpit renders migrated data, **valid TLS** (LE `*.fxhnt.ai`).
|
||||
`https://git.fxhnt.ai` serves the bizworx gitea; `git ls-remote ssh://git@git.fxhnt.ai/...` works.
|
||||
- **ROLLBACK:** `scw dns record set ... data=100.95.225.27` (back to the still-live foxhunt proxy) + C1-up +
|
||||
un-suspend foxhunt cronjobs. Fully reversible until Phase-2 teardown.
|
||||
|
||||
### C5. Arm the fund on bizworx (start conservative)
|
||||
- [ ] Un-suspend ONLY the UCITS rebalancer on bizworx (already the armed one; US + bybit stay suspended):
|
||||
`kubectl --context $BIZ -n foxhunt patch cronjob fxhnt-ucits-rebalancer -p '{"spec":{"suspend":false}}'`
|
||||
- [ ] **VERIFY C5:** its next scheduled run (09:00 UTC EU-hours) succeeds — a real fill or a clean gate WAIT,
|
||||
not a crash. Watch the Job pod: git-sync OK, DB reachable, ib-gateway reachable, recon-gate evaluated.
|
||||
- **EVAL C5:** first armed run green. If it errors → re-suspend, investigate; foxhunt still intact for rollback.
|
||||
|
||||
---
|
||||
|
||||
## POST-CUTOVER = Task 9 (verify) → Task 10 (observation window) → Task 11 (triage) → Task 12 (teardown)
|
||||
- **Rotate the pasted tailscale auth-key** (was in transcript) once the bizworx pods are settled.
|
||||
- **DELETE NOTHING** on foxhunt until T10 observation passes. foxhunt = suspended-but-intact through T12.
|
||||
@@ -101,6 +101,15 @@ out of 0b scope.
|
||||
`features` table are retired. Verify no residual reader (e.g. `bybit_edges_eval.py`'s Binance imports —
|
||||
remove or repoint) before dropping the table.
|
||||
|
||||
### F. Remove the vestigial registry `state_file` field
|
||||
The `state_file` field on every registry entry (the old JSON-tracker-on-PVC path) is vestigial since the
|
||||
single-DB-SSOT forward engine — `_run_paper_tracker` documents it as "retained ONLY for the one-time
|
||||
migration seed, unused at steady state." Remove the `state_file` field from all registry entries and drop
|
||||
the parameter from `_run_paper_tracker` / any `entry["state_file"]` reads. **Guard:** the JSON-based
|
||||
`ForwardTracker` (+ its `.bak` crash-recovery) is still used by the standalone `fxhnt forward-track` CLI —
|
||||
that command supplies its OWN path and must keep working; only the *registry field* + the Dagster-asset
|
||||
plumbing of it are removed (verify the CLI path is independent before deleting).
|
||||
|
||||
## §7 — 0a interaction (must-carry)
|
||||
0a Task 6 wired the `xsfunding` reconciliation ref via `get_feature_store(s)` (= `features` = **Binance**),
|
||||
deliberately, to match the *then*-live `XsFundingForward(BinanceXsFundingLive())`. 0b **moves xsfunding onto
|
||||
@@ -114,8 +123,6 @@ only inline sleeve sid, xsfunding, becomes a Bybit report-book sid). The 0a inva
|
||||
crash-amplifier finding) — a real-capital portfolio decision, separate from venue cleanup. Flagged for a
|
||||
later, deliberate decision with the crash-protection question (does trend protect out-of-sample?) resolved.
|
||||
- **A Bybit-native stablecoin edge** — fresh research + its own gate, not a migration.
|
||||
- **The vestigial registry `state_file` field** — a "DB-not-files" remnant; confirm include-or-defer during
|
||||
planning (small, orthogonal to venue consolidation).
|
||||
- Alpaca/Bybit rebalancer *code* that is venue-generic and reused by IBKR stays; only Alpaca/Binance-specific
|
||||
execution is removed.
|
||||
|
||||
|
||||
@@ -0,0 +1,267 @@
|
||||
# IBKR Paper Cockpit Parity — Design
|
||||
|
||||
**Date:** 2026-07-14
|
||||
**Status:** Approved (brainstorming) → ready for implementation plan
|
||||
**Branch:** `feat/ibkr-paper-cockpit-parity`
|
||||
|
||||
## Goal
|
||||
|
||||
Bring the IBKR paper strategies to cockpit parity with the Bybit book: show their **backtest** in the sim
|
||||
and their **real paper-account trading** (actual DU-account fills / positions / NAV) across the cockpit —
|
||||
overview (at-a-glance), strategy detail (the slice), and an account-level view (the whole DU pot) — so the
|
||||
paper trading that is already working is *visible and verifiable* at every level, as the honest pre-live
|
||||
confidence path. UX is a first-class requirement, and the design scales to multiple executing strategies by
|
||||
adding a row, not a redesign.
|
||||
|
||||
## Scope
|
||||
|
||||
**In:** (information architecture VALIDATED via UX mockup — consolidated, plain-language, click-through-to-holdings)
|
||||
- D1 — the IBKR strategies' honest-cost backtest curve on the **Backtests** page (both venues, one place).
|
||||
- D2 — capture the **real** IBKR DU paper-account state (fills, positions, NAV), render a **holdings detail
|
||||
page** (what it's holding: assets + shares + value + %; recent trades; value-over-time), and reconcile it
|
||||
against the recomputed (idealized) sim ("how closely the real trades follow the plan").
|
||||
- D3 — surface the executed reality on the **overview**: a "real trades" sub-line under an executing strategy
|
||||
(reusing the console's existing exec-twin sub-row) with a "behind/ahead of plan" pill.
|
||||
- D4 — a **consolidated "Paper books" page** (`/paper`) showing BOTH books side by side — the crypto 4-edge
|
||||
book (simulated) and the IBKR paper account (real) — each a clickable card leading to its holdings detail.
|
||||
The IBKR book's detail carries the per-strategy breakdown that scales by row as more strategies trade.
|
||||
|
||||
**UX rules (VALIDATED, binding):**
|
||||
- **Plain language, no dev jargon** in ANY user-facing copy. Human names only: "Crypto 4-edge book",
|
||||
"Multi-asset ETF portfolio", "Options income (put spreads)", "real trades", "0.3% behind plan", "cost to
|
||||
trade 0.04%", "building", "not trading — options access pending". NEVER surface internal IDs/terms
|
||||
(`bybit_4edge`, `multistrat`, `vrp`, the DU account number, `bps`, "sim", "recompute-replay", "exec-twin").
|
||||
Keep only the fund's own vocabulary (Sharpe, backtest, worst drop). A single `display_name` mapping is the
|
||||
source of every rendered name.
|
||||
- **`REAL PAPER`** badge (blue) marks the one real-execution surface (the IBKR paper account); **`PAPER`**
|
||||
(grey) marks the simulated crypto book. Reuse the existing `.badge-exec` style.
|
||||
- The crypto book and the IBKR account are the SAME kind of thing (paper books you watch) — consolidate onto
|
||||
one Paper-books page; click either for its holdings.
|
||||
|
||||
Designed for scale (UX is a first-class requirement): today only `multistrat` executes on DU (vrp exec is
|
||||
suspended on IBKR options permission; sixtyforty is a benchmark), so the account currently *is* the multistrat
|
||||
book. But ALL captured data is tagged by strategy and the views are built at both the per-strategy AND the
|
||||
account level, so a second executor arming is a new ROW, never a redesign.
|
||||
|
||||
**Out (explicit):**
|
||||
- No live / real-capital execution, no arm step, no promotion→allocation lifecycle changes ("still on paper,
|
||||
no live yet"). Gate/promotion/allocation semantics are UNCHANGED.
|
||||
- Bybit stays exactly as-is (a pure simulated book — no EU Bybit-testnet API keys are available, so there is
|
||||
no real Bybit account to ingest). This design does not touch the Bybit `/paper` book or its sim.
|
||||
|
||||
## Current state (grounded)
|
||||
|
||||
- **Backtesting IS integrated:** `multistrat` has a real 0a `backtest_ref` (Sharpe ~1.67), `vrp` has one; both
|
||||
reconcile against the gate. The `/paper/sim` honest-cost curve, however, reads ONLY `bybit_sleeve_ret`
|
||||
(`app.py` `_SIM_BOOKS = ("bybit_4edge", "bybit_4edge_levered")`, `_sim_returns` →
|
||||
`_paper_repo().bybit_sleeve_returns()`) — the IBKR backtests exist but are not shown there.
|
||||
- **Real paper execution happens but is not captured:** `IbkrBroker` (`adapters/broker/ibkr.py:39-40`) already
|
||||
reads real `accountSummary()` (NLV/cash/gross) + `ib.positions()`. `execute-multistrat` (`cli.py:248`) →
|
||||
`ExecutionService.rebalance` → `multistrat_exec_record.plan_and_record` books an *observed* return for the
|
||||
`multistrat_exec` track. BUT it records the **intended** target book (`weights·nlv/price`), not the actual
|
||||
fills; the `exec_fills` table (`ExecFillRow`: client_id, rebalance_id, symbol, side, qty, price, fee, status,
|
||||
shortfall_bps, at) is **empty**; and there is **no cockpit view** of the DU account's real positions/NAV.
|
||||
|
||||
So this is a faithful EXTENSION of existing machinery, not greenfield: populate `exec_fills`, snapshot the real
|
||||
account NAV, and add the cockpit view + reconciliation.
|
||||
|
||||
## D1 — IBKR backtest curve in `/paper/sim`
|
||||
|
||||
Generalize the sim's data source so a book draws its honest-cost backtest curve from its registry
|
||||
`backtest.kind` engine (the 0a `backtest_ref` dispatch: `recompute-replay` for `multistrat`/`vrp`) instead of
|
||||
only `bybit_sleeve_ret`.
|
||||
|
||||
- **Book list:** replace the hardcoded Bybit-only `_SIM_BOOKS` with a source that includes the IBKR recompute
|
||||
books (`multistrat`, `vrp`) alongside the Bybit books. Each book declares (via the registry) which engine
|
||||
produces its curve.
|
||||
- **Curve source:** `_sim_returns(book)` dispatches on the book's `backtest.kind`: Bybit books keep reading
|
||||
`bybit_sleeve_ret` (unchanged); `recompute-replay` books source their series from the 0a
|
||||
`backtest_ref(sid, ...)` / the recompute engine that already produces their backtest. NO new backtest
|
||||
compute on the request path — read precomputed/cached series as the Bybit path does (persist the IBKR
|
||||
recompute-replay curve where the sim can read it cheaply, mirroring `bybit_sleeve_ret`; the
|
||||
`fxhnt-backtest-refs` Job / the nightly asset is the natural writer).
|
||||
- **UI:** the existing `/paper/sim` book selector renders the added IBKR books; the honest-cost curve + summary
|
||||
stats render identically. No template redesign — a data-source generalization.
|
||||
|
||||
**Boundary:** D1 changes only the sim's book list + curve-source dispatch. It does not touch the gate,
|
||||
`forward_summary`, or the Bybit sim behavior.
|
||||
|
||||
## D2 — Real DU paper-account book + reconciliation
|
||||
|
||||
### D2.1 Capture (extend the exec path)
|
||||
At each `execute-multistrat` run (post-execution, when not blocked by a safety gate), capture the REAL account:
|
||||
- **Fills:** collect the actual fills from the placed trades (ib-async trade `.fills` / `ib.reqExecutions`) and
|
||||
persist to the existing `exec_fills` table — one row per fill (symbol, side, qty, filled price, fee/commission,
|
||||
status, `shortfall_bps` = filled-vs-intended slippage, `rebalance_id`, `at`). This populates the currently-empty
|
||||
table.
|
||||
- **Account snapshot:** persist a per-run snapshot of the real account — `NetLiquidation`, cash, gross position
|
||||
value, and the real held positions (`ib.positions()`) — into a NEW idempotent table `ibkr_account_nav`
|
||||
(run_date PK; nlv, cash, gross, positions_json, at). One row per run_date (idempotent replace), Timescale/
|
||||
Postgres-safe migrate following the `forward_nav`/`strategy_health` create-if-not-exists pattern.
|
||||
- **Reuse:** the broker already reads NLV+positions; add fill collection + the two persistence calls. Do NOT
|
||||
change the `multistrat_exec` return basis (it is gate-relevant, C1-designed) — the capture is additive.
|
||||
|
||||
### D2.2 Holdings detail page
|
||||
Clicking a book (from Paper books or the overview) opens its detail page. For the IBKR paper account book:
|
||||
- **What it's holding right now** — a holdings table: asset (SPY/IEF/…), a plain "what it is" description
|
||||
(US stocks / US Treasuries / Gold / Commodities / Managed futures), shares, value, % of book (from the
|
||||
latest `ibkr_account_nav` positions snapshot).
|
||||
- **Recent trades** — from `exec_fills`: time, asset, action (Buy/Sell), shares, price, "cost to trade"
|
||||
(slippage as a %, NOT bps), fee.
|
||||
- **Value over time** — the **real** account NAV curve (from `ibkr_account_nav`) with the plan it's trying to
|
||||
follow overlaid as a dashed line.
|
||||
Empty/absent data → "no trades recorded yet" (never a 500, never a fabricated number). The crypto book's detail
|
||||
renders its (simulated) holdings the same way.
|
||||
|
||||
### D2.3 Reconciliation — "how closely the real trades follow the plan"
|
||||
The one shared signal, in plain language. Compute the divergence between the **real DU account NAV**
|
||||
(`ibkr_account_nav`, new) and the recomputed idealized sim (`forward_summary`) → a "X% behind/ahead of plan"
|
||||
pill (reuse the existing amber "decaying"/green pill vocabulary). Surface it: a pill on the overview real-trades
|
||||
line, the "Following the plan" card + headline on the book detail, and the per-strategy column on the account
|
||||
detail. This gap — real fill prices, whole-share rounding, cost-to-trade, fees — is the pre-live verification.
|
||||
`multistrat_exec` (the observed envelope-return track) remains the gated exec twin, unchanged.
|
||||
|
||||
## D3 — Overview page: the executed reality at a glance
|
||||
|
||||
The overview (`/`, `dashboard_service.fleet()`/`overview()`) already renders each strategy with gate/health/
|
||||
backtest badges and — for a modeled edge with an executed twin — a nested sub-row. For a strategy executing on
|
||||
the real paper account, render that sub-row as a **"real trades"** line (reuse the existing `.exec-tag` +
|
||||
`tr.exrow` style): what actually filled (real return, trade count, holdings), a **"X% behind/ahead of plan"**
|
||||
pill (the shared D2.3 signal), and a plain "cost to trade / fees" note. Mark the strategy's row with the
|
||||
**`REAL PAPER`** badge. Clicking the strategy (or "N trades · M holdings →") opens its holdings detail (D2.2).
|
||||
No separate rollup card — the consolidated Paper-books page (D4) is the account home.
|
||||
|
||||
## D4 — Consolidated "Paper books" page (`/paper`)
|
||||
|
||||
Replace the current Bybit-only `/paper` landing with a page that shows BOTH books side by side, each a
|
||||
clickable card:
|
||||
- **Crypto 4-edge book** (`PAPER`, simulated) — value / since / Sharpe / worst drop / status, a mini curve.
|
||||
- **IBKR paper account** (`REAL PAPER`, real) — account value / strategies trading / holdings count / the
|
||||
"X% behind plan" pill / a mini curve with the plan overlaid.
|
||||
Both read identically (value, track record, status). Clicking a card opens that book's holdings detail (D2.2).
|
||||
|
||||
The IBKR book's detail page carries the **per-strategy breakdown** (one row per executing strategy: its real
|
||||
return, "behind/ahead of plan", holdings + trade counts) — one row today, N as strategies arm ("scales by row,
|
||||
not redesign"). The account is one pot; the per-strategy rows are its slices, driven by the strategy-tagged
|
||||
capture. A dimmed "not started yet" row illustrates the scale path.
|
||||
|
||||
## UX & information architecture
|
||||
|
||||
Progressive disclosure, one consistent "following the plan" signal at every level, plain language throughout:
|
||||
|
||||
```
|
||||
Overview / -> fleet scan; an executing strategy shows a "real trades" sub-line
|
||||
| + a "X% behind plan" pill (REAL PAPER badge on the row)
|
||||
|
|
||||
Paper books /paper -> BOTH books as clickable cards: Crypto 4-edge (PAPER, simulated)
|
||||
| + IBKR paper account (REAL PAPER, real). Same layout for each.
|
||||
|-- Book detail -> click a book: WHAT IT'S HOLDING (assets + shares + value + %),
|
||||
| recent trades (cost-to-trade %), value-over-time (real vs plan),
|
||||
| and — IBKR book — the per-strategy breakdown (1 row now, N later)
|
||||
|
|
||||
Backtests /paper/sim -> both venues' honest-cost curves in one place (human book names)
|
||||
```
|
||||
|
||||
UX principles (professional ops UI):
|
||||
- **At-a-glance first:** the overview must answer "which strats are actually paper-trading, and is execution
|
||||
tracking the model?" from badges/chips alone — no drill-in required.
|
||||
- **Progressive disclosure:** overview (fleet) → strategy detail (one slice) → account view (the pot). Each
|
||||
level adds resolution, none duplicates another's job.
|
||||
- **One divergence signal, rendered consistently** at all three levels (chip on overview, headline on detail,
|
||||
breakdown on account) — the same computation, the same color language.
|
||||
- **Reuse the existing visual system:** the gate/health badge palette + `exec_badge` macro; do NOT invent a new
|
||||
palette. The EXEC badge and divergence chip are new *states* in the existing badge vocabulary.
|
||||
- **Scale by row, not redesign:** every view renders a *set* of executing strats (size 1 today); adding a strat
|
||||
never changes the layout, only adds a row/chip.
|
||||
- **Honest empty/degraded states:** no captured account yet, or ib-gateway unreachable, renders a clear "no
|
||||
executed data yet" state — never a 500, never a fabricated number. Consistent with the health axis (a stalled
|
||||
exec track already surfaces as STALE).
|
||||
|
||||
## Data model
|
||||
|
||||
- **Reuse** `exec_fills` (`ExecFillRow`) — populate it (currently empty). It already carries `rebalance_id`;
|
||||
ensure each rebalance's `rebalance_id` maps to its `strategy_id` (a `strategy_id` column or a rebalance→strat
|
||||
mapping) so fills are attributable per strategy for D4's breakdown. No breaking schema change (add the tag).
|
||||
- **New** `ibkr_account_nav` (per-run real account snapshot): `run_date` PK, `nlv`, `cash`, `gross`,
|
||||
`positions_json`, `at`. Account-level (the whole DU pot). Idempotent create + replace-per-run; migrate follows
|
||||
the existing `forward_nav`/`paper_repo`/`strategy_health` pattern (no heavy startup DDL).
|
||||
- **Per-strategy executed return** already exists as the `*_exec` observed forward tracks (`multistrat_exec`
|
||||
today; each future executor gets its own, same pattern) — the account view's per-strategy breakdown reads
|
||||
these + the strategy-tagged `exec_fills`. The account NAV rolls up the account; per-strat return stays on the
|
||||
`*_exec` tracks. No new per-strat NAV store.
|
||||
- **No change** to `forward_summary`, `backtest_summary`, `strategy_health`, gate/promotion/allocation.
|
||||
|
||||
## Architecture / components
|
||||
|
||||
- `adapters/broker/ibkr.py` — add fill collection from placed trades; expose fills + the account snapshot to the
|
||||
caller (keep the broker a thin adapter; no persistence here).
|
||||
- `application/multistrat_exec_record.py` (or a small sibling) — persist the captured fills → `exec_fills` and the
|
||||
account snapshot → `ibkr_account_nav`. Additive to `plan_and_record`; gated behind `account_is_paper` (matches
|
||||
the existing paper-only guard). Does not alter the observed-return basis.
|
||||
- `adapters/persistence/` — the `ibkr_account_nav` repo (create/replace/read a NAV series + latest positions),
|
||||
following `forward_nav`/`paper_repo` conventions.
|
||||
- `adapters/web/app.py` + templates — D1 sim book-list/curve-source dispatch; D2 the multistrat-detail "IBKR
|
||||
paper account" section (real NAV curve + positions + fills + reconciliation headline); D3 the overview EXEC
|
||||
badge + divergence chip + account-rollup card (via `dashboard_service.fleet()`/`overview()` + the badge
|
||||
macros); D4 the `/paper/ibkr` account view (total NAV + all positions/fills + per-strategy breakdown table).
|
||||
- `application/dashboard_service.py` — compute the per-strategy exec status + real-vs-sim divergence (one shared
|
||||
function feeding overview chip, detail headline, account breakdown — the "one signal, three places" rule) and
|
||||
the account rollup.
|
||||
- `application/backtest_refs*` / the nightly asset — persist the IBKR recompute-replay curve where the sim can
|
||||
read it cheaply (mirrors `bybit_sleeve_ret`).
|
||||
|
||||
## Error handling
|
||||
|
||||
- Capture is BEST-EFFORT and paper-only: a fills/snapshot persistence failure logs (does not fail the rebalance
|
||||
or block execution) — the rebalance already succeeded; the capture is observability. Follows the existing
|
||||
"ref persist is best-effort" pattern.
|
||||
- Empty/absent data degrades to an empty section (never 500), matching the Bybit `/paper` "missing precomputed
|
||||
table → []" guard.
|
||||
- ib-gateway unreachable at ingest → no snapshot that run; the health axis already surfaces a stalled exec track.
|
||||
|
||||
## Testing
|
||||
|
||||
- **D1:** `/paper/sim` book list includes `multistrat`; `_sim_returns("multistrat")` sources the recompute-replay
|
||||
curve (not `bybit_sleeve_ret`); the sim curve + stats render for an IBKR book (TestClient); Bybit sim
|
||||
unchanged.
|
||||
- **D2.1:** captured fills persist to `exec_fills` with correct fields incl. `shortfall_bps`; the account snapshot
|
||||
persists to `ibkr_account_nav`; capture is paper-only + best-effort (a persistence error does not fail the
|
||||
rebalance; verified via a raising fake).
|
||||
- **D2.2/2.3:** the multistrat detail page renders the real NAV curve, positions, fills; the sim-vs-executed
|
||||
reconciliation computes the divergence on a fixture (known sim series vs known executed fills → expected
|
||||
divergence + median slippage); empty data → empty section, no 500.
|
||||
- **D3:** the shared divergence function returns the expected value on a fixture; the overview renders the EXEC
|
||||
badge + divergence chip for an executing strat and NOT for a pure-sim strat; the account-rollup card shows
|
||||
total NAV + strat count; no-executed-data → no badge/empty card, no 500.
|
||||
- **D4:** `/paper/ibkr` renders total NAV + all positions/fills + a per-strategy breakdown row for `multistrat`;
|
||||
the breakdown is a SET (assert it renders correctly for one strat and would render N — e.g. a two-strat
|
||||
fixture yields two rows); empty account → clean empty state, no 500.
|
||||
- **Scale test:** a fixture with a second (synthetic) executing strat proves overview/account views render both
|
||||
without layout change — the "new row not redesign" invariant.
|
||||
|
||||
## Constraints
|
||||
|
||||
- **Plain language, no dev jargon** in any user-facing copy (see UX rules). A single `display_name` mapping
|
||||
(internal `strategy_id`/book id → human name) is the source of every rendered name — no raw ids, no `bps`,
|
||||
no "sim"/"recompute-replay". `REAL PAPER`/`PAPER` badges as specified.
|
||||
- Paper only — no live/real-capital path, no arm, no gate/promotion/allocation change.
|
||||
- The consolidated `/paper` page shows BOTH books; the Bybit book's underlying data/behavior is untouched
|
||||
(only its landing is now shared with the IBKR book). Bybit stays a simulated book.
|
||||
- DB migrations idempotent + Postgres-safe (no heavy startup DDL — crashloops the cockpit).
|
||||
- Verify the cockpit LOCALLY in a real browser before deploy (read the rendered numbers; never claim "works"
|
||||
off synthetic events).
|
||||
- No silent degradation: capture failures log loudly; a stalled exec track already surfaces via the health axis.
|
||||
|
||||
## Sequencing
|
||||
|
||||
1. **D1** — sim curve for IBKR (low-risk, backtests exist, immediate parity).
|
||||
2. **D2.1** — capture the real account (fills → `exec_fills` tagged by strategy; snapshot → `ibkr_account_nav`).
|
||||
The data foundation everything else reads.
|
||||
3. **D2.2 + D2.3** — the multistrat detail "IBKR paper account" section + the sim-vs-executed reconciliation
|
||||
(build the ONE shared divergence function here).
|
||||
4. **D3** — overview EXEC badge + divergence chip (reusing the shared function) + account-rollup card.
|
||||
5. **D4** — the `/paper/ibkr` account view (total NAV + all positions/fills + per-strategy breakdown).
|
||||
|
||||
Each step is independently testable and shippable; D2.1 is the pivot (it fills `exec_fills` + the account
|
||||
snapshot that D2.2/D3/D4 all render). The shared divergence function (built in step 3) is reused by D3 and D4
|
||||
so the signal is computed once and rendered three places.
|
||||
@@ -0,0 +1,148 @@
|
||||
# Capacity-Aware Capital Allocator — Design
|
||||
|
||||
**Date:** 2026-07-15
|
||||
**Status:** Design. A focused UPGRADE of the existing B2a allocation engine — not a new system.
|
||||
**Author:** jgrusewski (+ Claude)
|
||||
**Builds on:** `docs/superpowers/specs/2026-07-15-crypto-honest-reference-pipeline-design.md` (merged 20437d7).
|
||||
|
||||
## 1. Problem
|
||||
|
||||
We are running a small, growing multi-edge fund across two venues (Bybit crypto sleeves + an IBKR equity
|
||||
book). The honest-reference pipeline (merged) proved the edges are real at low capital but revealed hard
|
||||
**capacity ceilings** — e.g. `xsfunding` is strong at $35k (Sh 3.5) but **breaks by ~$350k–$1M** (Sh → −2.5).
|
||||
The existing allocation engine (`allocation_engine.compute_allocation`, sub-project B2a) is **capacity-blind**:
|
||||
it sizes edges equal-weight (capped at a flat `max_strategy_weight`) + an ex-ante vol-target, with no knowledge
|
||||
that an edge stops working above a dollar figure. A growing fund would keep pouring capital into a
|
||||
capacity-limited edge past its break.
|
||||
|
||||
## 2. Goal
|
||||
|
||||
Make the allocation **capacity-aware and risk-parity weighted**, so the fund allocates its pot correctly across
|
||||
edges within each edge's honest capacity — and, as the pot grows past an edge's ceiling, shifts automatically to
|
||||
the capacity-robust edges. Minimal, SSOT-respecting upgrade of the existing engine; **no new allocation
|
||||
system**. Scope stays PAPER (the `capital_ceiling` human gate + forward-gate promotion already enforce this).
|
||||
|
||||
## 3. What already exists (verified 2026-07-15) — reuse, do not rebuild
|
||||
|
||||
`allocation_ingest.record_allocation(dsn, equity)` already:
|
||||
- allocates over **deploy-tier + forward-promoted** strategies (`evaluate_promotions` promotes on gate PASS) —
|
||||
**this IS the forward-gating**; a not-yet-passed edge gets $0;
|
||||
- is **cross-venue** — `returns = {sid: {date: ret}}` keyed by strategy id spans crypto + IBKR uniformly;
|
||||
- **de-funds decaying edges** (B3 `compute_funding`), applies the **rare drawdown killswitch**, and an **ex-ante
|
||||
vol-target** leverage `K` (`AllocationPolicy`: target_vol 0.12, kelly_fraction 0.5, max_leverage 1.5,
|
||||
cvar_alpha 0.10, `max_strategy_weight` 0.5, `capital_ceiling` $35k) sized from **full-history** vol (never
|
||||
trailing — the anti-reactive invariant);
|
||||
- writes `StrategyAllocationRow(target_weight, target_dollars)` which the rebalancers (`execute-multistrat`,
|
||||
`execute-bybit`) consume to size.
|
||||
|
||||
So the allocator's plumbing, gating, cross-venue combination, decay, killswitch, and $ conversion are DONE.
|
||||
|
||||
## 3.5 Current gate reality (investigated 2026-07-15) — why the sizing basis must change
|
||||
|
||||
Live state of the paper edges:
|
||||
- **Deploy-tier = only `bybit_4edge` (book) + `positioning`** (both crypto, both currently `WAIT`, building
|
||||
8/21 and 8/14 days). `multistrat` (IBKR) is tier=research, `WAIT` 6/20 — it joins the allocation only when
|
||||
promoted. `unlock` is `PASS` but tier=research with no `promote_on_pass`, so it is not in the deploy set
|
||||
(it lives inside the `bybit_4edge` book).
|
||||
- **`strategy_allocation` is currently EMPTY (0 rows).** Root cause: `compute_allocation` requires
|
||||
`min_history_days = 60` FORWARD observations for a stable ex-ante vol, but the 2026-07-14 re-warm reset the
|
||||
forward tracks to ~8 days. 8 < 60 → not eligible → empty allocation. Under the current design the allocation
|
||||
would not populate until ~60 forward days (~mid-September) — far later than the forward gates themselves
|
||||
(14–21 days, late July / early August).
|
||||
|
||||
**Consequence:** sizing off the 8-day forward record is both impossible (too short) and, after any future
|
||||
re-inception, self-defeating. But the honest-reference has 1,290–1,992 days of stable, capacity-adjusted
|
||||
history. So the sizing basis must move to the backtest (§4c).
|
||||
|
||||
## 4. The three changes
|
||||
|
||||
### 4a. Weighting: equal-weight → inverse-vol (risk-parity), capped
|
||||
In `compute_allocation`, replace the equal-weight `base` with **inverse-full-history-vol** weights, still capped
|
||||
at `max_strategy_weight` (the concentration prudence cap) and still NO-renormalise when the cap binds. Rationale
|
||||
(decided with the user): risk-parity gives better risk-adjusted return and is adaptive/not-tuned (reuses the
|
||||
existing `full_history_vol`), but a **firm concentration cap (~0.4–0.5)** is kept because this is a young fund
|
||||
whose edges are not yet forward-proven — do not bet the fund on one edge (the honest-reference's unconstrained
|
||||
risk-parity put 0.89 in `xsfunding`; that is a backtest weight, not a fund weight). Diversify across edges,
|
||||
concentrate only as the forward tracks confirm which edge is real.
|
||||
|
||||
### 4b. Capacity: a per-strategy dollar ceiling from the honest-reference
|
||||
Add a per-strategy `capacity_ceiling[sid]` and cap each edge's dollars at `min(target_dollars,
|
||||
capacity_ceiling[sid])` in `target_capital` (or as a post-step). The ceiling per edge = the AUM at which the
|
||||
honest-reference Sharpe crosses a viability threshold (e.g. the largest AUM on the capacity curve with honest
|
||||
Sharpe ≥ `capacity_min_sharpe`, default 1.0). This makes growth automatic: below the ceiling the cap is
|
||||
non-binding; as equity grows into it, the capped edge holds cash / the allocation shifts to capacity-robust
|
||||
edges. Un-capped (no honest curve) → no ceiling (unbounded), logged.
|
||||
|
||||
### 4c. Sizing basis: the honest-reference backtest, not the 8-day forward record
|
||||
`compute_allocation`'s ex-ante vol (and the new inverse-vol weighting) must be computed from the
|
||||
**honest-reference backtest return series** (capacity-adjusted, at the target AUM — 1,290–1,992 days, stable),
|
||||
NOT from `nav_history` (the forward record, ~8 days post-re-warm → fails `min_history_days`). The **forward
|
||||
gate stays the LIVE-DEPLOY switch**: `record_allocation` still allocates real (paper) dollars only to deploy +
|
||||
forward-promoted edges. So the honest target allocation is computable and visible NOW (backtest-sized +
|
||||
capacity-capped), while capital is only deployed to an edge once its forward gate passes — exactly "honest
|
||||
weights now, live per gate". Implementation: `record_allocation` builds `returns` from the honest-reference
|
||||
series (via a provider) rather than `nav_history`; the `min_history_days` eligibility is then trivially met by
|
||||
the long backtest. Keep `nav_history`-based sizing available behind the provider so a mature forward track
|
||||
(≥ `min_history_days` real days) can later blend/replace the backtest (a documented follow-on, not v1).
|
||||
|
||||
## 5. Granularity & cross-asset capacity (the load-bearing design decision)
|
||||
|
||||
The allocation runs over deploy **strategies** (`bybit_4edge` book, `positioning`, `multistrat`), but the
|
||||
honest-reference capacity is per **sleeve** (`xsfunding`/`positioning`/`unlock`) and is **crypto-specific**
|
||||
(Bybit `turnover` ADV). Resolution for v1:
|
||||
- **Crypto book (`bybit_4edge`):** extend the honest-reference to emit a **book-level** capacity ceiling (run
|
||||
the honest-cost pipeline over the book's constituent sleeves, take the book's honest capacity curve). Its
|
||||
ceiling is the min of the constituents' effective ceilings, roughly (the book breaks when its biggest
|
||||
capacity-limited sleeve does).
|
||||
- **`positioning`** (deploy sleeve, tracked directly): use its own sleeve capacity curve (ceiling >> $10M in the
|
||||
tested range — effectively unbounded at fund scale).
|
||||
- **`multistrat` (IBKR equity):** the crypto ADV model does NOT apply (equities, different data). US-ETF ADV is
|
||||
deep, so v1 assigns a **high default ceiling** (a documented constant, e.g. $10M) and flags that a proper
|
||||
equity-ADV capacity model is a follow-on. Do NOT fabricate a crypto-style number for it.
|
||||
|
||||
This mapping (deploy-strategy → capacity ceiling) is the crux; it must be explicit and auditable, not implicit.
|
||||
|
||||
## 6. Forward-gating (user decision: "honest weights now, live switch per gate")
|
||||
|
||||
Two separate surfaces, already aligned:
|
||||
- **Honest target for ALL edges (now, paper):** the honest-reference report shows the full honest picture across
|
||||
every edge (already built).
|
||||
- **Deployed allocation (live/paper $):** `record_allocation` only allocates to deploy + forward-**promoted**
|
||||
edges. As each edge's forward gate PASSes (`evaluate_promotions`), it enters the allocation and earns capital.
|
||||
Real money follows proven forward performance; nothing here changes that.
|
||||
|
||||
## 7. Scope
|
||||
|
||||
**In:** the two `compute_allocation`/`target_capital` changes; the honest-reference emitting per-deploy-strategy
|
||||
capacity ceilings; wiring the ceilings into `record_allocation`; policy knobs (`capacity_min_sharpe`, the
|
||||
equity default ceiling); tests. **Out (follow-on):** a real equity-ADV capacity model for IBKR; raising the
|
||||
`capital_ceiling` above $35k / going live (a deliberate human gate, per-edge on gate PASS + creds); any new
|
||||
execution path.
|
||||
|
||||
## 8. Critical review — where this could go wrong
|
||||
|
||||
- **Sharpe-4 is not yet forward-proven.** The honest backtest is strong but the forward tracks just re-warmed.
|
||||
The concentration cap (§4a) and forward-gating (§6) are the two guards; do not remove them to chase the
|
||||
backtest weight.
|
||||
- **Capacity ceiling is threshold-sensitive.** "Sharpe ≥ 1.0" is a choice; the curve is steep for `xsfunding`
|
||||
(3.5 → 0.28 → −2.5 across $35k/$350k/$1M), so the ceiling lands ~$350k either way, but report the ceiling
|
||||
next to the curve so the choice is visible, not buried.
|
||||
- **Book-vs-sleeve capacity is an approximation.** The `bybit_4edge` book capacity ≠ a clean sum of sleeve
|
||||
capacities; state the derivation and its limits.
|
||||
- **Equity capacity is a stub in v1** (high default). Be honest that the IBKR ceiling is not yet modeled.
|
||||
- **Sizing from the BACKTEST, not the live record (§4c).** Necessary today (8-day forward is too short), but it
|
||||
means the ex-ante vol/weights reflect the honest backtest, not live performance — the two can diverge. The
|
||||
forward gate (live-deploy switch) + the funding-decay layer are the guards that catch a live divergence; the
|
||||
sizing basis should switch to live vol once a forward track is long enough (documented follow-on). Do not
|
||||
read the backtest-sized weights as "what the live book is doing".
|
||||
- **Still a backtest-derived ceiling on a live allocation.** The capacity cap protects against over-scaling a
|
||||
known-limited edge; it cannot protect against an edge decaying — that is the funding-decay layer's job (which
|
||||
already exists). Don't conflate the two.
|
||||
|
||||
## 9. Decisions locked (from brainstorming)
|
||||
|
||||
- Weighting: **inverse-vol (risk-parity) capped at `max_strategy_weight`**. Sizing basis: **the honest-reference
|
||||
backtest series (§4c), NOT the 8-day forward record**; forward gate = live-deploy switch only. Forward-gating:
|
||||
**honest weights shown now for all edges; deployed $ only to forward-promoted edges** (existing). Capacity:
|
||||
**per-deploy-strategy ceiling from the honest-reference (crypto), high default for equity (v1)**. Scope:
|
||||
**paper**, an upgrade of the B2a engine, no new system.
|
||||
@@ -0,0 +1,172 @@
|
||||
# Honest-Reference Pipeline for the Bybit Crypto Book — Design
|
||||
|
||||
**Date:** 2026-07-15
|
||||
**Status:** Design (research/validation scope). Production gate-replacement is an explicit follow-on, NOT this spec.
|
||||
**Author:** jgrusewski (+ Claude)
|
||||
|
||||
## 1. Problem
|
||||
|
||||
The Bybit 4-edge book is graded (recon gate) and sized against a backtest that is **optimistic on
|
||||
exactly the days that drive its numbers**. Three linked modeling problems, all quantified 2026-07-15:
|
||||
|
||||
1. **Tail/capacity inflation.** Return is concentrated in a handful of microcap-pump days. Top-10 days
|
||||
(of ~1,300–2,000) drive **58%** of `unlock`'s return, **29%** of `positioning`'s. Under a P98 upside
|
||||
winsorize, `unlock` collapses (Sharpe 1.03 → **0.19**), `positioning` holds a real core (2.62 → 2.02),
|
||||
`xsfunding` is untouched (3.61 → 3.05). The equal-weight book drops from Sh 2.17 → 1.71.
|
||||
2. **Leverage modeling.** The "levered" backtest shows a *lower* maxDD (−11.8%) than the naive book
|
||||
(−15.4%) — impossible from leverage; it is `carry-static` **re-weighting** flattering the curve, plus an
|
||||
implausibly low modeled vol (13.7% for a 1.5× crypto book). The smooth modeled drawdown understates the
|
||||
real fat-tail path.
|
||||
3. **Cost/slippage.** A flat 5.5 bps applied to all history is a *current-liquidity snapshot* on 2021–22
|
||||
thinner markets, and — critically — is most wrong on the pump days (blown-out spreads), i.e. exactly the
|
||||
return-driving days.
|
||||
|
||||
All three inflate the **same** thing: the tail days. "The forward reconciles with the backtest" is only
|
||||
meaningful if the backtest is honest on those days. It is not.
|
||||
|
||||
## 2. Goal
|
||||
|
||||
Build a **capacity-, cost-, and leverage-honest reference** for the crypto book at the coin level, and use
|
||||
it to: (a) decide the honest sleeve **allocation**; (b) set an **ex-ante risk-management** layer; (c)
|
||||
**falsification-test** candidate edges — including an "anomaly/pump-detection" edge — through the same
|
||||
honest-cost machinery. **No live cockpit change in this spec** (see §9 scope).
|
||||
|
||||
Success = we can state, per sleeve and for the book, at a target AUM: honest **net** Sharpe / CAGR / vol /
|
||||
maxDD, the honest allocation, the **capacity curve** (edge vs AUM), the **leverage ceiling**, and a
|
||||
pass/fail verdict on the candidate pump edge — all reproducible from one pipeline.
|
||||
|
||||
## 3. Data feasibility (verified 2026-07-15)
|
||||
|
||||
- **Volume/ADV:** `bybit_features` feature `turnover` = USD volume per coin per day, **2021-01-01 →
|
||||
2026-07-14, 818 coins** (BTCUSDT ~$5.2 bn/day sanity-checks). → real ADV capacity + market-impact model.
|
||||
- **Spread:** `bybit_real_spread(coin, spread_bps, fetched_at)` is a **single current snapshot**, NOT a
|
||||
daily history. Usable as a per-coin slippage floor; the pump-day spread must be **inferred**, not measured
|
||||
(§8 — the largest honesty gap).
|
||||
- **Sleeve returns (aggregated):** `bybit_sleeve_ret(run_date, sleeve, ret)` — the *input* we are replacing
|
||||
with a coin-level recompute.
|
||||
- **Coin features:** `bybit_features` also has `close/high/low/funding/open_interest/long_ratio/worst_basis`
|
||||
per coin — enough to reconstruct sleeve coin-weights.
|
||||
|
||||
## 4. Architecture
|
||||
|
||||
Instead of taking the aggregated `bybit_sleeve_ret` as given, **recompute each sleeve at the coin level**,
|
||||
apply per-coin capacity + impact, re-aggregate, then risk-parity into a book, then grade against it.
|
||||
|
||||
```
|
||||
coin features ──▶ [0a sleeve coin-weights] ──▶ [0b ADV capacity cap] ──▶ [0c impact slippage]
|
||||
│
|
||||
honest daily sleeve returns ◀────────────────────────┘
|
||||
│
|
||||
[2 risk-parity weights] ──▶ honest book ──▶ [1 falsification gates]
|
||||
│ │
|
||||
[3 ex-ante risk layer] ┘ [4 candidate edge #4 through the same pipeline]
|
||||
```
|
||||
|
||||
## 5. Phase 0 — Honest-reference pipeline
|
||||
|
||||
- **0a — Coin-level sleeve recompute.** Surface each sleeve's *daily target coin weights* from the existing
|
||||
sleeve logic (`sleeve_returns_from_store` / the per-sleeve strategy). **Validation gate:** the recompute,
|
||||
*before* any haircut, must reproduce the stored `bybit_sleeve_ret` within tolerance (else the honest
|
||||
numbers are apples-to-oranges). If a sleeve's logic is not cleanly separable, that is a task blocker to
|
||||
surface, not paper over.
|
||||
- **0b — Capacity model.** For each coin-day, cap the position at `min(target_notional,
|
||||
participation_cap × turnover_coin_day)`. **Parameterize by AUM** → output the **capacity curve** (honest
|
||||
Sharpe vs AUM). `participation_cap` = **3%** (a single global knob, documented; not per-sleeve tuned).
|
||||
**AUM series = {$35k, $100k, $350k}** (the realistic growth path) **+ {$1M, $10M}** (anchor points to
|
||||
*locate the ceiling* — at $35k–350k the 3% cap likely never binds on liquid coins, so the curve may be
|
||||
flat there; the high anchors reveal where the edge actually breaks). Report the full curve.
|
||||
- **0c — Slippage model.** Market-impact per coin-position: `slippage_bps = k · sqrt(order_notional /
|
||||
turnover_coin_day)` (square-root impact law), floored at that coin's quoted spread from
|
||||
`bybit_real_spread`. `k` is one global constant, calibrated to the snapshot spreads (not per-sleeve).
|
||||
Applied to each coin-position's traded notional.
|
||||
- **0d — Re-aggregate** to honest daily sleeve returns → honest sleeve stats (Sharpe/CAGR/vol/maxDD, net of
|
||||
capacity + cost, at the target AUM).
|
||||
|
||||
## 6. Phase 1 — Falsification gates (predetermined, before looking)
|
||||
|
||||
- **G1 (edge survives):** each sleeve's honest net Sharpe > **1.0** at the target AUM. (`unlock` expected to
|
||||
fail — do not rescue it.)
|
||||
- **G2 (honest LOO):** each sleeve adds marginal Sharpe to the honest **risk-parity** book (drop-one test).
|
||||
- **G3 (beats concentration):** honest book Sharpe > the best single sleeve (diversification is real, not a
|
||||
label). Prior estimate: xsf+pos ≈ 3.14 > xsfunding-alone 3.06 — must survive the coin-level model.
|
||||
- **G4 (anti-reactive risk-management):** any proposed *de-risk* rule must beat "do nothing" in an honest
|
||||
A/B, else it is excluded. This gate exists specifically to keep reactive de-sizing out (§7).
|
||||
|
||||
## 7. Phase 2+3 — Allocation & ex-ante risk-management layer
|
||||
|
||||
**Phase 2 (allocation)** = the risk-parity/inverse-vol row below: sleeve weights are set by honest vol, not
|
||||
by an in-sample Sharpe grid; `unlock` receives near-zero weight naturally if G1/G2 reject it.
|
||||
|
||||
Risk management is **baked into sizing**, not a reactive overlay bolted on top. Reactive de-sizing
|
||||
(vol-targeting, drawdown-triggered cuts) has been **A/B-rejected 3× on this book** (pro-cyclical, cuts
|
||||
before the bounce, vol-shrinkage not return) — see `pearl_fxhnt_reactive_risk_controllers_rejected`,
|
||||
`pearl_fxhnt_voltarget_overlay_is_reactive_desizing_rejected`. That is out of scope by construction; G4
|
||||
enforces it.
|
||||
|
||||
| Layer | Ex-ante hendel | Source |
|
||||
|---|---|---|
|
||||
| Position | per-coin **ADV cap** (hard backstop) **and** impact-slippage penalty (soft — sizing self-adjusts) | 0b/0c |
|
||||
| Sleeve | **risk-parity / inverse-vol** on honest vol (adaptive, not tuned) | §2 decision |
|
||||
| Book | **leverage ceiling anchored to the honest real DD** — O3: −9.7% @1×, ~linear → ~**1.5–1.75×** with a 2× stress buffer under a −35% hard limit. This is ex-ante K(D*). | O3 |
|
||||
| Tail | `unlock` dropped/minimized per honest LOO (dead-weight fat-downside tail) | G1/G2 |
|
||||
| Regime | **rare killswitch** for genuine structural breaks only (existing accepted design) | existing |
|
||||
|
||||
## 8. Phase 4 — Candidate edge #4: anomaly / pump-detection
|
||||
|
||||
The pump days that inflate the backtest are also a *hypothesised* edge: detect the anomaly, profit from it.
|
||||
**Recognition ≠ profit** — the gap is the slippage/tail we just measured, and the recent precedent is
|
||||
sobering: `unlock` *is* an explicit "detect the scheduled anomaly and short it" edge, and it collapses under
|
||||
the honest haircut. P&D is a harder version (less predictable, worse execution, adversarial,
|
||||
short-borrow-constrained). It is the **maximally tail-exposed** strategy — the opposite of §7.
|
||||
|
||||
Therefore: run it through the **same honest-cost pipeline** as a candidate, with predetermined skepticism:
|
||||
- **G-detection:** a detectable, **out-of-sample** pump signal (volume/price/OI anomaly), not in-sample fit
|
||||
and not look-ahead (the signal must be knowable *before* the move it trades).
|
||||
- **G-execution (the killer):** the edge survives the **real pump-day slippage** (§0c) + any borrow cost. ~90%
|
||||
of such ideas die here.
|
||||
- **G-marginal:** it adds Sharpe *beyond* what `positioning` already harvests diffusely (same source →
|
||||
otherwise duplicate risk).
|
||||
|
||||
Verdict rule: passes all three → a **separate, hard-capped** sleeve (never risk-parity-core weight; too
|
||||
tail-heavy). Fails → excluded, documented. Expectation is high bar / likely fail — but cheap to falsify on
|
||||
the pipeline we build anyway.
|
||||
|
||||
## 9. Scope & deliverables
|
||||
|
||||
**In scope (this spec):** the Phase-0 pipeline, the capacity curve, the honest allocation recommendation,
|
||||
gate verdicts G1–G4, the leverage-ceiling number, and the candidate-edge-#4 verdict. Pure research output.
|
||||
|
||||
**Out of scope (follow-on spec):** replacing the cockpit `backtest_summary` reference / recon-gate,
|
||||
reweighting the live book, changing any live sizing. This spec must not alter live behavior.
|
||||
|
||||
**Deliverables:** a reproducible report (numbers + gate verdicts) + the pipeline code + tests. No cockpit
|
||||
diff.
|
||||
|
||||
## 10. Critical review — where this could still fool us (be skeptical)
|
||||
|
||||
- **Recompute fidelity (0a).** If the coin-weight reconstruction doesn't reproduce `bybit_sleeve_ret`
|
||||
pre-haircut, every honest number is invalid. This is the load-bearing validation; treat a mismatch as a
|
||||
hard stop.
|
||||
- **Slippage is modeled, not measured.** We have no historical spread — only a snapshot. The pump-day
|
||||
slippage (the crux of all three problems) is *inferred* via the impact law + snapshot floor. This is the
|
||||
biggest remaining honesty gap; the spec must state the slippage results as *model-dependent* and run a
|
||||
sensitivity (k × {1, 2, 4}) rather than a single number.
|
||||
- **Survivorship.** If delisted microcaps (pumped-then-died coins) are absent from `bybit_features`, the
|
||||
pump edge is survivorship-inflated even after haircut. **Verify** `bybit_features` is survivorship-free
|
||||
(memory `project_fxhnt_bybit_ingestion_and_book` claims yes — confirm, don't assume).
|
||||
- **AUM relevance.** At $35–100k the ADV cap barely binds — the honest picture is dominated by *slippage +
|
||||
repeatability*, not capacity. The capacity model's real value is the **future-growth ceiling**, not
|
||||
today's haircut. Say this plainly; don't oversell capacity at current size.
|
||||
- **Still a backtest.** The honest reference de-inflates *known* problems; it cannot de-inflate unknown
|
||||
future regime change. The **forward gate remains the real arbiter** — this improves the reference it
|
||||
grades against, nothing more.
|
||||
- **In-sample weighting risk.** Risk-parity uses realized vol; on the same sample it can still overfit.
|
||||
Weights must be set on a rule and validated on a held-out window, not grid-searched on Sharpe.
|
||||
|
||||
## 11. Decisions locked (from brainstorming)
|
||||
|
||||
- Capacity: **B** (coin-level ADV/impact), `participation_cap` **3%**, AUM curve **{35k, 100k, 350k, 1M,
|
||||
10M}**. Weighting: **risk-parity/inverse-vol**. Scope: **research now**.
|
||||
- Exit-liquidity: **hard ADV cap + slippage penalty** (backstop + adaptive). `unlock`: **not hard-cut** —
|
||||
let G1/G2 decide (expected ≈ 0). Edge #4: **candidate, gated**, separate capped sleeve if it survives.
|
||||
- Anti-reactive risk-management enforced by **G4**.
|
||||
@@ -0,0 +1,213 @@
|
||||
# Deploy-the-Fund Knobs — CLI-configurable capital + risk, per-sleeve allocation — Design
|
||||
|
||||
**Date:** 2026-07-16
|
||||
**Status:** Design. A focused UPGRADE of the merged capacity-aware allocator — not a new system.
|
||||
**Author:** jgrusewski (+ Claude)
|
||||
**Builds on:** `docs/superpowers/specs/2026-07-15-capacity-aware-capital-allocator-design.md` (merged) and
|
||||
`docs/superpowers/specs/2026-07-15-crypto-honest-reference-pipeline-design.md` (merged 20437d7).
|
||||
|
||||
## 1. Problem
|
||||
|
||||
The nightly allocation deploys only ~$8,840 of the $100k paper equity (≈9% — 91% cash). Two structural
|
||||
causes, both confirmed by an in-cluster smoke:
|
||||
|
||||
1. **The marginal-prune (leave-one-out) gate keeps only the volatile `positioning` sleeve** and drops the
|
||||
diversified `bybit_4edge` book: `survivors_after_marginal_prune:[positioning]` (the book adds no marginal
|
||||
Sharpe — it *contains* positioning plus lower-Sharpe drag sleeves).
|
||||
2. **The conservative 12% vol-target de-levers `positioning` hard**: its honest full-history vol is ~34% →
|
||||
`K = kelly 0.5 × target_vol 0.12 / 0.34 ≈ 0.18`, base capped at `max_strategy_weight 0.5` → deployed
|
||||
weight ≈ 0.088.
|
||||
|
||||
Compounding this: the fund's **capital and risk knobs are hardcoded defaults** — `paper_capital` in
|
||||
`config.py` and `target_vol` / `kelly_fraction` in `allocation_policy.py`. Changing them requires an env-var
|
||||
edit + redeploy. A small, actively-managed, growing fund needs to (a) tune its capital + risk appetite live
|
||||
via CLI, and (b) allocate across the individual capacity-bounded crypto sleeves rather than double-counting a
|
||||
book that overlaps a sleeve it already deploys.
|
||||
|
||||
## 2. Goal
|
||||
|
||||
Two coupled changes, paper scope:
|
||||
|
||||
- **Fund-config knobs (CLI-settable, single SSOT):** the fund's `capital`, `target_vol`, and `kelly_fraction`
|
||||
become a single persisted DB record, set via `fxhnt fund-config`, read by the nightly allocation — live, no
|
||||
redeploy, not hand-editable.
|
||||
- **Per-sleeve crypto allocation:** the allocation set switches from the `bybit_4edge` book (double-counted
|
||||
with the `positioning` sleeve) to the individual crypto sleeves (`positioning`, `xsfunding`, `unlock`), each
|
||||
sized within its own honest capacity ceiling. The book stays a tracked *reference* forward track, not an
|
||||
allocation target.
|
||||
|
||||
The forward gate (live-deploy switch) and the `capital_ceiling` human gate are unchanged. This is a minimal,
|
||||
SSOT-respecting upgrade — **no new allocation system**.
|
||||
|
||||
## 3. What already exists (verified 2026-07-16) — reuse, do not rebuild
|
||||
|
||||
- `allocation_ingest.record_allocation(dsn, equity, *, store, spread_bps, unlock_events)` allocates over
|
||||
`deploy_sids = {sid : tier == "deploy"} | repo.promoted_strategy_ids()`, honest-sizes via
|
||||
`allocation_honest_inputs.honest_allocation_inputs` when `store` is injected (else nav fallback), applies the
|
||||
killswitch on the forward record, de-funds decayers, and writes `StrategyAllocationRow`.
|
||||
- `allocation_engine.compute_allocation` does eligibility → LOO marginal-prune → inverse-vol (risk-parity)
|
||||
base capped at `max_strategy_weight` → killswitch → ex-ante vol-target `K` (kelly × target_vol /
|
||||
full-history-vol, CVaR-floored, max_leverage-capped). `target_capital` converts to dollars with per-strategy
|
||||
capacity ceilings + the `capital_ceiling` gross cap.
|
||||
- `allocation_honest_inputs.honest_allocation_inputs` maps each deploy sid → its honest-reference series +
|
||||
capacity ceiling. Today `_CRYPTO_DEPLOY_SIDS = {"positioning", "bybit_4edge"}`; `_deploy_series` special-cases
|
||||
those two; every other sid gets `equity_default_ceiling` and no series. `_sleeve_series` already computes ANY
|
||||
sleeve's honest series (it is called per-sleeve inside `_book_series`).
|
||||
- `config.py`: `paper_capital = 100_000`, `capital_ceiling_paper = 1_000_000` (the $35k figure is
|
||||
`capital_ceiling_live` — the LIVE gate, not paper). `allocation_policy.py`: `AllocationPolicy(target_vol
|
||||
0.12, kelly_fraction 0.5, max_leverage 1.5, max_strategy_weight 0.5, capital_ceiling 35_000 default,
|
||||
min_history_days 60, ...)`; `make_policy()` picks the paper/live ceiling from settings;
|
||||
`ALLOCATION_POLICY_VERSION = 3`.
|
||||
- `cockpit_forward` (Dagster asset, `assets.py:1046`) is where the nightly allocation runs:
|
||||
`equity = get_settings().paper_capital` → `record_allocation(...)`, wrapped in a graceful try/except that
|
||||
falls back to the nav path on any store failure.
|
||||
- The registry (`registry.py`) drives both the allocation set (`tier`) and the cockpit grouping. `bybit_4edge`
|
||||
is `tier: deploy`; `positioning` is `tier: deploy`; `xsfunding` / `unlock` are `tier: research`;
|
||||
`bybit_4edge_exec` / `bybit_4edge_levered` are already `tier: research`. `multistrat` uses
|
||||
`promote_on_pass: True` to auto-flip research→deploy on gate PASS (`forward_ingest.evaluate_promotions`).
|
||||
|
||||
So the plumbing, honest-sizing seam, killswitch, capacity ceilings, and $ conversion are DONE. This spec adds a
|
||||
config SSOT + read path, and re-points the crypto allocation from the book to its sleeves.
|
||||
|
||||
## 4. Design
|
||||
|
||||
### 4a. `fund_config` — the single SSOT for the tunable knobs
|
||||
|
||||
A **single-row** table (id fixed to 1), migration-created:
|
||||
|
||||
```sql
|
||||
CREATE TABLE IF NOT EXISTS fund_config (
|
||||
id INTEGER PRIMARY KEY CHECK (id = 1),
|
||||
capital DOUBLE PRECISION NOT NULL,
|
||||
target_vol DOUBLE PRECISION NOT NULL,
|
||||
kelly_fraction DOUBLE PRECISION NOT NULL,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
```
|
||||
|
||||
A small module `application/fund_config.py`:
|
||||
|
||||
- `@dataclass(frozen=True) FundConfig(capital: float, target_vol: float, kelly_fraction: float)`.
|
||||
- `read_fund_config(dsn) -> FundConfig` — reads row id=1. **Graceful fallback**: if the table or row is
|
||||
absent, returns `FundConfig(capital=settings.paper_capital, target_vol=AllocationPolicy().target_vol,
|
||||
kelly_fraction=AllocationPolicy().kelly_fraction)` — byte-identical to today's behaviour, so the nightly
|
||||
never breaks on a fresh DB (mirrors the existing store-fallback pattern in `cockpit_forward`).
|
||||
- `write_fund_config(dsn, *, capital=None, target_vol=None, kelly_fraction=None)` — **upsert with partial
|
||||
override**: reads the current effective config, overlays only the provided fields, writes row id=1. So
|
||||
`set --capital 250000` changes only capital and leaves vol/kelly as they were.
|
||||
- **Validation** (fail loud on nonsense — every input validated at the boundary): `capital > 0`,
|
||||
`0 < target_vol <= 1.0`, `0 < kelly_fraction <= 1.0`. Reject with a clear message, do not silently clamp.
|
||||
|
||||
### 4b. `fxhnt fund-config` CLI (typer, `cli.py`)
|
||||
|
||||
- `fxhnt fund-config show` — prints the effective config (capital, target_vol, kelly_fraction) and whether it
|
||||
came from the DB row or the settings fallback (so the operator can see "is this the default or a set value").
|
||||
- `fxhnt fund-config set [--capital FLOAT] [--target-vol FLOAT] [--kelly FLOAT]` — validates, upserts, prints
|
||||
the new effective config. At least one flag required (a no-op `set` errors).
|
||||
|
||||
### 4c. Wire the nightly to read `fund_config`
|
||||
|
||||
In `cockpit_forward` (`assets.py`), replace `equity = get_settings().paper_capital` with
|
||||
`cfg = read_fund_config(dsn); equity = cfg.capital`, and pass the risk knobs down so `record_allocation`
|
||||
builds its `AllocationPolicy` with `target_vol = cfg.target_vol`, `kelly_fraction = cfg.kelly_fraction` (all
|
||||
other policy fields unchanged, via `make_policy()` then a dataclass `replace`). `record_allocation` gains an
|
||||
optional `policy: AllocationPolicy | None = None` parameter (default `None` → `make_policy()`, byte-identical
|
||||
to today); `cockpit_forward` passes the fund-config-derived policy. Both the honest path and the nav-fallback
|
||||
path use the same policy.
|
||||
|
||||
### 4d. Per-sleeve allocation — resolve the book/sleeve overlap
|
||||
|
||||
Three coordinated changes:
|
||||
|
||||
1. **Registry (`registry.py`):**
|
||||
- `xsfunding`: `tier: research` → **`promote_on_pass: True`** (NOT a hard flip to deploy). It enters the
|
||||
allocation when its reconciliation gate PASSes — respecting the forward gate as the deploy switch, exactly
|
||||
as `multistrat` does. Same for **`unlock`**.
|
||||
- `bybit_4edge`: `tier: deploy` → **`tier: research`**. The book becomes a tracked *reference* aggregate
|
||||
(it already sits alongside its `bybit_4edge_exec` / `bybit_4edge_levered` research twins), no longer an
|
||||
allocation target. This removes the positioning/book double-count. `positioning` stays `tier: deploy`.
|
||||
- Net deploy set today: `{positioning}` + any of `{unlock, xsfunding}` whose gate has PASSed. As each
|
||||
sleeve gate passes, the allocation gains that sleeve — a gated per-sleeve rollout.
|
||||
|
||||
2. **Provider (`allocation_honest_inputs.py`):** generalize the crypto mapping so ANY crypto sleeve is
|
||||
honest-sized (not just `positioning`):
|
||||
- `_CRYPTO_DEPLOY_SIDS = frozenset({"positioning", "xsfunding", "unlock"})` (drop `bybit_4edge` — it is no
|
||||
longer in the deploy set; `_book_series` / `_BOOK_SLEEVES` stay for the reference report, unused by the
|
||||
allocation path).
|
||||
- `_deploy_series(store, sid, ...)`: for `sid in {"positioning", "xsfunding", "unlock"}` →
|
||||
`_sleeve_series(store, sid, ...)` (the existing per-sleeve engine); else `{}`. The `sid == "bybit_4edge"`
|
||||
book branch is removed from the allocation path.
|
||||
|
||||
3. **No execution change today.** Both rebalancers are SUSPENDED (no live crypto creds). The per-sleeve
|
||||
allocation writes per-sleeve `StrategyAllocationRow`s (paper/display sizing). When crypto execution is later
|
||||
armed, `execute-bybit` must size the book as `Σ(sleeve target_dollars)` over the deploy sleeves — a
|
||||
documented **follow-on**, out of scope here (nothing executes today, so no live risk).
|
||||
|
||||
### 4e. Seed the initial `fund_config` row (migration data step)
|
||||
|
||||
Insert row id=1: `capital = 100_000` (current), `target_vol = 0.20` (the chosen risk appetite — moderately
|
||||
conservative), `kelly_fraction = 0.5` (unchanged). Idempotent (`ON CONFLICT (id) DO NOTHING`) so a re-run does
|
||||
not clobber a value the operator has since set via CLI.
|
||||
|
||||
## 5. Falsification gate — does per-sleeve actually deploy more? (the honest check)
|
||||
|
||||
Per-sleeve is **not guaranteed** to raise deployment. The LOO marginal-prune gate — correctly — drops sleeves
|
||||
that do not add marginal Sharpe, and the diversifier-hunt (2026-07-15) already found **`unlock` fails the LOO
|
||||
gate** (co-crash / immaterial). So the deploy set may honestly collapse toward `{positioning}` or
|
||||
`{positioning, xsfunding}`. This spec does NOT weaken or remove the marginal gate to force diversification —
|
||||
co-crashers *should* be pruned.
|
||||
|
||||
**Predetermined gate (run in-cluster after implementation, before any go-live):**
|
||||
|
||||
- **G1 (survival):** report `survivors_after_marginal_prune` for the per-sleeve deploy set on the honest
|
||||
series. Expected: `{positioning}` and, if it adds marginal Sharpe, `{xsfunding}`; `unlock` expected pruned.
|
||||
- **G2 (deployment):** report the resulting deployment % of equity at `target_vol = 0.20`. Compare against the
|
||||
positioning-alone baseline (8.84% at 12% vol). The honest lever ranking is: **vol-target 0.12→0.20 raises
|
||||
deployment regardless of survival; per-sleeve raises it further ONLY if xsfunding survives LOO.**
|
||||
- **G3 (no reactive de-sizing):** confirm the ex-ante `K` is still computed from full-history vol (never
|
||||
trailing) — the anti-reactive invariant is untouched.
|
||||
- **Verdict rule:** if the deploy set collapses to `{positioning}` alone, the honest conclusion is
|
||||
*concentrate + use the vol-target as the deployment lever* (per-sleeve added nothing) — report that plainly,
|
||||
do NOT override the gate. If `{positioning, xsfunding}` survives, per-sleeve is the diversified win.
|
||||
|
||||
## 6. Scope
|
||||
|
||||
**In:** the `fund_config` table + migration + seed row; `application/fund_config.py` (read/write/fallback +
|
||||
validation); `fxhnt fund-config show|set` CLI; `record_allocation` optional `policy` param + `cockpit_forward`
|
||||
wiring; the registry tier/promote changes; the provider crypto-sleeve generalization; the falsification-gate
|
||||
in-cluster run + report; tests; local cockpit verification that `bybit_4edge`-as-reference renders sanely.
|
||||
|
||||
**Out (follow-on):** `execute-bybit` per-sleeve book sizing (execution is suspended); a real equity-ADV
|
||||
capacity model for IBKR; raising `capital_ceiling_live` above $35k / going live (the deliberate human gate);
|
||||
any new execution path; making `max_strategy_weight` / `max_leverage` / `capital_ceiling` CLI-tunable (only
|
||||
capital + target_vol + kelly are in scope — the three the user is actively choosing).
|
||||
|
||||
## 7. Critical review — where this could go wrong
|
||||
|
||||
- **Per-sleeve may not diversify** — §5 is the guard. `unlock` is a known LOO failure; do not force it in. The
|
||||
gate reports the truth; accept it.
|
||||
- **`bybit_4edge` → research is a cockpit change.** The fund's crypto headline book moves from the deploy
|
||||
section to research (with its exec/levered twins). Verify locally that the cockpit renders this without a
|
||||
broken headline or an empty deploy section before merge (the deploy section then shows the sleeves).
|
||||
- **Promoting sleeves via `promote_on_pass`, not a hard tier flip.** This respects the forward gate. It means
|
||||
the per-sleeve allocation is *gradual* (sleeves join on gate PASS), not instant — a feature, not a bug, but
|
||||
state it so "why is xsfunding not allocated yet" is answerable (its gate is WAIT 8/14).
|
||||
- **20% vol is a real risk increase** over 12%. At risk-parity the low-vol `xsfunding` (if it survives)
|
||||
dominates the base weight (capped 0.5) and `K` can run toward `max_leverage 1.5`. Report the actual
|
||||
deployed gross + implied book vol in the G2 run so the number is visible, not assumed.
|
||||
- **`fund_config` fallback must be byte-identical** to today when the table/row is absent — a fresh DB or a
|
||||
failed read must not change the allocation. Test this explicitly (fallback == settings default).
|
||||
- **Partial-override upsert** must not reset unspecified fields to defaults (a classic upsert bug):
|
||||
`set --capital X` must leave target_vol/kelly at their current values. Test it.
|
||||
- **Single-writer / idempotent seed:** the seed uses `ON CONFLICT DO NOTHING`; a re-applied migration must not
|
||||
overwrite an operator-set value.
|
||||
|
||||
## 8. Decisions locked (from the design dialogue)
|
||||
|
||||
- Knobs: **capital + target_vol + kelly_fraction**, one persisted `fund_config` DB row, CLI-set only (never
|
||||
hand-edited), read by the nightly with a settings fallback. Initial: **capital $100k, target_vol 0.20, kelly
|
||||
0.5**.
|
||||
- Per-sleeve: **promote `xsfunding` + `unlock` via `promote_on_pass`; demote `bybit_4edge` to a research
|
||||
reference track; generalize the provider to honest-size any crypto sleeve.** Marginal gate KEPT.
|
||||
- A **predetermined falsification gate** (§5) decides whether per-sleeve actually deploys more; the honest
|
||||
verdict is accepted either way. Scope stays **paper**; execution wiring is a follow-on.
|
||||
@@ -0,0 +1,81 @@
|
||||
# Equity Leg Deployable — Design
|
||||
|
||||
**Date:** 2026-07-16
|
||||
**Status:** Design. Makes the IBKR US-ETF `multistrat` book an honest-sized, capacity-bounded deploy edge — the growth vehicle above the crypto book's ~$350k break.
|
||||
**Author:** jgrusewski (+ Claude)
|
||||
**Builds on:** the capacity-aware allocator + honest-reference pipeline (crypto) + the load-once/reprice provider (merged 2026-07-16).
|
||||
|
||||
## 1. Problem
|
||||
|
||||
The fund's crypto book caps at ~$350k (measured; xsfunding-driven). To grow past that toward $1M the **IBKR
|
||||
US-ETF `multistrat` book** must carry the additional capital. But `multistrat` is NOT deployable today:
|
||||
1. **It can't be honest-sized.** The crypto provider (`honest_allocation_inputs`) gives equity sids no series,
|
||||
so `record_allocation` falls back to `multistrat`'s ~7-day forward NAV record, which is < `min_history_days`
|
||||
(60) → it is dropped from the allocation entirely (gets $0).
|
||||
2. **Its capacity is a $10M stub** (`equity_default_ceiling`) — not a real, data-derived ceiling.
|
||||
|
||||
User decision (2026-07-16): build it the rigorous way — **ingest real ETF volume** for a data-driven ADV
|
||||
capacity model, and charge **real trading costs** (not the inflated 15bp research stub).
|
||||
|
||||
## 2. Goal
|
||||
|
||||
An **equity honest-reference** analog (mirroring the crypto one) that produces, for `multistrat`:
|
||||
- a **cost-netted honest backtest sizing series** (~441 days on Yahoo PIT → passes `min_history_days`), and
|
||||
- a **real ADV-derived capacity ceiling** (from ingested ETF volume),
|
||||
merged into `record_allocation` so the equity book is honest-sized + capacity-bounded exactly like crypto.
|
||||
Scope stays PAPER; the forward gate (`multistrat` promote_on_pass, currently 7/20) remains the live-deploy
|
||||
switch. As equity grows into it, the capacity-aware allocator already shifts capital correctly.
|
||||
|
||||
## 3. What exists (verified 2026-07-16) — reuse
|
||||
|
||||
- `multistrat.book_series(closes, ...)` → the GROSS unlevered daily book return (~441d). `multistrat.target_weights(closes, ...)` → the per-instrument position vector per day (for turnover). `FUND_INSTRUMENTS = [SPY, IEF, GLD, PDBC, DBMF]`.
|
||||
- `equity_backtest_runner._turnover(prev, cur)` (two-way round-trip turnover) + its `cost_bps_per_turnover` idiom — the cost-charging pattern to mirror (but with REAL, per-instrument costs, not a flat 15bp).
|
||||
- Yahoo ingestion: `YahooDailyClient.adj_closes(sym)` extracts `ind["quote"][0].close`/`adjclose`; the SAME Yahoo response already carries `ind["quote"][0].volume` — volume is fetchable by extending the extraction, no new data source. The nightly asset (`assets.py:408-421`) snapshots `adj_closes` into `YahooPitCloseRow` via `snapshot_yahoo_closes`; `YahooPitClient.adj_closes` reads it back PIT.
|
||||
- `honest_reference.honest_sleeve_returns` (turnover-basis cost, sqrt-impact on ADV) + `capacity_curve` + `allocation_engine.capacity_ceiling_from_curve` — the crypto templates to mirror for equity.
|
||||
- `record_allocation` merges `honest_returns` (provider) with a nav fallback; the load-once provider pattern.
|
||||
|
||||
## 4. Design
|
||||
|
||||
### 4a. Ingest ETF volume (the real ADV input)
|
||||
- `YahooDailyClient.volumes(symbol) -> dict[str, float]`: extract `ind["quote"][0]["volume"]` (raw daily share volume), same fetch/retry path as `adj_closes`.
|
||||
- `YahooPitVolumeRow` (cockpit_models, `yahoo_pit_volume`, cols: symbol, date, volume, first_seen_at) + `ForwardNavRepo.snapshot_yahoo_volumes` / `yahoo_pit_volumes(symbol)` — PIT store parallel to closes (append-only, first-seen frozen — same PIT discipline).
|
||||
- Nightly asset (`assets.py`, next to the close snapshot): also `snapshot_yahoo_volumes(sym, live.volumes(sym), at)`.
|
||||
- **Backfill**: a `fxhnt backfill-etf-volume` CLI — one Yahoo fetch per FUND_INSTRUMENT (25y range), snapshot the volume history, so ADV is available immediately (not only forward). PIT: first-seen frozen.
|
||||
- **Dollar-ADV**: `adv_usd[sym][day] = adj_close[sym][day] * volume[sym][day]`, then a trailing **median** over a window (e.g. 63d) for a stable, outlier-robust ADV (median, not mean — a single volume spike must not inflate capacity). Missing volume → that day excluded (no fabricated ADV).
|
||||
|
||||
### 4b. Real equity cost model (`equity_honest.py` — the equity analog of honest_reference)
|
||||
Per instrument, per rebalance day, charge on TRADED weight (turnover-basis, mirroring the crypto model):
|
||||
- **Market impact** = `IMPACT_COEFF_BPS * impact_k * sqrt(traded_notional / adv_usd)` bps — REAL, from ingested ADV. This is what makes capacity bite: as AUM grows, `traded_notional` rises → impact rises → Sharpe degrades.
|
||||
- **Spread** = a realistic PER-ETF half-spread constant (bps), documented from published typical ETF spreads (SPY ~0.3, IEF ~1.0, GLD ~1.0, PDBC ~4.0, DBMF ~8.0 bps). This is the ONE estimated input (no ETF bid/ask feed); it is a documented constant, NOT the inflated flat 15bp. `slip_bps = max(impact, spread_bps[sym])` (same max-with-spread-floor as crypto).
|
||||
- **Commission** = IBKR real per-share (tiered ~ $0.0035/share, min/max) converted to bps at the day's price: `commission_bps = 100 * per_share / price`. Charged per traded share.
|
||||
- `equity_honest_returns(book_ret_by_day, weights_by_day, adv_usd, spread_bps, *, aum, participation_cap, impact_k, commission_per_share) -> dict[str,float]`: for each day, net = `book_ret[day] - Σ_sym traded_w * (slip_bps + commission_bps)/1e4`, where `traded_w = |w[sym,day] - w[sym,day-1]|`. Also apply the **participation cap** to positions (a position that would exceed `participation_cap * adv_usd` is scaled down — capacity, not just cost), consistent with `honest_sleeve_returns`.
|
||||
|
||||
### 4c. Equity capacity curve + ceiling
|
||||
- `equity_capacity_curve(book_ret, weights, adv_usd, spread_bps, *, aums, ...)` → `{aum: equity_honest_returns(...at aum)}` (load-once: book_ret/weights/adv computed once, repriced per AUM — mirrors the crypto load-once).
|
||||
- Ceiling = `capacity_ceiling_from_curve({aum: sharpe_of(series)}, min_sharpe)` — the largest AUM whose net Sharpe stays viable. For these liquid ETFs the ceiling is high but REAL (DBMF/PDBC ADV-limited); report it next to the curve.
|
||||
|
||||
### 4d. Equity provider + wiring
|
||||
- `equity_allocation_inputs(repo, deploy_sids, *, target_aum, aums, participation_cap, impact_k) -> (returns_by_strategy, ceilings)`: for each equity deploy sid (venue `ibkr-equity`, i.e. `multistrat`), build closes+volume from the PIT store, compute `book_series`/`target_weights`, then the cost-netted series at `target_aum` (ISO-keyed) + the ADV capacity ceiling. Non-equity sids ignored (crypto provider handles them).
|
||||
- `record_allocation`: when `store is not None`, ALSO call `equity_allocation_inputs` for the equity deploy sids and MERGE: `honest_returns |= eq_returns`, `ceilings |= eq_ceilings`. Then the existing `returns = {sid: honest_returns.get(sid) or nav_fallback}` picks up `multistrat`'s 441-day cost-netted series → eligible (>60) → sized; `target_capital` caps it at the real ADV ceiling. `store=None`/no-equity → byte-identical to today.
|
||||
|
||||
## 5. Falsification / validation gates (in-cluster, after build)
|
||||
- **G1 (sizes):** `multistrat` now has ≥ `min_history_days` in the allocation returns (not dropped). Report its cost-netted Sharpe/vol vs the gross book_series (costs must REDUCE Sharpe, not increase it).
|
||||
- **G2 (real capacity):** the ADV ceiling is finite and data-derived (report it + the tightest-ETF binding). Sanity: it must be well above the fund's current $100k and reflect DBMF/PDBC being the tightest.
|
||||
- **G3 (cost realism):** the equity cost must be materially BELOW the old 15bp stub for the liquid ETFs (SPY/IEF/GLD) — report the effective bps/turnover; if it lands ~15bp we mis-modeled.
|
||||
- **G4 (combined marginal gate):** with `multistrat` sized, re-run the marginal gate for {crypto book, multistrat}. Report whether the uncorrelated equity book SURVIVES alongside the crypto book (it should add marginal Sharpe vs the crypto-book-alone, unlike vs volatile positioning-alone) and the combined deployment. Accept the honest verdict.
|
||||
- **PIT integrity:** the volume snapshot is first-seen-frozen (a re-fetch must not rewrite history).
|
||||
|
||||
## 6. Scope
|
||||
**In:** volume extraction + PIT store + snapshot + backfill CLI; dollar-ADV (median); `equity_honest.py` (real cost model + capacity curve); `equity_allocation_inputs` + `record_allocation` merge; tests; in-cluster gates.
|
||||
**Out (follow-on):** live intraday spread/quote feed for ETFs (spread stays a documented per-ETF constant); options/VRP equity; raising the live `capital_ceiling` / going live (human gate); execute-side IBKR sizing beyond what already exists.
|
||||
|
||||
## 7. Critical review — where this could go wrong
|
||||
- **Spread is the one estimated input.** No ETF bid/ask feed → per-ETF published constants. Document them; they dominate cost for liquid ETFs at fund scale (impact ~0 there), so a wrong spread mis-sizes. Keep them conservative-realistic and visible, not buried.
|
||||
- **book_series is gross + unlevered.** Confirm the cost netting uses the SAME weights `book_series` applies (via `target_weights` = `L·tw·scale`), else cost/return bases mismatch. The turnover must be the ACTUAL traded weight, not the raw signal.
|
||||
- **ADV median window + PIT.** A too-short ADV window is noisy; too-long lags real liquidity. 63d median is a choice — report it. Volume PIT-frozen so a Yahoo revision can't retroactively inflate past capacity.
|
||||
- **Capacity for liquid ETFs is high → the ceiling may not bind until >$1M.** That is correct (deep ETFs), but it means the equity ceiling won't shape sizing at $100k–$350k — it is the GROWTH guard, not a near-term constraint. State it.
|
||||
- **Combined Sharpe dilution (§5 G4).** Equity is lower-Sharpe; the blended book Sharpe drops as equity's weight grows. The marginal gate + risk-parity handle WHETHER to include it; do not force it if it fails the gate against the crypto book.
|
||||
- **Equity turnover cost realism.** multistrat is adaptive (daily reweight) → non-trivial turnover; the cost model must charge it honestly (two-way turnover) or it flatters the equity Sharpe.
|
||||
|
||||
## 8. Decisions locked
|
||||
- **Ingest real ETF volume** (extend the Yahoo extraction + PIT snapshot + backfill) for a data-driven dollar-ADV (63d median). **Real costs**: sqrt-impact on real ADV + per-ETF documented spread + real IBKR commission (NOT the flat 15bp). Sizing series = `book_series` NET of these. Provider merged into `record_allocation`; forward gate unchanged; PAPER scope. Live spread feed = follow-on.
|
||||
@@ -0,0 +1,185 @@
|
||||
# Foxhunt → Bizworx Platform Consolidation — Design
|
||||
|
||||
**Date:** 2026-07-18
|
||||
**Status:** DRAFT (awaiting operator review)
|
||||
**Goal:** Retire the separate **foxhunt** Scaleway Kapsule cluster by moving the live fxhnt fund onto the
|
||||
**bizworx** Kapsule cluster and consolidating to a **single gitea**, then destroying the foxhunt Kapsule —
|
||||
eliminating a whole managed cluster + its nodes (cost saving).
|
||||
|
||||
> Written durably to git because this session suffered heavy context loss (a stale 2026-07-13 replay + a
|
||||
> dev-machine move). Ground truth in this doc was verified live against both clusters on 2026-07-18.
|
||||
|
||||
---
|
||||
|
||||
## 1. Current state (verified live 2026-07-18)
|
||||
|
||||
Two Scaleway Kapsule clusters, **same org (`a55098a3`), different projects**, both `fr-par`, k8s 1.34.6, cilium:
|
||||
|
||||
| Platform | Kapsule | Project | Nodes | Role |
|
||||
|---|---|---|---|---|
|
||||
| **bizworx** (survivor) | `05e1dae3` | `96d0717d` | 3× gp1_xs (+large) | MSP platform (msp-portal, tacticalrmm, argocd, vaultwarden, gitea…) |
|
||||
| **foxhunt** (to retire) | `34a1e3c4` | `c293eb98` | 3× dev1_l (+ci 0-4) | the fxhnt fund (ns `foxhunt`) + lots of legacy |
|
||||
|
||||
**Access note (resolved):** the foxhunt Kapsule API has an IP allowlist (`acl_available: true`). After the
|
||||
dev-machine move, this box's egress `51.158.111.225` was not allowed → HTTP 000. Fixed by appending it to the
|
||||
ACL (`scw k8s acl add … bizworx-devbox-v4`; existing tailscale-cgnat + admin entries preserved). kubeconfig
|
||||
re-installed via `scw k8s kubeconfig install 34a1e3c4…`. Both contexts now in `~/.kube/config`.
|
||||
|
||||
**xps-ubnt** (tailnet `100.91.132.125`) = the OLD dev machine. k3s inactive; runs some Docker (a dev
|
||||
`foxhunt-postgres` Timescale, a buildx builder, unrelated lexpdf/PDF tools). NOT the fund host — just where
|
||||
the foxhunt kubeconfig still lived. Not part of this migration except as a source of the old kubeconfig.
|
||||
|
||||
### 1a. What actually runs in ns `foxhunt` (empirical)
|
||||
|
||||
**Live (READY=1) — the fund's real footprint:** `postgres`, `dagster`, `ib-gateway`, `fxhnt-dashboard`
|
||||
(cockpit), `gitea`, `argo` (workflows + events + gitea push/deploy sensors), plus `minio`, `redis`,
|
||||
`grafana`/`prometheus` (the last four are NOT fund dependencies — see below).
|
||||
|
||||
**Scaled to zero / dead (READY none) — legacy foxhunt, Phase-2 triage:** `api`, `backtesting-service`,
|
||||
`broker-gateway`, `data-acquisition-service`, `ml-training-service` (old Rust microservice architecture),
|
||||
`kanidm` (SSO), `mattermost`, `stalwart` (mail), `netbird-*`, `questdb`, `loki`, `pushgateway`.
|
||||
|
||||
### 1b. Stateful data — what migrates vs drops (empirical: mounted-by-running-pod)
|
||||
|
||||
| PVC | Size | Mounted by | Decision |
|
||||
|---|---|---|---|
|
||||
| `postgres-pvc` | 100Gi | postgres | **MIGRATE** (operational DB + Timescale warehouse = the SSOT) |
|
||||
| `ib-gateway-settings` | 1Gi | ib-gateway | migrate OR re-auth fresh (trivial) |
|
||||
| `fxhnt-surfer-data` | 5Gi | dagster | holds only `unlock_calendar.json` (DB-first + JSON fallback) → copy the file or let it rebuild |
|
||||
| `gitea-shared-storage` | 5Gi | gitea | migrates via the gitea repo-migration (§3), not as a raw PVC |
|
||||
| `feature-cache-pvc` | 100Gi | — (orphan) | DROP (rebuildable cache) |
|
||||
| `fxhnt-backtest-data` | 20Gi | — (orphan) | DROP (B3 DuckDB warehouse, parked in 0a) |
|
||||
| `questdb-pvc` | 10Gi | — (questdb dead) | DROP |
|
||||
| `multistrat-state` | 1Gi | — (orphan) | DROP (vestigial state_file; forward state is DB-SSOT) |
|
||||
| `minio-data-new` | 150Gi | minio | Phase-2 triage (NOT a fund dep — `grep` of fxhnt src/manifests finds no minio/s3/redis runtime use) |
|
||||
| `training-data-pvc` | 500Gi | — | Phase-2 (decommissioned Rust ML) → DROP |
|
||||
| `repo-data-gitlab-gitaly-0` | 50Gi | — | Phase-2 (legacy GitLab, replaced by gitea) → DROP |
|
||||
| `kanidm/mattermost/stalwart/netbird/test-data/tempo/…` | ~120Gi | — | Phase-2 triage (drop unless a non-fund need surfaces) |
|
||||
|
||||
**Net fund data to physically migrate ≈ 100Gi (one Postgres), everything else is drop-or-rebuild.**
|
||||
"fxhnt heeft minder nodig" — confirmed.
|
||||
|
||||
### 1c. Gitea reality
|
||||
|
||||
- `git.fxhnt.ai` → `100.95.225.27` (tailnet node `foxhunt-gitlab`) → the **foxhunt-cluster gitea** (repos
|
||||
`gitadmin/fxhnt`, `gitadmin/foxhunt`). Canonical today.
|
||||
- **bizworx** already runs its own gitea (ns `gitea`, 50Gi, ClusterIP, no ingress).
|
||||
- The fund's in-cluster git-sync clones from `gitea-sshd.foxhunt.svc.cluster.local:22`.
|
||||
|
||||
### 1d. IaC form
|
||||
|
||||
Two layers, both in the fxhnt repo: **Terragrunt/Terraform** (`infra/live/production/{kapsule,
|
||||
public-gateway,block-storage}`, scw S3 tfstate backend) for the cloud infra (the Kapsule itself), and
|
||||
**k8s manifests** (`infra/k8s/{argo,databases,gitea,jobs,monitoring,…}`) applied via **argo** (git-sync from
|
||||
gitea; `reference_fxhnt_prod_deploy_mechanics`: `argo submit … -p mode=deploy`, NOT image-baked).
|
||||
|
||||
---
|
||||
|
||||
## 2. Target architecture (after Phase 1)
|
||||
|
||||
The fund runs in **ns `foxhunt` on the bizworx Kapsule** (name reused → minimal manifest churn):
|
||||
- Workloads: `postgres` (Timescale), `dagster`, `ib-gateway`, `fxhnt-dashboard`, the rebalancer/backtest
|
||||
CronJobs, git-sync from the **bizworx gitea**.
|
||||
- **Reuse bizworx's existing `argo` + observability** (do NOT migrate prometheus/grafana/loki/tempo).
|
||||
- Single **bizworx gitea** holds `gitadmin/fxhnt` + `gitadmin/foxhunt` (archive); `git.fxhnt.ai` re-points to it.
|
||||
- `dashboard.fxhnt.ai` ingress → bizworx (ACME TLS, per `reference_fxhnt_dashboard_proxy`).
|
||||
|
||||
Decisions locked with the operator (2026-07-18):
|
||||
- **Survivor gitea = bizworx**; migrate both `fxhnt` + `foxhunt` (archive); keep hostname `git.fxhnt.ai`,
|
||||
re-point to bizworx (no remote-URL changes in repos/CI).
|
||||
- **ib-gateway:** no 2FA, no IP allowlist on the IBKR side → re-auth = IBKR paper creds in a secret + reconnect.
|
||||
- **Downtime:** flexible (fund runs ~once/day) — schedule a convenient quiet window; no tight SLA.
|
||||
- Reuse bizworx argo + observability.
|
||||
|
||||
---
|
||||
|
||||
## 3. Phase 1 — migration mechanics
|
||||
|
||||
### 3.1 Prep bizworx (no downtime)
|
||||
- Create ns `foxhunt` on bizworx; confirm a suitable StorageClass (bizworx uses `sbs-5k`/`sbs-default`;
|
||||
foxhunt used `sbs-default`/`scw-bssd`).
|
||||
- Recreate secrets on bizworx (source via bizworx external-secrets + openbao where possible): `db-credentials`,
|
||||
`databento-credentials`, IBKR/IB-gateway creds, `scw-registry` (image pull), `argo-git-ssh-key` /
|
||||
`git-ssh-key`, tailscale auth. **Never** commit secrets; use `! kubectl`/openbao, never the transcript.
|
||||
- Migrate gitea repos: create `gitadmin/fxhnt` + `gitadmin/foxhunt` on the bizworx gitea and
|
||||
`git push --mirror` (or gitea's built-in migration) all refs/LFS. Verify commit counts + tags match.
|
||||
- Re-target the k8s manifests' cluster/namespace/gitea endpoint to bizworx; `kubectl apply --dry-run` on the
|
||||
bizworx `foxhunt` ns to validate before cutover.
|
||||
|
||||
### 3.2 Cutover (the downtime window)
|
||||
1. **Freeze foxhunt:** `kubectl patch cronjob … suspend:true` for all fund CronJobs; scale dagster/cockpit
|
||||
writers down so Postgres is quiescent.
|
||||
2. **Migrate the DB:** `pg_dump` the foxhunt Postgres (`fxhnt` DB, all schemas incl. TimescaleDB hypertables —
|
||||
use `pg_dump`-with-timescaledb guidance / `timescaledb_pre_restore`/`post_restore`) → `pg_restore` into the
|
||||
bizworx Postgres. Verify row counts on the key tables (`forward_nav`, `backtest_summary`, `bybit_sleeve_ret`,
|
||||
`xsp_option_bars`, the feature tables).
|
||||
3. **Stand up the fund on bizworx:** apply the re-targeted manifests; bring up postgres (restored),
|
||||
dagster, cockpit, ib-gateway (re-auth the IBKR paper session).
|
||||
4. **Re-point DNS:** `git.fxhnt.ai` and `dashboard.fxhnt.ai` → bizworx (tailnet node / ingress). Fund git-sync
|
||||
now resolves the bizworx gitea `gitea-sshd`.
|
||||
5. **Copy the small bits:** `unlock_calendar.json` (or let it rebuild from the DB-first loader);
|
||||
ib-gateway-settings if not re-authing fresh.
|
||||
|
||||
### 3.3 Verify (before decommissioning anything)
|
||||
- Cockpit renders real data — **locally in a real browser** (`reference_fxhnt_local_cockpit_dev_loop`) AND on
|
||||
`dashboard.fxhnt.ai`; read the numbers (`feedback_verify_cockpit_locally_not_prod`).
|
||||
- A `fxhnt-ucits-rebalancer` **dry-run** on bizworx fills/plans correctly against IBKR (EU hours).
|
||||
- Reconciliation gate + forward tracks intact (the migrated `forward_nav`/`backtest_summary` reconcile;
|
||||
no false re-inception — `forward_engine` recompute is deterministic from the DB anchor).
|
||||
- Dagster nightly asset graph runs green on bizworx.
|
||||
- Run **in parallel/observe** for a short window (foxhunt suspended, bizworx live) before teardown.
|
||||
|
||||
### 3.4 Rollback
|
||||
Until the foxhunt Kapsule is destroyed (Phase 3), rollback = re-point DNS back + un-suspend foxhunt CronJobs
|
||||
(its Postgres still holds the pre-cutover state). The foxhunt cluster is left intact-but-suspended through the
|
||||
observation window precisely to keep this cheap.
|
||||
|
||||
---
|
||||
|
||||
## 4. Phase 2 — triage the rest of ns `foxhunt`
|
||||
|
||||
Produce a definitive **keep/drop** list for every non-fund service + orphan PVC, so the teardown is
|
||||
accountable (nothing silently lost). Expected outcome (from §1a/1b, mostly DROP):
|
||||
- **DROP (confirmed dead):** the Rust-era microservices (api/backtesting-service/broker-gateway/
|
||||
data-acquisition-service/ml-training-service), `questdb`, `loki`, `pushgateway`, `training-data-pvc` (500Gi),
|
||||
`repo-data-gitlab-gitaly-0` (50Gi), `feature-cache`, `fxhnt-backtest-data`, `multistrat-state`, `test-data`.
|
||||
- **DECIDE (non-fund, may still be wanted somewhere):** `minio` (150Gi), `redis`, `kanidm` (SSO),
|
||||
`mattermost`, `stalwart` (mail), `netbird`. For each: drop, or migrate to bizworx / keep external. The
|
||||
operator confirms per item before its data is deleted (`feedback` — look before deleting).
|
||||
|
||||
## 5. Phase 3 — teardown (cost saving)
|
||||
|
||||
After Phase-1 verification holds and the Phase-2 list is resolved:
|
||||
- Remove the foxhunt k8s manifests; delete the foxhunt Kapsule via **Terragrunt** (`infra/live/production/
|
||||
kapsule` for project `c293eb98`) — control plane + the 3 dev1_l nodes + the ci pool.
|
||||
- Release orphaned block volumes (retain-class PVCs: `gitea-shared-storage`, `feature-cache` are
|
||||
`*-retain` → must be deleted explicitly), the load balancer, the private network, and the foxhunt-project
|
||||
scw resources.
|
||||
- Remove the temporary foxhunt API ACL entry for this box (or keep if still useful).
|
||||
- Update memory: mark the foxhunt Kapsule retired; the fund now lives on bizworx.
|
||||
|
||||
---
|
||||
|
||||
## 6. Risks / open items
|
||||
- **TimescaleDB dump/restore** needs the hypertable-aware procedure (`timescaledb_pre_restore()` /
|
||||
`timescaledb_post_restore()`), not a naive `pg_dump`. Verify the extension version matches on bizworx.
|
||||
- **ib-gateway** must re-establish its IBKR paper session on bizworx (no 2FA/IP-limit per operator, so
|
||||
expected simple — but confirm it connects + a whatIf order is accepted before arming the rebalancer).
|
||||
- **Secrets provenance:** confirm every secret can be re-sourced on bizworx (openbao/external-secrets) before
|
||||
cutover — a missing secret wedges a workload.
|
||||
- **Cross-project scw:** LB/DNS/tailnet re-pointing spans the two scw projects; confirm the tailnet ACLs +
|
||||
ingress + ACME cert issuance work from the bizworx side.
|
||||
- **Deploy pipeline:** the fund deploys via argo git-sync from gitea (`mode=deploy`); confirm the bizworx argo
|
||||
+ the migrated gitea + the eventsource/sensor wiring reproduce the deploy flow.
|
||||
|
||||
## 7. Success criteria
|
||||
- The fxhnt fund runs entirely on the bizworx Kapsule (cockpit, dagster, ib-gateway, rebalancers) against the
|
||||
migrated Postgres, verified in a real browser + a live rebalancer dry-run.
|
||||
- One gitea (bizworx) serves `git.fxhnt.ai` with both repos; fund git-sync + CI unchanged.
|
||||
- The foxhunt Kapsule + its nodes are destroyed; no fund data or wanted service lost (Phase-2 list resolved).
|
||||
- Monthly scw cost drops by the foxhunt cluster's control-plane + 3 dev1_l nodes + its block storage.
|
||||
|
||||
---
|
||||
*Related memory:* `project_phase2_gitlab_to_gitea` (GitLab→Gitea + foxhunt→fxhnt IaC), `netbird-mesh`,
|
||||
`reference_fxhnt_prod_deploy_mechanics`, `reference_fxhnt_dashboard_proxy`,
|
||||
`reference_fxhnt_local_cockpit_dev_loop`, `reference_ci_pool_and_tfstate`, `infrastructure`.
|
||||
85
docs/superpowers/specs/2026-07-20-eur-fund-ledger-design.md
Normal file
85
docs/superpowers/specs/2026-07-20-eur-fund-ledger-design.md
Normal file
@@ -0,0 +1,85 @@
|
||||
# EUR Fund Ledger — Design Spec (2026-07-20)
|
||||
|
||||
**Status:** design approved (scope + cost-basis decided with operator). Implementation plan follows separately.
|
||||
|
||||
**> NOT tax advice.** This is an engineering design for a bookkeeping system. The tax *treatment* it records
|
||||
(Box 3 passive capital, FIFO cost basis, EUR-functional) reflects the operator's resolved structure and must be
|
||||
confirmed by a Dutch belastingadviseur before it is relied on for a filing. See
|
||||
`docs/superpowers/2026-07-20-fund-tax-accounting-research-brief.md` and memory
|
||||
`project_fxhnt_fund_tax_accounting_research`.
|
||||
|
||||
## Goal
|
||||
|
||||
A tax-ready, append-only, double-entry **EUR ledger** for the fxhnt fund's **passive Box-3 investment book**,
|
||||
living in the existing Postgres/Timescale warehouse, fed by an automated **IBKR Flex** ingestion Dagster asset.
|
||||
Phase 1 books **the fund only** — the passive IBKR investment book. The tech-BV bookkeeping and the arm's-length
|
||||
BV→fund service stay in the operator's existing BV administration (explicitly out of scope).
|
||||
|
||||
## Why this shape (context)
|
||||
|
||||
- The operator's structure separates **arbeid** (tech-BV, taxed there) from **kapitaal** (private fund, Box 3).
|
||||
Phase 1 books only the kapitaal side → a clean Box-3 record with no BV entanglement.
|
||||
- **EUR is the functional currency**, not just presentation. IBKR DU9600528 is EUR-native; if crypto ever goes
|
||||
live it is Bybit-EU (EUR, not USDT). There is **no USD leg**, so there is **no conversion pipeline** — only a
|
||||
dormant `currency` + `fx_rate_to_eur` seam (default EUR/1.0) so a stray non-EUR line is representable and
|
||||
correct rather than silently mis-booked. A pure-EUR book has **no FX P&L** — only trading P&L.
|
||||
- Reuses existing patterns: SQLAlchemy 2.0 ORM (`Base`/`Mapped`, Postgres+SQLite), a `*Repo` with `migrate()`,
|
||||
a Dagster asset alongside the nightly ETL. No new infrastructure.
|
||||
|
||||
## Global Constraints
|
||||
|
||||
1. **EUR-functional.** Every posting stored in EUR. `currency` defaults `"EUR"`, `fx_rate_to_eur` defaults
|
||||
`1.0`. Build NO conversion pipeline — the FX columns are dormant insurance only.
|
||||
2. **Append-only.** `journal_entries` / `journal_postings` are immutable. Corrections are reversing entries
|
||||
(never UPDATE/DELETE of a posted row). Ingestion is idempotent (re-running a Flex pull must not double-book).
|
||||
3. **Balanced in EUR.** A DB-level trigger (or repo-enforced invariant on both Postgres and SQLite) rejects any
|
||||
entry whose postings do not sum to zero in `eur_amount`.
|
||||
4. **FIFO** cost basis, applied consistently. Realized gains come from lot-matching, not cash-flow deltas.
|
||||
5. **Scope = fund only.** No tech-BV, dividend, or arm's-length-service postings in Phase 1.
|
||||
6. **Runs on paper now.** The IBKR Flex asset must run against the *paper* account (DU9600528) — the schema and
|
||||
the EUR-sum-to-zero invariant are validated on paper data before any real capital exists.
|
||||
7. **Same test discipline as the codebase:** models work on SQLite (unit tests) and Postgres (prod); full suite
|
||||
green before finishing.
|
||||
|
||||
## Schema (SQLAlchemy 2.0 ORM, in a new `ledger` module)
|
||||
|
||||
- **`accounts`** — chart of accounts. `account_id` (str PK, e.g. `IBKR-EUR-cash`, `IBKR-securities`,
|
||||
`contributed-capital`, `trading-pnl`, `fees`), `kind` (asset|liability|equity|income|expense), `venue`,
|
||||
`currency` (default EUR). Per-venue sub-ledgers each balance standalone.
|
||||
- **`journal_entries`** — header. `entry_id` (str PK), `entry_date` (date), `description`, `source_venue`,
|
||||
`strategy_tag` (nullable — the Dagster attribution dimension), `source_ref` (the idempotency key: the IBKR
|
||||
Flex trade/cash id, so re-ingest is a no-op). Immutable.
|
||||
- **`journal_postings`** — lines. `posting_id` (PK), `entry_id` (FK), `account_id` (FK), `side` (debit|credit),
|
||||
`amount`, `currency` (default EUR), `fx_rate_to_eur` (default 1.0), `eur_amount`. Invariant: per `entry_id`,
|
||||
Σ `eur_amount`(debit) == Σ `eur_amount`(credit).
|
||||
- **`cost_basis_lots`** — `lot_id` (PK), `account_id`, `asset`, `open_date`, `qty_open`, `qty_remaining`,
|
||||
`unit_cost_eur`, and on close a link to the realizing entry. FIFO consumption.
|
||||
- **`fx_rates`** — `value_date` + `pair` → `rate`. **Dormant** — populated only if a non-EUR asset appears.
|
||||
Present so the seam exists; no ingestion job in Phase 1.
|
||||
|
||||
## IBKR Flex ingestion (new adapter surface + Dagster asset)
|
||||
|
||||
The existing `IbkrBroker` is a live `ib_insync` socket connection — NOT the Flex API. Flex is a separate
|
||||
**Flex Web Service**: a per-query token + query id, a 2-step `SendRequest` → `GetStatement` poll (~1 req/sec),
|
||||
returning an XML statement. Add a read-only `IbkrFlexClient` (new file) that fetches and parses:
|
||||
trades, cash transactions, dividends, corporate actions, positions, NAV.
|
||||
|
||||
- Land raw XML/rows in an append-only `raw` schema (mirrors the DB-anchor-SSOT pattern), then a transform
|
||||
normalizes into `journal_entries`/`journal_postings` (idempotent on `source_ref`).
|
||||
- One Dagster asset (`fund_ledger_ingest` or similar) alongside the nightly ETL; reuses monitoring patterns.
|
||||
- W-8BEN note: as a non-US person the account gets a 1042-S, never a Consolidated 1099 — irrelevant to the
|
||||
EUR ledger math, but archive the annual Activity Statement as an audit doc.
|
||||
|
||||
## Out of scope (Phase 1) — deliberately deferred
|
||||
|
||||
- Bybit ingestion (no live crypto book; Bybit-EU is EUR anyway → add later, same schema).
|
||||
- The tech-BV → fund arm's-length service postings, dividend, capital-contribution accounting (stays in BV admin).
|
||||
- The Box-3 *werkelijk-rendement* parallel tax-basis calc (unrealized + fee-disallow) — only needed if the
|
||||
tegenbewijsregeling is elected; add when the advisor confirms it's worth it.
|
||||
- Any FX conversion pipeline (there is no non-EUR leg).
|
||||
|
||||
## First build step
|
||||
|
||||
The EUR-functional ledger core (models + repo + the EUR-sum-to-zero invariant) **plus** the IBKR Flex ingestion
|
||||
asset — validated on the paper account — before anything else. This exercises every load-bearing decision:
|
||||
append-only immutability, the EUR-balanced invariant, the `strategy_tag` dimension, FIFO lots.
|
||||
134
docs/superpowers/specs/2026-07-21-fxhnt-cicd-pipeline-design.md
Normal file
134
docs/superpowers/specs/2026-07-21-fxhnt-cicd-pipeline-design.md
Normal file
@@ -0,0 +1,134 @@
|
||||
# fxhnt CI/CD Pipeline on bizworx — Design Spec (2026-07-21)
|
||||
|
||||
**Status:** design approved (approach + build-trigger decided with operator). Implementation plan follows.
|
||||
|
||||
## Goal
|
||||
|
||||
Restore automated build+deploy for fxhnt on the bizworx Kapsule so a push to `master` on
|
||||
`gitadmin/fxhnt` (Gitea) lands live without a manual `kubectl rollout restart` — the exact gap that left the
|
||||
cockpit on 2.5-day-old code (the CBU0/CSBGU0 fix). **Git-sync-first, image-on-demand:** code changes trigger a
|
||||
fast rollout (git-sync re-pulls src); only dependency/Dockerfile changes trigger a Kaniko image rebuild.
|
||||
|
||||
## Why this shape
|
||||
|
||||
The operator's insight: **code changes often, the image changes rarely.** git-sync already pulls fresh `src/`
|
||||
on every (re)start, so a code-only push needs only a rollout — not a 5-minute Kaniko build. A rebuild is needed
|
||||
only when `pyproject.toml` / `fxhnt.Dockerfile` change. This is exactly the `mode=build` vs `mode=deploy` split
|
||||
the retired `infra/argo/cockpit-build-deploy.yaml` already encoded — we reanimate and fix it, rather than adopt
|
||||
the generic `build-image` template (which lacks the split and fxhnt's deploy specifics).
|
||||
|
||||
## Governing principle: shared engine, separate projects
|
||||
|
||||
fxhnt and bizworx are **different projects**. fxhnt should *use* bizworx's heavy shared platform infra but not
|
||||
*pollute* the platform namespaces with fxhnt-specific objects or secrets:
|
||||
|
||||
- **SHARE (bizworx platform, use as-is):** the one Gitea (webhook source), the Argo Events eventbus +
|
||||
`gitea-webhook` eventsource, the Argo Workflows controller (the build *engine*), external-secrets/openbao,
|
||||
the `large` node-pool, the Scaleway registry.
|
||||
- **SEPARATE (fxhnt-owned, live in `foxhunt`):** the `WorkflowTemplate`, all build+deploy pods, fxhnt's
|
||||
secrets (`argo-git-ssh-key`, `scw-registry` — already in `foxhunt`), and fxhnt's RBAC.
|
||||
- **The ONE unavoidable object in a platform ns:** the Sensor. A Sensor must live in `argo-events` to hear the
|
||||
shared eventbus. Keep its footprint minimal — it only matches fxhnt pushes and *submits* a Workflow that runs
|
||||
in `foxhunt`. No fxhnt secrets, no fxhnt logic beyond the trigger, live in `argo-events`.
|
||||
|
||||
This is NOT the old "everything in foxhunt because the secrets were there" (path of least resistance) — it's the
|
||||
same placement for the *right* reason: respecting the project boundary. It also means **no cross-namespace RBAC**
|
||||
(the deploy runs in `foxhunt`, where the app + its permissions already are) and **no fxhnt secrets replicated
|
||||
into `argo`**.
|
||||
|
||||
## Verified current state (all components already exist on bizworx)
|
||||
|
||||
- **One Gitea** (ns `gitea`): `git.fxhnt.ai` and `gitea-http.gitea.svc` are the SAME instance; it holds
|
||||
`gitadmin/fxhnt` (code) alongside `bizworx-ops/infrastructure` (the msp GitOps monorepo). No second Gitea.
|
||||
- **Argo Events** (`gitea-webhook` eventsource, `/push`, repo-agnostic) + **Argo Workflows** (Kaniko) — already
|
||||
driving `build-msp-*` via path-filtered sensors. Reusable.
|
||||
- **Registry:** fxhnt's image is `rg.fr-par.scw.cloud/bizworx/fxhnt-cockpit` (Scaleway) — DIFFERENT from msp's
|
||||
`registry.bizworx.nl/bizworx-ops/`. Secret `scw-registry` (ns foxhunt) holds the push creds.
|
||||
- **Dockerfile EXISTS:** `infra/docker/fxhnt.Dockerfile` (python:3.11-slim, `fxhnt serve` :8080). Reusable as-is.
|
||||
- **SAs exist:** `argo-workflow` (ns argo), `argo-events-sa` (ns argo-events); `argo-git-ssh-key` (ns foxhunt).
|
||||
- **What's broken in the old pipeline:** (1) `nodeSelector: ci-compile-cpu` — that pool is DECOMMISSIONED
|
||||
(only `large` + `sfs` pools exist now) → build/deploy pods won't schedule; (2) the Sensor that triggered it
|
||||
is gone; (3) no webhook on `gitadmin/fxhnt` → the eventsource never hears fxhnt pushes.
|
||||
- **No fxhnt ArgoCD app** — fxhnt manifests are applied by the workflow's `kubectl apply`, not GitOps-synced.
|
||||
We keep it that way (see Non-goals).
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
git push → gitadmin/fxhnt (master) [SHARED: Gitea]
|
||||
│ Gitea webhook ──────────────► argo-events `gitea-webhook` eventsource [SHARED: platform eventbus]
|
||||
▼
|
||||
Sensor `build-fxhnt` (ns argo-events — the ONE fxhnt object in a platform ns, kept minimal)
|
||||
│ filters: ref==refs/heads/master, pusher != argocd-image-updater (SINGLE trigger, like the msp sensors)
|
||||
▼ submits ONE Workflow into ns `foxhunt`, injecting commit-sha ─────── [FXHNT-OWNED from here down]
|
||||
Argo Workflow (WorkflowTemplate `fxhnt-cockpit`, ns foxhunt, SA argo-workflow, node-pool `large`) — DAG:
|
||||
├─ decide-mode: git-clone(sha) → `git diff --name-only sha~1 sha` →
|
||||
│ touched pyproject.toml OR infra/docker/fxhnt.Dockerfile → mode=build ; else mode=deploy
|
||||
│ (mode decided IN the workflow, not a fragile sensor filter — the CRD schema is open, so a
|
||||
│ sensor-side path-negation can't be validated; a git-diff step is robust + testable. No lock file
|
||||
│ exists → those two paths are the complete rebuild-trigger set.)
|
||||
├─ mode=build: Kaniko → rg.fr-par.scw.cloud/bizworx/fxhnt-cockpit:<sha7>+:latest
|
||||
└─ (build skipped when mode=deploy)
|
||||
→ deploy step (in foxhunt, no cross-ns RBAC): kubectl apply manifests + refresh daemons (git-sync re-pulls src)
|
||||
```
|
||||
|
||||
## Components to build/fix
|
||||
|
||||
### 1. WorkflowTemplate `fxhnt-cockpit` (reanimate `infra/argo/cockpit-build-deploy.yaml`, ns `foxhunt`)
|
||||
- Fix `nodeSelector` `ci-compile-cpu` → `large` (both build + deploy templates — `ci-compile-cpu` pool is gone;
|
||||
only `large`/`sfs` exist).
|
||||
- Keep the `mode` DAG: `build` (Kaniko rebuild + deploy) vs `deploy` (skip build, deploy only).
|
||||
- Keep the Kaniko dest `rg.fr-par.scw.cloud/bizworx/fxhnt-cockpit` but ADD an immutable `:<sha7>` tag next to
|
||||
`:latest` (matches the msp pattern). Cache stays on.
|
||||
- Clone via the in-cluster Gitea using `argo-git-ssh-key`; push via `scw-registry`. Both already exist in
|
||||
`foxhunt` — the workflow runs there, so they resolve locally (no replication into a platform ns).
|
||||
- **Reproducibility despite git-sync:** the deploy step stamps the deployed commit sha onto the daemon
|
||||
Deployments as an annotation (`fxhnt.io/deployed-sha=<sha7>`), so "which commit is this pod running" is always
|
||||
answerable — closing the one real gap of git-sync-first without adopting full image-based deploys.
|
||||
|
||||
### 2. Sensor `build-fxhnt` (new, model on `build-msp-portal`)
|
||||
- Dependency: `gitea-webhook` push, filter `body.ref == refs/heads/master` AND
|
||||
`body.pusher.username != argocd-image-updater` (loop-guard; harmless here since no image-updater writes fxhnt,
|
||||
but kept for parity/safety).
|
||||
- **Mode decision via path-filter on `body.commits.#.modified`:** if any modified path matches
|
||||
`pyproject.toml` | `infra/docker/fxhnt.Dockerfile` → submit workflow with `mode=build`; else `mode=deploy`.
|
||||
(No lock file exists, so those two paths are the complete rebuild-trigger set.) Mechanism: two triggers with
|
||||
complementary data-filters (one matching the dep-paths → build, one matching their negation → deploy), or one
|
||||
trigger whose param is computed — the plan picks whichever Argo Events supports cleanly.
|
||||
- Inject `commit-sha` from `body.after`.
|
||||
|
||||
### 3. Gitea webhook on `gitadmin/fxhnt`
|
||||
- Add a repo (or org) webhook POSTing push events to the `gitea-webhook` eventsource endpoint (same target the
|
||||
infrastructure repo already uses). This is the one genuinely-missing wire.
|
||||
|
||||
### 4. The deploy step — VERIFY, don't assume (open design point)
|
||||
The old deploy used **`scale 0→1` for fxhnt-dashboard, NOT `rollout restart`**, because the tailscale sidecar
|
||||
served `dashboard.fxhnt.ai` and a rolling restart overlapped two tailscale sessions → external proxy read
|
||||
"down". BUT: the dashboard now also has a `fxhnt-ingress`, and a manual `rollout restart` on 2026-07-21
|
||||
succeeded cleanly. **The plan must verify which external route is live** before choosing:
|
||||
- If `dashboard.fxhnt.ai` still resolves through the tailscale sidecar → keep `scale 0→1` (the race is real).
|
||||
- If it now resolves through `fxhnt-ingress` (tailscale sidecar vestigial) → a plain `rollout restart` is
|
||||
simpler and correct; drop the scale-dance.
|
||||
Dagster has no external-proxy race → `rollout restart deploy/dagster` either way.
|
||||
|
||||
## Non-goals (YAGNI) — deliberately NOT doing, with the reason
|
||||
|
||||
- **No ArgoCD GitOps app for fxhnt.** Chosen over declarative GitOps: it would add a second source of truth for
|
||||
the pod's src (ArgoCD-synced manifests vs git-sync'd code) and needs a git.fxhnt.ai repo registration. The
|
||||
imperative `kubectl apply` from the foxhunt workflow fits git-sync-first and keeps manifests in the fxhnt repo.
|
||||
- **No registry unification.** fxhnt stays on Scaleway `rg.fr-par.scw.cloud/bizworx` (not the shared
|
||||
`registry.bizworx.nl/bizworx-ops`). Migrating would mean re-pointing the dashboard + reconciling old images
|
||||
for zero pipeline benefit — a separate, riskier task.
|
||||
- **No CI workloads in a platform namespace.** Only the Sensor lives in `argo-events` (unavoidable). The
|
||||
WorkflowTemplate + all build/deploy pods run in `foxhunt` — no fxhnt secrets in `argo`, no cross-ns RBAC.
|
||||
- **No image-updater for fxhnt.** The sensor writes the `:sha7`+`:latest` tags; the deploy step points the
|
||||
daemons at the fresh image. No git-write-back loop needed.
|
||||
- **No change to the execution CronJobs** (ucits/bybit/ibkr rebalancers) — they git-sync per run already.
|
||||
|
||||
## Success criteria
|
||||
|
||||
1. A code-only push to `master` → cockpit reflects it within ~1–2 min WITHOUT a manual restart, NO Kaniko build.
|
||||
2. A `pyproject.toml`/Dockerfile push → a Kaniko rebuild produces `fxhnt-cockpit:<sha7>`+`:latest`, then deploys.
|
||||
3. The dashboard external route (`dashboard.fxhnt.ai`) stays UP across a deploy (no stale-proxy "down").
|
||||
4. No build-loop, no false trigger on unrelated pushes.
|
||||
5. Real-money safety unaffected — the pipeline only builds/deploys the cockpit+dagster daemons, never arms exec.
|
||||
@@ -1,128 +0,0 @@
|
||||
# Argo WorkflowTemplate — build + deploy the fxhnt cockpit, fully in-cluster.
|
||||
#
|
||||
# Reuses cluster primitives: the `argo-workflow` ServiceAccount, the `argo-git-ssh-key` deploy key (clone
|
||||
# the fxhnt repo over in-cluster Gitea SSH, gitea-sshd:22), and the `scw-registry` dockerconfig (push to
|
||||
# the external Scaleway Container Registry rg.fr-par.scw.cloud/bizworx). Build runs in-cluster.
|
||||
# (Phase 2B: migrated off the in-cluster GitLab shell + registry.)
|
||||
#
|
||||
# Submit: argo submit -n foxhunt --from=wftmpl/fxhnt-cockpit -p commit-sha=<sha> --watch
|
||||
# (or use scripts/argo-deploy-cockpit.sh)
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: WorkflowTemplate
|
||||
metadata:
|
||||
name: fxhnt-cockpit
|
||||
namespace: foxhunt
|
||||
labels: { app.kubernetes.io/name: fxhnt-cockpit, app.kubernetes.io/part-of: foxhunt }
|
||||
spec:
|
||||
serviceAccountName: argo-workflow
|
||||
entrypoint: pipeline
|
||||
archiveLogs: true # archive build/deploy logs to MinIO (argo-logs bucket). Reachable now that the
|
||||
# pods carry app.kubernetes.io/part-of: argo-workflows (MinIO ingress netpol allow).
|
||||
activeDeadlineSeconds: 1800
|
||||
ttlStrategy: { secondsAfterCompletion: 3600 }
|
||||
podGC: { strategy: OnPodCompletion }
|
||||
arguments:
|
||||
parameters:
|
||||
- { name: commit-sha, value: HEAD }
|
||||
- { name: mode, value: build } # "build" = kaniko rebuild + deploy; "deploy" = skip build, just apply + rollout restart (git-sync re-pulls code). The sensor routes src-only pushes to "deploy".
|
||||
templates:
|
||||
- name: pipeline
|
||||
dag:
|
||||
tasks:
|
||||
# Skip the ~5min kaniko build for code-only pushes (mode=deploy) — git-sync re-pulls src/ on the
|
||||
# rollout restart, so only dep/Dockerfile changes need a new image (mode=build).
|
||||
- { name: build, template: kaniko-build, when: "{{workflow.parameters.mode}} == build" }
|
||||
- { name: deploy, template: kubectl-deploy, depends: "build.Succeeded || build.Skipped" }
|
||||
|
||||
# --- build: Kaniko builds infra/docker/fxhnt.Dockerfile -> internal registry ---
|
||||
- name: kaniko-build
|
||||
metadata:
|
||||
labels: { app.kubernetes.io/part-of: argo-workflows } # MinIO ingress allows part-of in {foxhunt,foxhunt-ci,argo-workflows} → log archival works. NOT 'foxhunt' (that adds argo-base-egress which blocks registry:5000/git:2222).
|
||||
nodeSelector: { k8s.scaleway.com/pool-name: ci-compile-cpu, topology.kubernetes.io/zone: fr-par-2 }
|
||||
tolerations: [{ effect: NoSchedule, key: node.cilium.io/agent-not-ready, operator: Exists }]
|
||||
initContainers:
|
||||
- name: git-clone
|
||||
image: alpine/git:latest
|
||||
command: [/bin/sh, -c]
|
||||
args:
|
||||
- |
|
||||
set -ex
|
||||
mkdir -p /root/.ssh
|
||||
cp /etc/git-ssh/ssh-privatekey /root/.ssh/id_ed25519
|
||||
chmod 600 /root/.ssh/id_ed25519
|
||||
printf 'Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n' > /root/.ssh/config
|
||||
git clone --no-checkout --filter=blob:none \
|
||||
"ssh://git@gitea-sshd.foxhunt.svc.cluster.local:22/gitadmin/fxhnt.git" /workspace/src
|
||||
cd /workspace/src && git checkout "{{workflow.parameters.commit-sha}}"
|
||||
echo "checked out $(git rev-parse --short HEAD)"
|
||||
volumeMounts:
|
||||
- { name: git-ssh-key, mountPath: /etc/git-ssh, readOnly: true }
|
||||
- { name: workspace, mountPath: /workspace }
|
||||
container:
|
||||
image: gcr.io/kaniko-project/executor:debug
|
||||
command: [/busybox/sh, -c]
|
||||
args:
|
||||
- |
|
||||
/kaniko/executor \
|
||||
--context=/workspace/src \
|
||||
--dockerfile=/workspace/src/infra/docker/fxhnt.Dockerfile \
|
||||
--destination=rg.fr-par.scw.cloud/bizworx/fxhnt-cockpit:latest \
|
||||
--cache=true \
|
||||
--cache-repo=rg.fr-par.scw.cloud/bizworx/cache \
|
||||
--snapshot-mode=redo
|
||||
env:
|
||||
- { name: DOCKER_CONFIG, value: /kaniko/.docker }
|
||||
resources: { requests: { cpu: "2", memory: 4Gi }, limits: { cpu: "6", memory: 12Gi } }
|
||||
volumeMounts:
|
||||
- { name: registry-auth, mountPath: /kaniko/.docker, readOnly: true }
|
||||
- { name: workspace, mountPath: /workspace }
|
||||
volumes:
|
||||
- { name: workspace, emptyDir: {} }
|
||||
- { name: git-ssh-key, secret: { secretName: argo-git-ssh-key, defaultMode: 256 } }
|
||||
- name: registry-auth
|
||||
secret:
|
||||
secretName: scw-registry
|
||||
items: [{ key: .dockerconfigjson, path: config.json }]
|
||||
|
||||
# --- deploy: apply manifests from the repo + roll the dashboard to the fresh image ---
|
||||
- name: kubectl-deploy
|
||||
metadata:
|
||||
labels: { app.kubernetes.io/part-of: argo-workflows } # MinIO ingress allow → log archival works (see kaniko-build note)
|
||||
nodeSelector: { k8s.scaleway.com/pool-name: platform, topology.kubernetes.io/zone: fr-par-2 }
|
||||
serviceAccountName: argo-workflow
|
||||
initContainers:
|
||||
- name: git-clone
|
||||
image: alpine/git:latest
|
||||
command: [/bin/sh, -c]
|
||||
args:
|
||||
- |
|
||||
set -ex
|
||||
mkdir -p /root/.ssh
|
||||
cp /etc/git-ssh/ssh-privatekey /root/.ssh/id_ed25519
|
||||
chmod 600 /root/.ssh/id_ed25519
|
||||
printf 'Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n' > /root/.ssh/config
|
||||
git clone --no-checkout --filter=blob:none \
|
||||
"ssh://git@gitea-sshd.foxhunt.svc.cluster.local:22/gitadmin/fxhnt.git" /workspace/src
|
||||
cd /workspace/src && git checkout "{{workflow.parameters.commit-sha}}"
|
||||
volumeMounts:
|
||||
- { name: git-ssh-key, mountPath: /etc/git-ssh, readOnly: true }
|
||||
- { name: workspace, mountPath: /workspace }
|
||||
container:
|
||||
image: alpine/k8s:1.31.0 # Alpine-based (has /bin/sh + kubectl). NOT registry.k8s.io/kubectl — that's distroless (no shell) and the deploy runs a shell script. bitnami/kubectl was removed from Docker Hub (2025).
|
||||
command: [/bin/sh, -c]
|
||||
args:
|
||||
- |
|
||||
set -e
|
||||
cd /workspace/src
|
||||
kubectl apply -f infra/k8s/services/fxhnt-dashboard.yaml
|
||||
kubectl apply -f infra/k8s/orchestration/dagster.yaml
|
||||
kubectl apply -f infra/k8s/network-policies/fxhnt-cockpit.yaml
|
||||
kubectl -n foxhunt rollout restart deploy/fxhnt-dashboard
|
||||
kubectl -n foxhunt rollout restart deploy/dagster
|
||||
kubectl -n foxhunt rollout status deploy/fxhnt-dashboard --timeout=180s
|
||||
kubectl -n foxhunt rollout status deploy/dagster --timeout=300s
|
||||
volumeMounts:
|
||||
- { name: workspace, mountPath: /workspace }
|
||||
volumes:
|
||||
- { name: workspace, emptyDir: {} }
|
||||
- { name: git-ssh-key, secret: { secretName: argo-git-ssh-key, defaultMode: 256 } }
|
||||
57
infra/argo/fxhnt-build-sensor.yaml
Normal file
57
infra/argo/fxhnt-build-sensor.yaml
Normal file
@@ -0,0 +1,57 @@
|
||||
# Argo Events Sensor — hears the shared Gitea webhook for pushes to gitadmin/fxhnt master, and submits the
|
||||
# fxhnt-cockpit WorkflowTemplate (ns foxhunt). Structurally identical to build-msp-portal (single dependency,
|
||||
# single trigger) — the build-vs-deploy decision is made INSIDE the workflow via `git diff` (decide-mode step in
|
||||
# fxhnt-cockpit-workflowtemplate.yaml), not via a sensor-side path filter. See
|
||||
# docs/superpowers/plans/2026-07-21-fxhnt-cicd-pipeline.md Task 3 design note: the Sensor CRD here has an open
|
||||
# schema, so trigger-level path-filter negation can't be reliably validated — keeping the sensor simple is the
|
||||
# robust choice.
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Sensor
|
||||
metadata:
|
||||
name: build-fxhnt
|
||||
namespace: argo-events
|
||||
labels: { app.kubernetes.io/instance: build-pipeline }
|
||||
spec:
|
||||
dependencies:
|
||||
- name: push-to-master
|
||||
eventSourceName: gitea-webhook
|
||||
eventName: push
|
||||
filters:
|
||||
dataLogicalOperator: and
|
||||
data:
|
||||
- path: body.ref
|
||||
type: string
|
||||
comparator: "="
|
||||
value: ["refs/heads/master"]
|
||||
- path: body.pusher.username
|
||||
type: string
|
||||
comparator: "!="
|
||||
value: ["argocd-image-updater"]
|
||||
template:
|
||||
serviceAccountName: argo-events-sa
|
||||
triggers:
|
||||
- template:
|
||||
name: submit-build
|
||||
argoWorkflow:
|
||||
operation: submit
|
||||
source:
|
||||
resource:
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Workflow
|
||||
metadata:
|
||||
generateName: build-fxhnt-
|
||||
namespace: foxhunt
|
||||
spec:
|
||||
workflowTemplateRef:
|
||||
name: fxhnt-cockpit
|
||||
arguments:
|
||||
parameters:
|
||||
- name: commit-sha
|
||||
value: ""
|
||||
- name: mode
|
||||
value: auto
|
||||
parameters:
|
||||
- src:
|
||||
dependencyName: push-to-master
|
||||
dataKey: body.after
|
||||
dest: spec.arguments.parameters.0.value
|
||||
124
infra/argo/fxhnt-ci-rbac.yaml
Normal file
124
infra/argo/fxhnt-ci-rbac.yaml
Normal file
@@ -0,0 +1,124 @@
|
||||
# RBAC for the fxhnt CI pipeline — all fxhnt-owned, in the foxhunt namespace (project boundary).
|
||||
#
|
||||
# Fills the two gaps the first real workflow run surfaced (2026-07-21):
|
||||
# 1. The fxhnt-cockpit WorkflowTemplate references SA `argo-workflow`, which existed in `argo` (where msp
|
||||
# builds) but NOT in `foxhunt` (where fxhnt's project boundary places its build/deploy pods).
|
||||
# 2. The build-fxhnt Sensor (argo-events-sa) submits Workflows into `foxhunt`; the bizworx `workflow-submitter`
|
||||
# Role only granted that in `argo`, so a cross-namespace submit into foxhunt was forbidden.
|
||||
#
|
||||
# Least privilege: the workflow SA gets exactly what Argo needs to run pods + what the kubectl-deploy step does
|
||||
# (apply/scale/annotate/rollout-status the two daemon Deployments and their Services/NetworkPolicies), nothing more.
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: argo-workflow
|
||||
namespace: foxhunt
|
||||
labels: { app.kubernetes.io/part-of: fxhnt-ci }
|
||||
---
|
||||
# Argo needs the workflow SA to write task results (the minimum the workflow-controller requires per step).
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: fxhnt-argo-workflow
|
||||
namespace: foxhunt
|
||||
labels: { app.kubernetes.io/part-of: fxhnt-ci }
|
||||
rules:
|
||||
- apiGroups: ["argoproj.io"]
|
||||
resources: ["workflowtaskresults"]
|
||||
verbs: ["create", "patch"]
|
||||
---
|
||||
# The kubectl-deploy step's exact actions on the foxhunt daemons — scoped to what it runs, no wildcards.
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: fxhnt-ci-deployer
|
||||
namespace: foxhunt
|
||||
labels: { app.kubernetes.io/part-of: fxhnt-ci }
|
||||
rules:
|
||||
# apply + scale + annotate + rollout-status the daemon Deployments (dashboard, dagster).
|
||||
- apiGroups: ["apps"]
|
||||
resources: ["deployments"]
|
||||
verbs: ["get", "list", "watch", "create", "update", "patch"]
|
||||
- apiGroups: ["apps"]
|
||||
resources: ["deployments/scale"]
|
||||
verbs: ["get", "update", "patch"]
|
||||
- apiGroups: ["apps"]
|
||||
resources: ["replicasets"]
|
||||
verbs: ["get", "list", "watch"] # rollout status reads the RS
|
||||
# apply the Services the manifests contain, and `wait --for=delete pod` + rollout status read pods.
|
||||
- apiGroups: [""]
|
||||
resources: ["services"]
|
||||
verbs: ["get", "list", "watch", "create", "update", "patch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["pods"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
# the fxhnt-dashboard + dagster manifests also bundle ConfigMaps (the tailscale sidecar serve-config) —
|
||||
# kubectl apply of the whole file must be able to read/apply them. NOTE (CI review M1): the tailscale
|
||||
# ServiceAccount + Role + RoleBinding objects were MOVED out of these app manifests into
|
||||
# infra/k8s/rbac/tailscale-sidecars.yaml (applied once, out-of-band, by a human) specifically so this
|
||||
# deployer SA does not need roles/rolebindings/serviceaccounts create-update-patch — that combination on a
|
||||
# webhook-triggered SA is a privilege-escalation primitive. Neither manifest applied by CI creates a
|
||||
# ServiceAccount anymore, so `serviceaccounts` is dropped from this grant too.
|
||||
- apiGroups: [""]
|
||||
resources: ["configmaps"]
|
||||
verbs: ["get", "list", "watch", "create", "update", "patch"]
|
||||
# the manifests apply NetworkPolicies (fxhnt-cockpit.yaml + dagster's).
|
||||
- apiGroups: ["networking.k8s.io"]
|
||||
resources: ["networkpolicies"]
|
||||
verbs: ["get", "list", "watch", "create", "update", "patch"]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: fxhnt-argo-workflow
|
||||
namespace: foxhunt
|
||||
labels: { app.kubernetes.io/part-of: fxhnt-ci }
|
||||
subjects:
|
||||
- { kind: ServiceAccount, name: argo-workflow, namespace: foxhunt }
|
||||
roleRef:
|
||||
kind: Role
|
||||
name: fxhnt-argo-workflow
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: fxhnt-ci-deployer
|
||||
namespace: foxhunt
|
||||
labels: { app.kubernetes.io/part-of: fxhnt-ci }
|
||||
subjects:
|
||||
- { kind: ServiceAccount, name: argo-workflow, namespace: foxhunt }
|
||||
roleRef:
|
||||
kind: Role
|
||||
name: fxhnt-ci-deployer
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
---
|
||||
# Cross-namespace: let the shared argo-events-sa submit Workflows into foxhunt (the sensor lives in argo-events
|
||||
# but its Workflow runs here). Mirrors the bizworx `workflow-submitter` Role, scoped to foxhunt.
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: fxhnt-workflow-submitter
|
||||
namespace: foxhunt
|
||||
labels: { app.kubernetes.io/part-of: fxhnt-ci }
|
||||
rules:
|
||||
- apiGroups: ["argoproj.io"]
|
||||
resources: ["workflows"]
|
||||
verbs: ["create", "get", "list", "watch"]
|
||||
- apiGroups: ["argoproj.io"]
|
||||
resources: ["workflowtemplates"]
|
||||
verbs: ["get", "list"]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: argo-events-sa-fxhnt-submitter
|
||||
namespace: foxhunt
|
||||
labels: { app.kubernetes.io/part-of: fxhnt-ci }
|
||||
subjects:
|
||||
- { kind: ServiceAccount, name: argo-events-sa, namespace: argo-events }
|
||||
roleRef:
|
||||
kind: Role
|
||||
name: fxhnt-workflow-submitter
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
208
infra/argo/fxhnt-cockpit-workflowtemplate.yaml
Normal file
208
infra/argo/fxhnt-cockpit-workflowtemplate.yaml
Normal file
@@ -0,0 +1,208 @@
|
||||
# Argo WorkflowTemplate — build + deploy the fxhnt cockpit, fully in-cluster.
|
||||
#
|
||||
# Reuses cluster primitives: the `argo-workflow` ServiceAccount (foxhunt, see fxhnt-ci-rbac.yaml), the
|
||||
# `argo-git-ssh-key` deploy key (clone the fxhnt repo over in-cluster Gitea SSH, gitea-ssh.gitea.svc:22), and
|
||||
# the `scw-registry` dockerconfig (push to the external Scaleway Container Registry rg.fr-par.scw.cloud/bizworx).
|
||||
#
|
||||
# Submit: argo submit -n foxhunt --from=wftmpl/fxhnt-cockpit -p commit-sha=<sha> -p mode=<auto|build|deploy> --watch
|
||||
# Normally auto-triggered: a push to gitadmin/fxhnt master -> the build-fxhnt Sensor (argo-events) submits this.
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: WorkflowTemplate
|
||||
metadata:
|
||||
name: fxhnt-cockpit
|
||||
namespace: foxhunt
|
||||
labels: { app.kubernetes.io/name: fxhnt-cockpit, app.kubernetes.io/part-of: foxhunt }
|
||||
spec:
|
||||
serviceAccountName: argo-workflow
|
||||
entrypoint: pipeline
|
||||
archiveLogs: false # archive build/deploy logs to MinIO (argo-logs bucket). Reachable now that the
|
||||
# pods carry app.kubernetes.io/part-of: argo-workflows (MinIO ingress netpol allow).
|
||||
activeDeadlineSeconds: 1800
|
||||
ttlStrategy: { secondsAfterCompletion: 3600 }
|
||||
podGC: { strategy: OnPodCompletion }
|
||||
arguments:
|
||||
parameters:
|
||||
- { name: commit-sha, value: HEAD }
|
||||
- { name: mode, value: auto } # "auto" = decide-mode git-diffs commit-sha~1..commit-sha to pick build/deploy; "build" = force kaniko rebuild + deploy; "deploy" = force skip build, just apply + rollout restart (git-sync re-pulls code). A human can still force build/deploy via `argo submit -p mode=...`.
|
||||
templates:
|
||||
- name: pipeline
|
||||
dag:
|
||||
tasks:
|
||||
# decide-mode: when mode=auto, git-diffs the pushed commit against its parent to pick build vs deploy
|
||||
# (dep/Dockerfile changes -> build; anything else, or a missing parent, -> build/deploy per script logic
|
||||
# below). When mode is forced (build|deploy), decide-mode just echoes it back unchanged.
|
||||
- { name: decide-mode, template: decide-mode }
|
||||
# Skip the ~5min kaniko build for code-only pushes (mode=deploy) — git-sync re-pulls src/ on the
|
||||
# rollout restart, so only dep/Dockerfile changes need a new image (mode=build).
|
||||
- { name: build, template: kaniko-build, depends: decide-mode, when: "{{tasks.decide-mode.outputs.parameters.mode}} == build" }
|
||||
- { name: deploy, template: kubectl-deploy, depends: "decide-mode && (build.Succeeded || build.Skipped)" }
|
||||
|
||||
# --- decide-mode: resolve "auto" to build|deploy via git diff of the pushed commit vs its parent ---
|
||||
- name: decide-mode
|
||||
metadata:
|
||||
labels: { app.kubernetes.io/part-of: argo-workflows }
|
||||
nodeSelector: { k8s.scaleway.com/pool-name: large }
|
||||
tolerations: [{ effect: NoSchedule, key: node.cilium.io/agent-not-ready, operator: Exists }]
|
||||
outputs:
|
||||
parameters:
|
||||
- name: mode
|
||||
valueFrom: { path: /workspace/mode }
|
||||
container:
|
||||
image: alpine/git:latest
|
||||
command: [/bin/sh, -c]
|
||||
# SECURITY: commit-sha/mode arrive via ENV (not inline {{...}} substitution) so an untrusted webhook
|
||||
# payload cannot inject shell. commit-sha is validated hex before use; mode is checked against an allowlist.
|
||||
env:
|
||||
- { name: COMMIT_SHA, value: "{{workflow.parameters.commit-sha}}" }
|
||||
- { name: MODE, value: "{{workflow.parameters.mode}}" }
|
||||
# Absolute key path via GIT_SSH_COMMAND — Argo's emissary executor does not set HOME=/root, so the
|
||||
# old `cp key -> ~/.ssh` idiom left git unable to find the key (clone failed with exit 128). This is
|
||||
# HOME-independent and verified to clone under HOME=/nonexistent.
|
||||
- { name: GIT_SSH_COMMAND, value: "ssh -i /etc/git-ssh/ssh-privatekey -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" }
|
||||
args:
|
||||
- |
|
||||
set -e
|
||||
case "$COMMIT_SHA" in *[!0-9a-fA-F]*|"") echo "refusing non-hex commit-sha" >&2; exit 1 ;; esac
|
||||
case "$MODE" in auto|build|deploy) : ;; *) echo "refusing unknown mode: $MODE" >&2; exit 1 ;; esac
|
||||
git clone --filter=blob:none \
|
||||
"ssh://git@gitea-ssh.gitea.svc.cluster.local:22/gitadmin/fxhnt.git" /workspace/src
|
||||
cd /workspace/src && git checkout "$COMMIT_SHA"
|
||||
|
||||
if [ "$MODE" != "auto" ]; then
|
||||
printf '%s' "$MODE" > /workspace/mode
|
||||
else
|
||||
RESOLVED="$(git rev-parse "$COMMIT_SHA")"
|
||||
if CHANGED=$(git diff --name-only "${RESOLVED}~1" "${RESOLVED}" 2>/dev/null); then
|
||||
# diff succeeded (parent exists): build only if a dep/Dockerfile file changed, else deploy.
|
||||
if echo "$CHANGED" | grep -qE '^pyproject\.toml$|^infra/docker/fxhnt\.Dockerfile$'; then
|
||||
printf 'build' > /workspace/mode
|
||||
else
|
||||
printf 'deploy' > /workspace/mode
|
||||
fi
|
||||
else
|
||||
# diff failed (no parent -- first commit / shallow history): default to build (safe, never wrong).
|
||||
printf 'build' > /workspace/mode
|
||||
fi
|
||||
fi
|
||||
cat /workspace/mode
|
||||
volumeMounts:
|
||||
- { name: git-ssh-key, mountPath: /etc/git-ssh, readOnly: true }
|
||||
- { name: workspace, mountPath: /workspace }
|
||||
volumes:
|
||||
- { name: workspace, emptyDir: {} }
|
||||
- { name: git-ssh-key, secret: { secretName: argo-git-ssh-key, defaultMode: 256 } }
|
||||
|
||||
# --- build: Kaniko builds infra/docker/fxhnt.Dockerfile -> internal registry ---
|
||||
- name: kaniko-build
|
||||
metadata:
|
||||
labels: { app.kubernetes.io/part-of: argo-workflows } # MinIO ingress allows part-of in {foxhunt,foxhunt-ci,argo-workflows} → log archival works. NOT 'foxhunt' (that adds argo-base-egress which blocks registry:5000/git:2222).
|
||||
nodeSelector: { k8s.scaleway.com/pool-name: large }
|
||||
tolerations: [{ effect: NoSchedule, key: node.cilium.io/agent-not-ready, operator: Exists }]
|
||||
initContainers:
|
||||
- name: git-clone
|
||||
image: alpine/git:latest
|
||||
command: [/bin/sh, -c]
|
||||
# SECURITY: commit-sha via ENV + hex-validated (see decide-mode note) — no inline shell substitution.
|
||||
env:
|
||||
- { name: COMMIT_SHA, value: "{{workflow.parameters.commit-sha}}" }
|
||||
# Absolute key path via GIT_SSH_COMMAND — HOME-independent (Argo's emissary doesn't set HOME=/root).
|
||||
- { name: GIT_SSH_COMMAND, value: "ssh -i /etc/git-ssh/ssh-privatekey -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" }
|
||||
args:
|
||||
- |
|
||||
set -ex
|
||||
case "$COMMIT_SHA" in *[!0-9a-fA-F]*|"") echo "refusing non-hex commit-sha" >&2; exit 1 ;; esac
|
||||
git clone --no-checkout --filter=blob:none \
|
||||
"ssh://git@gitea-ssh.gitea.svc.cluster.local:22/gitadmin/fxhnt.git" /workspace/src
|
||||
cd /workspace/src && git checkout "$COMMIT_SHA"
|
||||
echo "checked out $(git rev-parse --short HEAD)"
|
||||
volumeMounts:
|
||||
- { name: git-ssh-key, mountPath: /etc/git-ssh, readOnly: true }
|
||||
- { name: workspace, mountPath: /workspace }
|
||||
container:
|
||||
image: gcr.io/kaniko-project/executor:v1.23.2
|
||||
# Kaniko's own entrypoint + a YAML args LIST (the msp build-image pattern) — NOT `sh -c` with one big
|
||||
# string. A shell wrapper made busybox try to parse the `{{=sprig.trunc(...)}}` expansion and choke on
|
||||
# the `(` (syntax error). As a plain args list, Argo evaluates the expression and hands it straight to
|
||||
# Kaniko, no shell in between.
|
||||
args:
|
||||
- --context=/workspace/src
|
||||
- --dockerfile=/workspace/src/infra/docker/fxhnt.Dockerfile
|
||||
- --destination=rg.fr-par.scw.cloud/bizworx/fxhnt-cockpit:latest
|
||||
- --destination=rg.fr-par.scw.cloud/bizworx/fxhnt-cockpit:{{workflow.parameters.commit-sha}}
|
||||
- --cache=true
|
||||
- --cache-repo=rg.fr-par.scw.cloud/bizworx/cache
|
||||
- --snapshot-mode=redo
|
||||
env:
|
||||
- { name: DOCKER_CONFIG, value: /kaniko/.docker }
|
||||
resources: { requests: { cpu: "1", memory: 3Gi }, limits: { cpu: "3", memory: 10Gi } }
|
||||
volumeMounts:
|
||||
- { name: registry-auth, mountPath: /kaniko/.docker, readOnly: true }
|
||||
- { name: workspace, mountPath: /workspace }
|
||||
volumes:
|
||||
- { name: workspace, emptyDir: {} }
|
||||
- { name: git-ssh-key, secret: { secretName: argo-git-ssh-key, defaultMode: 256 } }
|
||||
- name: registry-auth
|
||||
secret:
|
||||
secretName: scw-registry
|
||||
items: [{ key: .dockerconfigjson, path: config.json }]
|
||||
|
||||
# --- deploy: apply manifests from the repo + roll the dashboard to the fresh image ---
|
||||
- name: kubectl-deploy
|
||||
metadata:
|
||||
labels: { app.kubernetes.io/part-of: argo-workflows } # MinIO ingress allow → log archival works (see kaniko-build note)
|
||||
nodeSelector: { k8s.scaleway.com/pool-name: large }
|
||||
serviceAccountName: argo-workflow
|
||||
initContainers:
|
||||
- name: git-clone
|
||||
image: alpine/git:latest
|
||||
command: [/bin/sh, -c]
|
||||
# SECURITY: commit-sha via ENV + hex-validated — no inline shell substitution.
|
||||
env:
|
||||
- { name: COMMIT_SHA, value: "{{workflow.parameters.commit-sha}}" }
|
||||
# Absolute key path via GIT_SSH_COMMAND — HOME-independent (Argo's emissary doesn't set HOME=/root).
|
||||
- { name: GIT_SSH_COMMAND, value: "ssh -i /etc/git-ssh/ssh-privatekey -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" }
|
||||
args:
|
||||
- |
|
||||
set -ex
|
||||
case "$COMMIT_SHA" in *[!0-9a-fA-F]*|"") echo "refusing non-hex commit-sha" >&2; exit 1 ;; esac
|
||||
git clone --no-checkout --filter=blob:none \
|
||||
"ssh://git@gitea-ssh.gitea.svc.cluster.local:22/gitadmin/fxhnt.git" /workspace/src
|
||||
cd /workspace/src && git checkout "$COMMIT_SHA"
|
||||
volumeMounts:
|
||||
- { name: git-ssh-key, mountPath: /etc/git-ssh, readOnly: true }
|
||||
- { name: workspace, mountPath: /workspace }
|
||||
container:
|
||||
image: alpine/k8s:1.31.0 # Alpine-based (has /bin/sh + kubectl). NOT registry.k8s.io/kubectl — that's distroless (no shell) and the deploy runs a shell script. bitnami/kubectl was removed from Docker Hub (2025).
|
||||
command: [/bin/sh, -c]
|
||||
args:
|
||||
- |
|
||||
set -e
|
||||
cd /workspace/src
|
||||
kubectl apply -f infra/k8s/services/fxhnt-dashboard.yaml
|
||||
kubectl apply -f infra/k8s/orchestration/dagster.yaml
|
||||
kubectl apply -f infra/k8s/network-policies/fxhnt-cockpit.yaml
|
||||
kubectl -n foxhunt annotate deploy/fxhnt-dashboard deploy/dagster \
|
||||
fxhnt.io/deployed-sha="$(cd /workspace/src && git rev-parse --short HEAD)" --overwrite
|
||||
# Dashboard: scale 0 -> 1, NOT `rollout restart`. Its tailscale sidecar serves dashboard.fxhnt.ai;
|
||||
# a rolling restart briefly overlaps two tailscale sessions on the same identity -> session race ->
|
||||
# the external proxy link goes stale and the cockpit reads "down" post-deploy. Scaling to 0 cleanly
|
||||
# terminates the session before a fresh, non-overlapping one comes up; the git-sync initContainer
|
||||
# re-pulls src the same way, so code still updates. See reference_fxhnt_dashboard_proxy.
|
||||
#
|
||||
# SAFETY (CI review I1): once we scale to 0, a trap guarantees the deployment is scaled BACK to 1 on
|
||||
# ANY exit path — so a failure between scale-0 and scale-1 (transient apiserver error, throttle) can
|
||||
# never strand the production dashboard at 0 replicas. The trap is best-effort (its own failure is
|
||||
# swallowed) and cleared on success.
|
||||
trap 'kubectl -n foxhunt scale deploy/fxhnt-dashboard --replicas=1 || true' EXIT
|
||||
kubectl -n foxhunt scale deploy/fxhnt-dashboard --replicas=0
|
||||
kubectl -n foxhunt wait --for=delete pod -l app.kubernetes.io/name=fxhnt-dashboard --timeout=90s || true
|
||||
kubectl -n foxhunt scale deploy/fxhnt-dashboard --replicas=1
|
||||
trap - EXIT # scaled back up cleanly; no longer need the safety net
|
||||
kubectl -n foxhunt rollout restart deploy/dagster # dagster has no external-proxy race
|
||||
kubectl -n foxhunt rollout status deploy/fxhnt-dashboard --timeout=180s
|
||||
kubectl -n foxhunt rollout status deploy/dagster --timeout=300s
|
||||
volumeMounts:
|
||||
- { name: workspace, mountPath: /workspace }
|
||||
volumes:
|
||||
- { name: workspace, emptyDir: {} }
|
||||
- { name: git-ssh-key, secret: { secretName: argo-git-ssh-key, defaultMode: 256 } }
|
||||
@@ -7,10 +7,11 @@ metadata:
|
||||
labels:
|
||||
app.kubernetes.io/part-of: foxhunt
|
||||
rules:
|
||||
# apps: roll + apply Deployments. watch is required by `kubectl rollout status`.
|
||||
# apps: roll/scale + apply Deployments. watch is required by `kubectl rollout status`; deployments/scale +
|
||||
# update by the dashboard scale-0->1 that avoids the tailscale session race (reference_fxhnt_dashboard_proxy).
|
||||
- apiGroups: ["apps"]
|
||||
resources: [deployments]
|
||||
verbs: [get, list, watch, patch]
|
||||
resources: [deployments, deployments/scale]
|
||||
verbs: [get, list, watch, patch, update]
|
||||
- apiGroups: ["argoproj.io"]
|
||||
resources: [workflowtemplates, eventsources, sensors, eventbus]
|
||||
verbs: [get, list, create, update, patch]
|
||||
@@ -19,10 +20,14 @@ rules:
|
||||
- apiGroups: [""]
|
||||
resources: [services, configmaps, serviceaccounts, persistentvolumeclaims]
|
||||
verbs: [get, list, create, update, patch]
|
||||
# pods: read-only, for `kubectl wait --for=delete pod` during the dashboard scale-0->1
|
||||
- apiGroups: [""]
|
||||
resources: [pods]
|
||||
verbs: [get, list, watch]
|
||||
- apiGroups: ["networking.k8s.io"]
|
||||
resources: [networkpolicies]
|
||||
verbs: [get, list, create, update, patch]
|
||||
# batch: the fxhnt-forward CronJob
|
||||
# batch: the fxhnt CronJobs (bybit/ibkr rebalancers, compare-measured-precompute)
|
||||
- apiGroups: ["batch"]
|
||||
resources: [cronjobs]
|
||||
verbs: [get, list, create, update, patch]
|
||||
|
||||
@@ -35,10 +35,10 @@ spec:
|
||||
app.kubernetes.io/part-of: foxhunt
|
||||
spec:
|
||||
nodeSelector:
|
||||
k8s.scaleway.com/pool-name: infra
|
||||
k8s.scaleway.com/pool-name: large
|
||||
containers:
|
||||
- name: postgres
|
||||
image: timescale/timescaledb:latest-pg16
|
||||
image: timescale/timescaledb:2.25.1-pg16 # PINNED to match the foxhunt source exactly (dump/restore needs equal tsdb versions; latest gave 2.28.3 -> catalog-restore failure)
|
||||
ports:
|
||||
- containerPort: 5432
|
||||
name: postgres
|
||||
|
||||
@@ -1,95 +0,0 @@
|
||||
# Alpaca rebalancer — weekday CronJob running the adaptive MULTI-STRAT book on ALPACA (paper) via the
|
||||
# fractional Broker adapter. Mirrors the fxhnt-CLI Job pattern (git-sync + fxhnt-cockpit image); runs
|
||||
# `fxhnt execute-multistrat --execute`, which computes the book's current target weights (multistrat.
|
||||
# target_weights over FUND_INSTRUMENTS), reconciles vs the Alpaca account, and places FRACTIONAL market
|
||||
# orders through the ExecutionService safety gates — the 5 PURE-EQUITY ETFs (SPY/IEF/GLD/PDBC/DBMF). BTC-USD
|
||||
# is dropped from the fund book (crypto exposure comes from the Bybit book; BTC only added correlation), so
|
||||
# this is a pure-equity leg — no crypto order. Paper base_url; live separately gated by allow_live.
|
||||
#
|
||||
# SHIP SUSPENDED (suspend: true): the `alpaca-credentials` paper API keys were regenerated on the
|
||||
# Alpaca dashboard and currently 401 on auth. Refresh the secret with the new keys, then
|
||||
# `kubectl patch cronjob fxhnt-alpaca-rebalancer -n foxhunt -p '{"spec":{"suspend":false}}'` to arm it
|
||||
# (mirrors the Bybit-testnet leg awaiting its key).
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: fxhnt-alpaca-rebalancer
|
||||
namespace: foxhunt
|
||||
labels: { app.kubernetes.io/name: fxhnt-alpaca-rebalancer, app.kubernetes.io/part-of: foxhunt }
|
||||
spec:
|
||||
schedule: "35 14 * * 1-5" # weekdays 14:35 UTC (~1h after US open); hysteresis prevents churn
|
||||
suspend: true # ARM after alpaca-credentials is refreshed (current keys 401)
|
||||
concurrencyPolicy: Forbid
|
||||
successfulJobsHistoryLimit: 3
|
||||
failedJobsHistoryLimit: 3
|
||||
jobTemplate:
|
||||
spec:
|
||||
backoffLimit: 1
|
||||
ttlSecondsAfterFinished: 604800
|
||||
template:
|
||||
metadata:
|
||||
labels: { app.kubernetes.io/name: fxhnt-alpaca-rebalancer, app.kubernetes.io/part-of: foxhunt }
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
imagePullSecrets:
|
||||
- name: scw-registry
|
||||
initContainers:
|
||||
- name: git-sync
|
||||
image: alpine/git:latest
|
||||
command: [/bin/sh, -c]
|
||||
args:
|
||||
- |
|
||||
mkdir -p /root/.ssh && cp /etc/git-ssh/ssh-privatekey /root/.ssh/id_ed25519 && chmod 600 /root/.ssh/id_ed25519
|
||||
printf 'Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n' > /root/.ssh/config
|
||||
rm -rf /code/src
|
||||
git clone --depth 1 --branch master --filter=blob:none --sparse \
|
||||
"ssh://git@gitea-sshd.foxhunt.svc.cluster.local:22/gitadmin/fxhnt.git" /tmp/repo
|
||||
cd /tmp/repo && git sparse-checkout set src && [ -d /tmp/repo/src ]
|
||||
cp -a /tmp/repo/src /code/src && echo "git-sync OK: $(git rev-parse --short HEAD)"
|
||||
resources: { requests: { cpu: 50m, memory: 64Mi }, limits: { cpu: 500m, memory: 256Mi } }
|
||||
volumeMounts:
|
||||
- { name: code, mountPath: /code }
|
||||
- { name: git-ssh-key, mountPath: /etc/git-ssh, readOnly: true }
|
||||
containers:
|
||||
- name: rebalancer
|
||||
image: rg.fr-par.scw.cloud/bizworx/fxhnt-cockpit:latest
|
||||
command: ["fxhnt", "execute-multistrat", "--execute", "--paper-envelope", "1000000"]
|
||||
env:
|
||||
- { name: PYTHONPATH, value: /code/src }
|
||||
- { name: FXHNT_OPERATIONAL_DSN, value: "postgresql+psycopg://foxhunt@postgres.foxhunt.svc.cluster.local:5432/fxhnt" }
|
||||
- name: PGPASSWORD
|
||||
valueFrom: { secretKeyRef: { name: db-credentials, key: password } }
|
||||
- name: FXHNT_ALPACA_API_KEY
|
||||
valueFrom: { secretKeyRef: { name: alpaca-credentials, key: api-key } }
|
||||
- name: FXHNT_ALPACA_API_SECRET
|
||||
valueFrom: { secretKeyRef: { name: alpaca-credentials, key: api-secret } }
|
||||
# paper is the AlpacaSettings default; override FXHNT_ALPACA_BASE_URL for live (+ allow_live).
|
||||
- { name: PYTHONUNBUFFERED, value: "1" }
|
||||
resources:
|
||||
requests: { memory: "512Mi", cpu: "250m" }
|
||||
limits: { memory: "1Gi", cpu: "1" }
|
||||
volumeMounts:
|
||||
- { name: code, mountPath: /code, readOnly: true }
|
||||
volumes:
|
||||
- { name: code, emptyDir: {} }
|
||||
- { name: git-ssh-key, secret: { secretName: argo-git-ssh-key, defaultMode: 256 } }
|
||||
---
|
||||
# Egress: DNS + Postgres (5432, read the survivor book) + gitea-sshd (22, git-sync) + external 443
|
||||
# (Yahoo price data + *.alpaca.markets order/account API).
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: fxhnt-alpaca-rebalancer
|
||||
namespace: foxhunt
|
||||
labels: { app.kubernetes.io/part-of: foxhunt }
|
||||
spec:
|
||||
podSelector: { matchLabels: { app.kubernetes.io/name: fxhnt-alpaca-rebalancer } }
|
||||
policyTypes: [Egress]
|
||||
egress:
|
||||
- ports: [{ port: 53, protocol: UDP }, { port: 53, protocol: TCP }]
|
||||
- to: [{ podSelector: { matchLabels: { app.kubernetes.io/name: postgres } } }]
|
||||
ports: [{ port: 5432, protocol: TCP }]
|
||||
- to: [{ podSelector: { matchLabels: { app.kubernetes.io/name: gitea } } }]
|
||||
ports: [{ port: 2222, protocol: TCP }, { port: 22, protocol: TCP }]
|
||||
- to: [{ ipBlock: { cidr: 0.0.0.0/0, except: [10.32.0.0/16, 172.16.0.0/16] } }] # Yahoo data + alpaca.markets
|
||||
ports: [{ port: 443, protocol: TCP }]
|
||||
@@ -1,109 +0,0 @@
|
||||
# backtest-refs SCHEDULED (WRITER) — recompute + upsert every reconciliation-gated strategy's
|
||||
# `backtest_summary` reference, covering the HEAVY `report`-kind bybit-book strategies (bybit_4edge /
|
||||
# bybit_4edge_levered / positioning) that are too expensive to recompute inline on every nightly asset run.
|
||||
# The cheap `recompute-replay`/`sleeve`-kind refs (multistrat/vrp/crypto_tstrend/stablecoin/unlock/xsfunding)
|
||||
# are already persisted inline by their own nightly assets (`_persist_track_backtest_ref` in assets.py) —
|
||||
# this CronJob does NOT duplicate those.
|
||||
#
|
||||
# Fixes bug B: this REPLACES the manual, on-demand `fxhnt-bybit-sleeve-ret` Job — and is now a COMPLETE,
|
||||
# no-regression replacement (not just the 3 refs): the bybit-book sids go through the shared compute-once
|
||||
# `_persist_bybit_book` helper in cli.py, which ALSO writes `bybit_sleeve_ret` (the cockpit sim's
|
||||
# precomputed backtest data — read live in ~6 places in adapters/web/app.py), exactly as the retired Job's
|
||||
# `bybit-persist-sleeve-ret` CLI did, from the SAME sleeve-return series (no wasted recompute). Any OTHER
|
||||
# report-kind sid is then covered generically via the unified `backtest_ref(sid, store)` dispatcher
|
||||
# (fxhnt.application.backtest_refs). That Job had to be applied BY HAND — a missed manual run silently
|
||||
# starves the reconciliation gate of a ref (the gate then WITHHOLDS PASS forever, with no operator signal).
|
||||
# Scheduling `fxhnt backtest-refs --all` weekly turns that manual step into a standing guarantee: both
|
||||
# `bybit_sleeve_ret` and the report-kind refs are refreshed every Monday 02:00 UTC without anyone
|
||||
# remembering to run a Job.
|
||||
#
|
||||
# OWN MEMORY (the OOM-proofing payoff, unchanged from the retired Job): this heavy multi-symbol compute runs
|
||||
# in THIS CronJob with its OWN memory limit (4Gi req / 6Gi limit), never in the 4Gi dagster daemon — the same
|
||||
# reasoning that motivated the original sleeve-ret split (it previously OOM-killed the daemon, exit 137, and
|
||||
# because OOM is uncatchable that also skipped the forward-nav step and broke the nightly forward track).
|
||||
# The liquid-universe bound (`liquid_universe(min_dollar_vol=10M)` inside `_persist_bybit_book`) restricts the
|
||||
# READ to the liquid subset, so panels stay memory-bounded, never the whole ~794-coin table.
|
||||
#
|
||||
# FULLY-DECOUPLED BATCH: NO PVC, NO podAffinity — it reads TimescaleDB over the network (5432; both
|
||||
# `bybit_features` for the sleeve compute and the operational DB for `backtest_summary` + the unlock-calendar
|
||||
# DB-first read), and git-sync brings the code (the `backtest-refs` CLI). Schedules on ANY node with room.
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: fxhnt-backtest-refs
|
||||
namespace: foxhunt
|
||||
labels: { app.kubernetes.io/name: fxhnt-backtest-refs, app.kubernetes.io/part-of: foxhunt }
|
||||
spec:
|
||||
schedule: "0 2 * * 1" # weekly, Monday 02:00 UTC
|
||||
concurrencyPolicy: Forbid # never overlap a slow run with the next week's trigger
|
||||
successfulJobsHistoryLimit: 3
|
||||
failedJobsHistoryLimit: 3
|
||||
jobTemplate:
|
||||
metadata:
|
||||
labels: { app.kubernetes.io/name: fxhnt-backtest-refs, app.kubernetes.io/part-of: foxhunt }
|
||||
spec:
|
||||
backoffLimit: 2
|
||||
ttlSecondsAfterFinished: 604800
|
||||
template:
|
||||
metadata:
|
||||
labels: { app.kubernetes.io/name: fxhnt-backtest-refs, app.kubernetes.io/part-of: foxhunt }
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
imagePullSecrets:
|
||||
- name: scw-registry
|
||||
initContainers:
|
||||
- name: git-sync
|
||||
image: alpine/git:latest
|
||||
command: [/bin/sh, -c]
|
||||
args:
|
||||
- |
|
||||
mkdir -p /root/.ssh && cp /etc/git-ssh/ssh-privatekey /root/.ssh/id_ed25519 && chmod 600 /root/.ssh/id_ed25519
|
||||
printf 'Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n' > /root/.ssh/config
|
||||
rm -rf /code/src
|
||||
git clone --depth 1 --branch master --filter=blob:none --sparse \
|
||||
"ssh://git@gitea-sshd.foxhunt.svc.cluster.local:22/gitadmin/fxhnt.git" /tmp/repo
|
||||
cd /tmp/repo && git sparse-checkout set src && [ -d /tmp/repo/src ]
|
||||
cp -a /tmp/repo/src /code/src && echo "git-sync OK: $(git rev-parse --short HEAD)"
|
||||
resources: { requests: { cpu: 50m, memory: 64Mi }, limits: { cpu: 500m, memory: 256Mi } }
|
||||
volumeMounts:
|
||||
- { name: code, mountPath: /code }
|
||||
- { name: git-ssh-key, mountPath: /etc/git-ssh, readOnly: true }
|
||||
containers:
|
||||
- name: backtest-refs
|
||||
image: rg.fr-par.scw.cloud/bizworx/fxhnt-cockpit:latest
|
||||
command: ["fxhnt", "backtest-refs", "--all"]
|
||||
env:
|
||||
- { name: PYTHONPATH, value: /code/src }
|
||||
- { name: FXHNT_OPERATIONAL_DSN, value: "postgresql+psycopg://foxhunt@postgres.foxhunt.svc.cluster.local:5432/fxhnt" }
|
||||
- name: PGPASSWORD
|
||||
valueFrom: { secretKeyRef: { name: db-credentials, key: password } }
|
||||
- { name: PYTHONUNBUFFERED, value: "1" }
|
||||
resources:
|
||||
# OWN memory so it never OOMs the daemon: 4Gi req / 6Gi limit (liquid-subset multi-symbol compute).
|
||||
requests: { memory: "4Gi", cpu: "1" }
|
||||
limits: { memory: "6Gi", cpu: "2" } # no PVC
|
||||
volumeMounts:
|
||||
- { name: code, mountPath: /code, readOnly: true }
|
||||
volumes:
|
||||
- { name: code, emptyDir: {} }
|
||||
- { name: git-ssh-key, secret: { secretName: argo-git-ssh-key, defaultMode: 256 } }
|
||||
---
|
||||
# Egress: DNS + Postgres/Timescale (5432, read bybit_features + write backtest_summary) + external 443
|
||||
# (DefiLlama unlock calendar loader, when the operational DB read is unavailable) + gitea-sshd (22, git-sync).
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: fxhnt-backtest-refs
|
||||
namespace: foxhunt
|
||||
labels: { app.kubernetes.io/part-of: foxhunt }
|
||||
spec:
|
||||
podSelector: { matchLabels: { app.kubernetes.io/name: fxhnt-backtest-refs } }
|
||||
policyTypes: [Egress]
|
||||
egress:
|
||||
- ports: [{ port: 53, protocol: UDP }, { port: 53, protocol: TCP }]
|
||||
- to: [{ podSelector: { matchLabels: { app.kubernetes.io/name: postgres } } }]
|
||||
ports: [{ port: 5432, protocol: TCP }]
|
||||
- to: [{ podSelector: { matchLabels: { app.kubernetes.io/name: gitea } } }]
|
||||
ports: [{ port: 2222, protocol: TCP }, { port: 22, protocol: TCP }]
|
||||
- to: [{ ipBlock: { cidr: 0.0.0.0/0, except: [10.32.0.0/16, 172.16.0.0/16] } }] # external loaders
|
||||
ports: [{ port: 443, protocol: TCP }]
|
||||
@@ -35,7 +35,7 @@ spec:
|
||||
printf 'Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n' > /root/.ssh/config
|
||||
rm -rf /code/src
|
||||
git clone --depth 1 --branch master --filter=blob:none --sparse \
|
||||
"ssh://git@gitea-sshd.foxhunt.svc.cluster.local:22/gitadmin/fxhnt.git" /tmp/repo
|
||||
"ssh://git@gitea-ssh.gitea.svc.cluster.local:22/gitadmin/fxhnt.git" /tmp/repo
|
||||
cd /tmp/repo && git sparse-checkout set src && [ -d /tmp/repo/src ]
|
||||
cp -a /tmp/repo/src /code/src && echo "git-sync OK: $(git rev-parse --short HEAD)"
|
||||
resources: { requests: { cpu: 50m, memory: 64Mi }, limits: { cpu: 500m, memory: 256Mi } }
|
||||
@@ -78,7 +78,7 @@ spec:
|
||||
- ports: [{ port: 53, protocol: UDP }, { port: 53, protocol: TCP }]
|
||||
- to: [{ podSelector: { matchLabels: { app.kubernetes.io/name: postgres } } }]
|
||||
ports: [{ port: 5432, protocol: TCP }]
|
||||
- to: [{ podSelector: { matchLabels: { app.kubernetes.io/name: gitea } } }]
|
||||
- to: [{ namespaceSelector: { matchLabels: { kubernetes.io/metadata.name: gitea } }, podSelector: { matchLabels: { app.kubernetes.io/name: gitea } } }]
|
||||
ports: [{ port: 2222, protocol: TCP }, { port: 22, protocol: TCP }]
|
||||
- to: [{ ipBlock: { cidr: 0.0.0.0/0, except: [10.32.0.0/16, 172.16.0.0/16] } }] # external loaders
|
||||
ports: [{ port: 443, protocol: TCP }]
|
||||
|
||||
@@ -29,7 +29,7 @@ spec:
|
||||
printf 'Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n' > /root/.ssh/config
|
||||
rm -rf /code/src
|
||||
git clone --depth 1 --branch master --filter=blob:none --sparse \
|
||||
"ssh://git@gitea-sshd.foxhunt.svc.cluster.local:22/gitadmin/fxhnt.git" /tmp/repo
|
||||
"ssh://git@gitea-ssh.gitea.svc.cluster.local:22/gitadmin/fxhnt.git" /tmp/repo
|
||||
cd /tmp/repo && git sparse-checkout set src && [ -d /tmp/repo/src ]
|
||||
cp -a /tmp/repo/src /code/src && echo "git-sync OK: $(git rev-parse --short HEAD)"
|
||||
resources: { requests: { cpu: 50m, memory: 64Mi }, limits: { cpu: 500m, memory: 256Mi } }
|
||||
@@ -69,7 +69,7 @@ spec:
|
||||
- ports: [{ port: 53, protocol: UDP }, { port: 53, protocol: TCP }]
|
||||
- to: [{ podSelector: { matchLabels: { app.kubernetes.io/name: postgres } } }]
|
||||
ports: [{ port: 5432, protocol: TCP }]
|
||||
- to: [{ podSelector: { matchLabels: { app.kubernetes.io/name: gitea } } }]
|
||||
- to: [{ namespaceSelector: { matchLabels: { kubernetes.io/metadata.name: gitea } }, podSelector: { matchLabels: { app.kubernetes.io/name: gitea } } }]
|
||||
ports: [{ port: 2222, protocol: TCP }, { port: 22, protocol: TCP }]
|
||||
- to: [{ ipBlock: { cidr: 0.0.0.0/0, except: [10.32.0.0/16, 172.16.0.0/16] } }] # Bybit public API
|
||||
ports: [{ port: 443, protocol: TCP }]
|
||||
|
||||
@@ -29,7 +29,7 @@ spec:
|
||||
printf 'Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n' > /root/.ssh/config
|
||||
rm -rf /code/src
|
||||
git clone --depth 1 --branch master --filter=blob:none --sparse \
|
||||
"ssh://git@gitea-sshd.foxhunt.svc.cluster.local:22/gitadmin/fxhnt.git" /tmp/repo
|
||||
"ssh://git@gitea-ssh.gitea.svc.cluster.local:22/gitadmin/fxhnt.git" /tmp/repo
|
||||
cd /tmp/repo && git sparse-checkout set src && [ -d /tmp/repo/src ]
|
||||
cp -a /tmp/repo/src /code/src && echo "git-sync OK: $(git rev-parse --short HEAD)"
|
||||
resources: { requests: { cpu: 50m, memory: 64Mi }, limits: { cpu: 500m, memory: 256Mi } }
|
||||
@@ -69,7 +69,7 @@ spec:
|
||||
- ports: [{ port: 53, protocol: UDP }, { port: 53, protocol: TCP }]
|
||||
- to: [{ podSelector: { matchLabels: { app.kubernetes.io/name: postgres } } }]
|
||||
ports: [{ port: 5432, protocol: TCP }]
|
||||
- to: [{ podSelector: { matchLabels: { app.kubernetes.io/name: gitea } } }]
|
||||
- to: [{ namespaceSelector: { matchLabels: { kubernetes.io/metadata.name: gitea } }, podSelector: { matchLabels: { app.kubernetes.io/name: gitea } } }]
|
||||
ports: [{ port: 2222, protocol: TCP }, { port: 22, protocol: TCP }]
|
||||
- to: [{ ipBlock: { cidr: 0.0.0.0/0, except: [10.32.0.0/16, 172.16.0.0/16] } }] # Bybit public API
|
||||
ports: [{ port: 443, protocol: TCP }]
|
||||
|
||||
@@ -31,7 +31,7 @@ spec:
|
||||
printf 'Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n' > /root/.ssh/config
|
||||
rm -rf /code/src
|
||||
git clone --depth 1 --branch master --filter=blob:none --sparse \
|
||||
"ssh://git@gitea-sshd.foxhunt.svc.cluster.local:22/gitadmin/fxhnt.git" /tmp/repo
|
||||
"ssh://git@gitea-ssh.gitea.svc.cluster.local:22/gitadmin/fxhnt.git" /tmp/repo
|
||||
cd /tmp/repo && git sparse-checkout set src && [ -d /tmp/repo/src ]
|
||||
cp -a /tmp/repo/src /code/src && echo "git-sync OK: $(git rev-parse --short HEAD)"
|
||||
resources: { requests: { cpu: 50m, memory: 64Mi }, limits: { cpu: 500m, memory: 256Mi } }
|
||||
@@ -70,7 +70,7 @@ spec:
|
||||
- ports: [{ port: 53, protocol: UDP }, { port: 53, protocol: TCP }]
|
||||
- to: [{ podSelector: { matchLabels: { app.kubernetes.io/name: postgres } } }]
|
||||
ports: [{ port: 5432, protocol: TCP }]
|
||||
- to: [{ podSelector: { matchLabels: { app.kubernetes.io/name: gitea } } }]
|
||||
- to: [{ namespaceSelector: { matchLabels: { kubernetes.io/metadata.name: gitea } }, podSelector: { matchLabels: { app.kubernetes.io/name: gitea } } }]
|
||||
ports: [{ port: 2222, protocol: TCP }, { port: 22, protocol: TCP }]
|
||||
- to: [{ ipBlock: { cidr: 0.0.0.0/0, except: [10.32.0.0/16, 172.16.0.0/16] } }]
|
||||
ports: [{ port: 443, protocol: TCP }]
|
||||
|
||||
@@ -40,7 +40,7 @@ spec:
|
||||
printf 'Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n' > /root/.ssh/config
|
||||
rm -rf /code/src
|
||||
git clone --depth 1 --branch master --filter=blob:none --sparse \
|
||||
"ssh://git@gitea-sshd.foxhunt.svc.cluster.local:22/gitadmin/fxhnt.git" /tmp/repo
|
||||
"ssh://git@gitea-ssh.gitea.svc.cluster.local:22/gitadmin/fxhnt.git" /tmp/repo
|
||||
cd /tmp/repo && git sparse-checkout set src && [ -d /tmp/repo/src ]
|
||||
cp -a /tmp/repo/src /code/src && echo "git-sync OK: $(git rev-parse --short HEAD)"
|
||||
resources: { requests: { cpu: 50m, memory: 64Mi }, limits: { cpu: 500m, memory: 256Mi } }
|
||||
@@ -86,7 +86,7 @@ spec:
|
||||
- ports: [{ port: 53, protocol: UDP }, { port: 53, protocol: TCP }]
|
||||
- to: [{ podSelector: { matchLabels: { app.kubernetes.io/name: postgres } } }]
|
||||
ports: [{ port: 5432, protocol: TCP }]
|
||||
- to: [{ podSelector: { matchLabels: { app.kubernetes.io/name: gitea } } }]
|
||||
- to: [{ namespaceSelector: { matchLabels: { kubernetes.io/metadata.name: gitea } }, podSelector: { matchLabels: { app.kubernetes.io/name: gitea } } }]
|
||||
ports: [{ port: 2222, protocol: TCP }, { port: 22, protocol: TCP }]
|
||||
- to: [{ ipBlock: { cidr: 0.0.0.0/0, except: [10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16] } }] # api-testnet.bybit.com
|
||||
ports: [{ port: 443, protocol: TCP }]
|
||||
|
||||
@@ -29,7 +29,7 @@ spec:
|
||||
printf 'Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n' > /root/.ssh/config
|
||||
rm -rf /code/src
|
||||
git clone --depth 1 --branch master --filter=blob:none --sparse \
|
||||
"ssh://git@gitea-sshd.foxhunt.svc.cluster.local:22/gitadmin/fxhnt.git" /tmp/repo
|
||||
"ssh://git@gitea-ssh.gitea.svc.cluster.local:22/gitadmin/fxhnt.git" /tmp/repo
|
||||
cd /tmp/repo && git sparse-checkout set src && [ -d /tmp/repo/src ]
|
||||
cp -a /tmp/repo/src /code/src && echo "git-sync OK: $(git rev-parse --short HEAD)"
|
||||
resources: { requests: { cpu: 50m, memory: 64Mi }, limits: { cpu: 500m, memory: 256Mi } }
|
||||
@@ -69,7 +69,7 @@ spec:
|
||||
- ports: [{ port: 53, protocol: UDP }, { port: 53, protocol: TCP }]
|
||||
- to: [{ podSelector: { matchLabels: { app.kubernetes.io/name: postgres } } }]
|
||||
ports: [{ port: 5432, protocol: TCP }]
|
||||
- to: [{ podSelector: { matchLabels: { app.kubernetes.io/name: gitea } } }]
|
||||
- to: [{ namespaceSelector: { matchLabels: { kubernetes.io/metadata.name: gitea } }, podSelector: { matchLabels: { app.kubernetes.io/name: gitea } } }]
|
||||
ports: [{ port: 2222, protocol: TCP }, { port: 22, protocol: TCP }]
|
||||
- to: [{ ipBlock: { cidr: 0.0.0.0/0, except: [10.32.0.0/16, 172.16.0.0/16] } }] # Bybit public API
|
||||
ports: [{ port: 443, protocol: TCP }]
|
||||
|
||||
@@ -1,105 +0,0 @@
|
||||
# Measured-cost precompute (WRITER) — DAILY CronJob. Computes the per-COIN MEASURED-cost artifacts and
|
||||
# idempotently upserts them into `compare_measured_cache`: (a) the COMPARE row (Binance per-coin book from the
|
||||
# FULL shadow history × `features` Corwin–Schultz spreads, vs the Bybit 4-edge per-coin book × `bybit_features`
|
||||
# spreads) AND (b) the per-book SINGLE measured curves the Backtest's DEFAULT cost mode reads
|
||||
# (`binance_combined` + `bybit_4edge`). The cockpit `/paper/sim` + `book=compare` measured paths ONLY READ
|
||||
# these rows (falling back to the LIGHT flat-cost compare until/if absent — so the page never OOMs/breaks).
|
||||
#
|
||||
# WHY A CRONJOB (own memory, NOT a Dagster asset): this heavy per-coin compute needs up to 6Gi and OOM-killed
|
||||
# the 4Gi `fxhnt-dashboard` when it ran live; the Dagster daemon is ALSO 4Gi, so it can't run inside the
|
||||
# nightly asset graph either. So it runs as its OWN-memory Job (6Gi req / 8Gi limit) on a daily schedule
|
||||
# timed AFTER the Dagster combined-book ingests (23:30 UTC) so it reads FRESH `features`/`bybit_features` +
|
||||
# the refreshed bybit backfill. On-demand: `kubectl create job <name> --from=cronjob/fxhnt-compare-measured-precompute -n foxhunt`.
|
||||
#
|
||||
# FULLY-DECOUPLED BATCH: NO PVC — reads TimescaleDB over the network (5432: `features` + `bybit_features` +
|
||||
# the cockpit paper tables incl. the unlock calendar), writes `compare_measured_cache` to the same DB;
|
||||
# schedules on ANY node with room. git-sync brings the current code (compare-measured-precompute CLI).
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: fxhnt-compare-measured-precompute
|
||||
namespace: foxhunt
|
||||
labels: { app.kubernetes.io/name: fxhnt-compare-measured-precompute, app.kubernetes.io/part-of: foxhunt }
|
||||
spec:
|
||||
schedule: "0 1 * * *" # 01:00 UTC daily — AFTER the Dagster 23:30 combined-book ingests finish
|
||||
timeZone: "UTC"
|
||||
concurrencyPolicy: Forbid # never overlap runs (one heavy writer at a time)
|
||||
startingDeadlineSeconds: 3600
|
||||
successfulJobsHistoryLimit: 2
|
||||
failedJobsHistoryLimit: 2
|
||||
jobTemplate:
|
||||
spec:
|
||||
backoffLimit: 2
|
||||
ttlSecondsAfterFinished: 604800
|
||||
template:
|
||||
metadata:
|
||||
labels: { app.kubernetes.io/name: fxhnt-compare-measured-precompute, app.kubernetes.io/part-of: foxhunt }
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
# Pin to the 16GB POP2-4C-16G pool (the platform DEV1-L nodes are 8GB — too small for the 8Gi
|
||||
# limit). The pool autoscales min=0→up on demand for this Pending pod, runs it, scales back to 0
|
||||
# (cost = only the minutes it runs). Soft PreferNoSchedule autoscaler tag doesn't block scheduling.
|
||||
nodeSelector:
|
||||
k8s.scaleway.com/pool-name: ci-compile-cpu
|
||||
imagePullSecrets:
|
||||
- name: scw-registry
|
||||
initContainers:
|
||||
- name: git-sync
|
||||
image: alpine/git:latest
|
||||
command: [/bin/sh, -c]
|
||||
args:
|
||||
- |
|
||||
mkdir -p /root/.ssh && cp /etc/git-ssh/ssh-privatekey /root/.ssh/id_ed25519 && chmod 600 /root/.ssh/id_ed25519
|
||||
printf 'Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n' > /root/.ssh/config
|
||||
rm -rf /code/src
|
||||
git clone --depth 1 --branch master --filter=blob:none --sparse \
|
||||
"ssh://git@gitea-sshd.foxhunt.svc.cluster.local:22/gitadmin/fxhnt.git" /tmp/repo
|
||||
cd /tmp/repo && git sparse-checkout set src && [ -d /tmp/repo/src ]
|
||||
cp -a /tmp/repo/src /code/src && echo "git-sync OK: $(git rev-parse --short HEAD)"
|
||||
resources: { requests: { cpu: 50m, memory: 64Mi }, limits: { cpu: 500m, memory: 256Mi } }
|
||||
volumeMounts:
|
||||
- { name: code, mountPath: /code }
|
||||
- { name: git-ssh-key, mountPath: /etc/git-ssh, readOnly: true }
|
||||
containers:
|
||||
- name: compare-precompute
|
||||
image: rg.fr-par.scw.cloud/bizworx/fxhnt-cockpit:latest
|
||||
# Refresh the REAL per-coin L1 quoted spread FIRST (samples Bybit top-of-book → median per coin,
|
||||
# needs api.bybit.com:443), THEN run the measured-cost precompute which reads bybit_real_spread to
|
||||
# cost the liquid book at the REAL spread (not the 104×-too-harsh daily-high/low CS proxy). The
|
||||
# refresh is non-fatal (kept prior/CS on a fetch failure), so && never blocks the precompute.
|
||||
command: ["/bin/sh", "-c"]
|
||||
args:
|
||||
- fxhnt bybit-spread-refresh --samples 5 && fxhnt compare-measured-precompute
|
||||
env:
|
||||
- { name: PYTHONPATH, value: /code/src }
|
||||
- { name: FXHNT_OPERATIONAL_DSN, value: "postgresql+psycopg://foxhunt@postgres.foxhunt.svc.cluster.local:5432/fxhnt" }
|
||||
- name: PGPASSWORD
|
||||
valueFrom: { secretKeyRef: { name: db-credentials, key: password } }
|
||||
- { name: PYTHONUNBUFFERED, value: "1" }
|
||||
resources:
|
||||
requests: { memory: "6Gi", cpu: "1" }
|
||||
limits: { memory: "8Gi", cpu: "2" } # own memory, generous headroom; no PVC
|
||||
volumeMounts:
|
||||
- { name: code, mountPath: /code, readOnly: true }
|
||||
volumes:
|
||||
- { name: code, emptyDir: {} }
|
||||
- { name: git-ssh-key, secret: { secretName: argo-git-ssh-key, defaultMode: 256 } }
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: fxhnt-compare-measured-precompute
|
||||
namespace: foxhunt
|
||||
labels: { app.kubernetes.io/part-of: foxhunt }
|
||||
spec:
|
||||
podSelector: { matchLabels: { app.kubernetes.io/name: fxhnt-compare-measured-precompute } }
|
||||
policyTypes: [Egress]
|
||||
egress:
|
||||
- ports: [{ port: 53, protocol: UDP }, { port: 53, protocol: TCP }]
|
||||
- to: [{ podSelector: { matchLabels: { app.kubernetes.io/name: postgres } } }]
|
||||
ports: [{ port: 5432, protocol: TCP }]
|
||||
- to: [{ podSelector: { matchLabels: { app.kubernetes.io/name: gitea } } }]
|
||||
ports: [{ port: 2222, protocol: TCP }, { port: 22, protocol: TCP }]
|
||||
# api.bybit.com (443) — the `bybit-spread-refresh` step samples the live L1 top-of-book over the public
|
||||
# internet. No selector (external host); HTTPS only.
|
||||
- ports: [{ port: 443, protocol: TCP }]
|
||||
@@ -26,7 +26,7 @@ spec:
|
||||
printf 'Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n' > /root/.ssh/config
|
||||
rm -rf /code/src
|
||||
git clone --depth 1 --branch master --filter=blob:none --sparse \
|
||||
"ssh://git@gitea-sshd.foxhunt.svc.cluster.local:22/gitadmin/fxhnt.git" /tmp/repo
|
||||
"ssh://git@gitea-ssh.gitea.svc.cluster.local:22/gitadmin/fxhnt.git" /tmp/repo
|
||||
cd /tmp/repo && git sparse-checkout set src && [ -d /tmp/repo/src ]
|
||||
cp -a /tmp/repo/src /code/src && echo "git-sync OK: $(git rev-parse --short HEAD)"
|
||||
resources: { requests: { cpu: 50m, memory: 64Mi }, limits: { cpu: 500m, memory: 256Mi } }
|
||||
@@ -65,7 +65,7 @@ spec:
|
||||
- ports: [{ port: 53, protocol: UDP }, { port: 53, protocol: TCP }]
|
||||
- to: [{ podSelector: { matchLabels: { app.kubernetes.io/name: postgres } } }]
|
||||
ports: [{ port: 5432, protocol: TCP }]
|
||||
- to: [{ podSelector: { matchLabels: { app.kubernetes.io/name: gitea } } }]
|
||||
- to: [{ namespaceSelector: { matchLabels: { kubernetes.io/metadata.name: gitea } }, podSelector: { matchLabels: { app.kubernetes.io/name: gitea } } }]
|
||||
ports: [{ port: 2222, protocol: TCP }, { port: 22, protocol: TCP }]
|
||||
- to: [{ ipBlock: { cidr: 0.0.0.0/0, except: [10.32.0.0/16, 172.16.0.0/16] } }] # Deribit API (443)
|
||||
ports: [{ port: 443, protocol: TCP }]
|
||||
|
||||
@@ -18,7 +18,7 @@ spec:
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
nodeSelector:
|
||||
k8s.scaleway.com/pool-name: platform
|
||||
k8s.scaleway.com/pool-name: large
|
||||
imagePullSecrets:
|
||||
- name: scw-registry
|
||||
containers:
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
# IBKR rebalancer — runs the adaptive MULTI-STRAT book (5 pure-equity ETFs) on IBKR PAPER (DU9600528) at
|
||||
# $1M via `execute-multistrat --broker ibkr --execute --paper-envelope 1000000` (C3 machinery). This is the
|
||||
# FORWARD-TRACK-WHILE-WAITING-FOR-ALPACA venue: at $1M the whole-share rounding that killed IBKR at $35k is
|
||||
# <0.1%, so IBKR paper is a faithful real-fill proxy for the eventual Alpaca (fractional) deploy. It accrues
|
||||
# the OBSERVED multistrat_exec track (real executed-vs-modeled reconciliation) — see the cockpit "Executed
|
||||
# reality" block. When Alpaca keys are valid: arm fxhnt-alpaca-rebalancer + `suspend: true` this one (venue swap).
|
||||
# $1M via `execute-multistrat --execute --paper-envelope 1000000` (C3 machinery). IBKR is the PERMANENT
|
||||
# equity-execution venue (Phase 0b Task 5 removed the fractional-broker leg for good — 401'd keys, dead
|
||||
# code). At $1M the whole-share rounding that hurt IBKR at $35k is <0.1%, so this stays the faithful
|
||||
# forward-track venue. It accrues the OBSERVED multistrat_exec track (real executed-vs-modeled
|
||||
# reconciliation) — see the cockpit "Executed reality" block.
|
||||
#
|
||||
# ib_async is NOT baked into the cockpit image (only the Alpaca httpx path is) -> pip-install it at runtime,
|
||||
# like the one-off C3 execute Job did. Requires ib-gateway UP (kubectl scale deploy/ib-gateway --replicas=1).
|
||||
# ib_async is NOT baked into the cockpit image -> pip-install it at runtime, like the one-off C3 execute
|
||||
# Job did. Requires ib-gateway UP (kubectl scale deploy/ib-gateway --replicas=1).
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
@@ -15,7 +15,11 @@ metadata:
|
||||
labels: { app.kubernetes.io/name: fxhnt-ibkr-rebalancer, app.kubernetes.io/part-of: foxhunt }
|
||||
spec:
|
||||
schedule: "35 14 * * 1-5" # weekdays 14:35 UTC (~1h after US equity open); once-daily forward track
|
||||
suspend: false # ARMED — IBKR paper is the live forward venue while Alpaca keys are 401
|
||||
suspend: true # SUSPENDED 2026-07-15 — the US-ETF leg gets PRIIPs/KID-rejected on the
|
||||
# EU-domiciled DU paper account ("No Trading Permission, no KID"). The
|
||||
# PRIIPs-legal UCITS leg (fxhnt-ucits-rebalancer, --venue ucits) replaces
|
||||
# it. Re-arm only if trading from a non-EU account. (Modeled US track
|
||||
# is unchanged; only execution routes to UCITS.)
|
||||
concurrencyPolicy: Forbid
|
||||
successfulJobsHistoryLimit: 3
|
||||
failedJobsHistoryLimit: 3
|
||||
@@ -43,7 +47,7 @@ spec:
|
||||
printf 'Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n' > /root/.ssh/config
|
||||
rm -rf /code/src
|
||||
git clone --depth 1 --branch master --filter=blob:none --sparse \
|
||||
"ssh://git@gitea-sshd.foxhunt.svc.cluster.local:22/gitadmin/fxhnt.git" /tmp/repo
|
||||
"ssh://git@gitea-ssh.gitea.svc.cluster.local:22/gitadmin/fxhnt.git" /tmp/repo
|
||||
cd /tmp/repo && git sparse-checkout set src && [ -d /tmp/repo/src ]
|
||||
cp -a /tmp/repo/src /code/src && echo "git-sync OK: $(git rev-parse --short HEAD)"
|
||||
resources: { requests: { cpu: 50m, memory: 64Mi }, limits: { cpu: 500m, memory: 256Mi } }
|
||||
@@ -57,7 +61,7 @@ spec:
|
||||
args:
|
||||
- |
|
||||
pip install --quiet --no-cache-dir ib_async==2.1.0
|
||||
fxhnt execute-multistrat --broker ibkr --execute --paper-envelope 1000000
|
||||
fxhnt execute-multistrat --execute --paper-envelope 1000000
|
||||
env:
|
||||
- { name: PYTHONPATH, value: /code/src }
|
||||
- { name: FXHNT_OPERATIONAL_DSN, value: "postgresql+psycopg://foxhunt@postgres.foxhunt.svc.cluster.local:5432/fxhnt" }
|
||||
@@ -91,7 +95,7 @@ spec:
|
||||
- ports: [{ port: 53, protocol: UDP }, { port: 53, protocol: TCP }]
|
||||
- to: [{ podSelector: { matchLabels: { app.kubernetes.io/name: postgres } } }]
|
||||
ports: [{ port: 5432, protocol: TCP }]
|
||||
- to: [{ podSelector: { matchLabels: { app.kubernetes.io/name: gitea } } }]
|
||||
- to: [{ namespaceSelector: { matchLabels: { kubernetes.io/metadata.name: gitea } }, podSelector: { matchLabels: { app.kubernetes.io/name: gitea } } }]
|
||||
ports: [{ port: 2222, protocol: TCP }, { port: 22, protocol: TCP }]
|
||||
- to: [{ podSelector: { matchLabels: { app: ib-gateway } } }]
|
||||
ports: [{ port: 4004, protocol: TCP }, { port: 4002, protocol: TCP }]
|
||||
|
||||
@@ -1,99 +0,0 @@
|
||||
# ============================================================================================
|
||||
# ONE-TIME MANUAL APPLY ONLY. Do NOT add this file to any automated/bulk `kubectl apply -f infra/k8s/jobs/`
|
||||
# path or CI pipeline. It is a `kind: Job` (not a CronJob), so there is no `suspend` flag to gate it —
|
||||
# submitting this manifest starts the backfill immediately. Apply it by hand when ready:
|
||||
# kubectl apply -f infra/k8s/jobs/fxhnt-opra-backfill.yaml
|
||||
# and delete it (`kubectl delete job/fxhnt-opra-backfill -n foxhunt`) before ever re-applying.
|
||||
# ============================================================================================
|
||||
#
|
||||
# One-time historical freeze: `fxhnt backfill-xsp-opra --start 2013-04-01 --max-cost 25.0`, from 2013-04-01
|
||||
# (XSP's mini-SPX option listing era) through today, freezing the WIDE DTE[1,45]/moneyness[0.70,1.20] put
|
||||
# slice + near-ATM calls into `xsp_option_bars`. BATCHED per MONTH (`_backfill_month`): one range `definition`
|
||||
# + one range `ohlcv-1d` fetch per month, then every trading day processed LOCALLY — ~2 OPRA calls/month vs
|
||||
# the old ~6-8/day (~1h vs ~45h over 2013->). `--max-cost 25.0` is the per-MONTH cost cap (the guard stays
|
||||
# ACTIVE, never disabled; a month over budget is logged + skipped, never a silent partial-freeze).
|
||||
#
|
||||
# Idempotent/resumable: `upsert_bars` merges on (osi_symbol, date), so RE-APPLYING this exact manifest after
|
||||
# an interruption is safe (already-frozen days are silently re-written, not duplicated) — it just re-spends
|
||||
# the OPRA cost on the days already done. To resume WITHOUT re-paying for completed months, read the last
|
||||
# "reached YYYY-MM" line from `kubectl logs`, delete the Job, and re-apply with `--start` bumped to that
|
||||
# month before resubmitting.
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: fxhnt-opra-backfill
|
||||
namespace: foxhunt
|
||||
labels: { app.kubernetes.io/name: fxhnt-opra-backfill, app.kubernetes.io/part-of: foxhunt }
|
||||
spec:
|
||||
backoffLimit: 2
|
||||
ttlSecondsAfterFinished: 604800 # 7d — keep the finished Job around for log inspection before GC
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: fxhnt-opra-backfill # egress NetworkPolicy selects this
|
||||
app.kubernetes.io/part-of: foxhunt # gitea ingress (git-sync) + base egress
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
imagePullSecrets:
|
||||
- name: scw-registry
|
||||
initContainers:
|
||||
- name: git-sync
|
||||
image: alpine/git:latest
|
||||
command: [/bin/sh, -c]
|
||||
args:
|
||||
- |
|
||||
mkdir -p /root/.ssh && cp /etc/git-ssh/ssh-privatekey /root/.ssh/id_ed25519 && chmod 600 /root/.ssh/id_ed25519
|
||||
printf 'Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n' > /root/.ssh/config
|
||||
rm -rf /code/src
|
||||
git clone --depth 1 --branch master --filter=blob:none --sparse \
|
||||
"ssh://git@gitea-sshd.foxhunt.svc.cluster.local:22/gitadmin/fxhnt.git" /tmp/repo
|
||||
cd /tmp/repo && git sparse-checkout set src && [ -d /tmp/repo/src ]
|
||||
cp -a /tmp/repo/src /code/src && echo "git-sync OK: $(git rev-parse --short HEAD)"
|
||||
resources: { requests: { cpu: 50m, memory: 64Mi }, limits: { cpu: 500m, memory: 256Mi } }
|
||||
volumeMounts:
|
||||
- { name: code, mountPath: /code }
|
||||
- { name: git-ssh-key, mountPath: /etc/git-ssh, readOnly: true }
|
||||
containers:
|
||||
- name: backfill
|
||||
image: rg.fr-par.scw.cloud/bizworx/fxhnt-cockpit:latest
|
||||
command: ["/bin/sh", "-c"]
|
||||
args:
|
||||
- |
|
||||
pip install --quiet --no-cache-dir databento==0.79.0
|
||||
fxhnt backfill-xsp-opra --start 2013-04-01 --max-cost 25.0
|
||||
env:
|
||||
- { name: PYTHONPATH, value: /code/src }
|
||||
- { name: FXHNT_OPERATIONAL_DSN, value: "postgresql+psycopg://foxhunt@postgres.foxhunt.svc.cluster.local:5432/fxhnt" }
|
||||
- name: PGPASSWORD
|
||||
valueFrom: { secretKeyRef: { name: db-credentials, key: password } }
|
||||
- name: DATABENTO_API_KEY
|
||||
valueFrom: { secretKeyRef: { name: databento-credentials, key: api-key } }
|
||||
- { name: PYTHONUNBUFFERED, value: "1" } # stream monthly progress logs live
|
||||
resources:
|
||||
requests: { memory: "512Mi", cpu: "250m" }
|
||||
limits: { memory: "1Gi", cpu: "1" }
|
||||
volumeMounts:
|
||||
- { name: code, mountPath: /code, readOnly: true }
|
||||
volumes:
|
||||
- { name: code, emptyDir: {} }
|
||||
- { name: git-ssh-key, secret: { secretName: argo-git-ssh-key, defaultMode: 256 } }
|
||||
---
|
||||
# Egress: DNS + Postgres (5432, write xsp_option_bars) + gitea-sshd (22, git-sync) + external 443 (OPRA via
|
||||
# Databento historical API). No IBKR/ib-gateway needed — this Job never places an order.
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: fxhnt-opra-backfill
|
||||
namespace: foxhunt
|
||||
labels: { app.kubernetes.io/part-of: foxhunt }
|
||||
spec:
|
||||
podSelector: { matchLabels: { app.kubernetes.io/name: fxhnt-opra-backfill } }
|
||||
policyTypes: [Egress]
|
||||
egress:
|
||||
- ports: [{ port: 53, protocol: UDP }, { port: 53, protocol: TCP }]
|
||||
- to: [{ podSelector: { matchLabels: { app.kubernetes.io/name: postgres } } }]
|
||||
ports: [{ port: 5432, protocol: TCP }]
|
||||
- to: [{ podSelector: { matchLabels: { app.kubernetes.io/name: gitea } } }]
|
||||
ports: [{ port: 2222, protocol: TCP }, { port: 22, protocol: TCP }]
|
||||
- to: [{ ipBlock: { cidr: 0.0.0.0/0, except: [10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16] } }] # OPRA 443
|
||||
ports: [{ port: 443, protocol: TCP }]
|
||||
@@ -41,7 +41,7 @@ spec:
|
||||
printf 'Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n' > /root/.ssh/config
|
||||
rm -rf /code/src
|
||||
git clone --depth 1 --branch master --filter=blob:none --sparse \
|
||||
"ssh://git@gitea-sshd.foxhunt.svc.cluster.local:22/gitadmin/fxhnt.git" /tmp/repo
|
||||
"ssh://git@gitea-ssh.gitea.svc.cluster.local:22/gitadmin/fxhnt.git" /tmp/repo
|
||||
cd /tmp/repo && git sparse-checkout set src && [ -d /tmp/repo/src ]
|
||||
cp -a /tmp/repo/src /code/src && echo "git-sync OK: $(git rev-parse --short HEAD)"
|
||||
resources: { requests: { cpu: 50m, memory: 64Mi }, limits: { cpu: 500m, memory: 256Mi } }
|
||||
@@ -88,7 +88,7 @@ spec:
|
||||
- ports: [{ port: 53, protocol: UDP }, { port: 53, protocol: TCP }]
|
||||
- to: [{ podSelector: { matchLabels: { app.kubernetes.io/name: postgres } } }]
|
||||
ports: [{ port: 5432, protocol: TCP }]
|
||||
- to: [{ podSelector: { matchLabels: { app.kubernetes.io/name: gitea } } }]
|
||||
- to: [{ namespaceSelector: { matchLabels: { kubernetes.io/metadata.name: gitea } }, podSelector: { matchLabels: { app.kubernetes.io/name: gitea } } }]
|
||||
ports: [{ port: 2222, protocol: TCP }, { port: 22, protocol: TCP }]
|
||||
- to: [{ ipBlock: { cidr: 0.0.0.0/0, except: [10.32.0.0/16, 172.16.0.0/16] } }] # external HTTPS for sleeve fetches
|
||||
ports: [{ port: 443, protocol: TCP }]
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
# UCITS rebalancer — routes the adaptive MULTI-STRAT book (5 pure-equity ETFs) through the PRIIPs-legal
|
||||
# US->UCITS execution map (Component B) on IBKR PAPER via `execute-multistrat --broker ibkr --venue ucits
|
||||
# US->UCITS execution map (Component B) on IBKR PAPER via `execute-multistrat --venue ucits
|
||||
# --execute --paper-envelope 1000000`. The MODELED track is UNCHANGED (still modeled on the US underlyings);
|
||||
# only the broker order symbols are remapped (SPY->CSPX, IEF->IBTM, GLD->SGLN, PDBC->ICOM, DBMF->DBMF.L) —
|
||||
# an unmapped sleeve refuses rather than routing a wrong/absent order. SHIPPED SUSPENDED: this is a thin
|
||||
# execution-routing addition validated by unit tests only, not yet armed against a real UCITS-capable
|
||||
# account. Monthly cadence (1st of month) — a UCITS rebalance need not track the daily US forward-track pace.
|
||||
#
|
||||
# ib_async is NOT baked into the cockpit image (only the Alpaca httpx path is) -> pip-install it at runtime,
|
||||
# like fxhnt-ibkr-rebalancer. Requires ib-gateway UP (kubectl scale deploy/ib-gateway --replicas=1).
|
||||
# ib_async is NOT baked into the cockpit image -> pip-install it at runtime, like fxhnt-ibkr-rebalancer.
|
||||
# Requires ib-gateway UP (kubectl scale deploy/ib-gateway --replicas=1).
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
@@ -15,8 +15,14 @@ metadata:
|
||||
namespace: foxhunt
|
||||
labels: { app.kubernetes.io/name: fxhnt-ucits-rebalancer, app.kubernetes.io/part-of: foxhunt }
|
||||
spec:
|
||||
schedule: "40 14 1 * *" # monthly, 1st of month, 14:40 UTC
|
||||
suspend: true # SUSPENDED — thin routing addition, not yet armed
|
||||
schedule: "0 9 * * 1-5" # weekdays 09:00 UTC — inside LSE/Euronext hours (open ~07:00 UTC BST,
|
||||
# close ~15:30) so the UCITS ETFs (CSPX/IBTM/SGLN/ICOM/DBMF) actually
|
||||
# price + fill. (Was monthly 14:40 UTC-1st; moved to a live-market
|
||||
# weekday slot for the PRIIPs-legal paper validation.)
|
||||
suspend: false # ARMED (paper) — the PRIIPs-legal leg for the EU-domiciled DU account;
|
||||
# ISIN-resolved UCITS contracts + real UCITS pricing (2026-07-15 fix).
|
||||
# The US leg (fxhnt-ibkr-rebalancer) is suspended — EU retail can't trade
|
||||
# US ETFs (PRIIPs/KID). Paper only; no live capital.
|
||||
concurrencyPolicy: Forbid
|
||||
successfulJobsHistoryLimit: 3
|
||||
failedJobsHistoryLimit: 3
|
||||
@@ -44,7 +50,7 @@ spec:
|
||||
printf 'Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n' > /root/.ssh/config
|
||||
rm -rf /code/src
|
||||
git clone --depth 1 --branch master --filter=blob:none --sparse \
|
||||
"ssh://git@gitea-sshd.foxhunt.svc.cluster.local:22/gitadmin/fxhnt.git" /tmp/repo
|
||||
"ssh://git@gitea-ssh.gitea.svc.cluster.local:22/gitadmin/fxhnt.git" /tmp/repo
|
||||
cd /tmp/repo && git sparse-checkout set src && [ -d /tmp/repo/src ]
|
||||
cp -a /tmp/repo/src /code/src && echo "git-sync OK: $(git rev-parse --short HEAD)"
|
||||
resources: { requests: { cpu: 50m, memory: 64Mi }, limits: { cpu: 500m, memory: 256Mi } }
|
||||
@@ -58,7 +64,7 @@ spec:
|
||||
args:
|
||||
- |
|
||||
pip install --quiet --no-cache-dir ib_async==2.1.0
|
||||
fxhnt execute-multistrat --broker ibkr --venue ucits --execute --paper-envelope 1000000
|
||||
fxhnt execute-multistrat --venue ucits --execute --paper-envelope 100000
|
||||
env:
|
||||
- { name: PYTHONPATH, value: /code/src }
|
||||
- { name: FXHNT_OPERATIONAL_DSN, value: "postgresql+psycopg://foxhunt@postgres.foxhunt.svc.cluster.local:5432/fxhnt" }
|
||||
@@ -92,7 +98,7 @@ spec:
|
||||
- ports: [{ port: 53, protocol: UDP }, { port: 53, protocol: TCP }]
|
||||
- to: [{ podSelector: { matchLabels: { app.kubernetes.io/name: postgres } } }]
|
||||
ports: [{ port: 5432, protocol: TCP }]
|
||||
- to: [{ podSelector: { matchLabels: { app.kubernetes.io/name: gitea } } }]
|
||||
- to: [{ namespaceSelector: { matchLabels: { kubernetes.io/metadata.name: gitea } }, podSelector: { matchLabels: { app.kubernetes.io/name: gitea } } }]
|
||||
ports: [{ port: 2222, protocol: TCP }, { port: 22, protocol: TCP }]
|
||||
- to: [{ podSelector: { matchLabels: { app: ib-gateway } } }]
|
||||
ports: [{ port: 4004, protocol: TCP }, { port: 4002, protocol: TCP }]
|
||||
|
||||
@@ -1,25 +1,23 @@
|
||||
# VRP rebalancer — the EXECUTED XSP put-credit-spread VRP twin (Task 7). Runs `execute-vrp` weekly (mirrors
|
||||
# `vrp`'s ~weekly entry cadence) via `fxhnt execute-vrp --execute --paper-envelope 1000000` at IBKR paper
|
||||
# (DU9600528). Records the OBSERVED vrp_exec track (real executed-vs-modeled reconciliation), same shape as
|
||||
# fxhnt-ibkr-rebalancer's multistrat_exec twin.
|
||||
#
|
||||
# SUSPENDED (suspend: true) — the IBKR paper account currently has NO options trading permission, so
|
||||
# `place_combo` cannot fill. Copied from fxhnt-ibkr-rebalancer.yaml; do NOT flip `suspend: false` until
|
||||
# options permission is granted on the account AND this has been dry-run (--execute omitted) against a live
|
||||
# gateway to confirm OPRA/IBKR reachability.
|
||||
#
|
||||
# ib_async + databento are NOT baked into the cockpit image -> pip-install both at runtime (databento is
|
||||
# needed here for the live OPRA freeze `execute-vrp` performs each run, unlike the equity-only multistrat
|
||||
# twin). Requires ib-gateway UP (kubectl scale deploy/ib-gateway --replicas=1).
|
||||
# UCITS volume ingest (IBKR) — fetches the REAL on-exchange daily volume of the UCITS lines we execute
|
||||
# (CSPX/CBU0/IGLN/ICOM/DBMF USD lines, resolved by ISIN on LSEETF) via `reqHistoricalData TRADES` and writes
|
||||
# it to `ibkr_pit_volume` (PIT, first-seen-frozen). The nightly allocation reads this store for the UCITS-
|
||||
# regime ADV when FXHNT_ALLOCATION_UCITS_VOLUME_SOURCE=ibkr (authoritative for the exact line we trade —
|
||||
# Yahoo's LSE feed is coarse/gappy on the thin lines; probe 2026-07-16 confirmed DBMF-UCITS ~16 shares/day).
|
||||
# The daemon CANNOT reach ib-gateway (NetworkPolicy) — this fetch runs here (label app: multistrat-rebalancer
|
||||
# is allowed on ib-gateway:4004) and stores to Postgres; the daemon reads Postgres. Spread stays the estimated
|
||||
# per-line constant (IBKR bid/ask needs a market-data subscription — Error 354; a documented follow-on).
|
||||
# ib_async is NOT baked into the cockpit image -> pip-install it at runtime. Requires ib-gateway UP.
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: fxhnt-vrp-rebalancer
|
||||
name: fxhnt-ucits-volume-ibkr
|
||||
namespace: foxhunt
|
||||
labels: { app.kubernetes.io/name: fxhnt-vrp-rebalancer, app.kubernetes.io/part-of: foxhunt }
|
||||
labels: { app.kubernetes.io/name: fxhnt-ucits-volume-ibkr, app.kubernetes.io/part-of: foxhunt }
|
||||
spec:
|
||||
schedule: "40 14 * * 1" # weekly Monday 14:40 UTC (~1h after US equity open)
|
||||
suspend: true # SUSPENDED — no IBKR options permission yet; arm manually once granted
|
||||
schedule: "30 8 * * 1-5" # weekdays 08:30 UTC — before the 09:00 ucits-rebalancer, so the ADV
|
||||
# store is fresh for the day's sizing (volume is historical daily bars,
|
||||
# so exact timing is not critical; kept inside market hours for freshness).
|
||||
suspend: false # ARMED (read-only market-data fetch; no orders, paper-safe).
|
||||
concurrencyPolicy: Forbid
|
||||
successfulJobsHistoryLimit: 3
|
||||
failedJobsHistoryLimit: 3
|
||||
@@ -30,9 +28,9 @@ spec:
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: fxhnt-vrp-rebalancer # egress NetworkPolicy selects this
|
||||
app.kubernetes.io/part-of: foxhunt # gitea ingress (git-sync) + base egress
|
||||
app: multistrat-rebalancer # ib-gateway INGRESS allows this label on 4004
|
||||
app.kubernetes.io/name: fxhnt-ucits-volume-ibkr # egress NetworkPolicy selects this
|
||||
app.kubernetes.io/part-of: foxhunt # gitea ingress (git-sync) + base egress
|
||||
app: multistrat-rebalancer # ib-gateway INGRESS allows this label on 4004
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
imagePullSecrets:
|
||||
@@ -47,7 +45,7 @@ spec:
|
||||
printf 'Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n' > /root/.ssh/config
|
||||
rm -rf /code/src
|
||||
git clone --depth 1 --branch master --filter=blob:none --sparse \
|
||||
"ssh://git@gitea-sshd.foxhunt.svc.cluster.local:22/gitadmin/fxhnt.git" /tmp/repo
|
||||
"ssh://git@gitea-ssh.gitea.svc.cluster.local:22/gitadmin/fxhnt.git" /tmp/repo
|
||||
cd /tmp/repo && git sparse-checkout set src && [ -d /tmp/repo/src ]
|
||||
cp -a /tmp/repo/src /code/src && echo "git-sync OK: $(git rev-parse --short HEAD)"
|
||||
resources: { requests: { cpu: 50m, memory: 64Mi }, limits: { cpu: 500m, memory: 256Mi } }
|
||||
@@ -55,13 +53,13 @@ spec:
|
||||
- { name: code, mountPath: /code }
|
||||
- { name: git-ssh-key, mountPath: /etc/git-ssh, readOnly: true }
|
||||
containers:
|
||||
- name: rebalancer
|
||||
- name: volume-ibkr
|
||||
image: rg.fr-par.scw.cloud/bizworx/fxhnt-cockpit:latest
|
||||
command: ["/bin/sh", "-c"]
|
||||
args:
|
||||
- |
|
||||
pip install --quiet --no-cache-dir ib_async==2.1.0 databento==0.79.0
|
||||
fxhnt execute-vrp --execute --paper-envelope 1000000
|
||||
pip install --quiet --no-cache-dir ib_async==2.1.0
|
||||
fxhnt backfill-ucits-volume-ibkr
|
||||
env:
|
||||
- { name: PYTHONPATH, value: /code/src }
|
||||
- { name: FXHNT_OPERATIONAL_DSN, value: "postgresql+psycopg://foxhunt@postgres.foxhunt.svc.cluster.local:5432/fxhnt" }
|
||||
@@ -69,9 +67,7 @@ spec:
|
||||
valueFrom: { secretKeyRef: { name: db-credentials, key: password } }
|
||||
- { name: FXHNT_IBKR_HOST, value: "ib-gateway" }
|
||||
- { name: FXHNT_IBKR_PORT, value: "4004" } # socat-bridged pod-to-pod port (TrustedIPs-safe)
|
||||
- { name: FXHNT_IBKR_CLIENT_ID, value: "26" } # unique client id (25 = multistrat rebalancer)
|
||||
- name: DATABENTO_API_KEY
|
||||
valueFrom: { secretKeyRef: { name: databento-credentials, key: api-key } }
|
||||
- { name: FXHNT_IBKR_CLIENT_ID, value: "27" } # unique client id (25 ibkr-rebalancer, 26 ucits-rebalancer)
|
||||
- { name: PYTHONUNBUFFERED, value: "1" }
|
||||
resources:
|
||||
requests: { memory: "512Mi", cpu: "250m" }
|
||||
@@ -82,24 +78,24 @@ spec:
|
||||
- { name: code, emptyDir: {} }
|
||||
- { name: git-ssh-key, secret: { secretName: argo-git-ssh-key, defaultMode: 256 } }
|
||||
---
|
||||
# Egress: DNS + Postgres (5432, record vrp_exec) + gitea-sshd (22, git-sync) + ib-gateway (4004/4002) +
|
||||
# external 443 (OPRA via Databento historical API + IBKR). Same shape as fxhnt-ibkr-rebalancer's egress.
|
||||
# Egress: DNS + Postgres (5432, write ibkr_pit_volume) + gitea-sshd (22, git-sync) + ib-gateway (4004/4002,
|
||||
# historical volume) + external 443 (pip install ib_async from PyPI).
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: fxhnt-vrp-rebalancer
|
||||
name: fxhnt-ucits-volume-ibkr
|
||||
namespace: foxhunt
|
||||
labels: { app.kubernetes.io/part-of: foxhunt }
|
||||
spec:
|
||||
podSelector: { matchLabels: { app.kubernetes.io/name: fxhnt-vrp-rebalancer } }
|
||||
podSelector: { matchLabels: { app.kubernetes.io/name: fxhnt-ucits-volume-ibkr } }
|
||||
policyTypes: [Egress]
|
||||
egress:
|
||||
- ports: [{ port: 53, protocol: UDP }, { port: 53, protocol: TCP }]
|
||||
- to: [{ podSelector: { matchLabels: { app.kubernetes.io/name: postgres } } }]
|
||||
ports: [{ port: 5432, protocol: TCP }]
|
||||
- to: [{ podSelector: { matchLabels: { app.kubernetes.io/name: gitea } } }]
|
||||
- to: [{ namespaceSelector: { matchLabels: { kubernetes.io/metadata.name: gitea } }, podSelector: { matchLabels: { app.kubernetes.io/name: gitea } } }]
|
||||
ports: [{ port: 2222, protocol: TCP }, { port: 22, protocol: TCP }]
|
||||
- to: [{ podSelector: { matchLabels: { app: ib-gateway } } }]
|
||||
ports: [{ port: 4004, protocol: TCP }, { port: 4002, protocol: TCP }]
|
||||
- to: [{ ipBlock: { cidr: 0.0.0.0/0, except: [10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16] } }] # OPRA/IBKR 443
|
||||
- to: [{ ipBlock: { cidr: 0.0.0.0/0, except: [10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16] } }] # PyPI 443
|
||||
ports: [{ port: 443, protocol: TCP }]
|
||||
@@ -44,7 +44,7 @@ spec:
|
||||
printf 'Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n' > /root/.ssh/config
|
||||
rm -rf /code/src
|
||||
git clone --depth 1 --branch master --filter=blob:none --sparse \
|
||||
"ssh://git@gitea-sshd.foxhunt.svc.cluster.local:22/gitadmin/fxhnt.git" /tmp/repo
|
||||
"ssh://git@gitea-ssh.gitea.svc.cluster.local:22/gitadmin/fxhnt.git" /tmp/repo
|
||||
cd /tmp/repo && git sparse-checkout set src && [ -d /tmp/repo/src ]
|
||||
cp -a /tmp/repo/src /code/src && echo "git-sync OK: $(git rev-parse --short HEAD)"
|
||||
resources: { requests: { cpu: 50m, memory: 64Mi }, limits: { cpu: 500m, memory: 256Mi } }
|
||||
@@ -88,5 +88,5 @@ spec:
|
||||
- ports: [{ port: 53, protocol: UDP }, { port: 53, protocol: TCP }]
|
||||
- to: [{ podSelector: { matchLabels: { app.kubernetes.io/name: postgres } } }]
|
||||
ports: [{ port: 5432, protocol: TCP }]
|
||||
- to: [{ podSelector: { matchLabels: { app.kubernetes.io/name: gitea } } }]
|
||||
- to: [{ namespaceSelector: { matchLabels: { kubernetes.io/metadata.name: gitea } }, podSelector: { matchLabels: { app.kubernetes.io/name: gitea } } }]
|
||||
ports: [{ port: 2222, protocol: TCP }, { port: 22, protocol: TCP }]
|
||||
|
||||
@@ -28,40 +28,15 @@ spec:
|
||||
- ports: # DNS
|
||||
- { port: 53, protocol: UDP }
|
||||
- { port: 53, protocol: TCP }
|
||||
- to: # tailscale sidecar -> coordination/DERP (public internet)
|
||||
- ipBlock: { cidr: 0.0.0.0/0, except: [10.32.0.0/16, 100.64.0.0/15] }
|
||||
- to: [{ ipBlock: { cidr: 172.16.0.11/32 } }] # kube-apiserver REAL endpoint (kube-proxy DNATs kubernetes.default.svc 10.32.0.1:443 -> this before NP eval). The tailscale sidecar reads/writes its ts-state Secret via the apiserver — WITHOUT this it crashloops "context deadline exceeded" on a fresh start (e.g. after the deploy scale-0->1), causing a bad gateway. Mirrors the dagster NP fix (06d12fd).
|
||||
ports: [{ port: 6443, protocol: TCP }]
|
||||
- to: # tailscale sidecar -> coordination/DERP (public internet). except now includes 172.16.0.0/16 so the apiserver rule above is the only cluster-internal path.
|
||||
- ipBlock: { cidr: 0.0.0.0/0, except: [10.32.0.0/16, 100.64.0.0/15, 172.16.0.0/16] }
|
||||
ports:
|
||||
- { port: 443, protocol: TCP }
|
||||
- { port: 3478, protocol: UDP }
|
||||
- { port: 41641, protocol: UDP }
|
||||
---
|
||||
# Forward CronJob: postgres (ingest) + DNS + PUBLIC HTTPS (Binance + Databento fetch). The public-443 rule is
|
||||
# what argo-base-egress withholds; without it the in-cluster data fetch cannot reach the exchanges.
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: fxhnt-forward
|
||||
namespace: foxhunt
|
||||
labels: { app.kubernetes.io/part-of: foxhunt }
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels: { app.kubernetes.io/name: fxhnt-forward }
|
||||
policyTypes: [Egress]
|
||||
egress:
|
||||
- to:
|
||||
- podSelector: { matchLabels: { app.kubernetes.io/name: postgres } }
|
||||
ports:
|
||||
- { port: 5432, protocol: TCP }
|
||||
- ports: # DNS (self-contained; also granted via argo-base-egress)
|
||||
- { port: 53, protocol: UDP }
|
||||
- { port: 53, protocol: TCP }
|
||||
- to: # public HTTPS for the data fetch (Binance fapi, Databento)
|
||||
- ipBlock:
|
||||
cidr: 0.0.0.0/0
|
||||
except: [10.32.0.0/16, 172.16.0.0/16]
|
||||
ports:
|
||||
- { port: 443, protocol: TCP }
|
||||
---
|
||||
# Factory CronJob: reads/writes the cockpit DB only — no public internet needed.
|
||||
# DNS already granted by argo-base-egress via the part-of label.
|
||||
apiVersion: networking.k8s.io/v1
|
||||
|
||||
@@ -27,11 +27,77 @@ data:
|
||||
port: 5432
|
||||
telemetry:
|
||||
enabled: false
|
||||
# K8sRunLauncher: each Dagster run launches as its OWN K8s Job (own memory) instead of running in-process in
|
||||
# the 4Gi daemon — so the heavy bybit_book_precompute asset (op_tags request 6Gi/limit 8Gi) never OOMs the
|
||||
# daemon (which previously exit-137'd and skipped the forward-nav step). The launched Job git-syncs the SAME
|
||||
# code the daemon runs (initContainer, same gitea+key) into an emptyDir /code; it needs NO PVC (the compute
|
||||
# reads Postgres only — the unlock calendar is DB-first, so the RWO fxhnt-surfer-data PVC is NOT mounted,
|
||||
# avoiding a deadlock with the daemon that holds it). Per-op memory comes from the asset's dagster-k8s/config
|
||||
# op_tags; this block sets the launcher defaults + the shared pod scaffolding (git-sync, env, code volume).
|
||||
run_launcher:
|
||||
module: dagster_k8s
|
||||
class: K8sRunLauncher
|
||||
config:
|
||||
service_account_name: tailscale-dagster
|
||||
job_namespace: foxhunt
|
||||
job_image: rg.fr-par.scw.cloud/bizworx/fxhnt-cockpit:latest
|
||||
image_pull_policy: Always # :latest -> Always-pull so a dep-bump build reaches launched run Jobs (CI review C1)
|
||||
image_pull_secrets:
|
||||
- name: scw-registry
|
||||
load_incluster_config: true
|
||||
instance_config_map: dagster-instance
|
||||
dagster_home: /dagster-home
|
||||
env_config_maps: []
|
||||
env_secrets: []
|
||||
env_vars:
|
||||
- "PYTHONPATH=/code/src"
|
||||
- "FXHNT_FEATURE_STORE=postgres"
|
||||
- "FXHNT_SURFER_DATA_DIR=/data/surfer"
|
||||
- "FXHNT_OPERATIONAL_DSN=postgresql+psycopg://foxhunt@postgres.foxhunt.svc.cluster.local:5432/fxhnt"
|
||||
- "FXHNT_ALLOCATION_UCITS_VOLUME_SOURCE=ibkr"
|
||||
- "FXHNT_DATABENTO_MAX_COST_USD=40.0"
|
||||
- "DAGSTER_HOME=/dagster-home"
|
||||
resources:
|
||||
requests: { cpu: "100m", memory: "1Gi" }
|
||||
limits: { cpu: "2", memory: "3Gi" }
|
||||
run_k8s_config:
|
||||
pod_spec_config:
|
||||
nodeSelector: { k8s.scaleway.com/pool-name: large }
|
||||
initContainers:
|
||||
- name: git-sync
|
||||
image: alpine/git:latest
|
||||
command: ["/bin/sh", "-c"]
|
||||
args:
|
||||
- |
|
||||
mkdir -p /root/.ssh && cp /etc/git-ssh/ssh-privatekey /root/.ssh/id_ed25519 && chmod 600 /root/.ssh/id_ed25519
|
||||
printf 'Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n' > /root/.ssh/config
|
||||
rm -rf /code/src
|
||||
git clone --depth 1 --branch master --filter=blob:none --sparse \
|
||||
"ssh://git@gitea-ssh.gitea.svc.cluster.local:22/gitadmin/fxhnt.git" /tmp/repo \
|
||||
&& (cd /tmp/repo && git sparse-checkout set src) \
|
||||
&& cp -a /tmp/repo/src /code/src \
|
||||
&& echo "run git-sync OK: $(cd /tmp/repo && git rev-parse --short HEAD)"
|
||||
volumeMounts:
|
||||
- { name: code, mountPath: /code }
|
||||
- { name: git-ssh-key, mountPath: /etc/git-ssh, readOnly: true }
|
||||
volumes:
|
||||
- { name: code, emptyDir: {} }
|
||||
- { name: git-ssh-key, secret: { secretName: argo-git-ssh-key, defaultMode: 256 } }
|
||||
container_config:
|
||||
env:
|
||||
- name: PGPASSWORD
|
||||
valueFrom: { secretKeyRef: { name: db-credentials, key: password } }
|
||||
- name: DATABENTO_API_KEY
|
||||
valueFrom: { secretKeyRef: { name: databento-credentials, key: api-key } }
|
||||
- name: TIINGO_API_KEY
|
||||
valueFrom: { secretKeyRef: { name: tiingo-api, key: key } }
|
||||
volumeMounts:
|
||||
- { name: code, mountPath: /code, readOnly: true }
|
||||
workspace.yaml: |
|
||||
load_from:
|
||||
- python_module:
|
||||
module_name: fxhnt.adapters.orchestration.definitions
|
||||
working_directory: /app
|
||||
working_directory: /code/src # git-sync target + PYTHONPATH; /app was the baked-image dir (mismatch -> code-server couldn't heartbeat -> 23:30 schedule never fired)
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
@@ -49,25 +115,6 @@ data:
|
||||
}
|
||||
}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata: { name: tailscale-dagster, namespace: foxhunt }
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata: { name: tailscale-dagster, namespace: foxhunt }
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["secrets"]
|
||||
verbs: ["create", "get", "update", "patch", "list", "watch"]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata: { name: tailscale-dagster, namespace: foxhunt }
|
||||
subjects:
|
||||
- { kind: ServiceAccount, name: tailscale-dagster, namespace: foxhunt }
|
||||
roleRef: { kind: Role, name: tailscale-dagster, apiGroup: rbac.authorization.k8s.io }
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
@@ -83,7 +130,7 @@ spec:
|
||||
labels: { app.kubernetes.io/name: dagster, app.kubernetes.io/part-of: foxhunt } # part-of -> postgres ingress + base egress
|
||||
spec:
|
||||
serviceAccountName: tailscale-dagster
|
||||
nodeSelector: { k8s.scaleway.com/pool-name: platform }
|
||||
nodeSelector: { k8s.scaleway.com/pool-name: large }
|
||||
imagePullSecrets: [{ name: scw-registry }]
|
||||
initContainers:
|
||||
- name: dagster-home-init # DAGSTER_HOME must be writable; seed it from the configmap
|
||||
@@ -111,7 +158,7 @@ spec:
|
||||
printf 'Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n' > /root/.ssh/config
|
||||
rm -rf /code/src
|
||||
if timeout 120 git clone --depth 1 --branch master --filter=blob:none --sparse \
|
||||
"ssh://git@gitea-sshd.foxhunt.svc.cluster.local:22/gitadmin/fxhnt.git" /tmp/repo \
|
||||
"ssh://git@gitea-ssh.gitea.svc.cluster.local:22/gitadmin/fxhnt.git" /tmp/repo \
|
||||
&& (cd /tmp/repo && git sparse-checkout set src) && [ -d /tmp/repo/src ]; then
|
||||
cp -a /tmp/repo/src /code/src && echo "git-sync OK: $(cd /tmp/repo && git rev-parse --short HEAD)"
|
||||
else
|
||||
@@ -134,6 +181,10 @@ spec:
|
||||
- { name: FXHNT_FEATURE_STORE, value: "postgres" } # warehouse reads/writes -> TimescaleDB (networked). Unset/=duckdb to roll back.
|
||||
- { name: FXHNT_SURFER_DATA_DIR, value: /data/surfer }
|
||||
- { name: FXHNT_OPERATIONAL_DSN, value: "postgresql+psycopg://foxhunt@postgres.foxhunt.svc.cluster.local:5432/fxhnt" }
|
||||
# UCITS-regime ADV from the REAL IBKR on-exchange volume (fxhnt-ucits-volume-ibkr CronJob writes
|
||||
# ibkr_pit_volume). Authoritative for the exact lines we trade; Yahoo's LSE feed lacks CBU0.L (the
|
||||
# book dropped under yahoo, sized $1M-ceiling/Sharpe 1.30 under ibkr — validated 2026-07-16).
|
||||
- { name: FXHNT_ALLOCATION_UCITS_VOLUME_SOURCE, value: "ibkr" }
|
||||
- { name: FXHNT_DATABENTO_MAX_COST_USD, value: "40.0" }
|
||||
- name: PGPASSWORD
|
||||
valueFrom: { secretKeyRef: { name: db-credentials, key: password } }
|
||||
@@ -192,13 +243,15 @@ spec:
|
||||
podSelector: { matchLabels: { app.kubernetes.io/name: dagster } }
|
||||
policyTypes: [Ingress, Egress]
|
||||
ingress:
|
||||
- from: [{ podSelector: { matchLabels: { app.kubernetes.io/name: tailscale-gitlab-proxy } } }] # dagster.fxhnt.ai
|
||||
- from: [{ podSelector: { matchLabels: { app.kubernetes.io/name: fxhnt-ingress } } }] # dagster.fxhnt.ai via the bizworx fxhnt-ingress proxy (was tailscale-gitlab-proxy on foxhunt)
|
||||
ports: [{ port: 3000, protocol: TCP }]
|
||||
egress:
|
||||
- to: [{ podSelector: { matchLabels: { app.kubernetes.io/name: postgres } } }]
|
||||
ports: [{ port: 5432, protocol: TCP }]
|
||||
- to: [{ podSelector: { matchLabels: { app.kubernetes.io/name: gitea } } }] # git-sync clones from gitea-sshd (svc :22 -> rootless pod :2222)
|
||||
- to: [{ namespaceSelector: { matchLabels: { kubernetes.io/metadata.name: gitea } }, podSelector: { matchLabels: { app.kubernetes.io/name: gitea } } }] # git-sync clones from the bizworx gitea in ns gitea (cross-namespace)
|
||||
ports: [{ port: 2222, protocol: TCP }, { port: 22, protocol: TCP }]
|
||||
- ports: [{ port: 53, protocol: UDP }, { port: 53, protocol: TCP }]
|
||||
- to: [{ ipBlock: { cidr: 172.16.0.11/32 } }] # kube-apiserver REAL endpoint (kube-proxy DNATs kubernetes.default.svc 10.32.0.1:443 -> this before NP eval, and 172.16.0.0/16 is excepted below). The tailscale sidecar reads/writes its ts-state Secret via the apiserver.
|
||||
ports: [{ port: 6443, protocol: TCP }]
|
||||
- to: [{ ipBlock: { cidr: 0.0.0.0/0, except: [10.32.0.0/16, 172.16.0.0/16] } }] # Binance/Databento + tailscale
|
||||
ports: [{ port: 443, protocol: TCP }, { port: 3478, protocol: UDP }, { port: 41641, protocol: UDP }]
|
||||
|
||||
72
infra/k8s/rbac/tailscale-sidecars.yaml
Normal file
72
infra/k8s/rbac/tailscale-sidecars.yaml
Normal file
@@ -0,0 +1,72 @@
|
||||
# Tailscale sidecar RBAC (ServiceAccount + Role + RoleBinding) for the fxhnt-dashboard and dagster
|
||||
# Deployments. Moved out of infra/k8s/services/fxhnt-dashboard.yaml and infra/k8s/orchestration/dagster.yaml
|
||||
# (CI review M1): those app manifests are applied every deploy by the fxhnt-ci-deployer ServiceAccount via
|
||||
# the kubectl-deploy Argo template, and having RBAC objects bundled in there meant the deployer SA needed
|
||||
# create/update/patch on roles/rolebindings in its own namespace — a privilege-escalation primitive on a
|
||||
# webhook-triggered SA. This file is STATIC and applied ONCE, out-of-band, by a human:
|
||||
#
|
||||
# kubectl apply -f infra/k8s/rbac/tailscale-sidecars.yaml
|
||||
#
|
||||
# It is NOT applied by the CI pipeline. The Deployments still reference these ServiceAccounts via
|
||||
# `serviceAccountName: tailscale-dashboard` / `tailscale-dagster` — that reference resolves because the
|
||||
# SAs are managed here, separately, not because the app manifests create them anymore.
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: tailscale-dashboard
|
||||
namespace: foxhunt
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: tailscale-dashboard
|
||||
namespace: foxhunt
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["secrets"]
|
||||
verbs: ["create", "get", "update", "patch", "list", "watch"]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: tailscale-dashboard
|
||||
namespace: foxhunt
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: tailscale-dashboard
|
||||
namespace: foxhunt
|
||||
roleRef:
|
||||
kind: Role
|
||||
name: tailscale-dashboard
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata: { name: tailscale-dagster, namespace: foxhunt }
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata: { name: tailscale-dagster, namespace: foxhunt }
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["secrets"]
|
||||
verbs: ["create", "get", "update", "patch", "list", "watch"]
|
||||
# K8sRunLauncher: the daemon launches each Dagster run as its OWN K8s Job (own memory) so the heavy
|
||||
# bybit_book_precompute asset (6-8Gi) never OOMs the 4Gi daemon. Needs to manage Jobs + read their pods/logs.
|
||||
- apiGroups: ["batch"]
|
||||
resources: ["jobs"]
|
||||
verbs: ["create", "get", "list", "watch", "delete"]
|
||||
- apiGroups: [""]
|
||||
resources: ["pods", "pods/log"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["events"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata: { name: tailscale-dagster, namespace: foxhunt }
|
||||
subjects:
|
||||
- { kind: ServiceAccount, name: tailscale-dagster, namespace: foxhunt }
|
||||
roleRef: { kind: Role, name: tailscale-dagster, apiGroup: rbac.authorization.k8s.io }
|
||||
@@ -1,15 +0,0 @@
|
||||
# Alpaca paper-trading API credentials — SCAFFOLD. Populate from the Scaleway secret / Alpaca dashboard
|
||||
# (Paper account → API keys) before un-suspending the fxhnt-alpaca-rebalancer CronJob. Paper keys carry NO
|
||||
# real-money risk, but keep them out of git — replace the placeholders via a sealed secret / SCW secret sync,
|
||||
# never commit real values.
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: alpaca-credentials
|
||||
namespace: foxhunt
|
||||
labels:
|
||||
app.kubernetes.io/part-of: foxhunt
|
||||
type: Opaque
|
||||
stringData:
|
||||
api-key: "REPLACE_FROM_ALPACA_PAPER_DASHBOARD"
|
||||
api-secret: "REPLACE_FROM_ALPACA_PAPER_DASHBOARD"
|
||||
@@ -19,7 +19,7 @@ spec:
|
||||
spec:
|
||||
serviceAccountName: tailscale-dashboard # RBAC to read/write the TS state secret
|
||||
nodeSelector:
|
||||
k8s.scaleway.com/pool-name: platform
|
||||
k8s.scaleway.com/pool-name: large
|
||||
imagePullSecrets:
|
||||
- name: scw-registry
|
||||
initContainers:
|
||||
@@ -35,7 +35,7 @@ spec:
|
||||
printf 'Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n' > /root/.ssh/config
|
||||
rm -rf /code/src
|
||||
if timeout 120 git clone --depth 1 --branch master --filter=blob:none --sparse \
|
||||
"ssh://git@gitea-sshd.foxhunt.svc.cluster.local:22/gitadmin/fxhnt.git" /tmp/repo \
|
||||
"ssh://git@gitea-ssh.gitea.svc.cluster.local:22/gitadmin/fxhnt.git" /tmp/repo \
|
||||
&& (cd /tmp/repo && git sparse-checkout set src) && [ -d /tmp/repo/src ]; then
|
||||
cp -a /tmp/repo/src /code/src && echo "git-sync OK: $(cd /tmp/repo && git rev-parse --short HEAD)"
|
||||
else
|
||||
@@ -112,33 +112,3 @@ metadata:
|
||||
spec:
|
||||
selector: { app.kubernetes.io/name: fxhnt-dashboard }
|
||||
ports: [{ port: 80, targetPort: 8080, name: http }]
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: tailscale-dashboard
|
||||
namespace: foxhunt
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: tailscale-dashboard
|
||||
namespace: foxhunt
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["secrets"]
|
||||
verbs: ["create", "get", "update", "patch", "list", "watch"]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: tailscale-dashboard
|
||||
namespace: foxhunt
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: tailscale-dashboard
|
||||
namespace: foxhunt
|
||||
roleRef:
|
||||
kind: Role
|
||||
name: tailscale-dashboard
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
|
||||
217
infra/k8s/services/fxhnt-ingress-proxy.yaml
Normal file
217
infra/k8s/services/fxhnt-ingress-proxy.yaml
Normal file
@@ -0,0 +1,217 @@
|
||||
# fxhnt-ingress — the bizworx front for *.fxhnt.ai (fund-only; replaces the foxhunt tailscale-gitlab-proxy).
|
||||
# nginx terminates the *.fxhnt.ai wildcard (cert-manager secret fxhnt-wildcard-tls) and reverse-proxies the 3
|
||||
# fund vhosts by ClusterIP + a socat sidecar forwards git SSH. A tailscale sidecar joins the tailnet as
|
||||
# `fxhnt-ingress` so the Scaleway fxhnt.ai A-records can point at its stable tailnet IP (C4).
|
||||
#
|
||||
# WHY ClusterIP (not pod tailnet IPs like the foxhunt proxy did): on bizworx a tailscale-sidecar pod CAN reach
|
||||
# fund ClusterIPs (10.32.x.x) — verified: cockpit ClusterIP returned HTTP 200 from a TS sidecar — but CANNOT
|
||||
# reach pod IPs in 100.64.0.0/15 (Tailscale CGNAT overlap; verified: pod-IP request timed out). So proxy to
|
||||
# Services by name, never to pod IPs.
|
||||
# Non-headless ClusterIP Service for gitea (ns gitea). The Helm chart's gitea-http/gitea-ssh are HEADLESS
|
||||
# (ClusterIP: None) → DNS returns the gitea pod IP in the 100.64.0.0/15 CGNAT range, which the fxhnt-ingress
|
||||
# proxy's kernel-mode tailscale sidecar CANNOT reach. This gives a real 10.32.x ClusterIP the proxy can hit.
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: gitea-clusterip
|
||||
namespace: gitea
|
||||
labels: { app.kubernetes.io/name: gitea-clusterip, app.kubernetes.io/part-of: foxhunt }
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector: { app.kubernetes.io/instance: gitea, app.kubernetes.io/name: gitea }
|
||||
ports:
|
||||
- { port: 3000, targetPort: 3000, name: http }
|
||||
- { port: 22, targetPort: 22, name: ssh }
|
||||
---
|
||||
# ServiceAccount + Role + RoleBinding: the tailscale sidecar needs get/create/update/patch on its ts-state
|
||||
# Secret (fxhnt-ingress-ts-state). Same pattern as tailscale-dagster / tailscale-dashboard.
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata: { name: tailscale-fxhnt-ingress, namespace: foxhunt }
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata: { name: tailscale-fxhnt-ingress, namespace: foxhunt }
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["secrets"]
|
||||
verbs: ["create", "get", "update", "patch", "list", "watch"]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata: { name: tailscale-fxhnt-ingress, namespace: foxhunt }
|
||||
subjects:
|
||||
- { kind: ServiceAccount, name: tailscale-fxhnt-ingress, namespace: foxhunt }
|
||||
roleRef: { kind: Role, name: tailscale-fxhnt-ingress, apiGroup: rbac.authorization.k8s.io }
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: fxhnt-ingress-nginx
|
||||
namespace: foxhunt
|
||||
labels: { app.kubernetes.io/name: fxhnt-ingress, app.kubernetes.io/part-of: foxhunt }
|
||||
data:
|
||||
default.conf: |
|
||||
# HTTP -> HTTPS
|
||||
server {
|
||||
listen 80;
|
||||
server_name *.fxhnt.ai;
|
||||
return 301 https://$host$request_uri;
|
||||
}
|
||||
# git.fxhnt.ai -> bizworx gitea (ns gitea)
|
||||
server {
|
||||
listen 443 ssl;
|
||||
server_name git.fxhnt.ai;
|
||||
ssl_certificate /etc/nginx/certs/tls.crt;
|
||||
ssl_certificate_key /etc/nginx/certs/tls.key;
|
||||
ssl_protocols TLSv1.2 TLSv1.3;
|
||||
client_max_body_size 0;
|
||||
location / {
|
||||
proxy_pass http://gitea-clusterip.gitea.svc.cluster.local:3000;
|
||||
proxy_set_header Host $http_host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto https;
|
||||
proxy_buffering off;
|
||||
proxy_request_buffering off;
|
||||
proxy_http_version 1.1;
|
||||
proxy_read_timeout 3600s;
|
||||
proxy_send_timeout 3600s;
|
||||
proxy_connect_timeout 300s;
|
||||
}
|
||||
}
|
||||
# dashboard.fxhnt.ai + cockpit.fxhnt.ai -> bizworx cockpit (ns foxhunt)
|
||||
server {
|
||||
listen 443 ssl;
|
||||
server_name dashboard.fxhnt.ai cockpit.fxhnt.ai;
|
||||
ssl_certificate /etc/nginx/certs/tls.crt;
|
||||
ssl_certificate_key /etc/nginx/certs/tls.key;
|
||||
ssl_protocols TLSv1.2 TLSv1.3;
|
||||
location / {
|
||||
proxy_pass http://fxhnt-dashboard.foxhunt.svc.cluster.local:80;
|
||||
proxy_set_header Host $http_host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto https;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection "upgrade";
|
||||
}
|
||||
}
|
||||
# dagster.fxhnt.ai -> bizworx dagster (ns foxhunt)
|
||||
server {
|
||||
listen 443 ssl;
|
||||
server_name dagster.fxhnt.ai;
|
||||
ssl_certificate /etc/nginx/certs/tls.crt;
|
||||
ssl_certificate_key /etc/nginx/certs/tls.key;
|
||||
ssl_protocols TLSv1.2 TLSv1.3;
|
||||
location / {
|
||||
proxy_pass http://dagster.foxhunt.svc.cluster.local:80;
|
||||
proxy_set_header Host $http_host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto https;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection "upgrade";
|
||||
proxy_read_timeout 3600s;
|
||||
}
|
||||
}
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: fxhnt-ingress
|
||||
namespace: foxhunt
|
||||
labels: { app.kubernetes.io/name: fxhnt-ingress, app.kubernetes.io/part-of: foxhunt }
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels: { app.kubernetes.io/name: fxhnt-ingress }
|
||||
template:
|
||||
metadata:
|
||||
labels: { app.kubernetes.io/name: fxhnt-ingress, app.kubernetes.io/part-of: foxhunt }
|
||||
spec:
|
||||
serviceAccountName: tailscale-fxhnt-ingress
|
||||
nodeSelector:
|
||||
k8s.scaleway.com/pool-name: large
|
||||
containers:
|
||||
- name: nginx
|
||||
image: nginx:alpine
|
||||
ports:
|
||||
- { containerPort: 80, name: http }
|
||||
- { containerPort: 443, name: https }
|
||||
volumeMounts:
|
||||
- { name: nginx-conf, mountPath: /etc/nginx/conf.d, readOnly: true }
|
||||
- { name: wildcard-cert, mountPath: /etc/nginx/certs, readOnly: true }
|
||||
resources:
|
||||
requests: { cpu: 25m, memory: 32Mi }
|
||||
limits: { cpu: 200m, memory: 128Mi }
|
||||
# git SSH: git.fxhnt.ai:22 -> bizworx gitea-ssh (ns gitea). socat TCP forward on the same tailnet node.
|
||||
- name: ssh-proxy
|
||||
image: alpine/socat:latest
|
||||
args:
|
||||
- TCP-LISTEN:22,fork,reuseaddr
|
||||
- TCP:gitea-clusterip.gitea.svc.cluster.local:22
|
||||
resources:
|
||||
requests: { cpu: 10m, memory: 16Mi }
|
||||
limits: { cpu: 100m, memory: 64Mi }
|
||||
- name: tailscale
|
||||
image: ghcr.io/tailscale/tailscale:stable
|
||||
env:
|
||||
- { name: TS_HOSTNAME, value: "fxhnt-ingress" }
|
||||
- { name: TS_KUBE_SECRET, value: "fxhnt-ingress-ts-state" }
|
||||
- { name: TS_USERSPACE, value: "false" }
|
||||
- name: TS_AUTHKEY
|
||||
valueFrom: { secretKeyRef: { name: tailscale-auth, key: TS_AUTHKEY } }
|
||||
- name: POD_NAME
|
||||
valueFrom: { fieldRef: { fieldPath: metadata.name } }
|
||||
- name: POD_UID
|
||||
valueFrom: { fieldRef: { fieldPath: metadata.uid } }
|
||||
securityContext:
|
||||
capabilities: { add: ["NET_ADMIN"] }
|
||||
resources:
|
||||
requests: { cpu: 25m, memory: 64Mi }
|
||||
limits: { cpu: 200m, memory: 128Mi }
|
||||
volumes:
|
||||
- name: nginx-conf
|
||||
configMap: { name: fxhnt-ingress-nginx }
|
||||
- name: wildcard-cert
|
||||
secret: { secretName: fxhnt-wildcard-tls }
|
||||
---
|
||||
# ClusterIP Service (in-cluster reachability; the public entrypoint is the tailscale node, not this Service)
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: fxhnt-ingress
|
||||
namespace: foxhunt
|
||||
labels: { app.kubernetes.io/name: fxhnt-ingress, app.kubernetes.io/part-of: foxhunt }
|
||||
spec:
|
||||
selector: { app.kubernetes.io/name: fxhnt-ingress }
|
||||
ports:
|
||||
- { port: 443, targetPort: 443, name: https }
|
||||
- { port: 80, targetPort: 80, name: http }
|
||||
- { port: 22, targetPort: 22, name: ssh }
|
||||
---
|
||||
# NetworkPolicy: allow this proxy the egress it needs (postgres NOT needed; it only fronts HTTP/SSH to services).
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: fxhnt-ingress
|
||||
namespace: foxhunt
|
||||
labels: { app.kubernetes.io/name: fxhnt-ingress, app.kubernetes.io/part-of: foxhunt }
|
||||
spec:
|
||||
podSelector: { matchLabels: { app.kubernetes.io/name: fxhnt-ingress } }
|
||||
policyTypes: [Egress]
|
||||
egress:
|
||||
- ports: [{ port: 53, protocol: UDP }, { port: 53, protocol: TCP }] # DNS
|
||||
- to: [{ podSelector: { matchLabels: { app.kubernetes.io/name: fxhnt-dashboard } } }]
|
||||
ports: [{ port: 8080, protocol: TCP }, { port: 80, protocol: TCP }] # cockpit
|
||||
- to: [{ podSelector: { matchLabels: { app.kubernetes.io/name: dagster } } }]
|
||||
ports: [{ port: 3000, protocol: TCP }] # dagster webserver
|
||||
- to: [{ namespaceSelector: { matchLabels: { kubernetes.io/metadata.name: gitea } } }]
|
||||
ports: [{ port: 3000, protocol: TCP }, { port: 22, protocol: TCP }] # gitea web + ssh (cross-ns)
|
||||
- to: [{ ipBlock: { cidr: 172.16.0.11/32 } }] # kube-apiserver (tailscale sidecar ts-state Secret; post-DNAT dest)
|
||||
ports: [{ port: 6443, protocol: TCP }]
|
||||
- to: [{ ipBlock: { cidr: 0.0.0.0/0, except: [10.32.0.0/16, 172.16.0.0/16] } }] # tailscale coordination/DERP
|
||||
ports: [{ port: 443, protocol: TCP }, { port: 3478, protocol: UDP }, { port: 41641, protocol: UDP }]
|
||||
22
infra/k8s/services/fxhnt-wildcard-cert.yaml
Normal file
22
infra/k8s/services/fxhnt-wildcard-cert.yaml
Normal file
@@ -0,0 +1,22 @@
|
||||
# Wildcard TLS cert for the fxhnt.ai fund ingress on bizworx.
|
||||
# Issued by the existing bizworx letsencrypt-prod ClusterIssuer (Scaleway DNS-01 webhook), which already
|
||||
# issues real LE certs across ~10 namespaces. The fxhnt-ingress proxy (P2) terminates *.fxhnt.ai with this.
|
||||
# DNS-01 provisions a TXT on the Scaleway fxhnt.ai zone we control -> no HTTP reachability needed to issue.
|
||||
apiVersion: cert-manager.io/v1
|
||||
kind: Certificate
|
||||
metadata:
|
||||
name: fxhnt-wildcard
|
||||
namespace: foxhunt
|
||||
labels:
|
||||
app.kubernetes.io/part-of: foxhunt
|
||||
spec:
|
||||
commonName: fxhnt.ai
|
||||
dnsNames:
|
||||
- fxhnt.ai
|
||||
- "*.fxhnt.ai"
|
||||
issuerRef:
|
||||
kind: ClusterIssuer
|
||||
name: letsencrypt-prod
|
||||
secretName: fxhnt-wildcard-tls
|
||||
privateKey:
|
||||
rotationPolicy: Always
|
||||
@@ -7,9 +7,10 @@ metadata:
|
||||
app: ib-gateway
|
||||
app.kubernetes.io/part-of: foxhunt
|
||||
spec:
|
||||
# Dev-mode: paused while working on model development.
|
||||
# Restore with: kubectl -n foxhunt scale deployment ib-gateway --replicas=1
|
||||
replicas: 0
|
||||
# LIVE — the sole IBKR paper execution gateway (fxhnt-ibkr-rebalancer -> :4004). Keep replicas: 1.
|
||||
# (Was replicas: 0 "dev-mode paused" while the live cluster ran 1 — a manifest/cluster drift that would
|
||||
# have scaled IBKR execution to ZERO on any re-apply. Corrected 2026-07-14.)
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
@@ -24,10 +25,14 @@ spec:
|
||||
securityContext:
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
imagePullSecrets:
|
||||
- name: gitlab-registry
|
||||
# gnzsnz image runs as the non-root `ibgateway` user (UID/GID 1000); fsGroup makes the mounted
|
||||
# tws_settings PVC group-writable so IBC can write jts.ini (a fresh PVC is root-owned otherwise ->
|
||||
# "Permission denied" crashloop).
|
||||
fsGroup: 1000
|
||||
# image is public (ghcr.io/gnzsnz/ib-gateway) — no pull secret needed on bizworx
|
||||
# (the legacy gitlab-registry secret does not exist there).
|
||||
nodeSelector:
|
||||
k8s.scaleway.com/pool-name: platform
|
||||
k8s.scaleway.com/pool-name: large
|
||||
containers:
|
||||
- name: ib-gateway
|
||||
image: ghcr.io/gnzsnz/ib-gateway:stable
|
||||
@@ -66,6 +71,20 @@ spec:
|
||||
value: "restart"
|
||||
- name: EXISTING_SESSION_DETECTED_ACTION
|
||||
value: "primaryoverride"
|
||||
# Daily IN-PLACE restart via IBC — "does not require daily 2FA validation" (gnzsnz docs), so the
|
||||
# container no longer EXITS on IBKR's forced daily reset (the cause of the ~daily container churn +
|
||||
# full re-login each time). 08:00 UTC is clear of the 14:35 rebalancer + 23:30 nightly windows.
|
||||
- name: TIME_ZONE
|
||||
value: "Etc/UTC"
|
||||
# MUST be zero-padded hh:mm (IBC's parser rejects "8:00 AM" with "Auto restart time setting must be
|
||||
# hh:mm AM or hh:mm PM" — the rejection left the daily auto-restart UNCONFIGURED, so the gateway
|
||||
# churned ~16 restarts/day instead of one clean daily cycle. Verified in the pod log 2026-07-20.)
|
||||
- name: AUTO_RESTART_TIME
|
||||
value: "08:00 AM"
|
||||
# Persist jts.ini + IBC config across pod restarts (a fresh empty volume just does a clean first-run
|
||||
# login) so settings survive a deploy/reschedule instead of being rebuilt each time.
|
||||
- name: TWS_SETTINGS_PATH
|
||||
value: "/home/ibgateway/tws_settings"
|
||||
- name: VNC_SERVER_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
@@ -89,7 +108,30 @@ spec:
|
||||
port: 4002
|
||||
initialDelaySeconds: 120
|
||||
periodSeconds: 30
|
||||
failureThreshold: 5
|
||||
# 6×30s = 180s tolerance: the daily auto-restart + IBKR reauth window drops port 4002 for up to a
|
||||
# couple of minutes; too tight a threshold would k8s-kill the pod mid-reauth and compound the churn.
|
||||
failureThreshold: 6
|
||||
volumeMounts:
|
||||
- name: tws-settings
|
||||
mountPath: /home/ibgateway/tws_settings
|
||||
volumes:
|
||||
- name: tws-settings
|
||||
persistentVolumeClaim:
|
||||
claimName: ib-gateway-settings
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: ib-gateway-settings
|
||||
namespace: foxhunt
|
||||
labels:
|
||||
app: ib-gateway
|
||||
app.kubernetes.io/part-of: foxhunt
|
||||
spec:
|
||||
accessModes: ["ReadWriteOnce"]
|
||||
resources:
|
||||
requests:
|
||||
storage: 1Gi
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
|
||||
@@ -1,123 +0,0 @@
|
||||
# Multi-strat rebalancer — weekly CronJob running the adaptive book on IBKR (paper) via the
|
||||
# in-cluster ib-gateway. Production deployment of scripts/surfer/multistrat_bot.py.
|
||||
#
|
||||
# Code is delivered via ConfigMap (no image build needed — lean). Create/refresh it with:
|
||||
# kubectl create configmap multistrat-bot-code -n foxhunt \
|
||||
# --from-file=scripts/surfer/multistrat_bot.py \
|
||||
# --from-file=scripts/surfer/multistrat_paper.py \
|
||||
# --dry-run=client -o yaml | kubectl apply -f -
|
||||
#
|
||||
# SAFETY: MULTISTRAT_EXECUTE defaults to "false" (cluster dry-run). After validating a few dry-run
|
||||
# CronJob logs, flip to "true" to place paper orders. Account is paper (ib-gateway TRADING_MODE=paper);
|
||||
# the bot additionally refuses any non-DU (live) account unless MULTISTRAT_ALLOW_LIVE_CONFIRMED is set.
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: multistrat-state
|
||||
namespace: foxhunt
|
||||
labels:
|
||||
app: multistrat-rebalancer
|
||||
spec:
|
||||
accessModes: ["ReadWriteOnce"] # sequential CronJob runs (Forbid concurrency) — RWO is fine
|
||||
resources:
|
||||
requests:
|
||||
storage: 1Gi
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: multistrat-rebalancer
|
||||
namespace: foxhunt
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app: multistrat-rebalancer
|
||||
policyTypes: [Egress]
|
||||
egress:
|
||||
- to: [] # DNS
|
||||
ports:
|
||||
- {protocol: UDP, port: 53}
|
||||
- {protocol: TCP, port: 53}
|
||||
- to: # in-cluster ib-gateway (4002 API + 4004 socat bridge for pod-to-pod)
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
app: ib-gateway
|
||||
ports:
|
||||
- {protocol: TCP, port: 4002}
|
||||
- {protocol: TCP, port: 4004}
|
||||
- to: # internet 443: PyPI (pip) + Yahoo (prices), no internal ranges
|
||||
- ipBlock:
|
||||
cidr: 0.0.0.0/0
|
||||
except: ["10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"]
|
||||
ports:
|
||||
- {protocol: TCP, port: 443}
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: multistrat-rebalancer
|
||||
namespace: foxhunt
|
||||
labels:
|
||||
app: multistrat-rebalancer
|
||||
app.kubernetes.io/part-of: foxhunt
|
||||
spec:
|
||||
schedule: "35 14 * * 1-5" # weekdays 14:35 UTC (~1h after US open) — daily risk re-eval; hysteresis prevents churn
|
||||
concurrencyPolicy: Forbid
|
||||
startingDeadlineSeconds: 3600
|
||||
successfulJobsHistoryLimit: 5
|
||||
failedJobsHistoryLimit: 10
|
||||
jobTemplate:
|
||||
spec:
|
||||
backoffLimit: 1
|
||||
activeDeadlineSeconds: 600
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: multistrat-rebalancer
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
nodeSelector:
|
||||
k8s.scaleway.com/pool-name: platform
|
||||
securityContext:
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: rebalancer
|
||||
image: python:3.12-slim
|
||||
command: ["/bin/sh", "-c"]
|
||||
args:
|
||||
- >-
|
||||
pip install --quiet --no-cache-dir ib_async==2.1.0 numpy &&
|
||||
python /app/multistrat_bot.py run
|
||||
env:
|
||||
- {name: IB_HOST, value: "ib-gateway"}
|
||||
- {name: IB_PORT, value: "4004"} # socat bridge: relays via localhost so IB Gateway trusts it (4002 = pod-IP, untrusted -> handshake timeout)
|
||||
- {name: IB_CLIENT_ID, value: "11"}
|
||||
- {name: MULTISTRAT_MAXLEV, value: "1.0"}
|
||||
- {name: MULTISTRAT_HYST, value: "0.03"}
|
||||
- {name: MULTISTRAT_REBALANCE_DAYS, value: "1"} # daily eval (weekday cron); hysteresis (3%) gates actual trades
|
||||
- {name: MULTISTRAT_DD_HALT, value: "0.20"}
|
||||
- {name: MULTISTRAT_MAX_ORDER, value: "0.30"}
|
||||
- {name: MULTISTRAT_EXECUTE, value: "true"} # ARMED: places paper orders (account DU* paper-guarded; live needs MULTISTRAT_ALLOW_LIVE_CONFIRMED)
|
||||
- {name: MULTISTRAT_STATE, value: "/data/state.json"}
|
||||
- {name: MULTISTRAT_LOG, value: "/data/bot.log"}
|
||||
- {name: PIP_DISABLE_PIP_VERSION_CHECK, value: "1"}
|
||||
- {name: PYTHONUNBUFFERED, value: "1"}
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
resources:
|
||||
requests: {memory: "256Mi", cpu: "100m"}
|
||||
limits: {memory: "512Mi", cpu: "500m"}
|
||||
volumeMounts:
|
||||
- {mountPath: /app, name: code, readOnly: true}
|
||||
- {mountPath: /data, name: state}
|
||||
volumes:
|
||||
- name: code
|
||||
configMap:
|
||||
name: multistrat-bot-code
|
||||
- name: state
|
||||
persistentVolumeClaim:
|
||||
claimName: multistrat-state
|
||||
@@ -18,7 +18,7 @@ dependencies = [
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
dev = ["pytest>=8.0", "hypothesis>=6.100", "ruff>=0.5", "mypy>=1.10"]
|
||||
dev = ["pytest>=8.0", "pytest-xdist>=3.6", "hypothesis>=6.100", "ruff>=0.5", "mypy>=1.10"]
|
||||
crypto = ["ccxt>=4.4"]
|
||||
ibkr = ["ib-async>=2.0"]
|
||||
agent = ["langgraph>=0.2", "langchain-core>=0.3", "langchain-ollama>=0.2"]
|
||||
@@ -26,7 +26,7 @@ data = ["databento>=0.50"]
|
||||
web = ["fastapi>=0.110", "uvicorn[standard]>=0.29", "jinja2>=3.1", "httpx>=0.27"]
|
||||
pg = ["psycopg[binary]>=3.1"]
|
||||
factory = ["scipy>=1.11"]
|
||||
orchestration = ["dagster>=1.12,<1.13", "dagster-webserver>=1.12,<1.13", "dagster-postgres>=0.28,<0.29"]
|
||||
orchestration = ["dagster>=1.12,<1.13", "dagster-webserver>=1.12,<1.13", "dagster-postgres>=0.28,<0.29", "dagster-k8s>=0.28,<0.29"]
|
||||
# Surfer PoC paper-track (vendored crypto XS-momentum + VRP). torch is a faithful-port requirement
|
||||
# (signal_sweep.sharpe_t/validate + surfer_poc.backtest use it) — CPU wheel only (no GPU in the cockpit pod).
|
||||
poc = ["torch>=2.2"]
|
||||
@@ -76,3 +76,9 @@ follow_imports = "skip"
|
||||
[tool.pytest.ini_options]
|
||||
pythonpath = ["src"]
|
||||
testpaths = ["tests"]
|
||||
# Parallel by default: the suite is sqlite-per-test / tmp_path isolated, so it fans out cleanly across
|
||||
# cores (~7min serial -> ~1min on 16 cores). Override with `-n0` to force serial for debugging.
|
||||
addopts = "-n auto"
|
||||
markers = [
|
||||
"slow: heavy real-compute/DB tests (4-sleeve walk-forwards, gauntlet DSR, forward-engine recompute). Skip the fast inner loop with `-m 'not slow'`.",
|
||||
]
|
||||
|
||||
@@ -1,24 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Build + deploy the fxhnt cockpit via Argo Workflows (in-cluster Kaniko build, no laptop push).
|
||||
#
|
||||
# Prereqs (one-time): the `fxhnt-cockpit` WorkflowTemplate is registered
|
||||
# (kubectl -n foxhunt apply -f infra/argo/cockpit-build-deploy.yaml)
|
||||
# and this repo is pushed to the in-cluster GitLab as root/fxhnt.git.
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/argo-deploy-cockpit.sh # build+deploy HEAD of root/fxhnt master
|
||||
# ./scripts/argo-deploy-cockpit.sh <commit-sha> # a specific commit
|
||||
set -euo pipefail
|
||||
|
||||
SHA="${1:-HEAD}"
|
||||
NS="foxhunt"
|
||||
|
||||
# Resolve HEAD to the real master sha so the in-cluster clone checks out exactly what was pushed.
|
||||
if [ "$SHA" = "HEAD" ]; then
|
||||
SHA="$(git rev-parse HEAD)"
|
||||
fi
|
||||
|
||||
echo "Submitting fxhnt-cockpit build+deploy for ${SHA} ..."
|
||||
argo submit -n "$NS" --from=wftmpl/fxhnt-cockpit \
|
||||
-p commit-sha="$SHA" \
|
||||
--watch
|
||||
@@ -28,4 +28,6 @@ sleep 4
|
||||
export FXHNT_OPERATIONAL_DSN="postgresql+psycopg://foxhunt:${PW_ENC}@127.0.0.1:${LPORT}/fxhnt"
|
||||
export FXHNT_DEV_PORT="${DEV_PORT}"
|
||||
echo "serving READ-ONLY cockpit against PROD data on http://127.0.0.1:${DEV_PORT}"
|
||||
uv run python3 scripts/dev_cockpit_local.py
|
||||
# --extra web (uvicorn/fastapi/jinja2) + --extra pg (psycopg) — the cockpit needs both; a bare `uv run` on a
|
||||
# fresh env fails ModuleNotFoundError: uvicorn / psycopg.
|
||||
uv run --extra web --extra pg python3 scripts/dev_cockpit_local.py
|
||||
|
||||
@@ -13,6 +13,7 @@ import os
|
||||
import uvicorn
|
||||
|
||||
from fxhnt.adapters.persistence.forward_nav import ForwardNavRepo
|
||||
from fxhnt.adapters.persistence.ibkr_account_repo import IbkrAccountRepo
|
||||
from fxhnt.adapters.persistence.paper_repo import PaperRepo
|
||||
from fxhnt.adapters.web.app import create_app
|
||||
|
||||
@@ -20,10 +21,14 @@ dsn = os.environ["FXHNT_OPERATIONAL_DSN"]
|
||||
port = int(os.environ.get("FXHNT_DEV_PORT", "8801"))
|
||||
|
||||
# READ-ONLY: no .migrate() — never write DDL to a prod DB we're only browsing.
|
||||
# Wire EVERY repo prod wires (create_app_from_settings), or a page silently degrades to an empty state that
|
||||
# does NOT match prod — the IBKR paper-account card in particular reads through ibkr_account_repo; without it
|
||||
# the /paper card renders "not connected yet" even when the real account is live (a misleading local artifact).
|
||||
app = create_app(
|
||||
ForwardNavRepo(dsn),
|
||||
paper_repo=PaperRepo(dsn),
|
||||
bybit_paper_repo=PaperRepo(dsn, venue="bybit"),
|
||||
bybit_paper_repo=PaperRepo(dsn),
|
||||
ibkr_account_repo=IbkrAccountRepo(dsn),
|
||||
)
|
||||
print(f"cockpit (read-only) on http://127.0.0.1:{port}/paper/sim?book=bybit_4edge")
|
||||
uvicorn.run(app, host="127.0.0.1", port=port, log_level="warning")
|
||||
|
||||
@@ -1,89 +0,0 @@
|
||||
"""Alpaca broker adapter — implements the Broker port via the Alpaca REST API (httpx). The
|
||||
application/domain never see httpx; swapping brokers is just another adapter here.
|
||||
|
||||
Alpaca gives us what IBKR does not: COMMISSION-FREE, FRACTIONAL US-equity orders — so a small-capital
|
||||
book implements its target weights precisely instead of IBKR's whole-share rounding. `base_url`
|
||||
defaults to PAPER; live is additionally gated by `execution.allow_live` in the ExecutionService.
|
||||
Only the two REST endpoints the Broker port needs (account + orders). httpx is an optional dependency
|
||||
(extra: web). A `transport` seam lets tests stub the REST layer with `httpx.MockTransport` (no network).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from types import TracebackType
|
||||
from typing import Any
|
||||
|
||||
from fxhnt.ports.broker import AccountState, Order
|
||||
|
||||
|
||||
class AlpacaBroker:
|
||||
name = "alpaca"
|
||||
# The capability the ExecutionService reads to size FRACTIONAL shares (IBKR sets this False).
|
||||
supports_fractional = True
|
||||
|
||||
def __init__(self, api_key: str, api_secret: str,
|
||||
base_url: str = "https://paper-api.alpaca.markets", timeout: int = 15,
|
||||
transport: Any = None) -> None:
|
||||
self._base = base_url.rstrip("/")
|
||||
self._headers = {"APCA-API-KEY-ID": api_key, "APCA-API-SECRET-KEY": api_secret}
|
||||
self._timeout = timeout
|
||||
self._transport = transport # test seam: an httpx.MockTransport in unit tests, None in prod
|
||||
self._client: Any = None
|
||||
|
||||
def _c(self) -> Any:
|
||||
import httpx
|
||||
|
||||
if self._client is None:
|
||||
self._client = httpx.Client(base_url=self._base, headers=self._headers,
|
||||
timeout=self._timeout, transport=self._transport)
|
||||
return self._client
|
||||
|
||||
def _get(self, path: str) -> Any:
|
||||
r = self._c().get(path)
|
||||
if r.status_code >= 300:
|
||||
raise RuntimeError(f"alpaca GET {path} -> {r.status_code}: {r.text[:120]}")
|
||||
return r.json()
|
||||
|
||||
def account_state(self) -> AccountState:
|
||||
acct = self._get("/v2/account")
|
||||
positions = self._get("/v2/positions")
|
||||
pos = {p["symbol"]: float(p["qty"]) for p in positions}
|
||||
# Σ|market_value| feeds the ExecutionService's NLV-vs-(cash+positions) data-consistency gate.
|
||||
gpv = sum(abs(float(p.get("market_value", 0.0) or 0.0)) for p in positions)
|
||||
return AccountState(
|
||||
nlv=float(acct.get("portfolio_value", 0.0)),
|
||||
cash=float(acct.get("cash", 0.0)),
|
||||
positions=pos,
|
||||
is_paper="paper-api" in self._base,
|
||||
gross_position_value=gpv,
|
||||
)
|
||||
|
||||
def place_order(self, order: Order) -> str:
|
||||
# Alpaca crypto pairs carry a slash ("BTC/USD"); equities do not ("SPY"). Both are fractional market
|
||||
# orders, but time-in-force differs: crypto is 24/7 → gtc; equity fractional REQUIRES day. Fixing
|
||||
# these here means a fractional qty is never rejected for an incompatible tif. qty is a string
|
||||
# (Alpaca's fractional contract).
|
||||
crypto = "/" in order.symbol
|
||||
payload = {
|
||||
"symbol": order.symbol,
|
||||
"qty": str(order.quantity),
|
||||
"side": order.side.lower(),
|
||||
"type": "market",
|
||||
"time_in_force": "gtc" if crypto else "day",
|
||||
}
|
||||
r = self._c().post("/v2/orders", json=payload)
|
||||
if r.status_code >= 300:
|
||||
raise RuntimeError(
|
||||
f"alpaca order {order.side} {order.quantity:g} {order.symbol} -> {r.status_code}: {r.text[:120]}")
|
||||
return str(r.json().get("status", "?"))
|
||||
|
||||
def close(self) -> None:
|
||||
if self._client is not None:
|
||||
self._client.close()
|
||||
self._client = None
|
||||
|
||||
def __enter__(self) -> "AlpacaBroker":
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type: type[BaseException] | None, exc: BaseException | None,
|
||||
tb: TracebackType | None) -> None:
|
||||
self.close()
|
||||
@@ -7,13 +7,60 @@ Bakes in hard-won lessons: paper detection via the DU* account prefix; whole-sha
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import math
|
||||
from dataclasses import dataclass, field
|
||||
from types import TracebackType
|
||||
from typing import Any
|
||||
|
||||
from fxhnt.ports.broker import AccountState, Order
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_SUMMARY_TAGS = ("NetLiquidation", "TotalCashValue", "GrossPositionValue")
|
||||
|
||||
# IBKR market-data type: 1=Live, 2=Frozen, 3=Delayed, 4=Delayed-Frozen. Paper accounts lack live
|
||||
# subscriptions for most UCITS lines, so the UCITS price fetch requests DELAYED data (3).
|
||||
_MKTDATA_DELAYED = 3
|
||||
|
||||
|
||||
class ContractResolutionError(RuntimeError):
|
||||
"""An IBKR contract failed to resolve (empty qualifyContracts). Raised by place_order so the caller
|
||||
SKIPS that sleeve — never places a wrong or unqualified order."""
|
||||
|
||||
|
||||
def _usable_price(x: object) -> float | None:
|
||||
"""A positive, non-NaN number as a float, else None (a delayed-data field can be NaN or absent)."""
|
||||
if isinstance(x, (int, float)) and not math.isnan(float(x)) and float(x) > 0.0:
|
||||
return float(x)
|
||||
return None
|
||||
|
||||
|
||||
def _ticker_price(ticker: Any) -> float | None:
|
||||
"""Extract a usable price from an ib_async Ticker, tolerating NaN/empty delayed-data fields. Prefers the
|
||||
library's blended marketPrice(), then last, close, then the bid/ask midpoint."""
|
||||
mp = getattr(ticker, "marketPrice", None)
|
||||
candidates = [mp() if callable(mp) else mp,
|
||||
getattr(ticker, "last", None), getattr(ticker, "close", None)]
|
||||
for c in candidates:
|
||||
px = _usable_price(c)
|
||||
if px is not None:
|
||||
return px
|
||||
bid, ask = _usable_price(getattr(ticker, "bid", None)), _usable_price(getattr(ticker, "ask", None))
|
||||
if bid is not None and ask is not None:
|
||||
return (bid + ask) / 2.0
|
||||
return None
|
||||
|
||||
|
||||
@dataclass
|
||||
class AccountSnapshot:
|
||||
"""The REAL DU-paper-account state at a rebalance (D2.1) — captured additively alongside the existing
|
||||
intended-book return basis, for the `ibkr_account_nav` table."""
|
||||
nlv: float
|
||||
cash: float
|
||||
gross: float
|
||||
positions: dict[str, float] = field(default_factory=dict)
|
||||
|
||||
|
||||
class IbkrBroker:
|
||||
name = "ibkr"
|
||||
@@ -22,6 +69,7 @@ class IbkrBroker:
|
||||
def __init__(self, host: str = "127.0.0.1", port: int = 4002, client_id: int = 17, timeout: int = 15) -> None:
|
||||
self._host, self._port, self._cid, self._timeout = host, port, client_id, timeout
|
||||
self._ib: Any = None
|
||||
self._trades: list[Any] = [] # ib-async Trade objects from place_order, popped by pop_trades()
|
||||
|
||||
def _ensure(self) -> Any:
|
||||
from ib_async import IB
|
||||
@@ -46,20 +94,179 @@ class IbkrBroker:
|
||||
gross_position_value=summ.get("GrossPositionValue"),
|
||||
)
|
||||
|
||||
def _resolve_contract(self, ib: Any, symbol: str, exchange: str, currency: str,
|
||||
sec_id_type: str | None, sec_id: str | None) -> Any | None:
|
||||
"""Build the IBKR contract and qualify it. With an ISIN (sec_id_type/sec_id) it builds a
|
||||
Contract(secIdType=ISIN, secId=..., exchange=..., currency=...) with NO symbol — the ISIN is the
|
||||
unambiguous key and the map's cosmetic ticker may differ from the USD line's real symbol (IEF's USD
|
||||
line is CBU0, GLD's is IGLN), so pinning `symbol` would fight the ISIN. Probed live on DU9600528
|
||||
(2026-07-15): SMART+ISIN returns 0 for these LSE-listed UCITS lines; LSEETF+ISIN+USD resolves each.
|
||||
The US path stays a plain Stock(symbol, SMART, USD). Returns the qualified contract, or None."""
|
||||
from ib_async import Contract, Stock
|
||||
|
||||
if sec_id_type and sec_id:
|
||||
contract = Contract(secType="STK", exchange=exchange or "LSEETF",
|
||||
currency=currency, secIdType=sec_id_type, secId=sec_id)
|
||||
else:
|
||||
contract = Stock(symbol, exchange or "SMART", currency)
|
||||
qualified = ib.qualifyContracts(contract)
|
||||
return qualified[0] if qualified else None
|
||||
|
||||
def market_price(self, symbol: str, exchange: str = "SMART", currency: str = "USD",
|
||||
sec_id_type: str | None = None, sec_id: str | None = None) -> float | None:
|
||||
"""Qualify the contract and return a REAL market price in `currency` via DELAYED market data (paper
|
||||
accounts lack live subscriptions). Returns None — caller SKIPS the sleeve — if the contract does not
|
||||
resolve or no usable price comes back. Used to size the UCITS route from the real UCITS price."""
|
||||
# A raise anywhere in the IB round-trip (connection blip, or an odd error on a thin/new line) must be
|
||||
# downgraded to a per-sleeve skip (None), NOT propagated — one bad sleeve can't wedge the rebalance.
|
||||
try:
|
||||
ib = self._ensure()
|
||||
contract = self._resolve_contract(ib, symbol, exchange, currency, sec_id_type, sec_id)
|
||||
except Exception:
|
||||
log.error("IBKR contract resolution RAISED for symbol=%r (secId=%r) — SKIPPING sleeve.",
|
||||
symbol, sec_id, exc_info=True)
|
||||
return None
|
||||
if contract is None:
|
||||
log.error("IBKR contract did NOT resolve for price: symbol=%r exchange=%r currency=%r "
|
||||
"secId=%r — SKIPPING sleeve (no price, no order).", symbol, exchange, currency, sec_id)
|
||||
return None
|
||||
# 1) delayed live snapshot — fast, works when the delayed stream is free.
|
||||
px = None
|
||||
try:
|
||||
ib.reqMarketDataType(_MKTDATA_DELAYED)
|
||||
ticker = ib.reqMktData(contract, "", True, False) # snapshot=True
|
||||
ib.sleep(2)
|
||||
px = _ticker_price(ticker)
|
||||
except Exception:
|
||||
log.warning("IBKR delayed market-data raised for symbol=%r — trying historical close.", symbol)
|
||||
# 2) FALLBACK: historical last close. Conflict-free — no live-session/subscription contention. Some LSE
|
||||
# lines return Error 10197 ("competing live session") / 354 on the live stream (probed 2026-07-15:
|
||||
# Euronext-primary DBMF and the physical-gold ETC IGLN) yet give a clean daily close here. A prior
|
||||
# close is fine for paper sizing (delayed-vs-live divergence already accepted on paper).
|
||||
if px is None or not (px > 0):
|
||||
try:
|
||||
bars = ib.reqHistoricalData(contract, "", "5 D", "1 day", "MIDPOINT", useRTH=True, formatDate=1)
|
||||
if bars and bars[-1].close and bars[-1].close > 0:
|
||||
px = float(bars[-1].close)
|
||||
log.info("IBKR: no live price for symbol=%r — using historical last close %.4f.", symbol, px)
|
||||
except Exception:
|
||||
log.error("IBKR historical-price fallback RAISED for symbol=%r (secId=%r) — SKIPPING sleeve.",
|
||||
symbol, sec_id, exc_info=True)
|
||||
if px is None or not (px > 0):
|
||||
log.error("IBKR returned no usable price (live + historical) for symbol=%r (secId=%r) — "
|
||||
"SKIPPING sleeve.", symbol, sec_id)
|
||||
return None
|
||||
return px
|
||||
|
||||
def historical_daily_volume(self, *, ticker: str, isin: str, exchange: str = "LSEETF",
|
||||
currency: str = "USD", days: int = 90) -> dict[str, float]:
|
||||
"""Real on-exchange daily volume for a UCITS line (reqHistoricalData TRADES, probe-verified
|
||||
2026-07-16) — keyed by ISO date string, feeding `snapshot_ibkr_volumes`. Integration-only (needs a
|
||||
live ib-gateway); never raises into the CronJob's per-line loop — resolve-fail/no-data/any error all
|
||||
downgrade to an empty dict (log + continue), same discipline as `market_price`."""
|
||||
try:
|
||||
ib = self._ensure()
|
||||
contract = self._resolve_contract(ib, ticker, exchange, currency, "ISIN", isin)
|
||||
if contract is None:
|
||||
log.error("IBKR contract did NOT resolve for historical volume: ticker=%r isin=%r "
|
||||
"exchange=%r currency=%r — returning empty.", ticker, isin, exchange, currency)
|
||||
return {}
|
||||
bars = ib.reqHistoricalData(contract, "", f"{days} D", "1 day", "TRADES",
|
||||
useRTH=True, formatDate=1)
|
||||
out: dict[str, float] = {}
|
||||
for bar in bars:
|
||||
if bar.volume in (None, -1):
|
||||
continue
|
||||
d = bar.date
|
||||
out[d.isoformat() if hasattr(d, "isoformat") else str(d)] = float(bar.volume)
|
||||
return out
|
||||
except Exception:
|
||||
log.error("IBKR historical volume fetch RAISED for ticker=%r (isin=%r) — returning empty.",
|
||||
ticker, isin, exc_info=True)
|
||||
return {}
|
||||
|
||||
def resolve_symbol_info(self, *, ticker: str, isin: str, exchange: str = "LSEETF",
|
||||
currency: str = "USD") -> dict[str, str] | None:
|
||||
"""READ-ONLY: qualify a UCITS line by ISIN and report the identity IBKR actually assigns it — the
|
||||
`symbol`, `localSymbol`, and `conId` that `positions()`/`accountSummary()` will report a HOLDING under.
|
||||
Used by the `probe-ucits-symbols` CLI to detect where a config `ticker` (cosmetic) diverges from what
|
||||
IBKR reports (e.g. IEF's `CBU0` -> canonical `CSBGU0`), so `_expected_book_symbols` can key the NLV
|
||||
stray-check on the REPORTED symbol instead of the display ticker. Returns None on resolve failure
|
||||
(logged), never raises — same discipline as `historical_daily_volume`."""
|
||||
try:
|
||||
ib = self._ensure()
|
||||
contract = self._resolve_contract(ib, ticker, exchange, currency, "ISIN", isin)
|
||||
if contract is None:
|
||||
log.error("IBKR contract did NOT resolve for symbol probe: ticker=%r isin=%r", ticker, isin)
|
||||
return None
|
||||
return {"ticker": ticker, "isin": isin,
|
||||
"ibkr_symbol": str(getattr(contract, "symbol", "") or ""),
|
||||
"local_symbol": str(getattr(contract, "localSymbol", "") or ""),
|
||||
"con_id": str(getattr(contract, "conId", "") or "")}
|
||||
except Exception:
|
||||
log.error("IBKR symbol probe RAISED for ticker=%r (isin=%r) — returning None.",
|
||||
ticker, isin, exc_info=True)
|
||||
return None
|
||||
|
||||
def place_order(self, order: Order) -> str:
|
||||
from ib_async import MarketOrder, Stock
|
||||
from ib_async import MarketOrder
|
||||
|
||||
ib = self._ensure()
|
||||
contract = Stock(order.symbol, "SMART", "USD")
|
||||
ib.qualifyContracts(contract)
|
||||
contract = self._resolve_contract(ib, order.symbol, order.exchange, order.currency,
|
||||
order.sec_id_type, order.sec_id)
|
||||
if contract is None:
|
||||
log.error("IBKR contract did NOT resolve for order %s %s (exchange=%r currency=%r secId=%r) "
|
||||
"— SKIPPING sleeve, NO order placed.", order.side, order.symbol, order.exchange,
|
||||
order.currency, order.sec_id)
|
||||
raise ContractResolutionError(
|
||||
f"contract not resolved for {order.symbol} (exchange={order.exchange}, "
|
||||
f"currency={order.currency}, secId={order.sec_id})")
|
||||
# Route via SMART, keeping the resolved conId. The UCITS contract resolves on LSEETF (see
|
||||
# _resolve_contract), but placing DIRECT to LSEETF trips IBKR's precautionary "direct-routed orders"
|
||||
# restriction -> Error 10311 -> order discarded. SMART is in every UCITS line's valid-exchange set
|
||||
# (probed 2026-07-15), so smart-routing the qualified conId fills without the direct-route penalty.
|
||||
# conId pins the exact instrument, so SMART cannot mis-route to the wrong (e.g. US grey-market) line.
|
||||
contract.exchange = "SMART"
|
||||
# Bug fix (Error 478, live 2026-07-20): the ISIN-qualified UCITS contract can retain a `secId`/
|
||||
# `localSymbol` that CONFLICTS with the conId IBKR pinned (e.g. requested localSymbol CBU0 vs the
|
||||
# canonical CSBGU0 for ISIN IE00B3VWN518) -> "Parameters in request conflicts with contract parameters"
|
||||
# -> the order is REJECTED and the leg gets 0 fills. The conId alone is unambiguous, so strip the
|
||||
# ambiguous identity fields after resolution; IBKR resolves purely on conId + SMART. Guarded getattr:
|
||||
# these fields may be absent on the plain-Stock US path.
|
||||
conid = getattr(contract, "conId", 0)
|
||||
if conid:
|
||||
for _f in ("secId", "secIdType", "localSymbol"):
|
||||
if getattr(contract, _f, None):
|
||||
setattr(contract, _f, "")
|
||||
market_order = MarketOrder(order.side, int(round(order.quantity))) # whole shares
|
||||
# Bug fix (Error 10349, live 2026-07-20): a MarketOrder with NO explicit TIF let the account's order
|
||||
# preset force TIF=DAY and cancel-then-reconfirm each order (a wasted round-trip; DBMF hung on
|
||||
# 'Submitted'). Set TIF=DAY explicitly so the order already matches the account preset — no override,
|
||||
# no cancel. DAY is correct for a market-hours rebalance (the run fires at the UCITS open).
|
||||
market_order.tif = "DAY"
|
||||
accts = ib.managedAccounts()
|
||||
if accts:
|
||||
market_order.account = accts[0]
|
||||
trade = ib.placeOrder(contract, market_order)
|
||||
ib.sleep(2)
|
||||
self._trades.append(trade) # collected by pop_trades() so fills are capturable (D2.1)
|
||||
return str(trade.orderStatus.status)
|
||||
|
||||
def pop_trades(self) -> list[Any]:
|
||||
"""Return and clear the ib-async `Trade` objects placed since the last pop (one rebalance's worth).
|
||||
`exec_capture.capture_fills` reads each trade's `.fills` to persist the realized executions."""
|
||||
trades, self._trades = self._trades, []
|
||||
return trades
|
||||
|
||||
def account_snapshot(self) -> AccountSnapshot:
|
||||
"""The REAL account state right now (NLV/cash/gross + positions), reusing the same `accountSummary`/
|
||||
`positions` reads as `account_state` (D2.1 — captured additively, not the gate-relevant basis)."""
|
||||
ib = self._ensure()
|
||||
summ = {v.tag: float(v.value) for v in ib.accountSummary() if v.tag in _SUMMARY_TAGS}
|
||||
positions = {p.contract.symbol: float(p.position) for p in ib.positions()}
|
||||
return AccountSnapshot(nlv=summ.get("NetLiquidation", 0.0), cash=summ.get("TotalCashValue", 0.0),
|
||||
gross=summ.get("GrossPositionValue", 0.0), positions=positions)
|
||||
|
||||
def place_combo(self, legs: list[tuple[str, str, int]], net_limit: float) -> str:
|
||||
"""Atomic XSP put-credit-spread as a BAG combo. legs = (osi_symbol, side, qty); net_limit<0 = credit.
|
||||
Requires options trading permission (account currently lacks it -> this path ships suspended)."""
|
||||
|
||||
@@ -1,86 +0,0 @@
|
||||
"""urllib BinanceFundingClient — Binance USDⓈ-M futures public API (no key, no capital). Faithful to
|
||||
foxhunt scripts/surfer/crypto_funding_paper.py's scan() (lines 48-92): the hedgeable, liquid,
|
||||
crypto-native universe + trailing-30d-mean funding (tf30) + today's booking rate (last24).
|
||||
|
||||
* crypto_native() -> symbols with underlyingType == "COIN" from /fapi/v1/exchangeInfo
|
||||
* spot_symbols() -> symbols with a Binance SPOT market (api.binance.com/api/v3/ticker/price)
|
||||
* universe() -> {sym: quoteVolume} for crypto-native, spot-hedgeable USDT perps with vol > LIQ_USD
|
||||
* scan(prev) -> (liq, tf30, last24) over union of the universe and prev positions
|
||||
|
||||
Retries with backoff; mirrors the original `get()` helper and the yahoo_daily `_get` style."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
import urllib.request
|
||||
from typing import Any
|
||||
|
||||
from fxhnt.domain.strategies.funding_carry import LIQ_USD
|
||||
|
||||
_FAPI = "https://fapi.binance.com"
|
||||
INTERVALS_PER_DAY = 3 # Binance funding settles every 8h (matches the original constant)
|
||||
|
||||
|
||||
class BinanceFundingClient:
|
||||
def __init__(self, liq_usd: float = LIQ_USD) -> None:
|
||||
self._liq_usd = liq_usd
|
||||
|
||||
def _get(self, url: str, tries: int = 4) -> Any:
|
||||
for a in range(tries):
|
||||
try:
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "curl/8"})
|
||||
return json.loads(urllib.request.urlopen(req, timeout=30).read())
|
||||
except Exception: # noqa: BLE001 — transient HTTP; retry then re-raise
|
||||
if a == tries - 1:
|
||||
raise
|
||||
time.sleep(2 * (a + 1))
|
||||
raise RuntimeError("unreachable")
|
||||
|
||||
def crypto_native(self) -> set[str]:
|
||||
"""Symbols whose underlyingType is COIN (original crypto_native(), line 48-50)."""
|
||||
info = self._get(f"{_FAPI}/fapi/v1/exchangeInfo")
|
||||
return {s["symbol"] for s in info["symbols"] if s.get("underlyingType") == "COIN"}
|
||||
|
||||
def spot_symbols(self) -> set[str]:
|
||||
"""Symbols with a Binance SPOT market — required to build the delta-neutral hedge
|
||||
(original spot_symbols(), line 53-57). Perp-only coins can't be cash-and-carry harvested."""
|
||||
return {x["symbol"] for x in self._get("https://api.binance.com/api/v3/ticker/price")}
|
||||
|
||||
def universe(self) -> dict[str, float]:
|
||||
"""Liquid, crypto-native, HEDGEABLE (spot+perp) USDT perps (original universe(), line 60-68)."""
|
||||
native = self.crypto_native()
|
||||
spot = self.spot_symbols()
|
||||
t = self._get(f"{_FAPI}/fapi/v1/ticker/24hr")
|
||||
return {
|
||||
x["symbol"]: float(x["quoteVolume"])
|
||||
for x in t
|
||||
if x["symbol"].endswith("USDT")
|
||||
and x["symbol"] in native
|
||||
and x["symbol"] in spot
|
||||
and float(x["quoteVolume"]) > self._liq_usd
|
||||
}
|
||||
|
||||
def _funding_hist(self, sym: str, limit: int = 90) -> list[float]:
|
||||
"""Funding-rate history (original funding_hist(), line 71-73)."""
|
||||
h = self._get(f"{_FAPI}/fapi/v1/fundingRate?symbol={sym}&limit={limit}")
|
||||
return [float(x["fundingRate"]) for x in h]
|
||||
|
||||
def scan(self, prev: set[str]) -> tuple[dict[str, float], dict[str, float], dict[str, float]]:
|
||||
"""(liq, tf30, last24) for the union of the universe and prev positions (original scan(),
|
||||
line 76-92)."""
|
||||
liq = self.universe()
|
||||
need = set(liq) | set(prev)
|
||||
tf30: dict[str, float] = {}
|
||||
last24: dict[str, float] = {}
|
||||
for i, sym in enumerate(sorted(need)):
|
||||
try:
|
||||
h = self._funding_hist(sym)
|
||||
except Exception: # noqa: BLE001 — skip transient per-symbol failures
|
||||
continue
|
||||
if len(h) < 30:
|
||||
continue
|
||||
tf30[sym] = (sum(h) / len(h)) * INTERVALS_PER_DAY
|
||||
last24[sym] = sum(h[-INTERVALS_PER_DAY:])
|
||||
if i % 50 == 49:
|
||||
time.sleep(0.5)
|
||||
return liq, tf30, last24
|
||||
@@ -1,40 +0,0 @@
|
||||
"""Binance SPOT daily-close history (the long leg of the cash-and-carry), paginated.
|
||||
Returns {epoch_day: close}. Live network in production; tests inject a fake `_get`."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
import urllib.request
|
||||
from typing import Any
|
||||
|
||||
_SPOT = "https://api.binance.com"
|
||||
|
||||
|
||||
class BinanceSpotHistory:
|
||||
def _get(self, url: str, tries: int = 4) -> Any:
|
||||
last: Exception | None = None
|
||||
for i in range(tries):
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=30) as r:
|
||||
return json.loads(r.read())
|
||||
except Exception as e:
|
||||
last = e
|
||||
time.sleep(0.5 * (i + 1))
|
||||
raise RuntimeError(f"spot fetch failed: {url}: {last}")
|
||||
|
||||
def daily_close(self, symbol: str, start_ms: int = 0) -> dict[int, float]:
|
||||
"""Full daily spot-close history {epoch_day: close}, forward-paginated (limit 1000).
|
||||
Terminates when a page returns < 1000 rows (or empty)."""
|
||||
out: dict[int, float] = {}
|
||||
st = start_ms
|
||||
while True:
|
||||
url = f"{_SPOT}/api/v3/klines?symbol={symbol}&interval=1d&startTime={st}&limit=1000"
|
||||
rows = self._get(url)
|
||||
if not rows:
|
||||
break
|
||||
for r in rows:
|
||||
out[int(r[0]) // 86_400_000] = float(r[4])
|
||||
if len(rows) < 1000:
|
||||
break
|
||||
st = int(rows[-1][0]) + 86_400_000
|
||||
return out
|
||||
@@ -1,110 +0,0 @@
|
||||
"""binance.vision 1m monthly-kline adapter — the FREE, survivorship-free source for the intraday
|
||||
`worst_basis` ingest. Fetches + parses a month of 1m klines for a (market, symbol) into {epoch_ms: close}.
|
||||
|
||||
* market is 'futures/um' (perp) or 'spot'.
|
||||
* URL: https://data.binance.vision/data/<market>/monthly/klines/<SYM>/1m/<SYM>-1m-<YYYY-MM>.zip
|
||||
* CSV (no header): open_time, open, high, low, close, volume, close_time, ... . open_time is ms, but
|
||||
some months ship µs (>1e14) — divided by 1000. We keep the CLOSE column.
|
||||
|
||||
RESUMABLE / cached: a parsed month is written to `cache_dir` as a small JSON; a re-run reads the cache and
|
||||
does NOT re-download. A MISSING month (delisted / not-listed-yet → 404) is parsed as EMPTY and the empty
|
||||
result is ALSO cached, so re-runs don't re-hammer the missing URL. The raw-bytes fetch is injected (tests
|
||||
pass a fake; production uses urllib) so unit tests never touch the network.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import zipfile
|
||||
from collections.abc import Callable
|
||||
|
||||
_log = logging.getLogger(__name__)
|
||||
_BASE = "https://data.binance.vision/data"
|
||||
_US_THRESHOLD = 1e14 # open_time above this is microseconds, not milliseconds
|
||||
|
||||
|
||||
def _http_fetch(url: str, tries: int = 4) -> bytes:
|
||||
"""Download the zip bytes with retry/backoff. A 404 (missing month) raises urllib HTTPError, which the
|
||||
caller treats as an empty month (delisted / not-yet-listed)."""
|
||||
last: Exception | None = None
|
||||
for a in range(tries):
|
||||
try:
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
|
||||
with urllib.request.urlopen(req, timeout=60) as r:
|
||||
return r.read()
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code == 404:
|
||||
raise # genuinely missing: don't retry, let the caller cache empty
|
||||
last = e
|
||||
time.sleep(2 * (a + 1))
|
||||
except Exception as e: # noqa: BLE001 — transient network; retry then re-raise
|
||||
last = e
|
||||
time.sleep(2 * (a + 1))
|
||||
raise RuntimeError(f"binance.vision fetch failed: {url}: {last}")
|
||||
|
||||
|
||||
class BinanceVisionKlines:
|
||||
def __init__(self, cache_dir: str, *, fetch: Callable[[str], bytes] = _http_fetch) -> None:
|
||||
self._cache_dir = cache_dir
|
||||
self._fetch = fetch
|
||||
|
||||
@staticmethod
|
||||
def _url(market: str, symbol: str, month: str) -> str:
|
||||
return f"{_BASE}/{market}/monthly/klines/{symbol}/1m/{symbol}-1m-{month}.zip"
|
||||
|
||||
def _cache_path(self, market: str, symbol: str, month: str) -> str:
|
||||
safe = market.replace("/", "_")
|
||||
return os.path.join(self._cache_dir, f"{safe}__{symbol}__{month}.json")
|
||||
|
||||
@staticmethod
|
||||
def _parse_zip(blob: bytes) -> dict[int, float]:
|
||||
"""Extract the single CSV from the monthly zip and parse {epoch_ms: close}. open_time in µs
|
||||
(>1e14) is divided to ms; the close is column index 4."""
|
||||
out: dict[int, float] = {}
|
||||
with zipfile.ZipFile(io.BytesIO(blob)) as z:
|
||||
name = z.namelist()[0]
|
||||
with z.open(name) as fh:
|
||||
text = io.TextIOWrapper(fh, encoding="utf-8")
|
||||
for row in csv.reader(text):
|
||||
if len(row) < 5:
|
||||
continue
|
||||
try:
|
||||
ot = int(float(row[0]))
|
||||
close = float(row[4])
|
||||
except (TypeError, ValueError):
|
||||
continue # header row or malformed line — skip
|
||||
if ot > _US_THRESHOLD:
|
||||
ot //= 1000 # microseconds -> milliseconds
|
||||
out[ot] = close
|
||||
return out
|
||||
|
||||
def monthly_closes(self, market: str, symbol: str, month: str) -> dict[int, float]:
|
||||
"""{epoch_ms: 1m close} for (market∈{'futures/um','spot'}, symbol, 'YYYY-MM'). Served from the
|
||||
on-disk cache when present (resumable); otherwise downloaded, parsed, and cached. A missing month
|
||||
(404) → {} (cached as empty so re-runs skip it). Never raises on a missing month."""
|
||||
cpath = self._cache_path(market, symbol, month)
|
||||
if os.path.exists(cpath):
|
||||
with open(cpath) as f:
|
||||
cached: dict[str, float] = json.load(f)
|
||||
return {int(k): float(v) for k, v in cached.items()}
|
||||
|
||||
url = self._url(market, symbol, month)
|
||||
try:
|
||||
blob = self._fetch(url)
|
||||
parsed = self._parse_zip(blob)
|
||||
except Exception as exc: # noqa: BLE001 — missing/delisted month or transient failure: cache empty
|
||||
_log.info("binance.vision: no klines for %s %s %s (%s) — caching empty", market, symbol, month, exc)
|
||||
parsed = {}
|
||||
|
||||
os.makedirs(self._cache_dir, exist_ok=True)
|
||||
tmp = cpath + ".tmp"
|
||||
with open(tmp, "w") as f:
|
||||
json.dump({str(k): v for k, v in parsed.items()}, f)
|
||||
os.replace(tmp, cpath) # atomic write so a crash mid-write never leaves a half cache
|
||||
return parsed
|
||||
@@ -1,52 +0,0 @@
|
||||
"""Live per-coin snapshot for the cross-sectional funding paper track: trailing funding,
|
||||
today's funding, perp price, spot price, quote-volume — for the liquid universe."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
import urllib.request
|
||||
from typing import Any
|
||||
|
||||
from fxhnt.adapters.data.binance_funding import BinanceFundingClient
|
||||
|
||||
_FAPI = "https://fapi.binance.com"
|
||||
_SPOT = "https://api.binance.com"
|
||||
|
||||
|
||||
class BinanceXsFundingLive:
|
||||
def __init__(self, funding: Any | None = None, *, max_basis: float = 0.30) -> None:
|
||||
self._f = funding if funding is not None else BinanceFundingClient()
|
||||
self._max_basis = max_basis
|
||||
|
||||
def _get(self, url: str, tries: int = 4) -> Any:
|
||||
"""Single live HTTP GET with backoff retry (mirrors BinanceFundingClient._get):
|
||||
a transient HTTP/DNS hiccup is retried, only the final failure re-raises."""
|
||||
for a in range(tries):
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=30) as r:
|
||||
return json.loads(r.read())
|
||||
except Exception: # noqa: BLE001 — transient HTTP; retry then re-raise
|
||||
if a == tries - 1:
|
||||
raise
|
||||
time.sleep(2 * (a + 1))
|
||||
raise RuntimeError("unreachable")
|
||||
|
||||
def _perp_prices(self) -> dict[str, float]:
|
||||
return {x["symbol"]: float(x["price"]) for x in self._get(f"{_FAPI}/fapi/v1/ticker/price")}
|
||||
|
||||
def _spot_prices(self) -> dict[str, float]:
|
||||
return {x["symbol"]: float(x["price"]) for x in self._get(f"{_SPOT}/api/v3/ticker/price")}
|
||||
|
||||
def snapshot(self, prev: dict[str, float]) -> dict[str, tuple[float, float, float, float, float]]:
|
||||
"""{sym: (trailing_funding, today_funding, perp_price, spot_price, qvol)} over the liquid
|
||||
universe; coins lacking a spot price (can't model basis) are omitted, and coins whose
|
||||
perp/spot basis exceeds max_basis (decoupled/relisted ticker-reuse artifacts, e.g.
|
||||
LITUSDT) are dropped — the historical panel applies the same |basis| guard."""
|
||||
liq, tf30, last24 = self._f.scan(set(prev))
|
||||
perp, spot = self._perp_prices(), self._spot_prices()
|
||||
out: dict[str, tuple[float, float, float, float, float]] = {}
|
||||
for s in liq:
|
||||
if s in perp and s in spot and perp[s] > 0 and spot[s] > 0 \
|
||||
and abs((perp[s] - spot[s]) / spot[s]) <= self._max_basis:
|
||||
out[s] = (tf30.get(s, 0.0), last24.get(s, 0.0), perp[s], spot[s], liq[s])
|
||||
return out
|
||||
@@ -1,9 +1,10 @@
|
||||
"""Bybit USDT-perp funding adapter — the carry signal, SURVIVORSHIP-FREE, off Bybit v5 public market data.
|
||||
|
||||
Why Bybit: live execution will be Bybit EU, but every edge is backtested on Binance. Carry is venue-sensitive
|
||||
(funding + universe differ), so we re-validate it on the venue we'll actually trade. Bybit's funding/history
|
||||
endpoint retains DELISTED symbols (e.g. LUNAUSDT returns rows) — strictly better than Binance REST, which
|
||||
purges delisted names — so we can build a truly survivorship-free funding panel.
|
||||
Why Bybit: live execution will be Bybit EU, but every edge was originally backtested on a legacy combined-book
|
||||
venue. Carry is venue-sensitive (funding + universe differ), so we re-validate it on the venue we'll actually
|
||||
trade. Bybit's funding/history endpoint retains DELISTED symbols (e.g. LUNAUSDT returns rows) — strictly
|
||||
better than the legacy venue's REST, which purges delisted names — so we can build a truly survivorship-free
|
||||
funding panel.
|
||||
|
||||
Three pieces (all HTTP is INJECTED so tests run with no network):
|
||||
* active_universe() -> the live USDT linear perps (instruments-info)
|
||||
@@ -11,8 +12,9 @@ Three pieces (all HTTP is INJECTED so tests run with no network):
|
||||
(3/day), paginated BACKWARD via endTime down to a floor.
|
||||
* ever_listed_universe(fetch_html) -> active ∪ delisted (public.bybit.com/trading dir listing) USDT perps
|
||||
|
||||
Daily aggregation matches the existing `crypto_funding_panel` convention (Binance ingest also sums the day's
|
||||
8h stamps into one daily carry number), so the same XsFundingReplay / carry accounting consumes either panel.
|
||||
Daily aggregation matches the existing `crypto_funding_panel` convention (the legacy combined-book ingest
|
||||
also summed the day's 8h stamps into one daily carry number), so the same XsFundingReplay / carry
|
||||
accounting consumes either panel.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -29,7 +31,7 @@ from fxhnt.application.bybit_concurrency import with_rate_limit_backoff
|
||||
|
||||
_API = "https://api.bybit.com"
|
||||
_PUBLIC_TRADING = "https://public.bybit.com/trading/"
|
||||
INTERVALS_PER_DAY = 3 # Bybit USDT perps settle funding every 8h (3/day) — matches the Binance constant.
|
||||
INTERVALS_PER_DAY = 3 # Bybit USDT perps settle funding every 8h (3/day) — matches the legacy venue's constant.
|
||||
_PAGE_LIMIT = 200 # max rows/page on funding/history
|
||||
|
||||
# Backstop floor for the backward pagination: ≈ 2020-01-01 UTC in ms. Bybit funding goes back to ~2022; the
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
"""Bybit daily-kline adapter — PERP + SPOT daily close, off Bybit v5 public market data.
|
||||
|
||||
Why Bybit: live execution will be Bybit EU, but every edge is backtested on Binance. Carry's return is
|
||||
funding + BASIS (spot_ret − perp_ret), so re-validating carry on Bybit needs both price legs on Bybit too.
|
||||
Why Bybit: live execution will be Bybit EU, but every edge was originally backtested on a legacy combined-book
|
||||
venue. Carry's return is funding + BASIS (spot_ret − perp_ret), so re-validating carry on Bybit needs both
|
||||
price legs on Bybit too.
|
||||
This pulls the daily close for the perp leg (category=linear) and the spot leg (category=spot); the ingest
|
||||
writes them as the `close` / `spot_close` features the carry/warehouse panels already consume.
|
||||
|
||||
@@ -179,7 +180,7 @@ class BybitKlines:
|
||||
"""{epoch_day: worst_basis} — the REAL intraday squeeze-tail: per UTC day, the deepest the delta-
|
||||
neutral carry leg (long spot / short perp) marked underwater intraday, from `interval`-minute
|
||||
(default 1m) perp (category=linear) AND spot (category=spot) klines, ALIGNED by bar startMs within
|
||||
each day. The per-day value reuses Binance's `worst_basis_by_day` UNCHANGED — the MINIMUM over the
|
||||
each day. The per-day value reuses the legacy venue's `worst_basis_by_day` UNCHANGED — the MINIMUM over the
|
||||
day's running cumulative Σ(spot_1m_ret − perp_1m_ret) from the day's first aligned minute, capped at
|
||||
−1.0 (total leg loss) — so the two venues' worst_basis are computed by the SAME definition and are
|
||||
directly comparable. A benign day where the basis never moves against the leg ≈ 0; only a genuine
|
||||
@@ -187,7 +188,7 @@ class BybitKlines:
|
||||
|
||||
ONLY days with BOTH perp AND spot 1m bars (≥2 aligned minutes) yield a value; a perp-only day (no
|
||||
spot leg) is OMITTED, so the carry accounting falls back to that day's daily-close basis — exactly
|
||||
like the Binance worst_basis fallback.
|
||||
like the legacy venue's worst_basis fallback.
|
||||
|
||||
MEMORY-BOUNDED via TIME-WINDOWING: instead of loading the whole multi-year 1m history of BOTH legs
|
||||
into memory at once (the old all-at-once path → ~6GB at 12 workers → OOM on a 6GB node), the fetch is
|
||||
|
||||
@@ -1,92 +0,0 @@
|
||||
"""Point-in-time crypto fetch for the combined-book momentum sleeve — Binance USDT-perp daily closes,
|
||||
survivorship-free (live top-N by 24h quote-volume UNION known-dead perps). No API key (public fapi).
|
||||
|
||||
Ported from foxhunt scripts/surfer/fetch_crypto_pit.py with two changes for daily forward use:
|
||||
* REFRESH (overwrite) instead of skip-existing — the forward tracker needs fresh days every run;
|
||||
* a SINGLE klines page (limit=1500) instead of deep endTime pagination — ~1500 daily bars dwarfs the
|
||||
30-day momentum lookback, and one request per symbol keeps a daily refresh cheap.
|
||||
The HTTP GET is injected so tests run offline."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
import urllib.request
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
_log = logging.getLogger(__name__)
|
||||
DAY_MS = 86_400_000
|
||||
# famous delisted/dead Binance USDT perps — klines remain available post-delist (the survivorship signal)
|
||||
DEAD = ["LUNAUSDT", "ANCUSDT", "SRMUSDT", "HNTUSDT", "MATICUSDT", "FTTUSDT", "RAYUSDT", "WAVESUSDT",
|
||||
"BNXUSDT", "SCUSDT", "OCEANUSDT", "AGIXUSDT", "CVCUSDT", "TLMUSDT", "DENTUSDT", "KEYUSDT",
|
||||
"CTKUSDT", "TOMOUSDT", "AKROUSDT", "BLZUSDT", "COMBOUSDT", "FTMUSDT", "DGBUSDT", "REEFUSDT",
|
||||
"SLPUSDT", "IDEXUSDT", "LINAUSDT", "NEBLUSDT", "RADUSDT", "BTSUSDT", "STMXUSDT", "MDTUSDT",
|
||||
"AMBUSDT", "GTCUSDT", "DARUSDT"]
|
||||
|
||||
|
||||
def _default_get(url: str) -> Any:
|
||||
try:
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "curl/8"})
|
||||
return json.load(urllib.request.urlopen(req, timeout=30))
|
||||
except Exception as e: # noqa: BLE001 — network best-effort; caller logs/skips
|
||||
_log.warning("crypto GET failed (%s): %s", url, e)
|
||||
return None
|
||||
|
||||
|
||||
def _universe(n: int, get: Callable[[str], Any]) -> list[str]:
|
||||
info = get("https://fapi.binance.com/fapi/v1/exchangeInfo")
|
||||
perps = {s["symbol"] for s in info["symbols"]
|
||||
if s.get("contractType") == "PERPETUAL" and s.get("quoteAsset") == "USDT"
|
||||
and s.get("status") == "TRADING" and s["symbol"].isascii()} # drop junk non-ASCII meme symbols
|
||||
tick = get("https://fapi.binance.com/fapi/v1/ticker/24hr")
|
||||
vol = {t["symbol"]: float(t["quoteVolume"]) for t in tick if t["symbol"] in perps}
|
||||
return sorted(vol, key=lambda s: -vol[s])[:n]
|
||||
|
||||
|
||||
def _klines(sym: str, get: Callable[[str], Any]) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
|
||||
k = get(f"https://fapi.binance.com/fapi/v1/klines?symbol={sym}&interval=1d&limit=1500")
|
||||
if not k:
|
||||
e = np.array([])
|
||||
return e, e, e, e, e
|
||||
# Binance kline row: [open_time, open, high, low, close, volume, close_time, quote_volume, ...]
|
||||
d = {int(r[0]): (float(r[4]), float(r[2]), float(r[3]), float(r[7])) for r in k} # close, high, low, quote-volume
|
||||
days = np.array(sorted(d))
|
||||
cl = np.array([d[t][0] for t in days])
|
||||
hi = np.array([d[t][1] for t in days])
|
||||
lo = np.array([d[t][2] for t in days])
|
||||
qv = np.array([d[t][3] for t in days])
|
||||
return days // DAY_MS, cl, hi, lo, qv
|
||||
|
||||
|
||||
def _funding_daily(sym: str, get: Callable[[str], Any]) -> dict[int, float]:
|
||||
f = get(f"https://fapi.binance.com/fapi/v1/fundingRate?symbol={sym}&limit=1000")
|
||||
daily: dict[int, float] = {}
|
||||
for r in f or []:
|
||||
d_key = int(r["fundingTime"]) // DAY_MS
|
||||
daily[d_key] = daily.get(d_key, 0.0) + float(r["fundingRate"])
|
||||
return daily
|
||||
|
||||
|
||||
def fetch_crypto_pit(data_dir: str, *, top_n: int = 120, get: Callable[[str], Any] = _default_get,
|
||||
sleep: Callable[[float], None] = time.sleep, dead: list[str] | None = None) -> tuple[int, int]:
|
||||
"""Fetch the survivorship-free crypto panel into <data_dir>/crypto_pit/<SYM>.npz (keys: day, close, high,
|
||||
low, qvol, funding). REFRESH semantics (overwrites). Returns (symbols_written, symbols_attempted)."""
|
||||
out_dir = os.path.join(data_dir, "crypto_pit")
|
||||
os.makedirs(out_dir, exist_ok=True)
|
||||
syms = sorted(set(_universe(top_n, get)) | set(DEAD if dead is None else dead))
|
||||
ok = 0
|
||||
for sym in syms:
|
||||
kd, cl, hi, lo, qv = _klines(sym, get)
|
||||
if len(kd) < 200:
|
||||
continue
|
||||
fund = _funding_daily(sym, get)
|
||||
funding = np.array([fund.get(int(d), 0.0) for d in kd])
|
||||
np.savez(os.path.join(out_dir, f"{sym}.npz"), day=kd, close=cl, high=hi, low=lo, qvol=qv, funding=funding)
|
||||
ok += 1
|
||||
sleep(0.1)
|
||||
_log.info("crypto fetch: %d/%d symbols -> %s", ok, len(syms), out_dir)
|
||||
return ok, len(syms)
|
||||
@@ -1,56 +0,0 @@
|
||||
"""Loader for the survivorship-free `crypto_pit` npz panel (Binance USDT-perp daily
|
||||
close/quote-volume/funding incl. delisted coins). Read-only; no live network."""
|
||||
from __future__ import annotations
|
||||
|
||||
import glob
|
||||
import os
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
class CryptoPitPanelSource:
|
||||
def __init__(self, pit_dir: str) -> None:
|
||||
self._dir = pit_dir
|
||||
self._cache: dict[str, dict[int, tuple[float, float, float]]] | None = None
|
||||
|
||||
def panel(self) -> dict[str, dict[int, tuple[float, float, float]]]:
|
||||
"""{symbol: {epoch_day: (close, qvol, funding)}}. Deterministic (sorted files).
|
||||
Loaded once per instance and cached (avoids double disk I/O via all_days)."""
|
||||
if self._cache is None:
|
||||
out: dict[str, dict[int, tuple[float, float, float]]] = {}
|
||||
for path in sorted(glob.glob(os.path.join(self._dir, "*.npz"))):
|
||||
sym = os.path.splitext(os.path.basename(path))[0]
|
||||
with np.load(path) as z: # numeric arrays only; no allow_pickle
|
||||
days = z["day"].astype(np.int64)
|
||||
close = z["close"].astype(float)
|
||||
qvol = z["qvol"].astype(float)
|
||||
funding = z["funding"].astype(float)
|
||||
out[sym] = {int(d): (float(c), float(q), float(f))
|
||||
for d, c, q, f in zip(days, close, qvol, funding)}
|
||||
self._cache = out
|
||||
return self._cache
|
||||
|
||||
def all_days(self) -> list[int]:
|
||||
return sorted({d for s in self.panel().values() for d in s})
|
||||
|
||||
@staticmethod
|
||||
def write_spot_panel(spot: dict[str, dict[int, float]], spot_dir: str) -> None:
|
||||
import os
|
||||
os.makedirs(spot_dir, exist_ok=True)
|
||||
for sym, series in spot.items():
|
||||
days = sorted(series)
|
||||
np.savez(os.path.join(spot_dir, f"{sym}.npz"),
|
||||
day=np.array(days, dtype=np.int64),
|
||||
spot=np.array([series[d] for d in days], dtype=float))
|
||||
|
||||
@staticmethod
|
||||
def load_spot_panel(spot_dir: str) -> dict[str, dict[int, float]]:
|
||||
import glob
|
||||
import os
|
||||
out: dict[str, dict[int, float]] = {}
|
||||
for path in sorted(glob.glob(os.path.join(spot_dir, "*.npz"))):
|
||||
sym = os.path.splitext(os.path.basename(path))[0]
|
||||
with np.load(path) as z:
|
||||
out[sym] = {int(d): float(s)
|
||||
for d, s in zip(z["day"].astype(np.int64), z["spot"].astype(float))}
|
||||
return out
|
||||
@@ -18,6 +18,38 @@ if TYPE_CHECKING:
|
||||
import pandas as pd
|
||||
|
||||
|
||||
def _to_utc_ts(value: object) -> "pd.Timestamp":
|
||||
"""`pd.Timestamp(value)` normalized to UTC whether the input parses tz-NAIVE (localize) or tz-AWARE
|
||||
(convert). The single tzinfo guard reused everywhere OPRA timestamps are compared — a bare
|
||||
`pd.Timestamp(x).tz_localize("UTC")` raises TypeError on an already-aware input (the C1b clamp gap)."""
|
||||
import pandas as pd
|
||||
ts = pd.Timestamp(value)
|
||||
return ts.tz_localize("UTC") if ts.tzinfo is None else ts.tz_convert("UTC")
|
||||
|
||||
|
||||
def _dataset_range_ends(rng: object) -> list[str]:
|
||||
"""Every available-END value carried by a databento `metadata.get_dataset_range` response, across ALL
|
||||
real shapes (the 0.79 return type is `dict[str, str | dict[str, str]]`):
|
||||
- a top-level dict `{"start","end"}` with ISO-string values → [rng["end"]]
|
||||
- a per-schema nested dict `{schema: {"start","end"}}` (values are dicts) → one end per schema
|
||||
- a `DatasetRange`-like object exposing a `.end` attribute → [obj.end]
|
||||
Returns [] ONLY for a genuinely empty/malformed response (no end anywhere) so the caller raises
|
||||
`DatabentoRangeUnavailable` on nothing-published, never on a shape that actually carries the range."""
|
||||
if not isinstance(rng, dict): # DatasetRange-like object (attribute access)
|
||||
end = getattr(rng, "end", None)
|
||||
return [str(end)] if end else []
|
||||
ends: list[str] = []
|
||||
top = rng.get("end")
|
||||
if isinstance(top, str) and top: # top-level {"start","end"}
|
||||
ends.append(top)
|
||||
elif isinstance(top, dict): # defensive: {"end": {schema: end_str}}
|
||||
ends.extend(str(v) for v in top.values() if v)
|
||||
for v in rng.values(): # per-schema nesting {schema: {"start","end"}}
|
||||
if isinstance(v, dict) and v.get("end"):
|
||||
ends.append(str(v["end"]))
|
||||
return ends
|
||||
|
||||
|
||||
def _df_to_series(market: Market, df: "pd.DataFrame") -> PriceSeries:
|
||||
"""Pure transform: an ohlcv-1d dataframe -> PriceSeries with NaN-guarded high/low."""
|
||||
dates = tuple(str(idx.date()) for idx in df.index)
|
||||
@@ -58,6 +90,37 @@ class DatabentoCostExceeded(RuntimeError):
|
||||
"""Raised when an estimated fetch cost exceeds the configured cap — the spend circuit-breaker."""
|
||||
|
||||
|
||||
class DatabentoRangeUnavailable(RuntimeError):
|
||||
"""Raised when a requested option window is not (yet) fully available on OPRA — EITHER it falls entirely
|
||||
PAST the dataset's available end (nothing published through `as_of`; a `data_end_after_available_end` /
|
||||
`start >= end` clamp), OR the session IS within the published range but its schema has not fully settled
|
||||
(`data_schema_not_fully_available`, a still-forming intraday session). ONE clear, CATCHABLE signal in
|
||||
place of the raw databento 422s so the nightly freeze / step-back can degrade instead of crash."""
|
||||
|
||||
|
||||
# A partial (still-settling) OPRA session is INSIDE the published range, so the C1a available-end clamp cannot
|
||||
# prevent it — databento answers with a `422 data_schema_not_fully_available` at query time instead. This is a
|
||||
# DIFFERENT 422 than the clamp's `data_end_after_available_end`. The freeze step-back needs ONE catchable type,
|
||||
# so we normalize this specific 422 into DatabentoRangeUnavailable (and ONLY this one — any other client error
|
||||
# must propagate raw, never be masked as a benign range-unavailable and silently degrade the freeze).
|
||||
_SESSION_INCOMPLETE_MARKER = "data_schema_not_fully_available"
|
||||
|
||||
|
||||
def _raise_if_session_unavailable(exc: Exception) -> None:
|
||||
"""If `exc` is the databento `422 data_schema_not_fully_available` (a still-settling session), raise
|
||||
DatabentoRangeUnavailable FROM it; otherwise return so the caller re-raises the original unchanged. The
|
||||
databento import is local because the package is an optional extra — with it absent there is no
|
||||
BentoClientError to normalize, so this is a no-op."""
|
||||
try:
|
||||
from databento.common.error import BentoClientError
|
||||
except Exception:
|
||||
return
|
||||
if (isinstance(exc, BentoClientError) and getattr(exc, "http_status", None) == 422
|
||||
and _SESSION_INCOMPLETE_MARKER in str(exc)):
|
||||
raise DatabentoRangeUnavailable(
|
||||
f"OPRA session not fully available yet ({_SESSION_INCOMPLETE_MARKER})") from exc
|
||||
|
||||
|
||||
class DatabentoDataProvider:
|
||||
name = "databento"
|
||||
|
||||
@@ -69,6 +132,52 @@ class DatabentoDataProvider:
|
||||
self._future_dataset = future_dataset
|
||||
self._option_dataset = option_dataset
|
||||
self._default_start = default_start
|
||||
# Cached OPRA range end: computed on first use and held for the PROVIDER INSTANCE's lifetime = one
|
||||
# nightly freeze run (fine for the one-shot nightly asset; a long-lived provider would go stale and
|
||||
# must recreate the provider to refresh — see the cache note on _option_available_end_ts).
|
||||
self._option_available_end: "pd.Timestamp | None" = None
|
||||
|
||||
def _option_available_end_ts(self, client) -> "pd.Timestamp": # type: ignore[no-untyped-def]
|
||||
"""The OPRA dataset's available END (exclusive), as a UTC Timestamp — fetched via the metadata range
|
||||
API ONCE per provider instance and cached (one metadata call per nightly run, not per contract). At
|
||||
the 23:30-UTC freeze OPRA only carries today's session to ~22:30 UTC, so this is BEFORE tomorrow —
|
||||
clamping queries to it avoids the 422 `data_end_after_available_end`.
|
||||
|
||||
Cache lifetime: `_option_available_end` is memoized for the provider INSTANCE's lifetime (= one
|
||||
nightly run). The nightly freeze creates a fresh provider each run, so it is always current; a
|
||||
long-lived/reused provider would serve a stale end and must be recreated to refresh."""
|
||||
if self._option_available_end is None:
|
||||
rng = client.metadata.get_dataset_range(dataset=self._option_dataset)
|
||||
ends = _dataset_range_ends(rng) # every real shape → the END value(s) it carries
|
||||
if not ends: # genuinely empty/malformed → catchable signal
|
||||
raise DatabentoRangeUnavailable(f"{self._option_dataset}: metadata returned no available end")
|
||||
self._option_available_end = max(_to_utc_ts(e) for e in ends) # latest published end, UTC
|
||||
return self._option_available_end
|
||||
|
||||
def last_available_option_date(self) -> "dt.date":
|
||||
"""OPRA's available-END as a calendar DATE (UTC) — the STARTING candidate for the freeze step-back
|
||||
(`_freeze_latest_session`). At the nightly freeze OPRA carries today's still-settling session, so this
|
||||
is today's date; the step-back then walks back to the last COMPLETE session. Creates its own client
|
||||
and reuses the per-instance `_option_available_end_ts` cache (one metadata round-trip per provider)."""
|
||||
import databento as db
|
||||
return self._option_available_end_ts(db.Historical()).date()
|
||||
|
||||
def _clamp_option_end(self, client, *, start: str, end_excl: str) -> str: # type: ignore[no-untyped-def]
|
||||
"""Clamp an EXCLUSIVE OPRA query `end` down to the dataset's available end so the request never asks
|
||||
for data past what OPRA has published. Returns the `end` string to pass to the client. Raises
|
||||
`DatabentoRangeUnavailable` if the (clamped) window is empty — `start >= end` — i.e. OPRA has nothing
|
||||
through `as_of` (a catchable signal, never a raw 422)."""
|
||||
avail_end = self._option_available_end_ts(client)
|
||||
end_ts = _to_utc_ts(end_excl) # tzinfo-guarded: tz-AWARE input no longer raises TypeError (C1b gap)
|
||||
start_ts = _to_utc_ts(start)
|
||||
if end_ts > avail_end:
|
||||
end_ts = avail_end
|
||||
end_excl = avail_end.isoformat()
|
||||
if start_ts >= end_ts:
|
||||
raise DatabentoRangeUnavailable(
|
||||
f"{self._option_dataset}: requested start {start} >= available end {end_excl} — "
|
||||
f"OPRA has no published data through {start}")
|
||||
return end_excl
|
||||
|
||||
def _resolve(self, market: Market) -> tuple[str, str, str]:
|
||||
ac = market.asset_class
|
||||
@@ -109,15 +218,20 @@ class DatabentoDataProvider:
|
||||
'XSP.OPT'). Cost-guarded."""
|
||||
import databento as db
|
||||
client = db.Historical()
|
||||
end = (dt.date.fromisoformat(as_of) + dt.timedelta(days=1)).isoformat()
|
||||
cost = client.metadata.get_cost(dataset=self._option_dataset, symbols=[underlying_parent],
|
||||
schema="definition", start=as_of, end=end, stype_in="parent")
|
||||
if cost > self._max_cost:
|
||||
raise DatabentoCostExceeded(
|
||||
f"estimated ${cost:.4f} > cap ${self._max_cost:.2f} for chain {underlying_parent}@{as_of}")
|
||||
data = client.timeseries.get_range(dataset=self._option_dataset, symbols=[underlying_parent],
|
||||
schema="definition", start=as_of, end=end, stype_in="parent")
|
||||
return _chain_from_definition_df(data.to_df())
|
||||
try:
|
||||
end = (dt.date.fromisoformat(as_of) + dt.timedelta(days=1)).isoformat()
|
||||
end = self._clamp_option_end(client, start=as_of, end_excl=end) # never query past OPRA's published end
|
||||
cost = client.metadata.get_cost(dataset=self._option_dataset, symbols=[underlying_parent],
|
||||
schema="definition", start=as_of, end=end, stype_in="parent")
|
||||
if cost > self._max_cost:
|
||||
raise DatabentoCostExceeded(
|
||||
f"estimated ${cost:.4f} > cap ${self._max_cost:.2f} for chain {underlying_parent}@{as_of}")
|
||||
data = client.timeseries.get_range(dataset=self._option_dataset, symbols=[underlying_parent],
|
||||
schema="definition", start=as_of, end=end, stype_in="parent")
|
||||
return _chain_from_definition_df(data.to_df())
|
||||
except Exception as e: # noqa: BLE001 — normalize the still-settling-session 422 to the catchable type
|
||||
_raise_if_session_unavailable(e) # raises DatabentoRangeUnavailable for that 422; else falls through
|
||||
raise
|
||||
|
||||
def fetch_option_chain_range(self, underlying_parent: str, start: str, end: str) -> list["OptionContract"]:
|
||||
"""ALL distinct contracts active in [start, end] (INCLUSIVE) via the OPRA `definition` schema — the
|
||||
@@ -126,6 +240,7 @@ class DatabentoDataProvider:
|
||||
import databento as db
|
||||
client = db.Historical()
|
||||
end_excl = (dt.date.fromisoformat(end) + dt.timedelta(days=1)).isoformat()
|
||||
end_excl = self._clamp_option_end(client, start=start, end_excl=end_excl) # never past OPRA's end
|
||||
cost = client.metadata.get_cost(dataset=self._option_dataset, symbols=[underlying_parent],
|
||||
schema="definition", start=start, end=end_excl, stype_in="parent")
|
||||
if cost > self._max_cost:
|
||||
@@ -150,18 +265,23 @@ class DatabentoDataProvider:
|
||||
return {}
|
||||
import databento as db
|
||||
client = db.Historical()
|
||||
end_excl = (dt.date.fromisoformat(end) + dt.timedelta(days=1)).isoformat()
|
||||
chunks = [osi_symbols[i:i + self._SYMBOL_LIMIT] for i in range(0, len(osi_symbols), self._SYMBOL_LIMIT)]
|
||||
total_cost = sum(
|
||||
client.metadata.get_cost(dataset=self._option_dataset, symbols=c, schema="ohlcv-1d",
|
||||
start=start, end=end_excl, stype_in="raw_symbol")
|
||||
for c in chunks)
|
||||
if total_cost > self._max_cost:
|
||||
raise DatabentoCostExceeded(
|
||||
f"estimated ${total_cost:.4f} > cap ${self._max_cost:.2f} for {len(osi_symbols)} option bars")
|
||||
out: dict[str, dict[str, float]] = {}
|
||||
for c in chunks:
|
||||
data = client.timeseries.get_range(dataset=self._option_dataset, symbols=c, schema="ohlcv-1d",
|
||||
start=start, end=end_excl, stype_in="raw_symbol")
|
||||
out.update(_bars_from_ohlcv_df(data.to_df()))
|
||||
return out
|
||||
try:
|
||||
end_excl = (dt.date.fromisoformat(end) + dt.timedelta(days=1)).isoformat()
|
||||
end_excl = self._clamp_option_end(client, start=start, end_excl=end_excl) # never past OPRA's end
|
||||
chunks = [osi_symbols[i:i + self._SYMBOL_LIMIT] for i in range(0, len(osi_symbols), self._SYMBOL_LIMIT)]
|
||||
total_cost = sum(
|
||||
client.metadata.get_cost(dataset=self._option_dataset, symbols=c, schema="ohlcv-1d",
|
||||
start=start, end=end_excl, stype_in="raw_symbol")
|
||||
for c in chunks)
|
||||
if total_cost > self._max_cost:
|
||||
raise DatabentoCostExceeded(
|
||||
f"estimated ${total_cost:.4f} > cap ${self._max_cost:.2f} for {len(osi_symbols)} option bars")
|
||||
out: dict[str, dict[str, float]] = {}
|
||||
for c in chunks:
|
||||
data = client.timeseries.get_range(dataset=self._option_dataset, symbols=c, schema="ohlcv-1d",
|
||||
start=start, end=end_excl, stype_in="raw_symbol")
|
||||
out.update(_bars_from_ohlcv_df(data.to_df()))
|
||||
return out
|
||||
except Exception as e: # noqa: BLE001 — normalize the still-settling-session 422 to the catchable type
|
||||
_raise_if_session_unavailable(e) # raises DatabentoRangeUnavailable for that 422; else falls through
|
||||
raise
|
||||
|
||||
@@ -1,134 +0,0 @@
|
||||
"""Deribit DVOL adapter — the DVOL annualised IMPLIED-vol index (the IV leg of the VRP edge).
|
||||
|
||||
Why this signal: Deribit's DVOL is a BTC/ETH 30-day implied-volatility index (annualised, in vol points, e.g.
|
||||
33–83 for BTC) computed off the live options order book — the market's forward-looking expected vol. Paired
|
||||
with the REALIZED vol of the underlying it is the volatility risk premium (VRP): IV systematically exceeds
|
||||
RV, so SELLING variance earns the spread, but with fat left tails when a vol spike makes RV >> IV. The
|
||||
backtest SIGNAL uses DVOL (IV) + close-to-close RV; live execution would be short Bybit option straddles
|
||||
(a documented cross-venue caveat — the DVOL index is Deribit's, the harvest is Bybit's options).
|
||||
|
||||
Endpoint (HTTP is INJECTED so tests run with no network):
|
||||
GET /api/v2/public/get_volatility_index_data?currency=BTC&start_timestamp=<ms>&end_timestamp=<ms>
|
||||
&resolution=86400
|
||||
-> result.data[] of rows [ts_ms, open, high, low, close], oldest-first. close (idx 4) = the DVOL value.
|
||||
resolution=86400 (seconds) = daily, so there is one row per UTC day. The API caps ~1000 rows/request, so
|
||||
we paginate BACKWARD via end_timestamp for the full ~5.3y of history (BTC/ETH DVOL begins 2021-03-24).
|
||||
|
||||
`dvol_history(currency, *, start_ms=None) -> {epoch_day: dvol_close}` paginates BACKWARD via end_timestamp
|
||||
(mirrors BybitAccountRatio.long_ratio_history's shape) down to a floor (default ≈ 2021-03, just below the
|
||||
DVOL inception 2021-03-24), stopping on the first empty page or once the oldest row falls below the floor.
|
||||
epoch_day =
|
||||
timestamp_ms // 86_400_000.
|
||||
|
||||
CLOCK SEAM (mirrors DeribitFunding): the wall clock is INJECTED (`clock: Callable[[], float] = time.time`).
|
||||
The first page anchors its end_timestamp at `now`; a bare `time.time()` in the loop would make the backward
|
||||
walk depend on the calendar date the test runs on — the determinism bug this seam exists to prevent. Tests
|
||||
pass a fixed clock so the walk is reproducible.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as _dt
|
||||
import json
|
||||
import time
|
||||
import urllib.request
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
_API = "https://www.deribit.com/api/v2/public/get_volatility_index_data"
|
||||
_PAGE_LIMIT = 1000 # Deribit caps ~1000 rows/request on the volatility-index endpoint
|
||||
_DAY_MS = 86_400_000
|
||||
_CLOSE_IX = 4 # row layout: [ts_ms, open, high, low, close]
|
||||
|
||||
# Backstop floor for the backward pagination: ≈ 2021-03-01 UTC in ms. The Deribit DVOL index for BTC and ETH
|
||||
# actually begins 2021-03-24 (verified against the live get_volatility_index_data endpoint); 2021-03-01 is the
|
||||
# natural lower bound just below inception at which the backward walk terminates. (The previous 2023-09 floor
|
||||
# silently discarded ~890 days — nearly half — of valid DVOL history, halving the VRP study's non-overlapping
|
||||
# hold count; this floor recovers the full ~5.3y back to inception.)
|
||||
DERIBIT_DVOL_FLOOR_MS = int(_dt.datetime(2021, 3, 1, tzinfo=_dt.UTC).timestamp() * 1000)
|
||||
|
||||
|
||||
def _http_get_json(url: str, tries: int = 4) -> Any:
|
||||
"""Default JSON fetcher (urllib, retry+backoff) — mirrors the other free-data adapters' `_get`."""
|
||||
for a in range(tries):
|
||||
try:
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
|
||||
with urllib.request.urlopen(req, timeout=40) as r:
|
||||
return json.loads(r.read())
|
||||
except Exception: # noqa: BLE001 — transient HTTP; retry then re-raise
|
||||
if a == tries - 1:
|
||||
raise
|
||||
time.sleep(2 * (a + 1))
|
||||
raise RuntimeError("unreachable")
|
||||
|
||||
|
||||
def _with_backoff(fetch: Callable[[str], Any], *, sleep: Callable[[float], None],
|
||||
base: float, tries: int) -> Callable[[str], Any]:
|
||||
"""Wrap `fetch(url)->dict` with exponential backoff on ANY transient exception (the Deribit endpoint has
|
||||
no Bybit-style retCode; a 429/5xx surfaces as a raised exception). Retries `base·2**attempt` up to
|
||||
`tries`; the final attempt re-raises. `sleep` is injectable (tests pass a no-op recorder)."""
|
||||
def wrapped(url: str) -> Any:
|
||||
for attempt in range(tries):
|
||||
try:
|
||||
return fetch(url)
|
||||
except Exception: # noqa: BLE001 — transient HTTP; backoff then re-raise on exhaustion
|
||||
if attempt == tries - 1:
|
||||
raise
|
||||
sleep(base * (2 ** attempt))
|
||||
raise RuntimeError("unreachable") # pragma: no cover — loop always returns or raises
|
||||
return wrapped
|
||||
|
||||
|
||||
class DeribitDVOL:
|
||||
def __init__(self, *, fetch: Callable[[str], Any] = _http_get_json,
|
||||
sleep: Callable[[float], None] = time.sleep,
|
||||
clock: Callable[[], float] = time.time,
|
||||
backoff_base: float = 0.5, backoff_tries: int = 5) -> None:
|
||||
# fetch(url) -> parsed JSON dict (injected for tests). Wrapped with exponential backoff so a transient
|
||||
# 429/5xx from Deribit is retried instead of failing — `sleep` is the (injectable) backoff sleeper
|
||||
# too, so tests passing a no-op sleep never actually wait.
|
||||
self._fetch = _with_backoff(fetch, sleep=sleep, base=backoff_base, tries=backoff_tries)
|
||||
self._sleep = sleep # throttle between pages in production; no-op-able in tests
|
||||
# `clock` is the wall-clock seam (epoch seconds; defaults to time.time). The first page anchors its
|
||||
# end_timestamp at `now`; injecting a FIXED clock makes the backward walk deterministic so tests do
|
||||
# not depend on the calendar date they run on (mirrors DeribitFunding — a bare time.time() in the loop
|
||||
# is the determinism bug this seam exists to prevent).
|
||||
self._clock = clock
|
||||
|
||||
def dvol_history(self, currency: str, *, start_ms: int | None = None) -> dict[int, float]:
|
||||
"""{epoch_day: dvol_close} for `currency` (BTC/ETH), where dvol_close = the daily DVOL index value
|
||||
(annualised implied vol, vol points). resolution=86400 → one row/day, no aggregation.
|
||||
|
||||
Paginates BACKWARD via end_timestamp: fetch the newest page, then set end_timestamp = oldest row's
|
||||
ts − 1 and loop, stopping on the first empty page or when the oldest row falls below `start_ms`
|
||||
(default DERIBIT_DVOL_FLOOR_MS ≈ 2021-03). Last-write-wins on a duplicate (currency, day) — there is
|
||||
at most one row per day, so this only guards page-boundary repeats. The infinite-loop guard breaks if
|
||||
the cursor stops advancing."""
|
||||
floor = DERIBIT_DVOL_FLOOR_MS if start_ms is None else int(start_ms)
|
||||
daily: dict[int, float] = {}
|
||||
seen: set[int] = set() # guard against duplicate ts across page boundaries
|
||||
end_ms: int | None = None # first page: no cursor (newest)
|
||||
while True:
|
||||
now_ms = int(self._clock() * 1000)
|
||||
url = (f"{_API}?currency={currency}&start_timestamp={floor}"
|
||||
f"&end_timestamp={end_ms if end_ms is not None else now_ms}&resolution=86400")
|
||||
data = self._fetch(url)
|
||||
rows = data.get("result", {}).get("data", [])
|
||||
if not rows:
|
||||
break
|
||||
oldest_ms = end_ms if end_ms is not None else None
|
||||
for row in rows:
|
||||
ts_ms = int(row[0])
|
||||
if oldest_ms is None or ts_ms < oldest_ms:
|
||||
oldest_ms = ts_ms
|
||||
if ts_ms < floor or ts_ms in seen:
|
||||
continue
|
||||
seen.add(ts_ms)
|
||||
daily[ts_ms // _DAY_MS] = float(row[_CLOSE_IX])
|
||||
if oldest_ms is None or oldest_ms <= floor:
|
||||
break # walked back to the floor → done
|
||||
next_end = oldest_ms - 1
|
||||
if end_ms is not None and next_end >= end_ms:
|
||||
break # cursor not advancing → avoid an infinite loop
|
||||
end_ms = next_end
|
||||
self._sleep(0.1) # gentle throttle; tests inject a no-op sleep
|
||||
return daily
|
||||
@@ -1,141 +0,0 @@
|
||||
"""Live data for the token-unlock dilution-short paper track.
|
||||
|
||||
Two pieces: (1) a CACHED unlock calendar (DefiLlama emissions + CoinGecko id->ticker), refreshed at most
|
||||
weekly — cliff dates are scheduled months ahead, so a 5-min 338-protocol pull does NOT belong on the
|
||||
nightly hot path; (2) live Binance perp prices + funding + qvol. `snapshot(prev_names, asof_day)` returns
|
||||
the names currently in their anticipatory short window plus current prices for any prior-book names (to
|
||||
book realized P&L). HTTP is injectable so tests run with NO live network.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import urllib.request
|
||||
from typing import Any, Callable
|
||||
|
||||
_FAPI = "https://fapi.binance.com"
|
||||
_DATASETS = "https://defillama-datasets.llama.fi"
|
||||
_CG = "https://api.coingecko.com/api/v3"
|
||||
_BEARISH = {"insiders", "privateSale"}
|
||||
|
||||
|
||||
def _http_get(url: str, tries: int = 4) -> Any:
|
||||
for a in range(tries):
|
||||
try:
|
||||
with urllib.request.urlopen(urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"}),
|
||||
timeout=40) as r:
|
||||
return json.loads(r.read())
|
||||
except Exception: # noqa: BLE001 — transient HTTP; retry then re-raise
|
||||
if a == tries - 1:
|
||||
raise
|
||||
time.sleep(2 * (a + 1))
|
||||
raise RuntimeError("unreachable")
|
||||
|
||||
|
||||
class UnlockCalendarLive:
|
||||
def __init__(self, cache_path: str, *, tradeable: set[str], refresh_days: int = 7,
|
||||
pre_window: int = 10, get: Callable[[str], Any] = _http_get,
|
||||
now_fn: Callable[[], float] = time.time, repo: Any | None = None) -> None:
|
||||
self._cache = cache_path
|
||||
self._tradeable = tradeable # our perp bases WITHOUT the USDT suffix (e.g. {"AAVE", ...})
|
||||
self._refresh_days = refresh_days
|
||||
self._pre = pre_window
|
||||
self._get = get
|
||||
self._now = now_fn
|
||||
# Optional operational-DB calendar repo. When supplied, a non-empty refresh ALSO upserts the cliff
|
||||
# events into `unlock_events` (networked, node-agnostic — no surfer-data PVC), while still writing
|
||||
# the json for back-compat. An empty refresh never touches the DB (transient-failure discipline).
|
||||
self._repo = repo
|
||||
|
||||
# ---- calendar (cached) ----
|
||||
def _stale(self) -> bool:
|
||||
if not os.path.exists(self._cache):
|
||||
return True
|
||||
return (self._now() - os.path.getmtime(self._cache)) > self._refresh_days * 86400
|
||||
|
||||
def _refresh_calendar(self) -> list[dict[str, Any]]:
|
||||
"""Pull DefiLlama emissions + CoinGecko map -> bearish cliff events for tradeable names. Heavy."""
|
||||
cg = {c["id"]: c["symbol"].upper() for c in self._get(f"{_CG}/coins/list")}
|
||||
events: list[dict[str, Any]] = []
|
||||
for slug in self._get(f"{_DATASETS}/emissionsProtocolsList"):
|
||||
try:
|
||||
d = self._get(f"{_DATASETS}/emissions/{slug}")
|
||||
except Exception: # noqa: BLE001 — skip a single bad protocol, keep going
|
||||
continue
|
||||
sym = cg.get(d.get("gecko_id") or "")
|
||||
if not sym or sym not in self._tradeable:
|
||||
continue
|
||||
cats = {k for k in (d.get("categories") or {}) if k in _BEARISH}
|
||||
for e in (d.get("metadata") or {}).get("events", []):
|
||||
if e.get("category") in cats and e.get("unlockType") == "cliff":
|
||||
try:
|
||||
ts = int(float(e.get("timestamp", 0)))
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
events.append({"sym": sym, "cliff_day": ts // 86400,
|
||||
"n": float(sum(e.get("noOfTokens") or [0])), "cat": e.get("category")})
|
||||
if not events:
|
||||
# An empty refresh is ~always a transient failure (DefiLlama/CoinGecko hiccup, or an
|
||||
# empty `tradeable` set) — there are always upcoming cliffs. NEVER cache []: keep the
|
||||
# last-known-good calendar if present, else return [] WITHOUT writing, so the cache stays
|
||||
# stale and the refresh retries next run instead of freezing for `refresh_days`.
|
||||
if os.path.exists(self._cache):
|
||||
with open(self._cache) as f:
|
||||
prev: list[dict[str, Any]] = json.load(f)
|
||||
if prev:
|
||||
return prev
|
||||
return []
|
||||
os.makedirs(os.path.dirname(self._cache) or ".", exist_ok=True)
|
||||
with open(self._cache, "w") as f:
|
||||
json.dump(events, f)
|
||||
if self._repo is not None:
|
||||
# Mirror the calendar into the operational DB so the unlock sleeve / evals / Jobs read it over
|
||||
# the network (no surfer-data PVC). write_events is an idempotent upsert; a no-op on []. A DB
|
||||
# failure must NOT lose the just-written json refresh.
|
||||
try:
|
||||
self._repo.write_events(events)
|
||||
except Exception: # noqa: BLE001 — json is already persisted; DB mirror is best-effort
|
||||
pass
|
||||
return events
|
||||
|
||||
def _calendar(self) -> list[dict[str, Any]]:
|
||||
if self._stale():
|
||||
return self._refresh_calendar()
|
||||
with open(self._cache) as f:
|
||||
cal: list[dict[str, Any]] = json.load(f)
|
||||
return cal
|
||||
|
||||
# ---- live prices ----
|
||||
def _perp(self) -> dict[str, float]:
|
||||
# Defensive: skip malformed entries (newly-listed/settling symbols can omit fields) — a single
|
||||
# bad row must NOT raise, or it cascades through cockpit_forward and freezes the whole gate.
|
||||
return {x["symbol"]: float(x.get("price") or 0.0) for x in self._get(f"{_FAPI}/fapi/v1/ticker/price") if "symbol" in x}
|
||||
|
||||
def _funding_qvol(self) -> tuple[dict[str, float], dict[str, float]]:
|
||||
fund: dict[str, float] = {}
|
||||
prem = self._get(f"{_FAPI}/fapi/v1/premiumIndex")
|
||||
for x in prem:
|
||||
fund[x["symbol"]] = float(x.get("lastFundingRate") or 0.0)
|
||||
qv = {x["symbol"]: float(x.get("quoteVolume") or 0.0) for x in self._get(f"{_FAPI}/fapi/v1/ticker/24hr") if "symbol" in x}
|
||||
return fund, qv
|
||||
|
||||
def snapshot(self, prev_names: set[str], asof_day: int) -> dict[str, Any]:
|
||||
"""Names currently in [cliff-pre, cliff) + current perp/funding for prior-book names + BTC.
|
||||
|
||||
Returns {"window": {sym: {cliff_day, n, perp, funding, qvol}}, "prices": {sym: (perp, funding)},
|
||||
"btc": price}. `window` is the new short book's candidate set (caller filters by shock/qvol);
|
||||
`prices` covers prev_names ∪ window so the caller can book realized P&L on names that exited."""
|
||||
cal = self._calendar()
|
||||
perp, (fund, qv) = self._perp(), self._funding_qvol()
|
||||
window: dict[str, dict[str, Any]] = {}
|
||||
for e in cal:
|
||||
if e["cliff_day"] - self._pre <= asof_day < e["cliff_day"]:
|
||||
sym = e["sym"] + "USDT"
|
||||
if sym in perp and perp[sym] > 0:
|
||||
window[e["sym"]] = {"cliff_day": e["cliff_day"], "n": e["n"], "perp": perp[sym],
|
||||
"funding": fund.get(sym, 0.0), "qvol": qv.get(sym, 0.0)}
|
||||
want = set(prev_names) | set(window)
|
||||
prices = {s: (perp[s + "USDT"], fund.get(s + "USDT", 0.0)) for s in want if s + "USDT" in perp}
|
||||
btc = perp.get("BTCUSDT", 0.0)
|
||||
return {"window": window, "prices": prices, "btc": btc}
|
||||
@@ -37,3 +37,13 @@ class YahooDailyClient:
|
||||
if c is not None:
|
||||
out[dt.datetime.fromtimestamp(t, dt.timezone.utc).strftime("%Y-%m-%d")] = float(c)
|
||||
return out
|
||||
|
||||
def volumes(self, symbol: str) -> dict[str, float]:
|
||||
res = self._get(_BASE.format(sym=symbol, rng=self._rng))["chart"]["result"][0]
|
||||
ts = res["timestamp"]
|
||||
vol = res["indicators"]["quote"][0].get("volume") or []
|
||||
out: dict[str, float] = {}
|
||||
for t, v in zip(ts, vol, strict=False):
|
||||
if v is not None:
|
||||
out[dt.datetime.fromtimestamp(t, dt.timezone.utc).strftime("%Y-%m-%d")] = float(v)
|
||||
return out
|
||||
|
||||
@@ -1,207 +0,0 @@
|
||||
"""Binance USDⓈ-M perp exchange adapter (ccxt binanceusdm). Owns ALL exchange I/O. Key production concerns
|
||||
baked in: a rate-limit governor (2400 weight/min per IP, 1200 orders/min per account; back off on 429, never
|
||||
hit the 418 ban), idempotent clientOrderId, exchange-filter parsing, and testnet via ccxt sandbox mode."""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from fxhnt.ports.crypto_exchange import (
|
||||
ChildOrder,
|
||||
CryptoAccount,
|
||||
Fill,
|
||||
OrderBook,
|
||||
PerpPosition,
|
||||
SymbolFilter,
|
||||
)
|
||||
|
||||
_log = logging.getLogger(__name__)
|
||||
_MAX_WEIGHT_PER_MIN = 2400
|
||||
_MAX_ORDERS_PER_MIN = 1200
|
||||
|
||||
|
||||
def parse_filter(symbol: str, market: dict[str, Any]) -> SymbolFilter:
|
||||
lim = market.get("limits", {})
|
||||
prec = market.get("precision", {})
|
||||
return SymbolFilter(
|
||||
symbol=symbol,
|
||||
step_size=float(prec.get("amount") or lim.get("amount", {}).get("min") or 0.0),
|
||||
tick_size=float(prec.get("price") or 0.0),
|
||||
min_notional=float((lim.get("cost", {}) or {}).get("min") or 5.0),
|
||||
)
|
||||
|
||||
|
||||
def client_id(rebalance_id: str, symbol: str, leg: str) -> str:
|
||||
"""Deterministic, <=36 chars (Binance clientOrderId limit). Retry-safe: same inputs -> same id."""
|
||||
h = hashlib.sha1(f"{rebalance_id}:{symbol}:{leg}".encode()).hexdigest()[:24]
|
||||
return f"fxhnt{h}"
|
||||
|
||||
|
||||
class RateBudget:
|
||||
"""Conservative per-minute governor. Orders are counted locally; weight is updated from the
|
||||
X-MBX-USED-WEIGHT response header the adapter reads after each call.
|
||||
The constructor accepts the effective limit directly (caller applies any safety margin).
|
||||
A 60-second rolling window resets the order count so long-running processes never
|
||||
permanently exhaust the budget."""
|
||||
_WINDOW_S = 60.0
|
||||
|
||||
def __init__(self, max_orders: int = _MAX_ORDERS_PER_MIN,
|
||||
max_weight: int = _MAX_WEIGHT_PER_MIN) -> None:
|
||||
self._max_orders = max_orders
|
||||
self._max_weight = max_weight
|
||||
self._orders = 0
|
||||
self._used_weight = 0
|
||||
self._window_start: float | None = None # lazily initialised on first try_order call
|
||||
|
||||
def try_order(self) -> bool:
|
||||
now = time.time()
|
||||
if self._window_start is None:
|
||||
self._window_start = now
|
||||
elif now - self._window_start >= self._WINDOW_S:
|
||||
# New minute window — reset the per-minute order counter
|
||||
self._orders = 0
|
||||
self._window_start = now
|
||||
if self._orders >= self._max_orders:
|
||||
return False
|
||||
self._orders += 1
|
||||
return True
|
||||
|
||||
def note_used_weight(self, used: int) -> None:
|
||||
self._used_weight = used
|
||||
|
||||
def weight_ok(self) -> bool:
|
||||
return self._used_weight < self._max_weight
|
||||
|
||||
|
||||
class BinanceUsdmExchange:
|
||||
"""Implements the CryptoExchange Protocol over ccxt binanceusdm."""
|
||||
name = "binanceusdm"
|
||||
|
||||
def __init__(self, api_key: str, api_secret: str, *, testnet: bool = True) -> None:
|
||||
import ccxt
|
||||
self._x = ccxt.binanceusdm({
|
||||
"apiKey": api_key, "secret": api_secret, "enableRateLimit": True,
|
||||
"options": {"defaultType": "future"},
|
||||
})
|
||||
if testnet:
|
||||
self._x.set_sandbox_mode(True)
|
||||
self._budget = RateBudget(max_orders=int(_MAX_ORDERS_PER_MIN * 0.9),
|
||||
max_weight=int(_MAX_WEIGHT_PER_MIN * 0.9))
|
||||
|
||||
def _backoff(self, fn: Callable[..., Any], *a: Any, **kw: Any) -> Any:
|
||||
for attempt in range(5):
|
||||
try:
|
||||
out = fn(*a, **kw)
|
||||
used = (self._x.last_response_headers or {}).get("X-MBX-USED-WEIGHT-1M")
|
||||
if used:
|
||||
self._budget.note_used_weight(int(used))
|
||||
return out
|
||||
except Exception as e: # noqa: BLE001 — ccxt raises many types; classify by msg
|
||||
msg = str(e).lower()
|
||||
if "429" in msg or "rate limit" in msg or "-1003" in msg:
|
||||
retry_after = None
|
||||
hdrs = getattr(self._x, "last_response_headers", None) or {}
|
||||
for k, v in hdrs.items():
|
||||
if k.lower() == "retry-after":
|
||||
try:
|
||||
retry_after = float(v)
|
||||
except (TypeError, ValueError):
|
||||
retry_after = None
|
||||
break
|
||||
wait = min(retry_after if retry_after is not None else 2 ** attempt, 60.0)
|
||||
_log.warning("rate limited; backoff %ss (attempt %d)", wait, attempt + 1)
|
||||
time.sleep(wait)
|
||||
continue
|
||||
raise
|
||||
raise RuntimeError("rate-limit backoff exhausted")
|
||||
|
||||
def exchange_info(self) -> dict[str, SymbolFilter]:
|
||||
markets = self._backoff(self._x.load_markets)
|
||||
out: dict[str, SymbolFilter] = {}
|
||||
for m in markets.values():
|
||||
if m.get("swap") and m.get("quote") == "USDT" and m.get("active"):
|
||||
out[m["id"]] = parse_filter(m["id"], m)
|
||||
return out
|
||||
|
||||
def order_book(self, symbol: str) -> OrderBook:
|
||||
ob = self._backoff(self._x.fetch_order_book, symbol, 5)
|
||||
bid, bq = (ob["bids"][0] if ob["bids"] else (0.0, 0.0))
|
||||
ask, aq = (ob["asks"][0] if ob["asks"] else (0.0, 0.0))
|
||||
return OrderBook(symbol=symbol, bid=float(bid), ask=float(ask), bid_qty=float(bq), ask_qty=float(aq))
|
||||
|
||||
def positions(self) -> list[PerpPosition]:
|
||||
ps = self._backoff(self._x.fetch_positions)
|
||||
out = []
|
||||
for p in ps:
|
||||
contracts = float(p.get("contracts") or 0.0)
|
||||
side = p.get("side")
|
||||
if contracts == 0.0 or side not in ("long", "short"):
|
||||
continue
|
||||
amt = contracts if side == "long" else -contracts
|
||||
out.append(PerpPosition(symbol=p["info"].get("symbol", p["symbol"]), qty=amt,
|
||||
entry_price=float(p.get("entryPrice") or 0.0),
|
||||
leverage=int(float(p.get("leverage") or 1))))
|
||||
return out
|
||||
|
||||
def account(self) -> CryptoAccount:
|
||||
bal = self._backoff(self._x.fetch_balance)
|
||||
nlv = float(bal["info"].get("totalMarginBalance") or bal.get("USDT", {}).get("total") or 0.0)
|
||||
avail = float(bal["info"].get("availableBalance") or bal.get("USDT", {}).get("free") or 0.0)
|
||||
pos = {p.symbol: p.qty for p in self.positions()}
|
||||
return CryptoAccount(nlv=nlv, available_balance=avail, positions=pos)
|
||||
|
||||
def set_leverage(self, symbol: str, leverage: int) -> None:
|
||||
try:
|
||||
self._backoff(self._x.set_leverage, leverage, symbol)
|
||||
except Exception as e: # noqa: BLE001 — idempotent; already-set is fine
|
||||
_log.debug("set_leverage %s=%d: %s", symbol, leverage, e)
|
||||
|
||||
def ensure_one_way_mode(self) -> None:
|
||||
try:
|
||||
self._backoff(self._x.set_position_mode, False) # hedged=False -> one-way
|
||||
except Exception as e: # noqa: BLE001 — already one-way raises; ignore
|
||||
_log.debug("set_position_mode one-way: %s", e)
|
||||
|
||||
def place(self, order: ChildOrder) -> Fill:
|
||||
if not self._budget.try_order() or not self._budget.weight_ok():
|
||||
_log.warning("rate budget exhausted; skipping %s", order.client_id)
|
||||
return Fill(order.client_id, order.symbol, order.side, 0.0, 0.0, 0.0, "REJECTED")
|
||||
params = {"newClientOrderId": order.client_id, "reduceOnly": order.reduce_only}
|
||||
if order.order_type == "LIMIT":
|
||||
params["timeInForce"] = order.time_in_force # "GTX" post-only or "IOC"
|
||||
try:
|
||||
r = self._backoff(self._x.create_order, order.symbol, order.order_type.lower(), order.side.lower(),
|
||||
order.qty, order.price, params)
|
||||
except Exception as e: # noqa: BLE001 — log + report REJECTED, never crash batch
|
||||
_log.warning("place %s rejected: %s", order.client_id, e)
|
||||
return Fill(order.client_id, order.symbol, order.side, 0.0, 0.0, 0.0, "REJECTED")
|
||||
filled = float(r.get("filled") or 0.0)
|
||||
status = "FILLED" if filled >= order.qty - 1e-12 else ("PARTIAL" if filled > 0 else
|
||||
(r.get("status") or "CANCELED").upper())
|
||||
return Fill(order.client_id, order.symbol, order.side, filled, float(r.get("average") or 0.0),
|
||||
float((r.get("fee") or {}).get("cost") or 0.0), status)
|
||||
|
||||
def cancel(self, symbol: str, client_id: str) -> None:
|
||||
try:
|
||||
self._backoff(self._x.cancel_order, client_id, symbol, {"origClientOrderId": client_id})
|
||||
except Exception as e: # noqa: BLE001 — already gone is fine
|
||||
_log.debug("cancel %s: %s", client_id, e)
|
||||
|
||||
def order_status(self, symbol: str, client_id: str) -> Fill:
|
||||
"""Fetch the authoritative post-cancel fill state by clientOrderId.
|
||||
After a cancel completes the exchange reports the FINAL filled quantity — use this
|
||||
(not the stale place-response) to size the IOC remainder and avoid over-filling."""
|
||||
try:
|
||||
r = self._backoff(self._x.fetch_order, None, symbol, {"origClientOrderId": client_id})
|
||||
except Exception as e: # noqa: BLE001 — unknown/already-gone -> treat as nothing filled (safe)
|
||||
_log.debug("order_status %s: %s", client_id, e)
|
||||
return Fill(client_id, symbol, "", 0.0, 0.0, 0.0, "CANCELED")
|
||||
filled = float(r.get("filled") or 0.0)
|
||||
side = (r.get("side") or "").upper()
|
||||
status = "FILLED" if filled > 0 and (r.get("status") or "").lower() == "closed" else \
|
||||
("PARTIAL" if filled > 0 else (r.get("status") or "CANCELED").upper())
|
||||
return Fill(client_id, symbol, side, filled, float(r.get("average") or 0.0),
|
||||
float((r.get("fee") or {}).get("cost") or 0.0), status)
|
||||
@@ -1,45 +0,0 @@
|
||||
"""Binance USDT-perp funding-rate history (public REST, paginated). Per-call never-raise: returns whatever
|
||||
it has collected (or []) on any error, so one bad symbol never aborts a batch ingest. No auth needed."""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
_BASE = "https://fapi.binance.com"
|
||||
_PATH = "/fapi/v1/fundingRate"
|
||||
_LIMIT = 1000
|
||||
_MAX_PAGES = 200 # runaway guard (~200k stamps / symbol)
|
||||
|
||||
|
||||
class BinanceFundingHistory:
|
||||
def __init__(self, client: Any | None = None, base: str = _BASE) -> None:
|
||||
if client is None:
|
||||
import httpx
|
||||
client = httpx.Client(timeout=20.0)
|
||||
self._c = client
|
||||
self._base = base
|
||||
|
||||
def fetch(self, symbol: str, start_ms: int, end_ms: int) -> list[tuple[int, float]]:
|
||||
"""Return [(fundingTime_ms, rate_float)] ascending for `symbol` in [start_ms, end_ms]. Paginates by
|
||||
advancing startTime past the last stamp; stops on a short/empty page, past end_ms, or the page cap.
|
||||
Never raises — returns what was collected so far on any error."""
|
||||
out: list[tuple[int, float]] = []
|
||||
cur = start_ms
|
||||
try:
|
||||
for _ in range(_MAX_PAGES):
|
||||
resp = self._c.get(self._base + _PATH,
|
||||
params={"symbol": symbol, "startTime": cur, "endTime": end_ms, "limit": _LIMIT},
|
||||
timeout=20.0)
|
||||
resp.raise_for_status()
|
||||
rows = resp.json()
|
||||
if not rows:
|
||||
break
|
||||
for r in rows:
|
||||
out.append((int(r["fundingTime"]), float(r["fundingRate"])))
|
||||
if len(rows) < _LIMIT:
|
||||
break
|
||||
cur = int(rows[-1]["fundingTime"]) + 1
|
||||
if cur > end_ms:
|
||||
break
|
||||
except Exception: # noqa: BLE001 — never abort the batch; keep what we have
|
||||
return out
|
||||
return out
|
||||
@@ -1,51 +0,0 @@
|
||||
"""Live-price adapter over ccxt binance spot (`fetch_tickers`) for the paper cockpit's mark-to-market.
|
||||
|
||||
Resilient by contract: a failed fetch NEVER raises to the caller — it returns the last-good cached prices so
|
||||
the web layer can mark positions and never 500 on a transient price hiccup. Sleeve symbols (e.g. 'USDTUSD')
|
||||
are mapped to binance pairs ('USDT/USD') where a mapping is known; unknowns are looked up verbatim and simply
|
||||
omitted if the exchange has no ticker for them (the view marks those at entry → 0 PnL)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from fxhnt.ports.live_price import LivePrice
|
||||
|
||||
_log = logging.getLogger(__name__)
|
||||
|
||||
# Sleeve symbol -> binance ccxt market symbol. Extend as sleeves add pairs.
|
||||
_SYMBOL_MAP: dict[str, str] = {
|
||||
"BTCUSDT": "BTC/USDT",
|
||||
"ETHUSDT": "ETH/USDT",
|
||||
"USDTUSD": "USDT/USD",
|
||||
"USDCUSD": "USDC/USD",
|
||||
}
|
||||
|
||||
|
||||
class CcxtLivePrice(LivePrice):
|
||||
def __init__(self, exchange: object | None = None) -> None:
|
||||
if exchange is None:
|
||||
import ccxt
|
||||
exchange = ccxt.binance({"enableRateLimit": True})
|
||||
self._x = exchange
|
||||
self._last_good: dict[str, float] = {}
|
||||
|
||||
def _market(self, symbol: str) -> str:
|
||||
return _SYMBOL_MAP.get(symbol, symbol)
|
||||
|
||||
def get_prices(self, symbols: list[str]) -> dict[str, float]:
|
||||
markets = {s: self._market(s) for s in symbols}
|
||||
try:
|
||||
tickers = self._x.fetch_tickers(list(set(markets.values())))
|
||||
except Exception as e: # noqa: BLE001 — any ccxt/network error: degrade to last-good, never raise
|
||||
_log.warning("live price fetch failed; serving last-good: %s", e)
|
||||
return {s: self._last_good[s] for s in symbols if s in self._last_good}
|
||||
out: dict[str, float] = {}
|
||||
for sym, mkt in markets.items():
|
||||
t = tickers.get(mkt)
|
||||
px = None if t is None else (t.get("last") or t.get("close"))
|
||||
if px:
|
||||
out[sym] = float(px)
|
||||
self._last_good[sym] = float(px)
|
||||
elif sym in self._last_good:
|
||||
out[sym] = self._last_good[sym]
|
||||
return out
|
||||
@@ -1,70 +1,12 @@
|
||||
"""Live-price adapter over Binance's public REST ticker via httpx (which IS in the cockpit image's `web`
|
||||
extra — unlike ccxt, the `crypto` extra). For the paper cockpit's mark-to-market.
|
||||
"""NullLivePrice — the startup-safe no-op LivePrice fallback for the paper cockpit's mark-to-market.
|
||||
|
||||
Resilient by contract: a failed fetch NEVER raises — it returns the last-good cached prices so the web
|
||||
layer marks positions and never 500s on a transient hiccup. Sleeve symbols that aren't Binance spot pairs
|
||||
(e.g. the stablecoin USD pegs 'USDTUSD') are simply omitted (the view marks those at entry → ~0 PnL, which
|
||||
is correct for near-pegged names). Crypto pairs ('BTCUSDT', 'ETHUSDT') pass through verbatim.
|
||||
|
||||
NullLivePrice is the startup-safe fallback: if httpx itself is somehow unavailable, the cockpit still boots
|
||||
and /paper degrades to entry-price marks."""
|
||||
(Phase 0b venue consolidation: the legacy-venue REST ticker adapter that used to live here was removed —
|
||||
the cockpit's only paper-book venue is Bybit now, marked via `WarehouseLivePrice` over `bybit_features`,
|
||||
never a live legacy-venue fetch.)"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
|
||||
from fxhnt.ports.live_price import LivePrice
|
||||
|
||||
_log = logging.getLogger(__name__)
|
||||
|
||||
_BINANCE_TICKER = "https://api.binance.com/api/v3/ticker/price"
|
||||
|
||||
# Sleeve symbol -> Binance REST symbol (no slash). Stablecoin USD pegs have no Binance USD pair → omitted.
|
||||
_SYMBOL_MAP: dict[str, str] = {
|
||||
"BTCUSDT": "BTCUSDT",
|
||||
"ETHUSDT": "ETHUSDT",
|
||||
}
|
||||
|
||||
|
||||
class HttpxBinanceLivePrice(LivePrice):
|
||||
def __init__(self, client: object | None = None, *, timeout: float = 6.0) -> None:
|
||||
self._client = client # injectable for tests; else httpx is imported lazily in get_prices
|
||||
self._timeout = timeout
|
||||
self._last_good: dict[str, float] = {}
|
||||
|
||||
def _market(self, symbol: str) -> str | None:
|
||||
return _SYMBOL_MAP.get(symbol) # None → not a Binance pair → skip (mark at entry)
|
||||
|
||||
def _fetch(self, binance_symbols: list[str]) -> list[dict]:
|
||||
params = {"symbols": json.dumps(binance_symbols)}
|
||||
client = self._client
|
||||
if client is None:
|
||||
import httpx
|
||||
with httpx.Client(timeout=self._timeout) as c:
|
||||
return c.get(_BINANCE_TICKER, params=params).json()
|
||||
return client.get(_BINANCE_TICKER, params=params).json()
|
||||
|
||||
def get_prices(self, symbols: list[str]) -> dict[str, float]:
|
||||
markets = {s: self._market(s) for s in symbols}
|
||||
wanted = sorted({m for m in markets.values() if m})
|
||||
if not wanted:
|
||||
return {s: self._last_good[s] for s in symbols if s in self._last_good}
|
||||
try:
|
||||
rows = self._fetch(wanted)
|
||||
by_mkt = {r["symbol"]: float(r["price"]) for r in rows if r.get("price")}
|
||||
except Exception as e: # noqa: BLE001 — any httpx/network/parse error: degrade to last-good, never raise
|
||||
_log.warning("live price fetch failed; serving last-good: %s", e)
|
||||
return {s: self._last_good[s] for s in symbols if s in self._last_good}
|
||||
out: dict[str, float] = {}
|
||||
for sym, mkt in markets.items():
|
||||
px = by_mkt.get(mkt) if mkt else None
|
||||
if px:
|
||||
out[sym] = px
|
||||
self._last_good[sym] = px
|
||||
elif sym in self._last_good:
|
||||
out[sym] = self._last_good[sym]
|
||||
return out
|
||||
|
||||
|
||||
class NullLivePrice(LivePrice):
|
||||
"""Startup-safe no-op: returns nothing → /paper marks every position at its entry price."""
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
"""WarehouseLivePrice — a LivePrice backed by the LATEST warehouse close, for the Bybit live paper book's
|
||||
mark-to-market. The Binance live view uses a public-REST price source (BTC/ETH only); the Bybit book holds
|
||||
many alt symbols whose live mark is the latest Bybit-perp close already in `bybit_features`. This adapter
|
||||
mark-to-market. The legacy-venue live view used a public-REST price source (BTC/ETH only); the Bybit book
|
||||
holds many alt symbols whose live mark is the latest Bybit-perp close already in `bybit_features`. This adapter
|
||||
serves `{symbol: latest_close}` for ONLY the requested symbols (a `WHERE symbol IN (...)` read — bounded by
|
||||
the open book, not the whole table). READ-ONLY; never raises on a read hiccup (degrades to {} → the view
|
||||
marks at entry), so it never 500s the cockpit."""
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,38 +1,43 @@
|
||||
"""Dagster Definitions for the B0 lifecycle graph — the 4 combined-book assets + a daily schedule that
|
||||
"""Dagster Definitions for the B0 lifecycle graph — the combined-book assets + a daily schedule that
|
||||
materializes the track (replaces the fxhnt-forward cron, same 23:30 UTC slot).
|
||||
|
||||
(Phase 0b venue consolidation: the legacy-venue crypto_bars/crypto_funding/crypto_spot ingest assets were
|
||||
retired here — crypto is 100% Bybit now (`bybit_warehouse_refresh` + the Bybit tracks below).
|
||||
`paper_book_snapshot` — the dead legacy-venue-era combined-crypto `/paper` book (redundant with the
|
||||
`bybit_4edge` deploy book below) — was RETIRED (Phase 0b Task 7d); sixtyforty_nav/multistrat_nav
|
||||
are their OWN independent live assets below, unaffected by that retirement.)
|
||||
|
||||
(VRP removed entirely 2026-07-20 — see project_fxhnt_diversifier_hunt_2026_07_15 for the
|
||||
falsification that led to the removal.)
|
||||
|
||||
NOTE: no `from __future__ import annotations` — Dagster 1.12 runtime type-hint inspection."""
|
||||
from dagster import DefaultScheduleStatus, Definitions, ScheduleDefinition, define_asset_job
|
||||
|
||||
from fxhnt.adapters.orchestration.assets import (
|
||||
bybit_4edge_levered_nav,
|
||||
bybit_4edge_nav,
|
||||
bybit_book_precompute,
|
||||
bybit_paper_book,
|
||||
bybit_warehouse_refresh,
|
||||
cockpit_forward,
|
||||
crypto_bars,
|
||||
crypto_funding,
|
||||
crypto_spot,
|
||||
crypto_tstrend_nav,
|
||||
deribit_funding_bars,
|
||||
futures_bars,
|
||||
multistrat_levered_nav,
|
||||
multistrat_nav,
|
||||
paper_book_snapshot,
|
||||
positioning_nav,
|
||||
sixtyforty_nav,
|
||||
stablecoin_rotation_nav,
|
||||
unlock_nav,
|
||||
vrp_nav,
|
||||
xsfunding_nav,
|
||||
)
|
||||
|
||||
combined_book_job = define_asset_job(
|
||||
name="combined_book_forward_job",
|
||||
selection=[
|
||||
crypto_bars, crypto_funding, crypto_spot, futures_bars, sixtyforty_nav,
|
||||
xsfunding_nav, unlock_nav, crypto_tstrend_nav, stablecoin_rotation_nav, multistrat_nav, vrp_nav,
|
||||
deribit_funding_bars, bybit_warehouse_refresh, bybit_4edge_nav, bybit_4edge_levered_nav,
|
||||
positioning_nav, bybit_paper_book, paper_book_snapshot, cockpit_forward,
|
||||
futures_bars, sixtyforty_nav,
|
||||
xsfunding_nav, unlock_nav, multistrat_nav, multistrat_levered_nav,
|
||||
deribit_funding_bars, bybit_warehouse_refresh, bybit_book_precompute,
|
||||
bybit_4edge_nav, bybit_4edge_levered_nav,
|
||||
positioning_nav, bybit_paper_book, cockpit_forward,
|
||||
],
|
||||
)
|
||||
|
||||
@@ -47,10 +52,11 @@ daily_combined_book_schedule = ScheduleDefinition(
|
||||
|
||||
defs = Definitions(
|
||||
assets=[
|
||||
crypto_bars, crypto_funding, crypto_spot, futures_bars, sixtyforty_nav,
|
||||
xsfunding_nav, unlock_nav, crypto_tstrend_nav, stablecoin_rotation_nav, multistrat_nav, vrp_nav,
|
||||
deribit_funding_bars, bybit_warehouse_refresh, bybit_4edge_nav, bybit_4edge_levered_nav,
|
||||
positioning_nav, bybit_paper_book, paper_book_snapshot, cockpit_forward,
|
||||
futures_bars, sixtyforty_nav,
|
||||
xsfunding_nav, unlock_nav, multistrat_nav, multistrat_levered_nav,
|
||||
deribit_funding_bars, bybit_warehouse_refresh, bybit_book_precompute,
|
||||
bybit_4edge_nav, bybit_4edge_levered_nav,
|
||||
positioning_nav, bybit_paper_book, cockpit_forward,
|
||||
],
|
||||
jobs=[combined_book_job],
|
||||
schedules=[daily_combined_book_schedule],
|
||||
|
||||
@@ -2,16 +2,28 @@
|
||||
|
||||
DRY against `assets.py`: each entry copies the EXACT `build_strategy` construction (same adapters, same
|
||||
dependency wiring) and the exact `state_file` basename its `*_nav` asset uses, so `migrate_track`'s recompute
|
||||
reconcile runs the strategy at the SAME basis the live forward track books. The observed track (`xsfunding`)
|
||||
is never recomputed by `migrate_track` — it maps to a harmless `lambda: None` since building its real
|
||||
strategy needs a live network adapter (`BinanceXsFundingLive`) that adds no value for an import that is never
|
||||
invoked. `combined` DOES need an entry despite being `record_mode="recompute"`: production has a pre-branch
|
||||
`forward_summary["combined"]` row (the old ingest wrote all tracks), so without a seeded anchor the first
|
||||
`forward_engine.run_track` sees `active_anchor("combined") is None` but `has_forward_history("combined") is
|
||||
True` and the lost-anchor guard raises. `migrate_track` will report `diverged` for it (the old
|
||||
ForwardTracker-level killswitch was dropped in Task 8c, so the recompute no longer matches the accumulated
|
||||
JSON nav) — that is expected/benign; the anchor still gets seeded from the JSON T0, and the engine then
|
||||
replaces the record on its next run."""
|
||||
reconcile runs the strategy at the SAME basis the live forward track books. `combined` DOES need an entry
|
||||
despite being `record_mode="recompute"`: production has a pre-branch `forward_summary["combined"]` row (the
|
||||
old ingest wrote all tracks), so without a seeded anchor the first `forward_engine.run_track` sees
|
||||
`active_anchor("combined") is None` but `has_forward_history("combined") is True` and the lost-anchor guard
|
||||
raises. `migrate_track` will report `diverged` for it (the old ForwardTracker-level killswitch was dropped in
|
||||
Task 8c, so the recompute no longer matches the accumulated JSON nav) — that is expected/benign; the anchor
|
||||
still gets seeded from the JSON T0, and the engine then replaces the record on its next run.
|
||||
|
||||
NOTE (Phase 0b): `xsfunding` was `record_mode="observed"` (legacy-venue-scoped `XsFundingForward`) when this
|
||||
module was written, so it mapped to a harmless `lambda: None` (an observed strategy is never recomputed by
|
||||
`migrate_track`). It is now `record_mode="recompute"` (its own Bybit single-sleeve track, mirroring
|
||||
`positioning` — see `build_bybit_sleeve_forward_nav`), so it gets a real builder below like every other
|
||||
recompute track; a stale `lambda: None` would crash `recompute_series` (`None.advance`) if this one-time
|
||||
migration is ever re-run against a leftover pre-migration `xsfunding_state.json`.
|
||||
|
||||
NOTE (Phase 0b, Task 2): `unlock` was ALREADY `record_mode="recompute"` (unlike xsfunding's pre-migration
|
||||
"observed"), so its builder here was never a `lambda: None` stub — but it built the OLD standalone legacy-venue
|
||||
`UnlockShortForward` construction. That construction is retired (its `unlock_nav`
|
||||
asset now uses the SAME Bybit single-sleeve `BybitFourEdgeStrategy(sleeves=["unlock"], unlock_events=...)`
|
||||
xsfunding/positioning use), so the builder below was updated to match — otherwise this one-time migration's
|
||||
recompute reconcile would run a DIFFERENT strategy than the live track books, producing a false `diverged`
|
||||
verdict (or worse, silently reconciling against the wrong basis)."""
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
@@ -27,9 +39,6 @@ def track_builders(settings: Settings, data_dir: str) -> TrackBuilders:
|
||||
"""strategy_id -> (state_file basename, build_strategy), one entry per `STRATEGY_REGISTRY` id."""
|
||||
builders: TrackBuilders = {}
|
||||
|
||||
# ---- observed tracks: migrate_track never calls build_strategy in observed mode -------------------
|
||||
builders["xsfunding"] = ("xsfunding_state", lambda: None)
|
||||
|
||||
# (the "combined" crypto-momentum book was RETIRED 2026-07-11 — removed from the registry, so
|
||||
# migrate_forward_anchors no longer iterates to it; its builder is gone.)
|
||||
|
||||
@@ -54,59 +63,27 @@ def track_builders(settings: Settings, data_dir: str) -> TrackBuilders:
|
||||
|
||||
builders["multistrat"] = ("multistrat_state", _build_multistrat)
|
||||
|
||||
# ---- crypto_tstrend_nav: crypto TS-momentum over the warehouse close panel -------------------------
|
||||
def _build_crypto_tstrend() -> Any:
|
||||
from fxhnt.adapters.warehouse import get_feature_store
|
||||
from fxhnt.application.ts_trend_strategy import TsTrendForward
|
||||
# (crypto_tstrend / stablecoin_rotation builders RETIRED 2026-07-13, Phase 0b Task 4 — both standalone
|
||||
# legacy-venue registry entries are gone, so `migrate_forward_anchors` no longer iterates to either sid; the
|
||||
# builders here are gone with them. crypto_tstrend the BYBIT BOOK SLEEVE is unaffected — that lives
|
||||
# under `bybit_4edge`'s own builder below, not here.)
|
||||
|
||||
def load_panel() -> dict: # type: ignore[type-arg]
|
||||
return get_feature_store(settings).crypto_close_panel()
|
||||
|
||||
return TsTrendForward(load_panel, book_target_vol=0.15)
|
||||
|
||||
builders["crypto_tstrend"] = ("crypto_tstrend_state", _build_crypto_tstrend)
|
||||
|
||||
# ---- stablecoin_rotation_nav: peg-reversion over the Binance-spot daily-close panel ------------------
|
||||
def _build_stablecoin_rotation() -> Any:
|
||||
from fxhnt.adapters.data.binance_spot_history import BinanceSpotHistory
|
||||
from fxhnt.application.stablecoin_strategy import StableRotationForward
|
||||
|
||||
def load_panel() -> dict: # type: ignore[type-arg]
|
||||
h = BinanceSpotHistory()
|
||||
panel: dict = {} # type: ignore[type-arg]
|
||||
for sym in ("FDUSDUSDT", "TUSDUSDT", "USDPUSDT"):
|
||||
try:
|
||||
series = h.daily_close(sym)
|
||||
except Exception: # noqa: BLE001 — one delisted/errored symbol must not kill the sleeve
|
||||
continue
|
||||
if series:
|
||||
panel[sym] = series
|
||||
return panel
|
||||
|
||||
return StableRotationForward(load_panel)
|
||||
|
||||
builders["stablecoin_rotation"] = ("stablecoin_rotation_state", _build_stablecoin_rotation)
|
||||
|
||||
# ---- vrp_nav: defined-risk XSP put-spread VRP on the frozen OPRA PIT table (recompute) ----------------
|
||||
def _build_vrp() -> Any:
|
||||
from fxhnt.adapters.persistence.xsp_option_bars import XspOptionBarsRepo
|
||||
from fxhnt.application.vrp_book import VrpStrategy
|
||||
|
||||
return VrpStrategy(XspOptionBarsRepo(settings.operational_dsn))
|
||||
|
||||
builders["vrp"] = ("vrp_state", _build_vrp)
|
||||
|
||||
# ---- unlock_nav: token-unlock dilution shorts over the warehouse close panel + DefiLlama calendar -----
|
||||
# ---- unlock_nav: single-sleeve unlock edge on the liquid universe (Phase 0b Task 2: own Bybit track, ----
|
||||
# ---- the SAME single-sleeve construction as xsfunding/positioning, mirrored here; carries the unlock ----
|
||||
# ---- calendar the sleeve needs, unlike xsfunding/positioning) --------------------------------------------
|
||||
def _build_unlock() -> Any:
|
||||
from fxhnt.adapters.data.unlock_calendar_live import UnlockCalendarLive
|
||||
from fxhnt.adapters.warehouse import get_feature_store
|
||||
from fxhnt.application.unlock_calendar_loader import operational_unlock_repo
|
||||
from fxhnt.application.unlock_short_strategy import UnlockShortForward
|
||||
from fxhnt.adapters.warehouse.timescale_feature_store import TimescaleFeatureStore
|
||||
from fxhnt.application.bybit_forward_track import BybitFourEdgeStrategy
|
||||
from fxhnt.application.bybit_liquidity import liquid_universe
|
||||
from fxhnt.application.unlock_calendar_loader import load_unlock_events, operational_unlock_repo
|
||||
|
||||
tradeable = {k[:-4] for k in get_feature_store(settings).crypto_close_panel() if k.endswith("USDT")}
|
||||
cal = UnlockCalendarLive(f"{data_dir}/unlock_calendar.json", tradeable=tradeable,
|
||||
repo=operational_unlock_repo(settings))
|
||||
return UnlockShortForward(cal)
|
||||
store = TimescaleFeatureStore(settings.operational_dsn, table="bybit_features")
|
||||
universe = liquid_universe(store) or None
|
||||
unlock_events = load_unlock_events(operational_unlock_repo(settings),
|
||||
f"{data_dir}/unlock_calendar.json")
|
||||
return BybitFourEdgeStrategy(
|
||||
store, universe=universe, cost_bps=settings.bybit_cost_bps, sleeves=["unlock"],
|
||||
unlock_events=unlock_events)
|
||||
|
||||
builders["unlock"] = ("unlock_state", _build_unlock)
|
||||
|
||||
@@ -116,28 +93,9 @@ def track_builders(settings: Settings, data_dir: str) -> TrackBuilders:
|
||||
# `strategy.advance()` later (inside `recompute_series`), which reads from `store`. It is intentionally
|
||||
# left open for the process lifetime of this one-time migration CLI.
|
||||
def _build_bybit_4edge() -> Any:
|
||||
from fxhnt.adapters.warehouse.timescale_feature_store import TimescaleFeatureStore
|
||||
from fxhnt.application.bybit_forward_track import POSITIONING_TAIL_CAP_K, BybitFourEdgeStrategy
|
||||
from fxhnt.application.bybit_liquidity import liquid_universe
|
||||
from fxhnt.application.unlock_calendar_loader import load_unlock_events, operational_unlock_repo
|
||||
|
||||
store = TimescaleFeatureStore(settings.operational_dsn, table="bybit_features")
|
||||
universe = liquid_universe(store) or None
|
||||
unlock_events = load_unlock_events(operational_unlock_repo(settings),
|
||||
f"{data_dir}/unlock_calendar.json")
|
||||
return BybitFourEdgeStrategy(
|
||||
store, universe=universe, cost_bps=settings.cost_bps_per_turnover,
|
||||
unlock_events=unlock_events, positioning_capacity_capital=settings.paper_capital,
|
||||
positioning_tail_cap_k=POSITIONING_TAIL_CAP_K)
|
||||
|
||||
builders["bybit_4edge"] = ("bybit_4edge_state", _build_bybit_4edge)
|
||||
|
||||
# ---- bybit_4edge_levered_nav: observe-only levered shadow (static leverage, book-gross <= 1.5x) -------
|
||||
def _build_bybit_4edge_levered() -> Any:
|
||||
from fxhnt.adapters.warehouse.timescale_feature_store import TimescaleFeatureStore
|
||||
from fxhnt.application.bybit_forward_track import (
|
||||
_LEVERED_SLEEVE_SHAPE,
|
||||
MAX_BOOK_GROSS,
|
||||
POSITIONING_COIN_GROSS_CAP,
|
||||
POSITIONING_TAIL_CAP_K,
|
||||
BybitFourEdgeStrategy,
|
||||
)
|
||||
@@ -149,26 +107,71 @@ def track_builders(settings: Settings, data_dir: str) -> TrackBuilders:
|
||||
unlock_events = load_unlock_events(operational_unlock_repo(settings),
|
||||
f"{data_dir}/unlock_calendar.json")
|
||||
return BybitFourEdgeStrategy(
|
||||
store, universe=universe, cost_bps=settings.cost_bps_per_turnover,
|
||||
store, universe=universe, cost_bps=settings.bybit_cost_bps,
|
||||
unlock_events=unlock_events, positioning_capacity_capital=settings.paper_capital,
|
||||
positioning_tail_cap_k=POSITIONING_TAIL_CAP_K,
|
||||
positioning_coin_gross_cap=POSITIONING_COIN_GROSS_CAP)
|
||||
|
||||
builders["bybit_4edge"] = ("bybit_4edge_state", _build_bybit_4edge)
|
||||
|
||||
# ---- bybit_4edge_levered_nav: observe-only levered shadow (static leverage, book-gross <= 1.5x) -------
|
||||
def _build_bybit_4edge_levered() -> Any:
|
||||
from fxhnt.adapters.warehouse.timescale_feature_store import TimescaleFeatureStore
|
||||
from fxhnt.application.bybit_forward_track import (
|
||||
_LEVERED_SLEEVE_SHAPE,
|
||||
MAX_BOOK_GROSS,
|
||||
POSITIONING_COIN_GROSS_CAP,
|
||||
POSITIONING_TAIL_CAP_K,
|
||||
BybitFourEdgeStrategy,
|
||||
)
|
||||
from fxhnt.application.bybit_liquidity import liquid_universe
|
||||
from fxhnt.application.unlock_calendar_loader import load_unlock_events, operational_unlock_repo
|
||||
|
||||
store = TimescaleFeatureStore(settings.operational_dsn, table="bybit_features")
|
||||
universe = liquid_universe(store) or None
|
||||
unlock_events = load_unlock_events(operational_unlock_repo(settings),
|
||||
f"{data_dir}/unlock_calendar.json")
|
||||
return BybitFourEdgeStrategy(
|
||||
store, universe=universe, cost_bps=settings.bybit_cost_bps,
|
||||
unlock_events=unlock_events, leverage_shape=_LEVERED_SLEEVE_SHAPE, max_gross=MAX_BOOK_GROSS,
|
||||
positioning_capacity_capital=settings.paper_capital,
|
||||
positioning_tail_cap_k=POSITIONING_TAIL_CAP_K)
|
||||
positioning_tail_cap_k=POSITIONING_TAIL_CAP_K,
|
||||
positioning_coin_gross_cap=POSITIONING_COIN_GROSS_CAP)
|
||||
|
||||
builders["bybit_4edge_levered"] = ("bybit_4edge_levered_state", _build_bybit_4edge_levered)
|
||||
|
||||
# ---- positioning_nav: single-sleeve positioning edge on the liquid universe --------------------------
|
||||
def _build_positioning() -> Any:
|
||||
from fxhnt.adapters.warehouse.timescale_feature_store import TimescaleFeatureStore
|
||||
from fxhnt.application.bybit_forward_track import POSITIONING_TAIL_CAP_K, BybitFourEdgeStrategy
|
||||
from fxhnt.application.bybit_forward_track import (
|
||||
POSITIONING_COIN_GROSS_CAP,
|
||||
POSITIONING_TAIL_CAP_K,
|
||||
BybitFourEdgeStrategy,
|
||||
)
|
||||
from fxhnt.application.bybit_liquidity import liquid_universe
|
||||
|
||||
store = TimescaleFeatureStore(settings.operational_dsn, table="bybit_features")
|
||||
universe = liquid_universe(store) or None
|
||||
return BybitFourEdgeStrategy(
|
||||
store, universe=universe, cost_bps=settings.cost_bps_per_turnover, sleeves=["positioning"],
|
||||
store, universe=universe, cost_bps=settings.bybit_cost_bps, sleeves=["positioning"],
|
||||
positioning_capacity_capital=settings.paper_capital,
|
||||
positioning_tail_cap_k=POSITIONING_TAIL_CAP_K)
|
||||
positioning_tail_cap_k=POSITIONING_TAIL_CAP_K,
|
||||
positioning_coin_gross_cap=POSITIONING_COIN_GROSS_CAP)
|
||||
|
||||
builders["positioning"] = ("positioning_state", _build_positioning)
|
||||
|
||||
# ---- xsfunding_nav: single-sleeve xsfunding edge on the liquid universe (Phase 0b: own Bybit track, -----
|
||||
# ---- the SAME single-sleeve construction as positioning, mirrored here) --------------------------------
|
||||
def _build_xsfunding() -> Any:
|
||||
from fxhnt.adapters.warehouse.timescale_feature_store import TimescaleFeatureStore
|
||||
from fxhnt.application.bybit_forward_track import BybitFourEdgeStrategy
|
||||
from fxhnt.application.bybit_liquidity import liquid_universe
|
||||
|
||||
store = TimescaleFeatureStore(settings.operational_dsn, table="bybit_features")
|
||||
universe = liquid_universe(store) or None
|
||||
return BybitFourEdgeStrategy(
|
||||
store, universe=universe, cost_bps=settings.bybit_cost_bps, sleeves=["xsfunding"])
|
||||
|
||||
builders["xsfunding"] = ("xsfunding_state", _build_xsfunding)
|
||||
|
||||
return builders
|
||||
|
||||
@@ -39,6 +39,22 @@ class ForwardSummaryRow(CockpitBase):
|
||||
src_updated_at: Mapped[dt.datetime] = mapped_column(DateTime)
|
||||
|
||||
|
||||
class StrategyHealthRow(CockpitBase):
|
||||
"""Runtime HEALTH verdict per track — a SEPARATE axis from the gate (a track can be legitimately
|
||||
WAIT/building AND healthy; STALE/NO_REF/BROKEN is orthogonal). Written each nightly run by
|
||||
`forward_health.evaluate_health`; the cockpit reads it to render a LOUD red badge so a live/gated track
|
||||
that stops updating surfaces on day 1 (the anti-silent-rot backbone that would have caught a dead
|
||||
strategy's silent staleness). One row
|
||||
per strategy, replaced each run (a track that recovers clears its stale row). Brand-new table →
|
||||
`create_all` (ForwardNavRepo.migrate) creates it; no ALTER, Postgres-safe."""
|
||||
__tablename__ = "strategy_health"
|
||||
strategy_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
health: Mapped[str] = mapped_column(String(12)) # HEALTHY | STALE | NO_REF | BROKEN
|
||||
stale_days: Mapped[int] = mapped_column(Integer, default=0) # business days stale (0 unless STALE)
|
||||
reason: Mapped[str] = mapped_column(String(256), default="")
|
||||
src_updated_at: Mapped[dt.datetime] = mapped_column(DateTime)
|
||||
|
||||
|
||||
class BacktestSummaryRow(CockpitBase):
|
||||
__tablename__ = "backtest_summary"
|
||||
strategy_id: Mapped[str] = mapped_column(String(64), primary_key=True) # eqfactor_long / eqfactor_ls / eqfactor_tilt
|
||||
@@ -93,6 +109,7 @@ class ExecFillRow(CockpitBase):
|
||||
__tablename__ = "exec_fills"
|
||||
client_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
rebalance_id: Mapped[str] = mapped_column(String(48), index=True)
|
||||
strategy_id: Mapped[str] = mapped_column(String(64), index=True, default="") # "" = back-compat/untagged
|
||||
symbol: Mapped[str] = mapped_column(String(32))
|
||||
side: Mapped[str] = mapped_column(String(4))
|
||||
qty: Mapped[float] = mapped_column(Float)
|
||||
@@ -103,13 +120,23 @@ class ExecFillRow(CockpitBase):
|
||||
at: Mapped[dt.datetime] = mapped_column(DateTime)
|
||||
|
||||
|
||||
class IbkrAccountNavRow(CockpitBase):
|
||||
"""Per-run REAL DU-paper-account snapshot (D2.1) — the actual NLV/cash/gross + positions IBKR reports at
|
||||
each `execute-multistrat` run, captured ADDITIVELY alongside the existing intended-book return basis
|
||||
(`multistrat_exec_pos`/`forward_record`, unchanged). `run_date` PK — idempotent replace (a same-day re-run
|
||||
overwrites, never appends). `positions_json` mirrors `ForwardBookStateRow.extra_json`'s JSON-as-text
|
||||
approach so this table works identically on SQLite (tests) and Postgres."""
|
||||
__tablename__ = "ibkr_account_nav"
|
||||
run_date: Mapped[dt.date] = mapped_column(Date, primary_key=True)
|
||||
nlv: Mapped[float] = mapped_column(Float)
|
||||
cash: Mapped[float] = mapped_column(Float)
|
||||
gross: Mapped[float] = mapped_column(Float)
|
||||
positions_json: Mapped[str] = mapped_column(Text)
|
||||
at: Mapped[dt.datetime] = mapped_column(DateTime)
|
||||
|
||||
|
||||
class PaperPositionRow(CockpitBase):
|
||||
__tablename__ = "paper_positions"
|
||||
# `venue` discriminates the live paper BOOK (binance vs bybit). Default "binance" + server_default so a
|
||||
# fresh create_all AND an in-place ALTER (PaperRepo.migrate) both backfill pre-existing rows to binance —
|
||||
# the binance book is bit-identical to before. Part of the PK so the two venues' books coexist.
|
||||
venue: Mapped[str] = mapped_column(String(16), primary_key=True, default="binance",
|
||||
server_default="binance")
|
||||
run_date: Mapped[dt.date] = mapped_column(Date, primary_key=True)
|
||||
sleeve: Mapped[str] = mapped_column(String(32), primary_key=True)
|
||||
symbol: Mapped[str] = mapped_column(String(32), primary_key=True)
|
||||
@@ -123,8 +150,6 @@ class PaperPositionRow(CockpitBase):
|
||||
class PaperTradeRow(CockpitBase):
|
||||
__tablename__ = "paper_trades"
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
|
||||
venue: Mapped[str] = mapped_column(String(16), index=True, default="binance",
|
||||
server_default="binance")
|
||||
run_date: Mapped[dt.date] = mapped_column(Date, index=True) # keyed so a re-run replaces this date's trades
|
||||
ts: Mapped[dt.datetime] = mapped_column(DateTime)
|
||||
sleeve: Mapped[str] = mapped_column(String(32))
|
||||
@@ -139,8 +164,6 @@ class PaperNavRow(CockpitBase):
|
||||
"""Equity-per-date series for the paper book (forward snapshot + backfill both write it). One row
|
||||
per run_date — idempotent. Timescale-friendly like forward_nav."""
|
||||
__tablename__ = "paper_nav"
|
||||
venue: Mapped[str] = mapped_column(String(16), primary_key=True, default="binance",
|
||||
server_default="binance")
|
||||
run_date: Mapped[dt.date] = mapped_column(Date, primary_key=True)
|
||||
capital: Mapped[float] = mapped_column(Float)
|
||||
realized: Mapped[float] = mapped_column(Float)
|
||||
@@ -150,17 +173,17 @@ class PaperNavRow(CockpitBase):
|
||||
|
||||
|
||||
class PaperNavSummaryRow(CockpitBase):
|
||||
"""MATERIALIZED headline nav-summary for one venue's paper book: the stats `nav_backfill_stats` returns
|
||||
"""MATERIALIZED headline nav-summary for the bybit paper book: the stats `nav_backfill_stats` returns
|
||||
(final_equity / total_return_pct / sharpe / max_dd_pct) PLUS `since` + `has_record` and the pre-rendered
|
||||
multi-year curve SVG. These change ONLY when the nav changes (nightly), yet the Overview + `/paper`
|
||||
headline used to re-read the full ~3650-row paper_nav series and re-run stats+curve on EVERY request.
|
||||
`refresh_nav_summary` computes this row ONCE (reusing the EXACT same helpers, so it is byte-for-byte
|
||||
identical to the live compute) keyed by `nav_run_date` = the nav's latest run_date (the version); the read
|
||||
path serves it when the version matches and self-heals otherwise. Authoritative + refresh-on-data-change +
|
||||
read-path self-heal — NOT a TTL cache. `venue` PK — one row per venue, replaced on each refresh. A brand-
|
||||
new table → `create_all` (PaperRepo.migrate) creates it; no ALTER needed."""
|
||||
read-path self-heal — NOT a TTL cache. Singleton row, id=1 (bybit is the sole paper book), replaced on each
|
||||
refresh. A brand-new table → `create_all` (PaperRepo.migrate) creates it; no ALTER needed."""
|
||||
__tablename__ = "paper_nav_summary"
|
||||
venue: Mapped[str] = mapped_column(String(16), primary_key=True)
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, default=1)
|
||||
nav_run_date: Mapped[str] = mapped_column(String(10)) # the nav's latest run_date = the version
|
||||
final_equity: Mapped[float] = mapped_column(Float)
|
||||
total_return_pct: Mapped[float] = mapped_column(Float)
|
||||
@@ -176,8 +199,6 @@ class PaperBookStateRow(CockpitBase):
|
||||
"""Cumulative book-level scalars (key/value) — currently the running realized PnL so closed-position
|
||||
gains/losses survive after a position is reduced or exited. key="realized", value=cumulative float."""
|
||||
__tablename__ = "paper_book_state"
|
||||
venue: Mapped[str] = mapped_column(String(16), primary_key=True, default="binance",
|
||||
server_default="binance")
|
||||
key: Mapped[str] = mapped_column(String(32), primary_key=True)
|
||||
value: Mapped[float] = mapped_column(Float, default=0.0)
|
||||
|
||||
@@ -186,8 +207,6 @@ class PaperSleeveRetRow(CockpitBase):
|
||||
"""One sleeve's simulated daily return for one run_date (backfill + snapshot both write). Feeds the
|
||||
ISV-adaptive trailing-vol window. (run_date, sleeve) PK — idempotent per date+sleeve."""
|
||||
__tablename__ = "paper_sleeve_ret"
|
||||
venue: Mapped[str] = mapped_column(String(16), primary_key=True, default="binance",
|
||||
server_default="binance")
|
||||
run_date: Mapped[dt.date] = mapped_column(Date, primary_key=True)
|
||||
sleeve: Mapped[str] = mapped_column(String(32), primary_key=True)
|
||||
ret: Mapped[float] = mapped_column(Float)
|
||||
@@ -206,9 +225,24 @@ class BybitSleeveRetRow(CockpitBase):
|
||||
src_updated_at: Mapped[dt.datetime] = mapped_column(DateTime)
|
||||
|
||||
|
||||
class SimCurveRetRow(CockpitBase):
|
||||
"""One recompute-replay strategy's (multistrat — the IBKR paper books the Backtest page's sim reads)
|
||||
daily honest-cost return for one run_date. Mirrors `BybitSleeveRetRow`'s shape/purpose for the Bybit-book
|
||||
sim: written nightly by `_persist_track_backtest_ref` (the `fxhnt-backtest-refs` Job) from the SAME
|
||||
inception rows `backtest_ref`'s recompute-replay branch derives the reconciliation-gate ref from, so
|
||||
/paper/sim NEVER re-runs an IBKR replay on a web request. `strategy_id` doubles as the sim's `sleeve` key
|
||||
(each IBKR book is single-sleeve — the whole book IS the strategy). (strategy_id, run_date) PK —
|
||||
idempotent per strategy+date."""
|
||||
__tablename__ = "sim_curve_ret"
|
||||
strategy_id: Mapped[str] = mapped_column(String(32), primary_key=True)
|
||||
run_date: Mapped[dt.date] = mapped_column(Date, primary_key=True)
|
||||
ret: Mapped[float] = mapped_column(Float)
|
||||
src_updated_at: Mapped[dt.datetime] = mapped_column(DateTime)
|
||||
|
||||
|
||||
class CompareMeasuredCacheRow(CockpitBase):
|
||||
"""The PRECOMPUTED measured-cost COMPARE artifact the cockpit reads (it NEVER recomputes the heavy
|
||||
per-coin Binance measured-cost path on a web request — that OOM-killed the 4Gi dashboard, exit 137).
|
||||
per-coin measured-cost path on a web request — that OOM-killed the 4Gi dashboard, exit 137).
|
||||
Written by the own-memory `fxhnt compare-measured-precompute` Job; the cockpit's `book=compare` measured
|
||||
path loads this row and renders it, falling back to the light FLAT-cost compare when the row is absent.
|
||||
Tiny: two normalized equity series + the per-book metrics rows + the window, all in one JSON payload.
|
||||
@@ -293,8 +327,6 @@ class PaperShadowPositionRow(CockpitBase):
|
||||
Persisted every date and NEVER flattened by the killswitch, so per-sleeve returns survive kill periods
|
||||
(re-entry detection + ISV vol estimate stay correct). (run_date, sleeve, symbol) PK — idempotent."""
|
||||
__tablename__ = "paper_shadow_positions"
|
||||
venue: Mapped[str] = mapped_column(String(16), primary_key=True, default="binance",
|
||||
server_default="binance")
|
||||
run_date: Mapped[dt.date] = mapped_column(Date, primary_key=True)
|
||||
sleeve: Mapped[str] = mapped_column(String(32), primary_key=True)
|
||||
symbol: Mapped[str] = mapped_column(String(32), primary_key=True)
|
||||
@@ -337,6 +369,27 @@ class YahooPitCloseRow(CockpitBase):
|
||||
first_seen_at: Mapped[dt.datetime] = mapped_column(DateTime)
|
||||
|
||||
|
||||
class YahooPitVolumeRow(CockpitBase):
|
||||
"""Point-in-time frozen Yahoo daily raw share volume — mirrors `YahooPitCloseRow` (spec §4). Feeds the
|
||||
equity-leg dollar-ADV capacity model. (symbol, date) PK — first-seen wins; NEVER overwritten in place."""
|
||||
__tablename__ = "yahoo_pit_volume"
|
||||
symbol: Mapped[str] = mapped_column(String(16), primary_key=True)
|
||||
date: Mapped[str] = mapped_column(String(10), primary_key=True)
|
||||
volume: Mapped[float] = mapped_column(Float)
|
||||
first_seen_at: Mapped[dt.datetime] = mapped_column(DateTime)
|
||||
|
||||
|
||||
class IbkrPitVolumeRow(CockpitBase):
|
||||
"""Point-in-time frozen IBKR daily on-exchange volume (reqHistoricalData TRADES) — mirrors
|
||||
`YahooPitVolumeRow` for the UCITS lines the fund actually executes. (symbol, date) PK — first-seen
|
||||
wins; NEVER overwritten in place."""
|
||||
__tablename__ = "ibkr_pit_volume"
|
||||
symbol: Mapped[str] = mapped_column(String(16), primary_key=True)
|
||||
date: Mapped[str] = mapped_column(String(10), primary_key=True)
|
||||
volume: Mapped[float] = mapped_column(Float)
|
||||
first_seen_at: Mapped[dt.datetime] = mapped_column(DateTime)
|
||||
|
||||
|
||||
class StrategyAllocationRow(CockpitBase):
|
||||
"""The latest B2a strategy-level allocation snapshot — one row per strategy, replaced each run.
|
||||
target_weight = the ex-ante-sized weight; target_dollars = capital-ceiling-bounded dollars. Stamped with
|
||||
@@ -367,6 +420,19 @@ class StrategyFundingRow(CockpitBase):
|
||||
computed_at: Mapped[dt.datetime] = mapped_column(DateTime)
|
||||
|
||||
|
||||
class FundConfigRow(CockpitBase):
|
||||
"""Single-row (id=1) fund-level knobs — the SSOT for capital + risk appetite, set ONLY via
|
||||
`fxhnt fund-config` (never hand-edited), read by the nightly allocation. Absent row/table → the nightly
|
||||
falls back to settings/policy defaults (see application/fund_config.read_fund_config)."""
|
||||
__tablename__ = "fund_config"
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True) # always 1
|
||||
capital: Mapped[float] = mapped_column(Float)
|
||||
target_vol: Mapped[float] = mapped_column(Float)
|
||||
kelly_fraction: Mapped[float] = mapped_column(Float)
|
||||
max_venue_weight: Mapped[float] = mapped_column(Float)
|
||||
updated_at: Mapped[dt.datetime] = mapped_column(DateTime)
|
||||
|
||||
|
||||
class XspOptionBarRow(CockpitBase):
|
||||
__tablename__ = "xsp_option_bars"
|
||||
osi_symbol: Mapped[str] = mapped_column(String(32), primary_key=True)
|
||||
|
||||
@@ -17,12 +17,16 @@ from fxhnt.adapters.persistence.cockpit_models import (
|
||||
ForwardNavRow,
|
||||
ForwardRecordRow,
|
||||
ForwardSummaryRow,
|
||||
IbkrPitVolumeRow,
|
||||
StrategyAllocationRow,
|
||||
StrategyAnchorRow,
|
||||
StrategyFundingRow,
|
||||
StrategyHealthRow,
|
||||
StrategyPromotionRow,
|
||||
StrategyRegistryRow,
|
||||
XspOptionBarRow,
|
||||
YahooPitCloseRow,
|
||||
YahooPitVolumeRow,
|
||||
)
|
||||
from fxhnt.application.forward_models import BacktestSummary, ForwardSummary
|
||||
from fxhnt.application.forward_models import ForwardNavRow as NavRowDTO
|
||||
@@ -123,6 +127,30 @@ class ForwardNavRepo:
|
||||
with Session(self._engine) as s:
|
||||
return list(s.scalars(select(BacktestSummaryRow).order_by(BacktestSummaryRow.strategy_id)))
|
||||
|
||||
def replace_health(self, rows: list[StrategyHealthRow], at: dt.datetime) -> None:
|
||||
"""Replace the ENTIRE strategy_health snapshot (delete-then-insert, atomic) so a track that RECOVERS
|
||||
clears its old STALE/NO_REF row and a retired track leaves no stale health row behind — the table
|
||||
reflects exactly this run's verdicts."""
|
||||
with Session(self._engine) as s:
|
||||
s.query(StrategyHealthRow).delete()
|
||||
for r in rows:
|
||||
r.src_updated_at = at
|
||||
s.add(r)
|
||||
s.commit()
|
||||
|
||||
def all_health(self) -> list[StrategyHealthRow]:
|
||||
with Session(self._engine) as s:
|
||||
return list(s.scalars(select(StrategyHealthRow).order_by(StrategyHealthRow.strategy_id)))
|
||||
|
||||
def xsp_freeze_max_date(self) -> str | None:
|
||||
"""The latest date present in the opra-pit-anchored freeze table (`xsp_option_bars`), ISO yyyy-mm-dd,
|
||||
or None when the table is empty. The freeze-staleness axis: a frozen-input recompute is bounded by
|
||||
this max date, so if the freeze stops advancing the track is STALE even though its forward 'updates'."""
|
||||
from sqlalchemy import func
|
||||
with Session(self._engine) as s:
|
||||
d = s.scalar(select(func.max(XspOptionBarRow.date)))
|
||||
return d.isoformat() if d else None
|
||||
|
||||
def promote(self, strategy_id: str, at: dt.datetime) -> None:
|
||||
"""Record a ONE-WAY research→deploy promotion (idempotent: merge on the PK, keeps the first
|
||||
promoted_at on re-runs since callers only promote a track that is not already promoted)."""
|
||||
@@ -158,9 +186,22 @@ class ForwardNavRepo:
|
||||
with Session(self._engine) as s:
|
||||
return list(s.scalars(select(ForwardSummaryRow).order_by(ForwardSummaryRow.strategy_id)))
|
||||
|
||||
def registry(self) -> list[StrategyRegistryRow]:
|
||||
def cum_return(self, strategy_id: str) -> float | None:
|
||||
"""The persisted `forward_summary.total_return` for `strategy_id`, or None if no summary row exists
|
||||
yet (no recorded forward days). Used by `exec_reconcile.exec_vs_sim` (Task 3) to compare a strategy's
|
||||
executed vs modeled cumulative forward return without re-deriving it from the nav history."""
|
||||
with Session(self._engine) as s:
|
||||
return list(s.scalars(select(StrategyRegistryRow).order_by(StrategyRegistryRow.strategy_id)))
|
||||
row = s.get(ForwardSummaryRow, strategy_id)
|
||||
return row.total_return if row is not None else None
|
||||
|
||||
def registry(self) -> list[StrategyRegistryRow]:
|
||||
"""The cockpit-visible registry rows. SINGLE SOURCE for the hide: an ARCHIVED sleeve (its
|
||||
`STRATEGY_REGISTRY` entry carries `archived: True` — e.g. a shelved/FALSIFIED strategy) is
|
||||
excluded here, so EVERY caller (fleet overview, per-strategy gate specs, `detail()`'s reg lookup)
|
||||
drops it automatically. The DB row is still seeded (code + data kept) — it is only hidden from the UI."""
|
||||
with Session(self._engine) as s:
|
||||
rows = s.scalars(select(StrategyRegistryRow).order_by(StrategyRegistryRow.strategy_id))
|
||||
return [r for r in rows if not STRATEGY_REGISTRY.get(r.strategy_id, {}).get("archived")]
|
||||
|
||||
def has_forward_history(self, strategy_id: str) -> bool:
|
||||
"""True if this track has ANY persisted forward history — a forward_summary row OR any forward_anchor
|
||||
@@ -256,6 +297,38 @@ class ForwardNavRepo:
|
||||
with Session(self._engine) as s:
|
||||
return {r.date: r.adj_close for r in s.scalars(stmt)}
|
||||
|
||||
def snapshot_yahoo_volumes(self, symbol: str, series: dict[str, float], at: dt.datetime) -> None:
|
||||
"""Insert only (symbol, date) pairs not already stored — first-seen wins so the PIT value is frozen."""
|
||||
with Session(self._engine) as s:
|
||||
have = {d for (d,) in s.execute(
|
||||
select(YahooPitVolumeRow.date).where(YahooPitVolumeRow.symbol == symbol)).all()}
|
||||
for d, v in series.items():
|
||||
if d not in have:
|
||||
s.add(YahooPitVolumeRow(symbol=symbol, date=d, volume=float(v), first_seen_at=at))
|
||||
s.commit()
|
||||
|
||||
def yahoo_pit_volumes(self, symbol: str) -> dict[str, float]:
|
||||
stmt = (select(YahooPitVolumeRow).where(YahooPitVolumeRow.symbol == symbol)
|
||||
.order_by(YahooPitVolumeRow.date))
|
||||
with Session(self._engine) as s:
|
||||
return {r.date: r.volume for r in s.scalars(stmt)}
|
||||
|
||||
def snapshot_ibkr_volumes(self, symbol: str, series: dict[str, float], at: dt.datetime) -> None:
|
||||
"""Insert only (symbol, date) pairs not already stored — first-seen wins so the PIT value is frozen."""
|
||||
with Session(self._engine) as s:
|
||||
have = {d for (d,) in s.execute(
|
||||
select(IbkrPitVolumeRow.date).where(IbkrPitVolumeRow.symbol == symbol)).all()}
|
||||
for d, v in series.items():
|
||||
if d not in have:
|
||||
s.add(IbkrPitVolumeRow(symbol=symbol, date=d, volume=float(v), first_seen_at=at))
|
||||
s.commit()
|
||||
|
||||
def ibkr_pit_volumes(self, symbol: str) -> dict[str, float]:
|
||||
stmt = (select(IbkrPitVolumeRow).where(IbkrPitVolumeRow.symbol == symbol)
|
||||
.order_by(IbkrPitVolumeRow.date))
|
||||
with Session(self._engine) as s:
|
||||
return {r.date: r.volume for r in s.scalars(stmt)}
|
||||
|
||||
def replace_allocations(self, rows: list[StrategyAllocationRow], at: dt.datetime) -> None:
|
||||
"""Replace the ENTIRE strategy_allocation snapshot with `rows` in one transaction (delete-then-insert),
|
||||
so a strategy that drops out of the allocation this run leaves no stale row behind — the table reflects
|
||||
|
||||
94
src/fxhnt/adapters/persistence/ibkr_account_repo.py
Normal file
94
src/fxhnt/adapters/persistence/ibkr_account_repo.py
Normal file
@@ -0,0 +1,94 @@
|
||||
"""Repository for the D2.1 REAL DU-paper-account capture: the per-run `ibkr_account_nav` snapshot (NLV/cash/
|
||||
gross/positions) plus strategy-tagged reads of the (pre-existing, previously-empty) `exec_fills` table.
|
||||
Mirrors `ForwardNavRepo`'s engine/StaticPool construction and its guarded idempotent ALTER pattern
|
||||
(`_ensure_summary_policy_columns`) — `migrate()` never runs heavy DDL on every startup."""
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
import json
|
||||
import uuid
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import create_engine, select, text
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy.pool import StaticPool
|
||||
|
||||
from fxhnt.adapters.persistence.cockpit_models import CockpitBase, ExecFillRow, IbkrAccountNavRow
|
||||
from fxhnt.application.exec_capture import FillDTO
|
||||
|
||||
|
||||
class IbkrAccountRepo:
|
||||
def __init__(self, dsn: str) -> None:
|
||||
# StaticPool keeps a single in-memory SQLite connection alive across sessions; only applied for the
|
||||
# in-memory SQLite case (tests). StaticPool is NOT used for Postgres.
|
||||
kw: dict[str, Any] = {"future": True}
|
||||
if dsn == "sqlite://" or (dsn.startswith("sqlite://") and ":memory:" in dsn):
|
||||
kw |= {"connect_args": {"check_same_thread": False}, "poolclass": StaticPool}
|
||||
self._engine = create_engine(dsn, **kw)
|
||||
self._is_pg = self._engine.dialect.name == "postgresql"
|
||||
|
||||
def migrate(self) -> None:
|
||||
CockpitBase.metadata.create_all(self._engine)
|
||||
self._ensure_exec_fills_strategy_column()
|
||||
|
||||
def _ensure_exec_fills_strategy_column(self) -> None:
|
||||
"""Idempotently add `exec_fills.strategy_id` to a table that pre-dates this task (`create_all` does
|
||||
NOT add columns to an already-existing table on prod Postgres). Postgres has `ADD COLUMN IF NOT
|
||||
EXISTS`; sqlite does not, so PRAGMA-check the columns first. A brand-new table already has the
|
||||
column from `create_all` — this is then a no-op on both dialects."""
|
||||
with self._engine.begin() as c:
|
||||
if self._is_pg:
|
||||
c.execute(text(
|
||||
"ALTER TABLE exec_fills ADD COLUMN IF NOT EXISTS strategy_id VARCHAR(64) DEFAULT ''"))
|
||||
else:
|
||||
cols = {row[1] for row in c.execute(text("PRAGMA table_info(exec_fills)"))}
|
||||
if "strategy_id" not in cols:
|
||||
c.execute(text(
|
||||
"ALTER TABLE exec_fills ADD COLUMN strategy_id VARCHAR(64) DEFAULT ''"))
|
||||
|
||||
def replace_snapshot(self, run_date: str, *, nlv: float, cash: float, gross: float,
|
||||
positions: dict[str, float], at: dt.datetime) -> None:
|
||||
"""Idempotent per run_date: delete-then-insert in one transaction, so a same-day re-run overwrites
|
||||
rather than double-books (mirrors `ForwardNavRepo.replace_nav`)."""
|
||||
rd = dt.date.fromisoformat(run_date)
|
||||
positions_json = json.dumps({str(k): float(v) for k, v in positions.items()})
|
||||
with Session(self._engine) as s:
|
||||
s.query(IbkrAccountNavRow).filter(IbkrAccountNavRow.run_date == rd).delete()
|
||||
s.add(IbkrAccountNavRow(run_date=rd, nlv=float(nlv), cash=float(cash), gross=float(gross),
|
||||
positions_json=positions_json, at=at))
|
||||
s.commit()
|
||||
|
||||
def nav_series(self) -> list[tuple[str, float]]:
|
||||
"""The account's (date, NLV) series ascending — the real-account equivalent of `ForwardNavRepo`'s
|
||||
nav history."""
|
||||
with Session(self._engine) as s:
|
||||
rows = s.scalars(select(IbkrAccountNavRow).order_by(IbkrAccountNavRow.run_date.asc()))
|
||||
return [(r.run_date.isoformat(), r.nlv) for r in rows]
|
||||
|
||||
def latest_positions(self) -> dict[str, float]:
|
||||
"""The most recently snapshotted real positions, or {} if none captured yet."""
|
||||
with Session(self._engine) as s:
|
||||
row = s.scalars(select(IbkrAccountNavRow)
|
||||
.order_by(IbkrAccountNavRow.run_date.desc()).limit(1)).first()
|
||||
if row is None:
|
||||
return {}
|
||||
return {str(k): float(v) for k, v in json.loads(row.positions_json).items()}
|
||||
|
||||
def record_fills(self, *, strategy_id: str, rebalance_id: str, fills: list[FillDTO],
|
||||
at: dt.datetime) -> None:
|
||||
"""Append each captured fill as an `exec_fills` row tagged `strategy_id`. `client_id` (the table's
|
||||
PK) is synthesized per fill since `FillDTO` carries no execution id of its own."""
|
||||
with Session(self._engine) as s:
|
||||
for f in fills:
|
||||
s.add(ExecFillRow(client_id=str(uuid.uuid4()), rebalance_id=rebalance_id,
|
||||
strategy_id=strategy_id, symbol=f.symbol, side=f.side, qty=f.qty,
|
||||
price=f.price, fee=f.fee, status=f.status,
|
||||
shortfall_bps=f.shortfall_bps, at=at))
|
||||
s.commit()
|
||||
|
||||
def fills_for(self, strategy_id: str, limit: int = 20) -> list[ExecFillRow]:
|
||||
"""The most recent `limit` fills tagged `strategy_id`, newest first. Read-only."""
|
||||
stmt = (select(ExecFillRow).where(ExecFillRow.strategy_id == strategy_id)
|
||||
.order_by(ExecFillRow.at.desc()).limit(limit))
|
||||
with Session(self._engine) as s:
|
||||
return list(s.scalars(stmt))
|
||||
76
src/fxhnt/adapters/persistence/paper_book_migration.py
Normal file
76
src/fxhnt/adapters/persistence/paper_book_migration.py
Normal file
@@ -0,0 +1,76 @@
|
||||
"""One-time reverse migration for the paper-book tables: drop the vestigial, single-valued `venue`
|
||||
discriminator column. bybit is fxhnt's sole paper book (Task 7f hardcoded every `PaperRepo` read/write to it),
|
||||
and the prod purge left every table bybit-only — so the new venue-less primary keys are collision-free.
|
||||
|
||||
`PaperRepo.migrate()` calls this on every cockpit-pod startup (git-sync deploy), so it MUST be idempotent and a
|
||||
pure NO-OP after the first successful run: heavy DDL on every startup crashloops the 4Gi dashboard. Each table's
|
||||
work is guarded on the live `venue` column still being present, so once dropped the whole pass short-circuits.
|
||||
|
||||
Postgres-only: a fresh `create_all` already emits the venue-less schema (new deployments / the sqlite test DBs),
|
||||
and sqlite cannot ALTER a primary key. This module is the sole home of the column-drop DDL, keeping `paper_repo`
|
||||
free of any discriminator logic."""
|
||||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import Connection, Engine, Inspector, inspect, text
|
||||
|
||||
# The five composite-PK paper tables that shed `venue` from their primary key → the venue-less PK column set.
|
||||
_PK_TABLES: dict[str, tuple[str, ...]] = {
|
||||
"paper_positions": ("run_date", "sleeve", "symbol"),
|
||||
"paper_nav": ("run_date",),
|
||||
"paper_book_state": ("key",),
|
||||
"paper_sleeve_ret": ("run_date", "sleeve"),
|
||||
"paper_shadow_positions": ("run_date", "sleeve", "symbol"),
|
||||
}
|
||||
|
||||
|
||||
def _drop_pk_and_column(c: Connection, insp: Inspector, tbl: str, new_pk: tuple[str, ...]) -> None:
|
||||
"""Drop `tbl`'s existing PK, drop its `venue` column, and re-add the PRIMARY KEY on `new_pk`. No-op when the
|
||||
column is already absent (already migrated), which keeps the whole pass a no-op after the first run."""
|
||||
if not insp.has_table(tbl):
|
||||
return
|
||||
cols = {col["name"] for col in insp.get_columns(tbl)}
|
||||
if "venue" not in cols:
|
||||
return
|
||||
pk_name = insp.get_pk_constraint(tbl).get("name") or f"{tbl}_pkey"
|
||||
c.execute(text(f'ALTER TABLE {tbl} DROP CONSTRAINT "{pk_name}"'))
|
||||
c.execute(text(f"ALTER TABLE {tbl} DROP COLUMN venue"))
|
||||
c.execute(text(f'ALTER TABLE {tbl} ADD PRIMARY KEY ({", ".join(new_pk)})'))
|
||||
|
||||
|
||||
def migrate_paper_book_schema(engine: Engine) -> None:
|
||||
"""Reverse-migrate the paper-book tables to the venue-less schema (see module docstring). Assumes
|
||||
`create_all` has already run (fresh tables are created venue-less; this only rewrites pre-existing ones)."""
|
||||
if engine.dialect.name == "sqlite":
|
||||
# Fresh sqlite `create_all` already has the venue-less schema, and sqlite can't ALTER a PK. The
|
||||
# newest-first trades read wants the (ts, id) index either way.
|
||||
with engine.begin() as c:
|
||||
c.execute(text("CREATE INDEX IF NOT EXISTS ix_paper_trades_ts_id ON paper_trades (ts, id)"))
|
||||
return
|
||||
|
||||
insp = inspect(engine)
|
||||
with engine.begin() as c:
|
||||
for tbl, new_pk in _PK_TABLES.items():
|
||||
_drop_pk_and_column(c, insp, tbl, new_pk)
|
||||
|
||||
# paper_nav_summary: its SOLE PK was `venue` (one materialized row). Replace it with a singleton
|
||||
# surrogate `id` defaulted to 1 for the existing bybit row, then PK on id.
|
||||
if insp.has_table("paper_nav_summary"):
|
||||
cols = {col["name"] for col in insp.get_columns("paper_nav_summary")}
|
||||
if "venue" in cols:
|
||||
pk_name = insp.get_pk_constraint("paper_nav_summary").get("name") or "paper_nav_summary_pkey"
|
||||
c.execute(text(f'ALTER TABLE paper_nav_summary DROP CONSTRAINT "{pk_name}"'))
|
||||
c.execute(text("ALTER TABLE paper_nav_summary ADD COLUMN id INTEGER NOT NULL DEFAULT 1"))
|
||||
c.execute(text("ALTER TABLE paper_nav_summary DROP COLUMN venue"))
|
||||
c.execute(text("ALTER TABLE paper_nav_summary ADD PRIMARY KEY (id)"))
|
||||
|
||||
# paper_trades: `venue` was a non-PK indexed column → drop its index then the column (the PK is the
|
||||
# autoincrement id, untouched). The newest-first read no longer filters on it.
|
||||
if insp.has_table("paper_trades"):
|
||||
cols = {col["name"] for col in insp.get_columns("paper_trades")}
|
||||
if "venue" in cols:
|
||||
c.execute(text("DROP INDEX IF EXISTS ix_paper_trades_venue_ts_id"))
|
||||
c.execute(text("ALTER TABLE paper_trades DROP COLUMN venue"))
|
||||
|
||||
# The newest-first trades read backward-scans (ts, id) for the top-N (no venue filter any more).
|
||||
# IF NOT EXISTS → idempotent. Replaces the old (venue, ts, id) index dropped just above.
|
||||
c.execute(text("CREATE INDEX IF NOT EXISTS ix_paper_trades_ts_id ON paper_trades (ts, id)"))
|
||||
@@ -8,7 +8,7 @@ import datetime as dt
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import create_engine, delete, inspect, select, text
|
||||
from sqlalchemy import create_engine, delete, select
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy.pool import StaticPool
|
||||
|
||||
@@ -24,7 +24,9 @@ from fxhnt.adapters.persistence.cockpit_models import (
|
||||
PaperShadowPositionRow,
|
||||
PaperSleeveRetRow,
|
||||
PaperTradeRow,
|
||||
SimCurveRetRow,
|
||||
)
|
||||
from fxhnt.adapters.persistence.paper_book_migration import migrate_paper_book_schema
|
||||
from fxhnt.domain.paper import Position, Trade
|
||||
|
||||
|
||||
@@ -47,84 +49,40 @@ def _parse_ts(ts: str) -> dt.datetime:
|
||||
return dt.datetime(1970, 1, 1, tzinfo=dt.UTC)
|
||||
|
||||
|
||||
# The six paper tables that carry the `venue` discriminator. Used by the in-place ALTER migration so a
|
||||
# pre-existing prod table (created before `venue` existed) gains the column + backfills to "binance".
|
||||
_VENUE_TABLES = (
|
||||
"paper_positions", "paper_trades", "paper_nav", "paper_book_state",
|
||||
"paper_sleeve_ret", "paper_shadow_positions",
|
||||
)
|
||||
|
||||
|
||||
class PaperRepo:
|
||||
def __init__(self, dsn: str, *, venue: str = "binance") -> None:
|
||||
def __init__(self, dsn: str) -> None:
|
||||
# StaticPool keeps a single in-memory SQLite connection alive across sessions; only applied for the
|
||||
# in-memory SQLite case (tests). StaticPool is NOT used for Postgres.
|
||||
kw: dict[str, Any] = {"future": True}
|
||||
if dsn == "sqlite://" or (dsn.startswith("sqlite://") and ":memory:" in dsn):
|
||||
kw |= {"connect_args": {"check_same_thread": False}, "poolclass": StaticPool}
|
||||
self._engine = create_engine(dsn, **kw)
|
||||
# The BOOK this repo reads/writes. "binance" (default) is bit-identical to the pre-venue repo; a
|
||||
# second instance with venue="bybit" operates the parallel Bybit live book on the SAME tables,
|
||||
# discriminated by the `venue` column. Every read/write below is scoped to `self._venue`.
|
||||
self._venue = venue
|
||||
|
||||
def migrate(self) -> None:
|
||||
CockpitBase.metadata.create_all(self._engine)
|
||||
# In-place ALTER for tables that already existed BEFORE the `venue` column was added (prod was created
|
||||
# by an earlier create_all). create_all never adds a column to an existing table, so add it here and
|
||||
# backfill the pre-existing rows to "binance". Idempotent: skipped when the column is already present.
|
||||
insp = inspect(self._engine)
|
||||
is_sqlite = self._engine.dialect.name == "sqlite"
|
||||
with self._engine.begin() as c:
|
||||
for tbl in _VENUE_TABLES:
|
||||
if not insp.has_table(tbl):
|
||||
continue
|
||||
cols = {col["name"] for col in insp.get_columns(tbl)}
|
||||
if "venue" not in cols:
|
||||
c.execute(text(
|
||||
f"ALTER TABLE {tbl} ADD COLUMN venue VARCHAR(16) NOT NULL DEFAULT 'binance'"))
|
||||
# ADD COLUMN does NOT change the PRIMARY KEY: a prod table created before `venue` existed keeps
|
||||
# its old PK (e.g. paper_positions = run_date,sleeve,symbol), so binance + bybit rows with the
|
||||
# same (run_date,sleeve,symbol) COLLIDE. Recreate the composite PK to match the ORM (include
|
||||
# venue) when the live PK omits it. Postgres only — sqlite/fresh create_all already has it
|
||||
# right, and sqlite can't ALTER a PK. Idempotent: skipped once venue is in the live PK. Safe on
|
||||
# existing rows (all venue='binance' → unique under the wider key).
|
||||
if is_sqlite:
|
||||
continue
|
||||
orm_pk = [col.name for col in CockpitBase.metadata.tables[tbl].primary_key.columns]
|
||||
if "venue" not in orm_pk:
|
||||
continue # PK legitimately excludes venue (e.g. autoincrement id on paper_trades)
|
||||
cur = insp.get_pk_constraint(tbl)
|
||||
if "venue" in (cur.get("constrained_columns") or []):
|
||||
continue # already migrated
|
||||
pk_name = cur.get("name") or f"{tbl}_pkey"
|
||||
c.execute(text(f'ALTER TABLE {tbl} DROP CONSTRAINT "{pk_name}"'))
|
||||
c.execute(text(f'ALTER TABLE {tbl} ADD PRIMARY KEY ({", ".join(orm_pk)})'))
|
||||
# Composite index for the live view's recent_trades (WHERE venue ORDER BY ts DESC, id DESC LIMIT n).
|
||||
# Without it that read SEQ-SCANs + top-N-sorts the whole ~150k-row paper_trades table (~400ms on
|
||||
# prod — a /paper + /paper/live hotspot). (venue, ts, id) lets the planner filter venue then
|
||||
# backward-index-scan ts/id and take the first n. IF NOT EXISTS → idempotent (PG + sqlite).
|
||||
c.execute(text("CREATE INDEX IF NOT EXISTS ix_paper_trades_venue_ts_id "
|
||||
"ON paper_trades (venue, ts, id)"))
|
||||
# Reverse migration: drop the vestigial single-valued discriminator column from the paper-book tables
|
||||
# so the schema is genuinely single-book (bybit is the sole paper book). Lives in its own module so
|
||||
# this repo carries no discriminator logic. Postgres-only + idempotent + a pure NO-OP after the first
|
||||
# run (heavy DDL on every startup crashloops the cockpit); it also (re)creates the (ts, id) index the
|
||||
# newest-first trades read backward-scans.
|
||||
migrate_paper_book_schema(self._engine)
|
||||
|
||||
def replace_positions(self, run_date: str, positions: list[Position],
|
||||
weight_by_sleeve: dict[str, float], at: dt.datetime) -> None:
|
||||
rd = dt.date.fromisoformat(run_date)
|
||||
with Session(self._engine) as s:
|
||||
s.execute(delete(PaperPositionRow).where(PaperPositionRow.venue == self._venue,
|
||||
PaperPositionRow.run_date == rd))
|
||||
s.execute(delete(PaperPositionRow).where(PaperPositionRow.run_date == rd))
|
||||
for p in positions:
|
||||
s.add(PaperPositionRow(
|
||||
venue=self._venue, run_date=rd, sleeve=p.sleeve, symbol=p.symbol, qty=p.qty,
|
||||
run_date=rd, sleeve=p.sleeve, symbol=p.symbol, qty=p.qty,
|
||||
entry_price=p.entry_price, entry_date=p.entry_date,
|
||||
weight=float(weight_by_sleeve.get(p.sleeve, 0.0)), src_updated_at=at))
|
||||
# Record the latest run_date so a full-exit (empty book) is still the current view, and never
|
||||
# roll the marker backwards on a replay of an older date.
|
||||
cur = s.scalar(select(PaperBookStateRow.value).where(
|
||||
PaperBookStateRow.venue == self._venue,
|
||||
PaperBookStateRow.key == self._LATEST_RD_KEY))
|
||||
if cur is None or rd.toordinal() >= int(cur):
|
||||
s.merge(PaperBookStateRow(venue=self._venue, key=self._LATEST_RD_KEY,
|
||||
s.merge(PaperBookStateRow(key=self._LATEST_RD_KEY,
|
||||
value=float(rd.toordinal())))
|
||||
s.commit()
|
||||
|
||||
@@ -134,11 +92,10 @@ class PaperRepo:
|
||||
"""The most recent rebalanced run_date. Read from state (set on every replace_positions, so an empty
|
||||
book — a full exit — is still recorded) and fall back to the max row date for legacy data."""
|
||||
v = s.scalar(select(PaperBookStateRow.value).where(
|
||||
PaperBookStateRow.venue == self._venue, PaperBookStateRow.key == self._LATEST_RD_KEY))
|
||||
PaperBookStateRow.key == self._LATEST_RD_KEY))
|
||||
if v is not None:
|
||||
return dt.date.fromordinal(int(v))
|
||||
return s.scalar(select(PaperPositionRow.run_date)
|
||||
.where(PaperPositionRow.venue == self._venue)
|
||||
.order_by(PaperPositionRow.run_date.desc()).limit(1))
|
||||
|
||||
def latest_positions(self) -> list[Position]:
|
||||
@@ -147,8 +104,7 @@ class PaperRepo:
|
||||
if rd is None:
|
||||
return []
|
||||
rows = s.scalars(select(PaperPositionRow)
|
||||
.where(PaperPositionRow.venue == self._venue,
|
||||
PaperPositionRow.run_date == rd)
|
||||
.where(PaperPositionRow.run_date == rd)
|
||||
.order_by(PaperPositionRow.sleeve, PaperPositionRow.symbol))
|
||||
return [Position(r.sleeve, r.symbol, r.qty, r.entry_price, r.entry_date) for r in rows]
|
||||
|
||||
@@ -161,20 +117,18 @@ class PaperRepo:
|
||||
rd = dt.date.fromisoformat(run_date)
|
||||
with Session(self._engine) as s:
|
||||
prev = s.scalar(select(PaperNavRow.run_date)
|
||||
.where(PaperNavRow.venue == self._venue, PaperNavRow.run_date < rd)
|
||||
.where(PaperNavRow.run_date < rd)
|
||||
.order_by(PaperNavRow.run_date.desc()).limit(1))
|
||||
if prev is None:
|
||||
# No nav anchor (e.g. positions persisted without a nav row) — fall back to the latest
|
||||
# position-row date strictly before, preserving the legacy behavior for those call paths.
|
||||
prev = s.scalar(select(PaperPositionRow.run_date)
|
||||
.where(PaperPositionRow.venue == self._venue,
|
||||
PaperPositionRow.run_date < rd)
|
||||
.where(PaperPositionRow.run_date < rd)
|
||||
.order_by(PaperPositionRow.run_date.desc()).limit(1))
|
||||
if prev is None:
|
||||
return []
|
||||
rows = s.scalars(select(PaperPositionRow)
|
||||
.where(PaperPositionRow.venue == self._venue,
|
||||
PaperPositionRow.run_date == prev)
|
||||
.where(PaperPositionRow.run_date == prev)
|
||||
.order_by(PaperPositionRow.sleeve, PaperPositionRow.symbol))
|
||||
return [Position(r.sleeve, r.symbol, r.qty, r.entry_price, r.entry_date) for r in rows]
|
||||
|
||||
@@ -183,10 +137,10 @@ class PaperRepo:
|
||||
rd = dt.date.fromisoformat(run_date)
|
||||
with Session(self._engine) as s:
|
||||
s.execute(delete(PaperShadowPositionRow).where(
|
||||
PaperShadowPositionRow.venue == self._venue, PaperShadowPositionRow.run_date == rd))
|
||||
PaperShadowPositionRow.run_date == rd))
|
||||
for p in positions:
|
||||
s.add(PaperShadowPositionRow(
|
||||
venue=self._venue, run_date=rd, sleeve=p.sleeve, symbol=p.symbol, qty=p.qty,
|
||||
run_date=rd, sleeve=p.sleeve, symbol=p.symbol, qty=p.qty,
|
||||
entry_price=p.entry_price, entry_date=p.entry_date, src_updated_at=at))
|
||||
s.commit()
|
||||
|
||||
@@ -195,14 +149,12 @@ class PaperRepo:
|
||||
rd = dt.date.fromisoformat(run_date)
|
||||
with Session(self._engine) as s:
|
||||
prev = s.scalar(select(PaperShadowPositionRow.run_date)
|
||||
.where(PaperShadowPositionRow.venue == self._venue,
|
||||
PaperShadowPositionRow.run_date < rd)
|
||||
.where(PaperShadowPositionRow.run_date < rd)
|
||||
.order_by(PaperShadowPositionRow.run_date.desc()).limit(1))
|
||||
if prev is None:
|
||||
return []
|
||||
rows = s.scalars(select(PaperShadowPositionRow)
|
||||
.where(PaperShadowPositionRow.venue == self._venue,
|
||||
PaperShadowPositionRow.run_date == prev)
|
||||
.where(PaperShadowPositionRow.run_date == prev)
|
||||
.order_by(PaperShadowPositionRow.sleeve, PaperShadowPositionRow.symbol))
|
||||
return [Position(r.sleeve, r.symbol, r.qty, r.entry_price, r.entry_date) for r in rows]
|
||||
|
||||
@@ -211,7 +163,6 @@ class PaperRepo:
|
||||
compare walks to surface within-sleeve weights. READ-ONLY."""
|
||||
with Session(self._engine) as s:
|
||||
rows = s.scalars(select(PaperShadowPositionRow.run_date)
|
||||
.where(PaperShadowPositionRow.venue == self._venue)
|
||||
.distinct().order_by(PaperShadowPositionRow.run_date.asc()))
|
||||
return [r.isoformat() for r in rows]
|
||||
|
||||
@@ -220,8 +171,7 @@ class PaperRepo:
|
||||
rd = dt.date.fromisoformat(run_date)
|
||||
with Session(self._engine) as s:
|
||||
rows = s.scalars(select(PaperShadowPositionRow)
|
||||
.where(PaperShadowPositionRow.venue == self._venue,
|
||||
PaperShadowPositionRow.run_date == rd)
|
||||
.where(PaperShadowPositionRow.run_date == rd)
|
||||
.order_by(PaperShadowPositionRow.sleeve, PaperShadowPositionRow.symbol))
|
||||
return [Position(r.sleeve, r.symbol, r.qty, r.entry_price, r.entry_date) for r in rows]
|
||||
|
||||
@@ -231,8 +181,7 @@ class PaperRepo:
|
||||
if rd is None:
|
||||
return 0.0
|
||||
w = s.scalar(select(PaperPositionRow.weight)
|
||||
.where(PaperPositionRow.venue == self._venue,
|
||||
PaperPositionRow.run_date == rd,
|
||||
.where(PaperPositionRow.run_date == rd,
|
||||
PaperPositionRow.sleeve == sleeve).limit(1))
|
||||
return float(w) if w is not None else 0.0
|
||||
|
||||
@@ -248,8 +197,7 @@ class PaperRepo:
|
||||
if rd is None:
|
||||
return out
|
||||
rows = s.execute(select(PaperPositionRow.sleeve, PaperPositionRow.weight)
|
||||
.where(PaperPositionRow.venue == self._venue,
|
||||
PaperPositionRow.run_date == rd,
|
||||
.where(PaperPositionRow.run_date == rd,
|
||||
PaperPositionRow.sleeve.in_(list(sleeves)))).all()
|
||||
for sleeve, w in rows: # all positions in a sleeve share its weight
|
||||
if w is not None:
|
||||
@@ -265,7 +213,7 @@ class PaperRepo:
|
||||
ts = _parse_ts(t.ts)
|
||||
rd = dt.date.fromisoformat(run_date) if run_date is not None else ts.date()
|
||||
s.add(PaperTradeRow(
|
||||
venue=self._venue, run_date=rd, ts=ts, sleeve=t.sleeve, symbol=t.symbol, side=t.side,
|
||||
run_date=rd, ts=ts, sleeve=t.sleeve, symbol=t.symbol, side=t.side,
|
||||
qty=t.qty, price=t.price, reason=t.reason))
|
||||
s.commit()
|
||||
|
||||
@@ -274,18 +222,16 @@ class PaperRepo:
|
||||
the same run_date thus replaces rather than appends (mirrors `replace_positions`)."""
|
||||
rd = dt.date.fromisoformat(run_date)
|
||||
with Session(self._engine) as s:
|
||||
s.execute(delete(PaperTradeRow).where(PaperTradeRow.venue == self._venue,
|
||||
PaperTradeRow.run_date == rd))
|
||||
s.execute(delete(PaperTradeRow).where(PaperTradeRow.run_date == rd))
|
||||
for t in trades:
|
||||
s.add(PaperTradeRow(
|
||||
venue=self._venue, run_date=rd, ts=_parse_ts(t.ts), sleeve=t.sleeve, symbol=t.symbol,
|
||||
run_date=rd, ts=_parse_ts(t.ts), sleeve=t.sleeve, symbol=t.symbol,
|
||||
side=t.side, qty=t.qty, price=t.price, reason=t.reason))
|
||||
s.commit()
|
||||
|
||||
def recent_trades(self, n: int) -> list[Trade]:
|
||||
with Session(self._engine) as s:
|
||||
rows = s.scalars(select(PaperTradeRow)
|
||||
.where(PaperTradeRow.venue == self._venue)
|
||||
.order_by(PaperTradeRow.ts.desc(), PaperTradeRow.id.desc())
|
||||
.limit(n))
|
||||
return [Trade(ts=r.ts.isoformat(), sleeve=r.sleeve, symbol=r.symbol, side=r.side,
|
||||
@@ -301,7 +247,6 @@ class PaperRepo:
|
||||
"""Cumulative realized PnL across all closed/reduced positions (0.0 if never set)."""
|
||||
with Session(self._engine) as s:
|
||||
rows = s.scalars(select(PaperBookStateRow).where(
|
||||
PaperBookStateRow.venue == self._venue,
|
||||
(PaperBookStateRow.key == self._REALIZED_KEY)
|
||||
| PaperBookStateRow.key.like(self._REALIZED_PREFIX + "%")))
|
||||
return float(sum(r.value for r in rows))
|
||||
@@ -315,7 +260,6 @@ class PaperRepo:
|
||||
display-only ledger. Retained for that ledger / back-compat; not the equity driver."""
|
||||
with Session(self._engine) as s:
|
||||
rows = s.scalars(select(PaperBookStateRow).where(
|
||||
PaperBookStateRow.venue == self._venue,
|
||||
(PaperBookStateRow.key == self._REALIZED_KEY)
|
||||
| PaperBookStateRow.key.like(self._REALIZED_PREFIX + "%")))
|
||||
total = 0.0
|
||||
@@ -327,16 +271,16 @@ class PaperRepo:
|
||||
def set_realized_for_run_date(self, run_date: str, value: float) -> None:
|
||||
"""Idempotent per run_date: set (overwrite) that run_date's realized contribution."""
|
||||
with Session(self._engine) as s:
|
||||
s.merge(PaperBookStateRow(venue=self._venue, key=self._REALIZED_PREFIX + run_date,
|
||||
s.merge(PaperBookStateRow(key=self._REALIZED_PREFIX + run_date,
|
||||
value=float(value)))
|
||||
s.commit()
|
||||
|
||||
def add_realized(self, delta: float) -> None:
|
||||
"""Add `delta` to the (non-dated) running realized total — read-modify-write under one session."""
|
||||
with Session(self._engine) as s:
|
||||
row = s.get(PaperBookStateRow, (self._venue, self._REALIZED_KEY))
|
||||
row = s.get(PaperBookStateRow, self._REALIZED_KEY)
|
||||
if row is None:
|
||||
s.add(PaperBookStateRow(venue=self._venue, key=self._REALIZED_KEY, value=float(delta)))
|
||||
s.add(PaperBookStateRow(key=self._REALIZED_KEY, value=float(delta)))
|
||||
else:
|
||||
row.value = float(row.value) + float(delta)
|
||||
s.commit()
|
||||
@@ -347,7 +291,7 @@ class PaperRepo:
|
||||
"""Idempotent per run_date: write (or overwrite) that date's equity row — a re-run replaces it."""
|
||||
rd = dt.date.fromisoformat(run_date)
|
||||
with Session(self._engine) as s:
|
||||
s.merge(PaperNavRow(venue=self._venue, run_date=rd, capital=float(capital),
|
||||
s.merge(PaperNavRow(run_date=rd, capital=float(capital),
|
||||
realized=float(realized), unrealized=float(unrealized),
|
||||
equity=float(equity), src_updated_at=at))
|
||||
s.commit()
|
||||
@@ -355,7 +299,7 @@ class PaperRepo:
|
||||
def nav_history(self) -> list[NavPoint]:
|
||||
"""The equity series ascending by run_date (one point per date)."""
|
||||
with Session(self._engine) as s:
|
||||
rows = s.scalars(select(PaperNavRow).where(PaperNavRow.venue == self._venue)
|
||||
rows = s.scalars(select(PaperNavRow)
|
||||
.order_by(PaperNavRow.run_date.asc()))
|
||||
return [NavPoint(run_date=r.run_date.isoformat(), equity=r.equity,
|
||||
realized=r.realized, unrealized=r.unrealized) for r in rows]
|
||||
@@ -365,14 +309,13 @@ class PaperRepo:
|
||||
rd = dt.date.fromisoformat(run_date)
|
||||
with Session(self._engine) as s:
|
||||
return s.scalar(select(PaperNavRow.equity)
|
||||
.where(PaperNavRow.venue == self._venue, PaperNavRow.run_date < rd)
|
||||
.where(PaperNavRow.run_date < rd)
|
||||
.order_by(PaperNavRow.run_date.desc()).limit(1))
|
||||
|
||||
def latest_nav_equity(self) -> float | None:
|
||||
"""The most recent paper_nav equity (for the live view's curve baseline)."""
|
||||
with Session(self._engine) as s:
|
||||
return s.scalar(select(PaperNavRow.equity)
|
||||
.where(PaperNavRow.venue == self._venue)
|
||||
.order_by(PaperNavRow.run_date.desc()).limit(1))
|
||||
|
||||
def latest_nav_run_date(self) -> str | None:
|
||||
@@ -380,18 +323,17 @@ class PaperRepo:
|
||||
incremental backfill resumes STRICTLY AFTER this date (the last fully-persisted date)."""
|
||||
with Session(self._engine) as s:
|
||||
rd = s.scalar(select(PaperNavRow.run_date)
|
||||
.where(PaperNavRow.venue == self._venue)
|
||||
.order_by(PaperNavRow.run_date.desc()).limit(1))
|
||||
return rd.isoformat() if rd is not None else None
|
||||
|
||||
# ---- paper_nav_summary: the MATERIALIZED headline (stats + curve), read 1 row not 3650 ----------
|
||||
def read_nav_summary(self, venue: str) -> dict[str, Any] | None:
|
||||
"""The MATERIALIZED headline nav-summary for `venue` (the stats + `since`/`has_record` + the
|
||||
pre-rendered curve SVG + the nav version `nav_run_date`), or None when none is stored yet. ONE 1-row
|
||||
read — NOT the full nav series. Read-only. The caller compares `nav_run_date` to the live latest
|
||||
def read_nav_summary(self) -> dict[str, Any] | None:
|
||||
"""The MATERIALIZED headline nav-summary for the (sole) bybit book — the stats + `since`/`has_record`
|
||||
+ the pre-rendered curve SVG + the nav version `nav_run_date` — or None when none is stored yet. ONE
|
||||
1-row read — NOT the full nav series. Read-only. The caller compares `nav_run_date` to the live latest
|
||||
run_date to decide fresh-vs-stale."""
|
||||
with Session(self._engine) as s:
|
||||
row = s.get(PaperNavSummaryRow, venue)
|
||||
row = s.get(PaperNavSummaryRow, 1)
|
||||
if row is None:
|
||||
return None
|
||||
return {"nav_run_date": row.nav_run_date, "final_equity": row.final_equity,
|
||||
@@ -399,13 +341,13 @@ class PaperRepo:
|
||||
"max_dd_pct": row.max_dd_pct, "since": row.since,
|
||||
"has_record": row.has_record, "curve_svg": row.curve_svg}
|
||||
|
||||
def upsert_nav_summary(self, venue: str, payload: dict[str, Any], *, at: dt.datetime) -> None:
|
||||
"""Idempotent per venue: write/overwrite the materialized headline nav-summary (the stats +
|
||||
`since`/`has_record` + rendered `curve_svg`, versioned by `payload['nav_run_date']`). Replaced on
|
||||
every refresh."""
|
||||
def upsert_nav_summary(self, payload: dict[str, Any], *, at: dt.datetime) -> None:
|
||||
"""Idempotent: write/overwrite the materialized headline nav-summary for the (sole) bybit book (the
|
||||
stats + `since`/`has_record` + rendered `curve_svg`, versioned by `payload['nav_run_date']`). Replaced
|
||||
on every refresh."""
|
||||
with Session(self._engine) as s:
|
||||
s.merge(PaperNavSummaryRow(
|
||||
venue=venue, nav_run_date=str(payload["nav_run_date"]),
|
||||
id=1, nav_run_date=str(payload["nav_run_date"]),
|
||||
final_equity=float(payload["final_equity"]),
|
||||
total_return_pct=float(payload["total_return_pct"]),
|
||||
sharpe=float(payload["sharpe"]), max_dd_pct=float(payload["max_dd_pct"]),
|
||||
@@ -427,23 +369,21 @@ class PaperRepo:
|
||||
return
|
||||
with Session(self._engine) as s:
|
||||
cur = s.scalar(select(PaperBookStateRow.value).where(
|
||||
PaperBookStateRow.venue == self._venue,
|
||||
PaperBookStateRow.key == self._LATEST_RD_KEY))
|
||||
cur_ord = int(cur) if cur is not None else None
|
||||
max_ord = cur_ord
|
||||
for run_date, (positions, weight_by_sleeve) in by_date.items():
|
||||
rd = dt.date.fromisoformat(run_date)
|
||||
s.execute(delete(PaperPositionRow).where(PaperPositionRow.venue == self._venue,
|
||||
PaperPositionRow.run_date == rd))
|
||||
s.execute(delete(PaperPositionRow).where(PaperPositionRow.run_date == rd))
|
||||
for p in positions:
|
||||
s.add(PaperPositionRow(
|
||||
venue=self._venue, run_date=rd, sleeve=p.sleeve, symbol=p.symbol, qty=p.qty,
|
||||
run_date=rd, sleeve=p.sleeve, symbol=p.symbol, qty=p.qty,
|
||||
entry_price=p.entry_price, entry_date=p.entry_date,
|
||||
weight=float(weight_by_sleeve.get(p.sleeve, 0.0)), src_updated_at=at))
|
||||
if max_ord is None or rd.toordinal() >= max_ord:
|
||||
max_ord = rd.toordinal()
|
||||
if max_ord is not None and (cur_ord is None or max_ord >= cur_ord):
|
||||
s.merge(PaperBookStateRow(venue=self._venue, key=self._LATEST_RD_KEY,
|
||||
s.merge(PaperBookStateRow(key=self._LATEST_RD_KEY,
|
||||
value=float(max_ord)))
|
||||
s.commit()
|
||||
|
||||
@@ -454,11 +394,10 @@ class PaperRepo:
|
||||
with Session(self._engine) as s:
|
||||
for run_date, trades in by_date.items():
|
||||
rd = dt.date.fromisoformat(run_date)
|
||||
s.execute(delete(PaperTradeRow).where(PaperTradeRow.venue == self._venue,
|
||||
PaperTradeRow.run_date == rd))
|
||||
s.execute(delete(PaperTradeRow).where(PaperTradeRow.run_date == rd))
|
||||
for t in trades:
|
||||
s.add(PaperTradeRow(
|
||||
venue=self._venue, run_date=rd, ts=_parse_ts(t.ts), sleeve=t.sleeve,
|
||||
run_date=rd, ts=_parse_ts(t.ts), sleeve=t.sleeve,
|
||||
symbol=t.symbol, side=t.side, qty=t.qty, price=t.price, reason=t.reason))
|
||||
s.commit()
|
||||
|
||||
@@ -468,7 +407,7 @@ class PaperRepo:
|
||||
return
|
||||
with Session(self._engine) as s:
|
||||
for run_date, value in by_date.items():
|
||||
s.merge(PaperBookStateRow(venue=self._venue, key=self._REALIZED_PREFIX + run_date,
|
||||
s.merge(PaperBookStateRow(key=self._REALIZED_PREFIX + run_date,
|
||||
value=float(value)))
|
||||
s.commit()
|
||||
|
||||
@@ -480,7 +419,7 @@ class PaperRepo:
|
||||
with Session(self._engine) as s:
|
||||
for r in rows:
|
||||
s.merge(PaperNavRow(
|
||||
venue=self._venue, run_date=dt.date.fromisoformat(r["run_date"]),
|
||||
run_date=dt.date.fromisoformat(r["run_date"]),
|
||||
capital=float(r["capital"]), realized=float(r["realized"]),
|
||||
unrealized=float(r["unrealized"]),
|
||||
equity=float(r["equity"]), src_updated_at=r["at"]))
|
||||
@@ -494,10 +433,10 @@ class PaperRepo:
|
||||
for run_date, positions in by_date.items():
|
||||
rd = dt.date.fromisoformat(run_date)
|
||||
s.execute(delete(PaperShadowPositionRow).where(
|
||||
PaperShadowPositionRow.venue == self._venue, PaperShadowPositionRow.run_date == rd))
|
||||
PaperShadowPositionRow.run_date == rd))
|
||||
for p in positions:
|
||||
s.add(PaperShadowPositionRow(
|
||||
venue=self._venue, run_date=rd, sleeve=p.sleeve, symbol=p.symbol, qty=p.qty,
|
||||
run_date=rd, sleeve=p.sleeve, symbol=p.symbol, qty=p.qty,
|
||||
entry_price=p.entry_price, entry_date=p.entry_date, src_updated_at=at))
|
||||
s.commit()
|
||||
|
||||
@@ -508,7 +447,7 @@ class PaperRepo:
|
||||
return
|
||||
with Session(self._engine) as s:
|
||||
for run_date, sleeve, ret in rows:
|
||||
s.merge(PaperSleeveRetRow(venue=self._venue, run_date=dt.date.fromisoformat(run_date),
|
||||
s.merge(PaperSleeveRetRow(run_date=dt.date.fromisoformat(run_date),
|
||||
sleeve=sleeve, ret=float(ret), src_updated_at=at))
|
||||
s.commit()
|
||||
|
||||
@@ -517,8 +456,7 @@ class PaperRepo:
|
||||
rd = dt.date.fromisoformat(run_date)
|
||||
with Session(self._engine) as s:
|
||||
rows = s.scalars(select(PaperPositionRow)
|
||||
.where(PaperPositionRow.venue == self._venue,
|
||||
PaperPositionRow.run_date == rd)
|
||||
.where(PaperPositionRow.run_date == rd)
|
||||
.order_by(PaperPositionRow.sleeve, PaperPositionRow.symbol))
|
||||
return [Position(r.sleeve, r.symbol, r.qty, r.entry_price, r.entry_date) for r in rows]
|
||||
|
||||
@@ -527,8 +465,7 @@ class PaperRepo:
|
||||
rd = dt.date.fromisoformat(run_date)
|
||||
with Session(self._engine) as s:
|
||||
rows = s.scalars(select(PaperTradeRow)
|
||||
.where(PaperTradeRow.venue == self._venue,
|
||||
PaperTradeRow.run_date == rd)
|
||||
.where(PaperTradeRow.run_date == rd)
|
||||
.order_by(PaperTradeRow.ts.asc(), PaperTradeRow.id.asc()))
|
||||
return [Trade(ts=r.ts.isoformat(), sleeve=r.sleeve, symbol=r.symbol, side=r.side,
|
||||
qty=r.qty, price=r.price, reason=r.reason) for r in rows]
|
||||
@@ -538,7 +475,7 @@ class PaperRepo:
|
||||
"""Idempotent per (run_date, sleeve): write/overwrite that date's per-sleeve simulated return."""
|
||||
rd = dt.date.fromisoformat(run_date)
|
||||
with Session(self._engine) as s:
|
||||
s.merge(PaperSleeveRetRow(venue=self._venue, run_date=rd, sleeve=sleeve, ret=float(ret),
|
||||
s.merge(PaperSleeveRetRow(run_date=rd, sleeve=sleeve, ret=float(ret),
|
||||
src_updated_at=at))
|
||||
s.commit()
|
||||
|
||||
@@ -549,8 +486,7 @@ class PaperRepo:
|
||||
out: dict[str, dict[int, float]] = {}
|
||||
with Session(self._engine) as s:
|
||||
rows = s.scalars(select(PaperSleeveRetRow)
|
||||
.where(PaperSleeveRetRow.venue == self._venue,
|
||||
PaperSleeveRetRow.run_date < cutoff)
|
||||
.where(PaperSleeveRetRow.run_date < cutoff)
|
||||
.order_by(PaperSleeveRetRow.run_date.asc()))
|
||||
for r in rows:
|
||||
out.setdefault(r.sleeve, {})[r.run_date.toordinal()] = r.ret
|
||||
@@ -577,12 +513,36 @@ class PaperRepo:
|
||||
out.setdefault(r.sleeve, {})[r.run_date.toordinal()] = r.ret
|
||||
return out
|
||||
|
||||
# ---- sim_curve_ret: the PRECOMPUTED recompute-replay (IBKR) book curve /paper/sim reads ------------
|
||||
def upsert_sim_curve_ret(self, strategy_id: str, run_date: str, ret: float, *, at: dt.datetime) -> None:
|
||||
"""Idempotent per (strategy_id, run_date): write/overwrite that recompute-replay book's (multistrat)
|
||||
per-day honest-cost return. `_persist_track_backtest_ref` writes the full inception series here
|
||||
from the SAME rows it derives the reconciliation-gate ref from, so /paper/sim NEVER re-runs an IBKR
|
||||
replay on a web request — mirrors `upsert_bybit_sleeve_ret`'s pattern for the Bybit-book sim."""
|
||||
rd = dt.date.fromisoformat(run_date)
|
||||
with Session(self._engine) as s:
|
||||
s.merge(SimCurveRetRow(strategy_id=strategy_id, run_date=rd, ret=float(ret), src_updated_at=at))
|
||||
s.commit()
|
||||
|
||||
def sim_curve_returns(self, strategy_id: str) -> dict[int, float]:
|
||||
"""The FULL precomputed per-day return series for one recompute-replay book (`strategy_id`), as
|
||||
{epoch_day: ret} (epoch_day = date.toordinal()) — the shape /paper/sim's IBKR curve reads. Read-only,
|
||||
scoped to this one strategy (unlike `bybit_sleeve_returns`, this table holds several IBKR books)."""
|
||||
out: dict[int, float] = {}
|
||||
with Session(self._engine) as s:
|
||||
rows = s.scalars(select(SimCurveRetRow)
|
||||
.where(SimCurveRetRow.strategy_id == strategy_id)
|
||||
.order_by(SimCurveRetRow.run_date.asc()))
|
||||
for r in rows:
|
||||
out[r.run_date.toordinal()] = r.ret
|
||||
return out
|
||||
|
||||
# ---- compare_measured_cache: the PRECOMPUTED measured-cost COMPARE artifact ----------------------
|
||||
def upsert_compare_measured_cache(self, key: str, payload: dict[str, Any], *,
|
||||
at: dt.datetime) -> None:
|
||||
"""Idempotent per key: write/overwrite the precomputed measured-cost COMPARE payload. The own-memory
|
||||
`fxhnt compare-measured-precompute` Job writes it; the cockpit ONLY reads it (the heavy per-coin
|
||||
measured-cost path OOM-killed the 4Gi dashboard when run live). Not venue-scoped (one global artifact)."""
|
||||
measured-cost path OOM-killed the 4Gi dashboard when run live). Not book-scoped (one global artifact)."""
|
||||
with Session(self._engine) as s:
|
||||
s.merge(CompareMeasuredCacheRow(key=key, payload=payload, computed_at=at))
|
||||
s.commit()
|
||||
@@ -602,7 +562,7 @@ class PaperRepo:
|
||||
own-memory `fxhnt bybit-spread-refresh` Job writes it (median of several samples a few seconds apart);
|
||||
the measured-cost precompute READS it to charge the Bybit liquid book the REAL spread instead of the
|
||||
high/low Corwin–Schultz proxy. Defensive: non-finite / negative spreads are skipped (never stored as a
|
||||
garbage cost). Not venue-scoped (one global per-coin table)."""
|
||||
garbage cost). Not book-scoped (one global per-coin table)."""
|
||||
import math as _math
|
||||
|
||||
with Session(self._engine) as s:
|
||||
|
||||
@@ -1,55 +0,0 @@
|
||||
"""Parquet SIDE-STORE for the intraday `worst_basis` panel: {symbol: {epoch_day: worst_basis}}.
|
||||
|
||||
Deliberately a standalone Parquet file (path from settings, default under the surfer data dir) and NOT the
|
||||
DuckDB warehouse — the warehouse is single-writer and the nightly ingest/book already hold it; a separate
|
||||
Parquet file sidesteps that contention entirely. I/O goes through DuckDB (already a core dependency, no new
|
||||
package): the panel is stored LONG (symbol, epoch_day, worst_basis), written atomically, read back into the
|
||||
nested dict the carry accounting consumes.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
import duckdb
|
||||
|
||||
_COLS = ("symbol", "epoch_day", "worst_basis")
|
||||
|
||||
|
||||
def write_worst_basis(path: str, data: dict[str, dict[int, float]]) -> None:
|
||||
"""Write `{symbol: {epoch_day: worst_basis}}` to `path` as a LONG Parquet (symbol, epoch_day,
|
||||
worst_basis). Overwrites any existing file atomically (write tmp, then replace). An empty panel still
|
||||
produces a valid (0-row) Parquet so the loader round-trips to {}."""
|
||||
os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
|
||||
rows: list[tuple[str, int, float]] = [
|
||||
(sym, int(day), float(val))
|
||||
for sym, days in data.items()
|
||||
for day, val in days.items()
|
||||
]
|
||||
tmp = path + ".tmp"
|
||||
con = duckdb.connect()
|
||||
try:
|
||||
con.execute(
|
||||
"CREATE TABLE wb (symbol VARCHAR, epoch_day BIGINT, worst_basis DOUBLE)")
|
||||
if rows:
|
||||
con.executemany("INSERT INTO wb VALUES (?, ?, ?)", rows)
|
||||
con.execute("COPY wb TO ? (FORMAT PARQUET)", [tmp])
|
||||
finally:
|
||||
con.close()
|
||||
os.replace(tmp, path)
|
||||
|
||||
|
||||
def load_worst_basis(path: str) -> dict[str, dict[int, float]]:
|
||||
"""Read the side-store back into `{symbol: {epoch_day: worst_basis}}` (native int days, float values).
|
||||
A missing file → {} (the carry accounting then falls back to the daily-close basis everywhere)."""
|
||||
if not os.path.exists(path):
|
||||
return {}
|
||||
con = duckdb.connect()
|
||||
try:
|
||||
result = con.execute(
|
||||
f"SELECT symbol, epoch_day, worst_basis FROM read_parquet('{path}')").fetchall()
|
||||
finally:
|
||||
con.close()
|
||||
out: dict[str, dict[int, float]] = {}
|
||||
for sym, day, val in result:
|
||||
out.setdefault(str(sym), {})[int(day)] = float(val)
|
||||
return out
|
||||
@@ -1,64 +0,0 @@
|
||||
"""PIT store for frozen XSP option daily bars — the deterministic substrate the VRP recompute reads. OPRA is
|
||||
hit only when FILLING this table (asset/backfill); VrpStrategy never calls OPRA. Same freeze discipline as the
|
||||
equity Yahoo-PIT snapshot."""
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
|
||||
from sqlalchemy import create_engine, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from fxhnt.adapters.persistence.cockpit_models import CockpitBase, XspOptionBarRow
|
||||
from fxhnt.domain.models import OptionContract
|
||||
|
||||
|
||||
class XspOptionBarsRepo:
|
||||
def __init__(self, dsn: str) -> None:
|
||||
kw = {} if dsn.startswith("postgresql") else {"connect_args": {"check_same_thread": False}}
|
||||
self._engine = create_engine(dsn, **kw)
|
||||
|
||||
def migrate(self) -> None:
|
||||
CockpitBase.metadata.create_all(self._engine)
|
||||
|
||||
def upsert_bars(self, rows: list[dict], at: dt.datetime) -> None:
|
||||
with Session(self._engine) as s:
|
||||
for r in rows:
|
||||
obj = XspOptionBarRow(
|
||||
osi_symbol=r["osi_symbol"], date=dt.date.fromisoformat(r["date"]),
|
||||
expiry=dt.date.fromisoformat(r["expiry"]), strike=float(r["strike"]),
|
||||
right=r["right"], close=float(r["close"]), src_updated_at=at)
|
||||
s.merge(obj) # merge = insert-or-overwrite on the (osi_symbol, date) PK
|
||||
s.commit()
|
||||
|
||||
def bars_for(self, osi_symbols: list[str], start: str, end: str) -> dict[str, dict[str, float]]:
|
||||
if not osi_symbols:
|
||||
return {}
|
||||
lo, hi = dt.date.fromisoformat(start), dt.date.fromisoformat(end)
|
||||
out: dict[str, dict[str, float]] = {}
|
||||
with Session(self._engine) as s:
|
||||
q = select(XspOptionBarRow).where(
|
||||
XspOptionBarRow.osi_symbol.in_(osi_symbols),
|
||||
XspOptionBarRow.date >= lo, XspOptionBarRow.date <= hi)
|
||||
for row in s.scalars(q):
|
||||
out.setdefault(row.osi_symbol, {})[row.date.isoformat()] = row.close
|
||||
return out
|
||||
|
||||
def trading_days(self, after: str | None) -> list[str]:
|
||||
"""Ascending distinct dates present in the table, strictly after `after` (or all if None)."""
|
||||
from sqlalchemy import distinct
|
||||
with Session(self._engine) as s:
|
||||
q = select(distinct(XspOptionBarRow.date)).order_by(XspOptionBarRow.date)
|
||||
days = [d.isoformat() for d in s.scalars(q)]
|
||||
return [d for d in days if after is None or d > after]
|
||||
|
||||
def chain_asof(self, date: str, dte_lo: int, dte_hi: int, right: str = "P") -> list[OptionContract]:
|
||||
d = dt.date.fromisoformat(date)
|
||||
exp_lo, exp_hi = d + dt.timedelta(days=dte_lo), d + dt.timedelta(days=dte_hi)
|
||||
with Session(self._engine) as s:
|
||||
q = select(XspOptionBarRow).where(
|
||||
XspOptionBarRow.date == d, XspOptionBarRow.right == right,
|
||||
XspOptionBarRow.expiry >= exp_lo, XspOptionBarRow.expiry <= exp_hi
|
||||
).order_by(XspOptionBarRow.strike) # deterministic strike order (no ORDER BY = arbitrary
|
||||
# row order on Postgres — the ATM pairing below relies on stable iteration)
|
||||
return [OptionContract(osi_symbol=r.osi_symbol, expiry=r.expiry, strike=r.strike, right=r.right)
|
||||
for r in s.scalars(q)]
|
||||
@@ -25,7 +25,7 @@ def get_feature_store(settings: "Settings") -> "FeatureStore":
|
||||
if settings.feature_store == "duckdb":
|
||||
return DuckDbFeatureStore(settings.warehouse_path)
|
||||
# `warehouse_table` (env FXHNT_WAREHOUSE_TABLE) selects WHICH table this store reads/writes — the
|
||||
# single knob that points the whole book/sim/edges stack at Binance `features` (default) or Bybit
|
||||
# `bybit_features`. Every get_feature_store consumer inherits it, so flipping the env runs all edges
|
||||
# single knob that points the whole book/sim/edges stack at the legacy `features` table (default) or
|
||||
# Bybit `bybit_features`. Every get_feature_store consumer inherits it, so flipping the env runs all edges
|
||||
# on Bybit data with no per-site change. DuckDb path above is unaffected (single warehouse file).
|
||||
return TimescaleFeatureStore(settings.operational_dsn, table=settings.warehouse_table)
|
||||
|
||||
@@ -122,7 +122,7 @@ class DuckDbFeatureStore:
|
||||
|
||||
def crypto_funding_panel(self, *, suffix: str = "USDT") -> dict[str, dict[int, float]]:
|
||||
"""{symbol: {epoch_day: daily_funding}} for every crypto symbol ending in `suffix` that has a
|
||||
'funding' feature (Binance perp funding history, ingested daily). Mirror of `crypto_close_panel`;
|
||||
'funding' feature (legacy-venue perp funding history, ingested daily). Mirror of `crypto_close_panel`;
|
||||
ts (epoch-seconds) folded to epoch-days (ts // 86400). Empty {} when no funding ingested yet."""
|
||||
rows = self._con.execute(
|
||||
"SELECT symbol, ts, value FROM features WHERE feature = 'funding' AND symbol LIKE ?",
|
||||
@@ -135,7 +135,7 @@ class DuckDbFeatureStore:
|
||||
|
||||
def crypto_spot_panel(self, *, suffix: str = "USDT") -> dict[str, dict[int, float]]:
|
||||
"""{symbol: {epoch_day: spot_close}} for every crypto symbol ending in `suffix` that has a
|
||||
'spot_close' feature (Binance SPOT daily closes — the long leg of the delta-neutral carry).
|
||||
'spot_close' feature (legacy-venue SPOT daily closes — the long leg of the delta-neutral carry).
|
||||
Mirror of crypto_close_panel/crypto_funding_panel; ts folded to epoch-days. {} when none ingested."""
|
||||
rows = self._con.execute(
|
||||
"SELECT symbol, ts, value FROM features WHERE feature = 'spot_close' AND symbol LIKE ?",
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user