Compare commits
926 Commits
chore/obse
...
master
| Author | SHA1 | Date | |
|---|---|---|---|
| c58a514d77 | |||
| 9e3c9cd649 | |||
| 341c911b0f | |||
| ff360a7799 | |||
| 10f3ae1ace | |||
| 88489fc5db | |||
| 9989220440 | |||
| 72e422294d | |||
| 6539beec8a | |||
| 6783932860 | |||
| 30ff8bddab | |||
| fc4f3c1f0e | |||
| 74ea6b2d93 | |||
| d58c8dc04d | |||
| c51e45806c | |||
| c6235140ff | |||
| 2901bee4d3 | |||
| f1f7f0e1a5 | |||
| 3f94328fde | |||
| 0da0d0fc02 | |||
| f938366c6a | |||
| 0d8ce554a3 | |||
| 7da51863a1 | |||
| 05f6d624c0 | |||
| 40c0b187a7 | |||
| ee077a2cc1 | |||
| e00bac067f | |||
| 9d35df6bfc | |||
| da28c049f7 | |||
| 6677f73a64 | |||
| 330bb56b52 | |||
| 9f2b47a789 | |||
| 80bf243023 | |||
| 6cf19ec379 | |||
| 7d708b03b1 | |||
| 7318f77054 | |||
| 6d8bcc3ae7 | |||
| 0069b75d0e | |||
| 603aaaabff | |||
| 9b2b25433e | |||
| 6e9dfec067 | |||
| c98b8d4b49 | |||
| fc8b1b7849 | |||
| 5d15b0f801 | |||
| bf99ce2bd1 | |||
| 2f957bb6a6 | |||
| aa85ffd3a3 | |||
| baf6ddbe9e | |||
| c7a40b70ec | |||
| 92b0871050 | |||
| adf3edb054 | |||
| fc663215ef | |||
| 5b5fbb0e0b | |||
| eb6ecf8272 | |||
| b71cd6b3b0 | |||
| 59cc06e594 | |||
| a263b676de | |||
| fb14ddc75d | |||
| 8df02c2198 | |||
| 39b313800d | |||
| a16b341d90 | |||
| 9baf1f8115 | |||
| 420cffa93c | |||
| 90fcbfde5c | |||
| 9080dffed5 | |||
| 3a26b87f15 | |||
| f325a37070 | |||
| dd527e6658 | |||
| fb4cb49b18 | |||
| 5d4ddcfb39 | |||
| f2730d195d | |||
| 46401afbd1 | |||
| 3d1c325d2c | |||
| 8bbe7fb123 | |||
| 6a109bfec7 | |||
| 9e94b3b645 | |||
| a455dc0fea | |||
| f8831a7502 | |||
| 1ec5afa0c4 | |||
| dcf333d97e | |||
| 0ed2cfdff3 | |||
| a00001e826 | |||
| 4e7adf42f2 | |||
| 1ecf3983d9 | |||
| 06156bdb1a | |||
| 3461f0e331 | |||
| 5b016b9c4d | |||
| 04305a1c72 | |||
| 25e3d7ca85 | |||
| 18cdf3f704 | |||
| 22a1a404ba | |||
| fa6c0c11f3 | |||
| dce4a72c45 | |||
| 2986c0d940 | |||
| aacc7eff8f | |||
| 2438294f48 | |||
| 33c1f62918 | |||
| 6ff522cf7c | |||
| 1240113d33 | |||
| 05c209b64c | |||
| e67c6e1612 | |||
| ce1c978d4b | |||
| 9396f53669 | |||
| 0d3fca6b0c | |||
| 06d12fd4e3 | |||
| 30e62675fc | |||
| 17c36d3afe | |||
| edeabc611b | |||
| 3844a9774d | |||
| de6f738719 | |||
| bd8ef0fed8 | |||
| c182c0209a | |||
| 543129a248 | |||
|
|
d66d11325b | ||
|
|
38b8ff26d2 | ||
|
|
8d66f37981 | ||
|
|
713424a89d | ||
|
|
8e0016a302 | ||
|
|
f6f1c9082b | ||
|
|
2055d0366f | ||
|
|
ebd31d7c07 | ||
|
|
c71a1d8a06 | ||
|
|
9e3ff64e1d | ||
|
|
ecc51cb2db | ||
|
|
5d584c5dbb | ||
|
|
fd2316b1e8 | ||
|
|
b1ceee39a5 | ||
|
|
8bd87e8ab1 | ||
|
|
ce464f82f4 | ||
|
|
c5758e30bb | ||
|
|
bd5f820697 | ||
|
|
5f5c032f24 | ||
|
|
d5f871af0a | ||
|
|
a71737eeca | ||
|
|
048e5ca26d | ||
|
|
9564305920 | ||
|
|
07e6091647 | ||
|
|
70d584251f | ||
|
|
d96b1ea666 | ||
|
|
cd5203a33c | ||
|
|
ee8694dffb | ||
|
|
2a7492fd07 | ||
|
|
875294d679 | ||
|
|
515117de2c | ||
|
|
5f520165a4 | ||
|
|
0c4ffc11fc | ||
|
|
c2dd057ce1 | ||
|
|
1ad41f5145 | ||
|
|
201bd3f217 | ||
|
|
514f17df6a | ||
|
|
797462bdf7 | ||
|
|
dde51b8641 | ||
|
|
3f2c114241 | ||
|
|
1f260b9ddf | ||
|
|
1322706080 | ||
|
|
0d3e06bb53 | ||
|
|
90cc809456 | ||
|
|
e5c2ac003a | ||
|
|
a16ccc4406 | ||
|
|
86e0035d46 | ||
|
|
c4102c7319 | ||
|
|
bc83fbdbe7 | ||
|
|
6ac1325d36 | ||
|
|
f40c44c873 | ||
|
|
2bae84e8c6 | ||
|
|
8b887af153 | ||
|
|
98dda4e835 | ||
|
|
c1d1cb80fb | ||
|
|
d2353ced52 | ||
|
|
b5a2b48de0 | ||
|
|
756b3bab47 | ||
|
|
b1cfdf1a7f | ||
|
|
6abf2a8289 | ||
|
|
efc304e3f9 | ||
|
|
242277db30 | ||
|
|
5c5171efc5 | ||
|
|
a27fc717e6 | ||
|
|
41913da027 | ||
|
|
74ae34a28b | ||
|
|
c5f4405570 | ||
|
|
5874c2d279 | ||
|
|
20437d7f67 | ||
|
|
b6e3f24ed2 | ||
|
|
dd55f331c3 | ||
|
|
91f25215f3 | ||
|
|
d1cba19133 | ||
|
|
255d5baa81 | ||
|
|
65a9ae1f90 | ||
|
|
5019e9ffbb | ||
|
|
edf7e41527 | ||
|
|
4dbaf90609 | ||
|
|
19e669664b | ||
|
|
ec972017ce | ||
|
|
1958315344 | ||
|
|
595dc5da0e | ||
|
|
97b530de10 | ||
|
|
206cde3ee5 | ||
|
|
cd6e554534 | ||
|
|
516499aa17 | ||
|
|
bcf58e703d | ||
|
|
67ca804395 | ||
|
|
0d191badac | ||
|
|
c5a5a0ba24 | ||
|
|
09ebc4c4e0 | ||
|
|
c493516c8f | ||
|
|
29b6c19167 | ||
|
|
8697e1c0ba | ||
|
|
38f5dfa784 | ||
|
|
768ce4a068 | ||
|
|
0a3308c3a6 | ||
|
|
33318eca7c | ||
|
|
42b5fe322f | ||
|
|
8af24f96c2 | ||
|
|
edf7a659f6 | ||
|
|
35bf7a25de | ||
|
|
c3c627ebfd | ||
|
|
fe9509c189 | ||
|
|
55c8644b51 | ||
|
|
524c43eebd | ||
|
|
29f60a8584 | ||
|
|
d25ff9659a | ||
|
|
4a6b5b8f2f | ||
|
|
ff7a81ceeb | ||
|
|
784cd27949 | ||
|
|
3532671892 | ||
|
|
49b6a102d8 | ||
|
|
2cbd2736e5 | ||
|
|
8a99aff76a | ||
|
|
5c0b9ddb27 | ||
|
|
9c51b754ba | ||
|
|
86db827740 | ||
|
|
d399982cd1 | ||
|
|
bb61323904 | ||
|
|
39fa034f49 | ||
|
|
fd5d97bc94 | ||
|
|
43afd24062 | ||
|
|
129409a282 | ||
|
|
7e7a657090 | ||
|
|
70708ca8a7 | ||
|
|
de9beb89ac | ||
|
|
01e437816b | ||
|
|
d070ec9127 | ||
|
|
6c9d43774e | ||
|
|
515dec4b55 | ||
|
|
3302ca5d04 | ||
|
|
afe488e6aa | ||
|
|
95d47aaa05 | ||
|
|
2677bcd02a | ||
|
|
94ad91977e | ||
|
|
649ea25000 | ||
|
|
f38b5ad55b | ||
|
|
375cd108bc | ||
|
|
68970a793f | ||
|
|
ea66df5d7f | ||
|
|
62357d6cc2 | ||
|
|
badded008c | ||
|
|
0be7eb2fc6 | ||
|
|
5fc25a428b | ||
|
|
994bb59641 | ||
|
|
86aba147c9 | ||
|
|
4fd0f9454d | ||
|
|
af68419909 | ||
|
|
fb9564c798 | ||
|
|
49e30b17ea | ||
|
|
4cf6de17ee | ||
|
|
d7a9c32690 | ||
|
|
a213edb800 | ||
|
|
47419b398b | ||
|
|
14f9894482 | ||
|
|
5a3fca0de3 | ||
|
|
810ebdc105 | ||
|
|
9a9fc431a2 | ||
|
|
22e35b39af | ||
|
|
31346cf4c5 | ||
|
|
976bd0089b | ||
|
|
c6923d053c | ||
|
|
ae3b906f64 | ||
|
|
b0901a8b1d | ||
|
|
dbc37900cf | ||
|
|
83a0e29894 | ||
|
|
c38f3372ab | ||
|
|
4203c1b882 | ||
|
|
bb2a3859e2 | ||
|
|
fa70f06416 | ||
|
|
c60b8f82ae | ||
|
|
c2d2557372 | ||
|
|
cf946f04f2 | ||
|
|
d035ff24b1 | ||
|
|
dc5bdc929d | ||
|
|
6ad4773376 | ||
|
|
b8bd43480d | ||
|
|
1b3dd58610 | ||
|
|
ce6d9eadb4 | ||
|
|
7ea817cba8 | ||
|
|
35302b4ba5 | ||
|
|
49fda9a02a | ||
|
|
972332ba52 | ||
|
|
0fe2448dbb | ||
|
|
ab4e6d4c42 | ||
|
|
ccfa40c3fe | ||
|
|
c9b04e9e31 | ||
|
|
641e11bc6f | ||
|
|
0208967825 | ||
|
|
c4fb3f73eb | ||
|
|
8d0c6c67f9 | ||
|
|
4c49d94b45 | ||
|
|
fe8d8c1ff2 | ||
|
|
e0db586bbf | ||
|
|
03e12004aa | ||
|
|
3c13ec8f5b | ||
|
|
9cdcb7b80b | ||
|
|
4cc401f83b | ||
|
|
5d7aa3a3f8 | ||
|
|
2cea5be70c | ||
|
|
e5e9f7e8de | ||
|
|
1c30e69a50 | ||
|
|
1ac3627c80 | ||
|
|
8da219dfa3 | ||
|
|
938ea3f0cd | ||
|
|
4114f647d2 | ||
|
|
a68a3ff929 | ||
|
|
edeb40b541 | ||
|
|
4a1d91885d | ||
|
|
31bc726053 | ||
|
|
689fd3672e | ||
|
|
4630491454 | ||
|
|
51369199dc | ||
|
|
441b4b381a | ||
|
|
8ec63cecb5 | ||
|
|
c0401f10e3 | ||
|
|
3bdb9c5d32 | ||
|
|
08477ac5f1 | ||
|
|
884183a959 | ||
|
|
58d16db332 | ||
|
|
55cd3af22b | ||
|
|
b73c3a5228 | ||
|
|
bcbeb44552 | ||
|
|
18d8d1e9cc | ||
|
|
d43ecce814 | ||
|
|
e8f0dd6c4e | ||
|
|
e5e17462e7 | ||
|
|
391400f233 | ||
|
|
bfc7bbdc4b | ||
|
|
9f39cc3633 | ||
|
|
da91c412ab | ||
|
|
7c05189aa2 | ||
|
|
732d6e9792 | ||
|
|
609fa1b7a5 | ||
|
|
b21cab0984 | ||
|
|
58e39955ba | ||
|
|
26e80df075 | ||
|
|
c832e94197 | ||
|
|
185e504028 | ||
|
|
59439bfcd9 | ||
|
|
1c9cec8e29 | ||
|
|
4837a1fe8e | ||
|
|
8e54d6dc1d | ||
|
|
5953272f88 | ||
|
|
e6713367b1 | ||
|
|
730d3dbf7d | ||
|
|
503cc47612 | ||
|
|
a88d017170 | ||
|
|
195955b2c8 | ||
|
|
5a601cad4f | ||
|
|
0a126754eb | ||
|
|
3488c69ff2 | ||
|
|
2b02942ef9 | ||
|
|
cec036e453 | ||
|
|
146e1733f3 | ||
|
|
7dd4a813e4 | ||
|
|
0c97f24ab8 | ||
|
|
e15b648f24 | ||
|
|
6899c2787f | ||
|
|
5bb652132b | ||
|
|
94674ddfe6 | ||
|
|
05f0b1439e | ||
|
|
c39d80745c | ||
|
|
e508f71aa5 | ||
|
|
110d1bd3b7 | ||
|
|
f6b42aa740 | ||
|
|
370c0be80b | ||
|
|
371f795161 | ||
|
|
9f0322c177 | ||
|
|
b3b3b71341 | ||
|
|
397f734f7a | ||
|
|
b791ecf7c3 | ||
|
|
1d2338a7df | ||
|
|
0e78a88c5c | ||
|
|
5a93176f12 | ||
|
|
6639db22fe | ||
|
|
33e79fe4a1 | ||
|
|
2713d739ca | ||
|
|
7515f088ec | ||
|
|
801d240c67 | ||
|
|
e00d2b2ad7 | ||
|
|
6f99901ace | ||
|
|
c19bdd1e8d | ||
|
|
72ba666dd2 | ||
|
|
a16c8e2eb0 | ||
|
|
7456e987fe | ||
|
|
02317a14ab | ||
|
|
e96f4c7eb0 | ||
|
|
5ca03bc704 | ||
|
|
140395ed87 | ||
|
|
45df3f2e5d | ||
|
|
6d4316c882 | ||
|
|
f2947c53e1 | ||
|
|
a5a78ffac9 | ||
|
|
65569d593f | ||
|
|
f0bd4c4405 | ||
|
|
8d904f521d | ||
|
|
6c5daefcbf | ||
|
|
fc0c08d66a | ||
|
|
e0225e3379 | ||
|
|
56f37aeed3 | ||
|
|
84c9fcd146 | ||
|
|
497d4022db | ||
|
|
5893604158 | ||
|
|
7b886f0a05 | ||
|
|
9ec8eaab9c | ||
|
|
3b43856621 | ||
|
|
3a91373ac4 | ||
|
|
a1190dddca | ||
|
|
178f0388d9 | ||
|
|
a8f1fa99f4 | ||
|
|
0817067dcf | ||
|
|
5eef3068b4 | ||
|
|
1adcd40ee0 | ||
|
|
f821929ddf | ||
|
|
cba9f6531b | ||
|
|
5a99bb6329 | ||
|
|
889ba74852 | ||
|
|
7e9f8520b9 | ||
|
|
7137d62bfa | ||
|
|
5c7a5c5c49 | ||
|
|
1745249cf4 | ||
|
|
a7533980da | ||
|
|
b1c736b35f | ||
|
|
9a2859ea88 | ||
|
|
a957821e55 | ||
|
|
965f286ca2 | ||
|
|
4bae18b4b7 | ||
|
|
07bc9a2da8 | ||
|
|
04723ba997 | ||
|
|
33c356a8df | ||
|
|
5af0668b78 | ||
|
|
a224127ab2 | ||
|
|
d07e70b5a5 | ||
|
|
1603114260 | ||
|
|
423aad7911 | ||
|
|
c3451560f5 | ||
|
|
b16e8b87fa | ||
|
|
15572f0e60 | ||
|
|
c88138aa1a | ||
|
|
4a50c88ea5 | ||
|
|
55b26f8d32 | ||
|
|
e20eae988c | ||
|
|
6b590ccf5e | ||
|
|
42c9234c81 | ||
|
|
a1f9b27822 | ||
|
|
ec9f660f97 | ||
|
|
c9d4bdda1d | ||
|
|
f1916404b4 | ||
|
|
f30b9c1871 | ||
|
|
fd9534e978 | ||
|
|
4ada050f7e | ||
|
|
5b2cb72500 | ||
|
|
d386c972d9 | ||
|
|
adc3d96d60 | ||
|
|
b737445a67 | ||
|
|
5b79c9416f | ||
|
|
765402062a | ||
|
|
12615cf90a | ||
|
|
f258222094 | ||
|
|
f34fa0b856 | ||
|
|
9e3040be04 | ||
|
|
eebebbf01e | ||
|
|
4108c28510 | ||
|
|
eaea1a03cb | ||
|
|
180bed28ee | ||
|
|
b51da1ec2d | ||
|
|
6b25e3fc40 | ||
|
|
abc5291164 | ||
|
|
406baa9eb4 | ||
|
|
89b2232aa1 | ||
|
|
2b95bc262d | ||
|
|
cde42b022f | ||
|
|
e706b6488c | ||
|
|
95b5d9e8b0 | ||
|
|
de87d715ff | ||
|
|
1fd0942c74 | ||
|
|
147f6d21ab | ||
|
|
b019204360 | ||
|
|
e712f234d0 | ||
|
|
9142c1e96f | ||
|
|
b1ceea23f2 | ||
|
|
0b34130222 | ||
|
|
d71e8b6671 | ||
|
|
384115a55e | ||
|
|
1cf2943e5b | ||
|
|
e863e1dea9 | ||
|
|
e02907684d | ||
|
|
85a4afa7ab | ||
|
|
09b6af4c75 | ||
|
|
e58557d7da | ||
|
|
eb0343b06c | ||
|
|
f89f9fd162 | ||
|
|
ea6a94f38a | ||
|
|
47fdd4f7e7 | ||
|
|
5b3a173411 | ||
|
|
e077521571 | ||
|
|
f649b5b267 | ||
|
|
8898774d5a | ||
|
|
882dd0e7eb | ||
|
|
99d1deade1 | ||
|
|
bf3a494c7e | ||
|
|
4036c4d4e8 | ||
|
|
c7eab9aea8 | ||
|
|
92a3fc3b5b | ||
|
|
bf0394c609 | ||
|
|
6b4ea8f298 | ||
|
|
14ba825454 | ||
|
|
2a49700883 | ||
|
|
039538fe71 | ||
|
|
d31d21cdf0 | ||
|
|
770eca6826 | ||
|
|
1db1a7bf45 | ||
|
|
78c625a99b | ||
|
|
be68876755 | ||
|
|
c04bc73c76 | ||
|
|
ec7092df45 | ||
|
|
d736b12353 | ||
|
|
d05b1697a7 | ||
|
|
06f2f4db78 | ||
|
|
3ed0cd83da | ||
|
|
d0cd7a204a | ||
|
|
421317ae00 | ||
|
|
7fa06ced65 | ||
|
|
bd73e02774 | ||
|
|
d64af9ab00 | ||
|
|
cea305f6e9 | ||
|
|
63c79523d3 | ||
|
|
c7c91726c9 | ||
|
|
dd5500a0ed | ||
|
|
7ed3926a13 | ||
|
|
54f610ce99 | ||
|
|
b89cbaf241 | ||
|
|
0461222872 | ||
|
|
d2d47c0f14 | ||
|
|
ad5b9dcc28 | ||
|
|
30e723c89e | ||
|
|
0b59e8cabe | ||
|
|
a669156059 | ||
|
|
5efb91123c | ||
|
|
e792dea1ee | ||
|
|
9afa752e47 | ||
|
|
6bd2ea61ad | ||
|
|
e90c57ab35 | ||
|
|
5ceede3258 | ||
|
|
dfe001f969 | ||
|
|
6e4c3eb2b8 | ||
|
|
b028ed3170 | ||
|
|
f41be8c39a | ||
|
|
2e50b04eff | ||
|
|
359f2f5c50 | ||
|
|
27c20050b1 | ||
|
|
109981eede | ||
|
|
b415052f70 | ||
|
|
5e8085d5d3 | ||
|
|
38147f87ed | ||
|
|
ee7e665eb3 | ||
|
|
0fee87cd28 | ||
|
|
f5260d21aa | ||
|
|
95e9a3a9ac | ||
|
|
c4cd4f4f1f | ||
|
|
be9a1cca9a | ||
|
|
cd640ef131 | ||
|
|
02938ff407 | ||
|
|
6ce1821e02 | ||
|
|
f8b5426a16 | ||
|
|
bc5e00bb9e | ||
|
|
4343cd9e13 | ||
|
|
58fb270d05 | ||
|
|
12d0f11945 | ||
|
|
edfa69890d | ||
|
|
8bd4d59383 | ||
|
|
76f20836ee | ||
|
|
a76f3b7418 | ||
|
|
11046441cf | ||
|
|
4725eb0078 | ||
|
|
37f0103c60 | ||
|
|
5e9ddf28a1 | ||
|
|
a9a3dd9b49 | ||
|
|
e82ccbba5d | ||
|
|
03c356d5d5 | ||
|
|
309638eac9 | ||
|
|
54d7a98c7a | ||
|
|
cc4566c901 | ||
|
|
182d5db717 | ||
|
|
5d6b5d33ca | ||
|
|
40ae2386e5 | ||
|
|
0c24369bdd | ||
|
|
451b1b9c45 | ||
|
|
20a4bedd90 | ||
|
|
050737e093 | ||
|
|
6de4e9a891 | ||
|
|
8a27963fdd | ||
|
|
68f4a6f341 | ||
|
|
3e28af31ed | ||
|
|
456e9a9102 | ||
|
|
0fea165253 | ||
|
|
fec9a9501f | ||
|
|
016ed92422 | ||
|
|
d483643f2a | ||
|
|
2e43c61e66 | ||
|
|
917216e358 | ||
|
|
f0ded1247f | ||
|
|
d2429ef233 | ||
|
|
72eb482565 | ||
|
|
0a7ca60722 | ||
|
|
ecde5794d5 | ||
|
|
19153c289a | ||
|
|
decfed9669 | ||
|
|
46d2d4abe8 | ||
|
|
ff3fe62644 | ||
|
|
faa109e40b | ||
|
|
6f0dec9a44 | ||
|
|
77d8def5c4 | ||
|
|
b706a74649 | ||
|
|
de613f8b7d | ||
|
|
8eb23f3878 | ||
|
|
c48c0dba95 | ||
|
|
0acf0cccf3 | ||
|
|
7fc038ae6d | ||
|
|
bdc56dc1ea | ||
|
|
5f67756747 | ||
|
|
5c77558e53 | ||
|
|
dbbb30b399 | ||
|
|
fd1d4c9cec | ||
|
|
af433d7af3 | ||
|
|
c2f472ad2d | ||
|
|
6181aea8ad | ||
|
|
3c6e927c2a | ||
|
|
c33f2a4646 | ||
|
|
146f0ed107 | ||
|
|
a83550429b | ||
|
|
1c8a7831ac | ||
|
|
7acb98ec7b | ||
|
|
2c9c3e2bfd | ||
|
|
c6adc1c8e8 | ||
|
|
1ac989912d | ||
|
|
21da364d35 | ||
|
|
205345ab29 | ||
|
|
76817c29b0 | ||
|
|
914df97f94 | ||
|
|
cfd338aaa0 | ||
|
|
aec18dedbc | ||
|
|
42d459a6d8 | ||
|
|
87b2db66b0 | ||
|
|
060d747c7a | ||
|
|
075df1c8f6 | ||
|
|
f7f221a96e | ||
|
|
ef6bafc6a0 | ||
|
|
f3fadcdfa8 | ||
|
|
830ff825d8 | ||
|
|
f41c7a04d7 | ||
|
|
57fcc51d21 | ||
|
|
d12ce59efc | ||
|
|
d95fb56b28 | ||
|
|
e089fc5340 | ||
|
|
3f887e6eaa | ||
|
|
bb299b13d9 | ||
|
|
86374125ba | ||
|
|
b077f2960c | ||
|
|
6d3e0203f0 | ||
|
|
983fdced69 | ||
|
|
537365c9fc | ||
|
|
92b875aab1 | ||
|
|
1c3bf27a9c | ||
|
|
824875ddf7 | ||
|
|
2ab0c44341 | ||
|
|
5284e8c58f | ||
|
|
98c8f950ad | ||
|
|
fe3df2aacc | ||
|
|
ff2c696d40 | ||
|
|
1db41e4189 | ||
|
|
6f4828dbb2 | ||
|
|
2c00fb33a3 | ||
|
|
f1e8194cd4 | ||
|
|
fad52abb8b | ||
|
|
249baee6d8 | ||
|
|
911560bd74 | ||
|
|
23f0c84045 | ||
|
|
12e66bc66c | ||
|
|
ef4fdec71b | ||
|
|
d34d0ca967 | ||
|
|
d96a0f492d | ||
|
|
a741ab7c40 | ||
|
|
8aa74c6d7c | ||
|
|
07e0daa669 | ||
|
|
9431a0e30a | ||
|
|
67d127e4f8 | ||
|
|
0ce8767061 | ||
|
|
36a143b060 | ||
|
|
3e17e2738e | ||
|
|
efdb240667 | ||
|
|
80c4c7a8ea | ||
|
|
3a95c3a746 | ||
|
|
446268bdfa | ||
|
|
f939dc30a6 | ||
|
|
4fd9ceaf1a | ||
|
|
7c343b655c | ||
|
|
5b12a0268f | ||
|
|
1023e9c77a | ||
|
|
8687e4d033 | ||
|
|
4b059046f5 | ||
|
|
a2e8516f77 | ||
|
|
b0cf2aa332 | ||
|
|
42f91cbcfc | ||
|
|
9c86d7cc9a | ||
|
|
0262980432 | ||
|
|
2ccb41ead3 | ||
|
|
7d61452979 | ||
|
|
6daf43b03c | ||
|
|
6f94869394 | ||
|
|
575ebcc0ae | ||
|
|
3ab56aeb6d | ||
|
|
7bda81d359 | ||
|
|
d446565950 | ||
|
|
d939df61f4 | ||
|
|
68a90c2e4d | ||
|
|
e048721c8d | ||
|
|
d43d83fd5d | ||
|
|
e9152abed8 | ||
|
|
8789bf5a6e | ||
|
|
5cdce4c6b8 | ||
|
|
0fa63dcf42 | ||
|
|
8ee511eede | ||
|
|
1338d9d44e | ||
|
|
94cf9adc46 | ||
|
|
d39b7b4e7a | ||
|
|
7e3f132f75 | ||
|
|
2aefd0fcba | ||
|
|
2aa41579db | ||
|
|
147fcfbf48 | ||
|
|
93eb96c2b2 | ||
|
|
333f3b1cc6 | ||
|
|
bbc2ffd2ac | ||
|
|
5d96e42c6d | ||
|
|
6a497bbd8c | ||
|
|
3de029eaf9 | ||
|
|
43b80b147f | ||
|
|
2483b9b327 | ||
|
|
d7cfc2c7b3 | ||
|
|
0384608514 | ||
|
|
62380758f3 | ||
|
|
095235d2c8 | ||
|
|
a5562a9235 | ||
|
|
dd5cf73e3b | ||
|
|
d3278d1d6a | ||
|
|
2da32e8305 | ||
|
|
2e68183375 | ||
|
|
5f995a36a9 | ||
|
|
11ded748bf | ||
|
|
0fa36ca454 | ||
|
|
a71a105d67 | ||
|
|
6d04c584f5 | ||
|
|
baa4682ec6 | ||
|
|
40c21af503 | ||
|
|
274e974854 | ||
|
|
611f3c08d2 | ||
|
|
9d803b7d2a | ||
|
|
089edff867 | ||
|
|
4e3938751b | ||
|
|
5563f772b5 | ||
|
|
542b2d5ca5 | ||
|
|
b5301a0fff | ||
|
|
f48aa7a641 | ||
|
|
caa4ae6f8e | ||
|
|
0ac43e8520 | ||
|
|
5bcf3d381a | ||
|
|
dff6737df9 | ||
|
|
7d96ff2a00 | ||
|
|
dfe58fbd9c | ||
|
|
773fa51196 | ||
|
|
8cad8909b2 | ||
|
|
9d22a90fcc | ||
|
|
8f00406517 | ||
|
|
0f8d00f207 | ||
|
|
b31eb50a18 | ||
|
|
0149de2535 | ||
|
|
af84a99ea6 | ||
|
|
fc25686631 | ||
|
|
9c6c798f30 | ||
|
|
fea24bb191 | ||
|
|
9ffc61bc53 | ||
|
|
b5ade841b6 | ||
|
|
c508edc5c6 | ||
|
|
5a763ca996 | ||
|
|
f6b1802306 | ||
|
|
1a930d610c | ||
|
|
a748d340c9 | ||
|
|
309bcc7475 | ||
|
|
d9ee043385 | ||
|
|
1cca29f2aa | ||
|
|
659e8846b3 | ||
|
|
030d55e83f | ||
|
|
66e9443647 | ||
|
|
bc035a02ea | ||
|
|
8eeb4fc027 | ||
|
|
f6aa028b12 | ||
|
|
6d5920b625 | ||
|
|
7d9c329921 | ||
|
|
dd83251933 | ||
|
|
026f4e0e65 | ||
|
|
dbf428f485 | ||
|
|
696915fb6d | ||
|
|
9b88805984 | ||
|
|
931dce4a8a | ||
|
|
39362263a7 | ||
|
|
2a60d0d9cc | ||
|
|
c6c950e73a | ||
|
|
219c818805 | ||
|
|
43ae01b650 | ||
|
|
3d23efd2a8 | ||
|
|
ffe62307ad | ||
|
|
fdb2c1700c | ||
|
|
906138539c | ||
|
|
2a83ccb839 | ||
|
|
0f9dc21006 | ||
|
|
449007018e | ||
|
|
65d3cd3f04 | ||
|
|
abdb8e212d | ||
|
|
5290d40f8d | ||
|
|
98af16ad78 | ||
|
|
66fa02b0cc | ||
|
|
104cc19774 | ||
|
|
09ddc94889 | ||
|
|
fcd2f9378a | ||
|
|
948f52546b | ||
|
|
ad5ae4917f | ||
|
|
a66c5eb7e0 | ||
|
|
6661e987d2 | ||
|
|
d5e930c8a0 | ||
|
|
03b61134d3 | ||
|
|
20cafc21fd | ||
|
|
d5f867adea | ||
|
|
fa2196820b | ||
|
|
864eda9f9f | ||
|
|
3ab4fec0cf | ||
|
|
c17dd5c96e | ||
|
|
490e6cd575 | ||
|
|
07a3841065 | ||
|
|
f5d0e7f31c | ||
|
|
833cec0d6a | ||
|
|
0c2872d536 | ||
|
|
4def4f0ef8 | ||
|
|
d85747b0f2 | ||
|
|
2239442687 | ||
|
|
d99db2e837 | ||
|
|
6c9dc00a54 | ||
|
|
a01875b7be | ||
|
|
30ab1b85e7 | ||
|
|
e9e6856489 | ||
|
|
95f8ed9f28 | ||
|
|
7d0dd4277b | ||
|
|
251eece8e2 | ||
|
|
aabe2e0b6a | ||
|
|
3b3c6e537d | ||
|
|
986b59f1cf | ||
|
|
53167748ce | ||
|
|
638bfde86f | ||
|
|
f52d893163 | ||
|
|
3e9323832c | ||
|
|
9338a0e3d1 | ||
|
|
12016ee60c | ||
|
|
44b3a65dcd | ||
|
|
ae56ba394e | ||
|
|
b727b677b5 | ||
|
|
c9f6df6736 | ||
|
|
5ed54a5ea4 | ||
|
|
fdc721e6d8 | ||
|
|
4bbae5cacc | ||
|
|
27accf0458 | ||
|
|
c7df907410 | ||
|
|
e8704d74e9 | ||
|
|
8e70289c8a | ||
|
|
d972a53593 | ||
|
|
75b0c35ebb | ||
|
|
d726d058c7 | ||
|
|
cdcb0f747a | ||
|
|
298dbb5c32 | ||
|
|
a8729b9dfb | ||
|
|
1f1d588e48 | ||
|
|
8be61fdce1 | ||
|
|
c606d285e0 | ||
|
|
a2665a0b6b | ||
|
|
aecffccff1 | ||
|
|
01f1291b6d | ||
|
|
a14f3910b3 | ||
|
|
763048702c | ||
|
|
8f7ffd39ba | ||
|
|
a5493d71cd | ||
|
|
3f83e31eea | ||
|
|
255f916b72 | ||
|
|
96244c05fd | ||
|
|
33a80bbbe6 | ||
|
|
c51509645a | ||
|
|
6a21c07d3e | ||
|
|
306702953d | ||
|
|
a521d16f30 | ||
|
|
623bbc696a | ||
|
|
d1f0120130 | ||
|
|
34e32551d2 | ||
|
|
590310d11c | ||
|
|
0bd62545a9 | ||
|
|
6cc9aafae4 | ||
|
|
aad97c2b27 | ||
|
|
a4b806b221 | ||
|
|
392ba67138 | ||
|
|
e193e806cc | ||
|
|
b7df984b18 | ||
|
|
06a7db2225 | ||
|
|
d816d2d709 | ||
|
|
672570e079 | ||
|
|
76494280bf |
22
.gitignore
vendored
22
.gitignore
vendored
@@ -17,3 +17,25 @@ dist/
|
||||
*.tfstate
|
||||
*.tfstate.*
|
||||
**/.terragrunt-cache/
|
||||
|
||||
# market data blobs live in PVCs/object storage, never git
|
||||
*.dbn
|
||||
|
||||
# downloaded 1m klines (binance.vision) + the worst_basis side-store — data, never git
|
||||
klines_cache/
|
||||
*.parquet
|
||||
|
||||
# local paper-trading API keys (never commit)
|
||||
alpacca-paper-keys.txt
|
||||
*-paper-keys.txt
|
||||
|
||||
# Playwright MCP scratch output (console logs + accessibility snapshots) written into the repo root when
|
||||
# the cockpit is driven in a browser during development — debug artifacts, never git.
|
||||
.playwright-mcp/
|
||||
# Screenshots taken while verifying the cockpit locally. Scoped to the repo ROOT (leading slash) so real
|
||||
# tracked assets — the cockpit logo at src/fxhnt/adapters/web/static/fxhnt.png — are never caught.
|
||||
/*.png
|
||||
|
||||
# process coredumps (a 62MB ELF core dump was committed by accident in 6ff522c) — never git
|
||||
core
|
||||
core.*
|
||||
|
||||
BIN
cockpit-overview-bizworx.png
Normal file
BIN
cockpit-overview-bizworx.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 173 KiB |
@@ -0,0 +1,119 @@
|
||||
# fxhnt — Fund / Tax / Accounting Research Brief (2026-07-20)
|
||||
|
||||
**> This is researched context, NOT legal, tax, or financial advice.** The Dutch tax rules that govern this the most (Box 3 wealth taxation and the FGR fund regime) are in **active reform** as of mid-2026: Box 3's actual-return system is legislatively unsettled (Eerste Kamer voting paused) and the FGR definition was rewritten Jan 2025 with further consultation open. Every rate, threshold, and classification below must be confirmed with a licensed Dutch belastingadviseur / fiscaal jurist against the finalized Belastingdienst publications for your actual filing year before you rely on it. The single highest-stakes item — whether your systematic/algorithmic trading is Box 3 or Box 1 — is genuinely unsettled case law and needs a written advisor opinion (and possibly a vooroverleg with the Belastingdienst) *before* you scale real capital.
|
||||
|
||||
**> STATE NOTE (2026-07-20): the fund is STILL PAPER — no live book yet** (IBKR DU9600528 is a paper account; Bybit exec never armed). This is go-live planning from euro zero, and the Bybit ~1 Jul 2026 EEA change (§5.15) is **not** a live-position emergency — just a reason to open the right Bybit-EU (EUR-not-USDT) account before funding crypto.
|
||||
|
||||
**> RESOLVED STRUCTURE (2026-07-20) — supersedes §1's "start personal, then decide" and §3/§4's BV-from-zero modelling.** The operator surfaced that they **already have an IT/tech-BV + holding + a DGA-salary**. That collapses the whole Box-1-vs-Box-3 gamble into a clean, already-in-place structure:
|
||||
|
||||
> **Tech-BV = ALL the arbeid** (build/run the system, buy/develop technology) — already taxed there (Vpb 19% + the existing DGA loon, so *no extra €58k gebruikelijk-loon is needed*). → BV pays **dividend** (Vpb 19% + Box 2 24.5%, cleanly taxed) to private. → that al-belaste private money is invested in the **fund as passive capital → Box 3** (~6% forfait × 36% ≈ 2.2%/yr of assets, regardless of real return).
|
||||
|
||||
This **removes the Box-1 reclassification risk entirely** — the only real threat — because the danger was ever *arbeid + winst in one hand* (what sank GHARL:2024:4613). Here the labour is taxed in the BV and the private capital does nothing but *renderen*: exactly the "normaal vermogensbeheer" the daytrader cases won on. **No new entity, no fund wrapper, no AIFMD (own money). Dividend is an option the operator HAS, not a forced cost.** Two design guards remain (not risks): keep the fund genuinely passive → price any tech-BV→fund service **at arm's length**; and time the dividend for the Box 2 bracket. So §1's staged "personal → BV → fund" path and §3/§4's break-even math are **background theory** for this operator — the resolved route above is what applies. The advisor conversation shifts from *"which box?"* to *"how do I price the tech-BV→fund service so the fund stays demonstrably passive?"*
|
||||
|
||||
**> ACCOUNTING NOTE: EUR is the functional currency, no conversion pipeline.** IBKR is EUR-native and Bybit-EU is EUR too → there is no USD leg. §2's FX-revaluation subsystem (realized/unrealized-FX split, ECB-rate join, USDT proxy) is **downgraded from core to dormant insurance**: keep a `currency` + `fx_rate_to_eur` column per posting (default EUR/1.0) so a stray non-EUR line is representable, but build no conversion pipeline. Pure-EUR book = no FX P&L = cleaner books and cleaner tax.
|
||||
|
||||
---
|
||||
|
||||
# Decision brief: bookkeeping, profit extraction, and structure for an NL-resident agentic quant fund
|
||||
|
||||
## 1. Bottom line up front
|
||||
|
||||
**Start personal (Box 3), keep your own capital, and build the accounting system now — but get a written advisor opinion on the Box 1 reclassification risk before you go live with real money.** At ~$35k pilot scale, a BV's fixed costs (incorporation, annual accounts, Vpb filing, and especially the ~€58k gebruikelijk-loon obligation that an actively-run trading BV likely cannot escape) are disproportionate, and every pooled-fund wrapper (FGR/VBI/FBI/RAIF/QIAIF) is either closed off post-2025 reform or needs tens of millions in AUM to be economical. The one serious caveat: your setup — a continuously-running, code-driven, systematic strategy — sits squarely on the fact pattern (ECLI:NL:GHARL:2024:4613, the self-built crypto bot with a stable 75–80% win rate) that Dutch courts pushed into **Box 1** (progressive rates to ~49.5% on actual profit) rather than Box 3, so "Box 3 is cheaper" is only true if Box 3 actually holds. **Move to a dedicated (ring-fenced) BV when either (a) your advisor judges Box 1 reclassification risk high enough that corporate-rate certainty is worth the overhead, or (b) profits are large and consistently retained/reinvested so Vpb-only compounding (19%) beats paying Box 3 tax on capital every year.** Move to a fund structure (de-minimis AIFM + FGR) only when external capital is a concrete, near-term plan — not before.
|
||||
|
||||
## 2. The accounting system (the part you can build)
|
||||
|
||||
**Recommendation: build the tax ledger inside your existing Postgres/Timescale warehouse, not in a plaintext tool (beancount/hledger).** You already run Postgres + Dagster with per-strategy tagging; a second, disconnected file-based ledger just creates a reconciliation surface between the warehouse and the books. beancount's one real advantage — a native FIFO/AVERAGE cost-basis engine — is a bounded amount of SQL to replicate, and neither tool has a Bybit importer (you build that either way). Keep beancount only as a *possible export target* if your accountant wants to inspect plain text.
|
||||
|
||||
### Schema (append-only double-entry)
|
||||
|
||||
- **`journal_entries`** (header): `entry_id, entry_date, description, source_venue, strategy_tag, source_ref` — immutable, corrections via reversing entries only (never UPDATE/DELETE).
|
||||
- **`journal_postings`** (lines): `entry_id, account_id, side ('debit'|'credit'), amount, currency, fx_rate_to_eur, eur_amount` — with a DB-level CHECK/trigger enforcing that each entry's postings sum to zero **in EUR-converted terms**.
|
||||
- **`accounts`**: per-venue sub-ledgers that each balance standalone — `IBKR-EUR-cash`, `IBKR-securities`, `Bybit-USD-cash`, `Bybit-crypto`, `Bybit-perp-margin` — plus **two distinct FX accounts per foreign balance**: `unrealized-FX-PnL` (hit at each period-end retranslation) and `realized-FX-PnL` (hit only on actual conversion/withdrawal). This split is what lets you resolve the accounting-vs-tax realization question later without re-deriving history.
|
||||
- **`cost_basis_lots`** (per venue/asset): matched lots so realized gains come from actual lot-matching (FIFO — the NL de-facto default — or whatever your advisor confirms, applied *consistently* across both venues), not cash-flow deltas.
|
||||
- **`fx_rates`**: daily ECB reference rates, keyed by value_date.
|
||||
- **`account_balances`**: a materialized view for fast reads.
|
||||
|
||||
The **strategy_tag** on every posting makes P&L-by-strategy a `GROUP BY`, reusing your existing Dagster attribution — a filter, not a rebuild.
|
||||
|
||||
### Ingestion pipeline (Dagster assets, mirroring your nightly ETL pattern)
|
||||
|
||||
1. **IBKR Flex Web Service API** (not PDF scraping): a per-query token + query ID, 2-step SendRequest→GetStatement poll, ~1 req/sec limit, 2–3 pulls/day is safe. Pull trades, cash transactions, dividends, corporate actions, positions, NAV. Archive the annual Activity Statement as an audit doc even though you compute P&L independently. Note: as a W-8BEN non-US person you get a **1042-S**, never a Consolidated 1099, and there is no IBKR "NL tax statement."
|
||||
2. **Bybit** transaction-history / Tax-API REST endpoints for fills, **funding fees**, and deposits/withdrawals. **Explicitly verify the Unified Trading Account API surfaces perp funding and mark-to-market events** — they are easy to miss and material to P&L.
|
||||
3. Land raw payloads in an append-only `raw` schema (mirrors your DB-anchor-SSOT pattern), then a transform layer normalizes both venues into the canonical `journal_postings`.
|
||||
|
||||
### FX conversion & reconciliation
|
||||
|
||||
- **Standard rate: ECB daily reference rate** (`eurofxref-daily.xml`, ~16:00 CET; `eurofxref-hist.xml` for one-time backfill), applied at each transaction's value_date (IAS 21 transaction-date convention).
|
||||
- **The ECB publishes no crypto crosses.** Two-step for Bybit: (1) treat USDT as a USD-proxy ~1:1 **and document that assumption explicitly** (or use Bybit's own USD-index for USDT legs), then (2) USD→EUR at the ECB rate for that value_date.
|
||||
- **Box 3 peildatum (1 January)** crypto valuation: use the **exchange's own displayed value at 00:00 on 1 Jan** (crypto is venue-priced), and snapshot it via API.
|
||||
- **Reconciliation target: DAC8/CARF.** From FY2026, in-scope CASPs report your crypto data straight to the Belastingdienst (first reporting Jan 2027). Your books must reconcile to what the venue reports. Given Bybit's current global entity is non-EU pre-migration, **assume no third-party feed and keep fully independent records** — this raises, not lowers, your self-reporting burden. (See §5 on the Bybit EU migration deadline.)
|
||||
|
||||
## 3. Tax-efficient profit extraction (ranked)
|
||||
|
||||
| Rank | Route | Box / rate implication | When it wins |
|
||||
|---|---|---|---|
|
||||
| **1** | **Stay personal, Box 3** | 36% on a *deemed* return (~6% "overige bezittingen" forfait for 2026 → effective ~2.2% of asset value/yr); tegenbewijsregeling lets you report actual return if lower | Pilot / own-capital, **if Box 3 classification holds** and actual returns exceed the forfait. Dramatically cheaper than any actual-profit regime in a strong year. |
|
||||
| **2** | **BV, retain earnings** | Vpb only: 19% ≤€200k, 25.8% above — no Box 2 layer until you distribute | Once profits are large and **reinvested/compounded** inside the BV; deferral of the Box 2 layer is the compounding advantage. Also removes the Box 1/3 question entirely. |
|
||||
| **3** | **BV, distribute as dividend** | Vpb (19/25.8%) **then** Box 2 (24.5% ≤~€68.8k / 31% above); combined effective ~38–44% on distributed profit | Only when you actually need the cash personally *and* Box 1 reclassification would otherwise have applied (~49.5%) — you pay for certainty. |
|
||||
| **4** | **DGA salary from a BV** | gebruikelijk loon ~€58k (2026), taxed as Box 1 wage income, progressive to ~49.5% | Not a "choice" — it's a *mandatory* cost of the BV route (an active trading BV is unlikely to qualify for the ~€5k limited-activity carve-out, and from 2026 needs an advisor-cosigned salary justification). Factor it into the BV break-even, don't treat it as extraction. |
|
||||
|
||||
**Key numbers to hold in mind:** Box 3 taxes *deemed* return on *capital*; the BV route taxes *actual* profit. So Box 3 is cheaper the more your real return exceeds ~6%, and the BV is cheaper mainly when Box 1 (~49.5% on actual) is the realistic alternative or when earnings are retained. Under the Box 3 **tegenbewijsregeling**, "actual return" includes **unrealized** mark-to-market gains and does **not** allow deduction of trading fees/funding (only actual Box-3 debt interest) — this is *not* how a trading P&L system naturally computes returns, so your ledger needs a parallel tax-basis calculation.
|
||||
|
||||
## 4. Fund-structure decision map
|
||||
|
||||
```
|
||||
STAGE 0 — Personal, Box 3 own capital only, any size
|
||||
trigger to advance ↓ external capital contemplated, OR Box 1 risk judged high, OR large retained profits
|
||||
STAGE 1 — Dedicated BV (ring-fenced) own capital; new sister/subsidiary BV, NOT commingled with bizworx
|
||||
trigger to advance ↓ first external euro is concrete & near-term
|
||||
STAGE 2 — BV(manager) + FGR + AIFMD "light" registration
|
||||
trigger to advance ↓ AUM approaches ~€20–30m (dedicated-fund fixed costs become proportionate)
|
||||
STAGE 3 — Licensed fund / RAIF (LU) or QIAIF (IE) institutional external capital, tens of €m+
|
||||
```
|
||||
|
||||
**Regulatory gates:**
|
||||
|
||||
- **"Own money only" is a genuine AIFMD safe harbor** (Art. 4(1)(a); Recital 7 family-office carve-out). No license, no registration while you trade solely your own capital — *this covers a solo BV too*. Also outside MiFID II via the Art. 2(1)(d) own-account exemption (subject to a technical check that your order/message rate doesn't hit the "high-frequency algorithmic trading" threshold), and outside MiCA CASP authorization for own-account crypto.
|
||||
- **The moment you accept one external euro**, AIF/AIFMD scope begins. Below **€100m AUM** (leveraged / redemption within 5yr) or **€500m** (unleveraged closed-end), you need only **AFM registration** (the "light"/de-minimis regime), *not* a full AIFM license — but that registration must be done **before** the first external euro, plus Wwft (AML) and Sanctiewet compliance and AFM/DNB reporting. Marketing to retail adds top-up rules (Wft 4:37p) and a <150-investor / €100k-minimum-ticket cap.
|
||||
- **Ring-fence, don't commingle.** If/when you do incorporate, put trading in a **new dedicated BV**, not inside bizworx.nl — standard NL practice isolates a volatile, potentially loss-making book from an operating business's balance sheet, credit, and any regulatory status.
|
||||
- **FGR is in flux and not fit-for-purpose at pilot scale.** The Jan-2025 reform abolished the open/closed test; a new fund must qualify directly under Wft-anchored rules and a further consultation (opened Dec 2025) is open. VBI is effectively closed to solo operators post-reform; FBI structurally disfavors active high-turnover trading; RAIF/QIAIF both *require* an external authorized AIFM and are uneconomical below ~€20–30m. So Stage 2 is a *later* decision, timed to land after the FGR rules settle.
|
||||
|
||||
## 5. What to take to a Dutch tax advisor / notaris
|
||||
|
||||
**Classification (highest stakes):**
|
||||
1. Given a fully agentic, continuously-running systematic strategy across IBKR UCITS ETFs + Bybit perps — is this **Box 3 or Box 1** (resultaat uit overige werkzaamheden / winst uit onderneming)? Reconcile the taxpayer-favorable *Gelderland/Noord-Holland* daytrading line against the *GHARL:2024:4613* bot line that landed in Box 1. **Is a vooroverleg / advance ruling advisable before scaling capital**, given the retroactive-assessment downside if reclassified after the fact?
|
||||
|
||||
**Entity & existing structure:**
|
||||
2. Is **bizworx.nl a BV or an eenmanszaak?** This determines whether housing trading is a fresh incorporation or a restructuring (eenmanszaak→BV has geruisloze inbreng considerations; a fiscale eenheid question if a second BV is created).
|
||||
3. If moving personally-held IBKR/Bybit positions into a BV — does that trigger a **deemed-disposal (afrekening)** for Box 3, and is any rollover/deferral available for a trading portfolio?
|
||||
4. **gebruikelijk loon**: does running an autonomous/agentic system (software does most of the "work") qualify for the ~€5k limited-activity carve-out, or does the algorithm-design labor count against it?
|
||||
|
||||
**Accounting method & FX:**
|
||||
5. **Cost-basis method** — FIFO or weighted-average, and must it be identical across IBKR ETF lots and Bybit crypto lots (goedkoopmansgebruik consistency)?
|
||||
6. **Unrealized FX/crypto revaluation** — book to P&L each period (RJ122/IAS21) for internal reporting while the tax return recognizes only realized (goedkoopmansgebruik; the NLFiscaal unrealized-FX-loss-denial precedent)? Do I need two parallel FX-recognition tracks (book vs tax)?
|
||||
7. **Functional-currency election** (EUR vs USD) — worth the 10-year lock-in, or is "EUR presentation + translate every Bybit leg at transaction-date ECB rate" sufficient without electing?
|
||||
8. **USDT** — treated as USD-equivalent, or its own crypto-asset (own EUR cross) for both ongoing bookkeeping and the Box 3 peildatum?
|
||||
|
||||
**Instrument-specific (a genuine gap in the research):**
|
||||
9. How are **perpetual-futures funding payments** characterized for Box 3 werkelijk-rendement — as income each period, or folded into position P&L? No authoritative Belastingdienst guidance was found.
|
||||
10. Under werkelijk rendement, are Bybit **trading fees / funding paid / margin costs non-deductible** (as the tegenbewijsregeling mechanics suggest, unlike normal trading P&L)?
|
||||
|
||||
**Reform & reporting:**
|
||||
11. Confirm the **finalized 2026 figures** at filing time (heffingsvrij vermogen — sources conflicted between ~€51,396 and ~€59,357 — forfait %, Box 2 thresholds).
|
||||
12. Does the fund trigger a formal **boekhoudplicht** (BW 2:10 / 3:15i — 7-year retention, audit-trail format), and does the Postgres pipeline satisfy it?
|
||||
13. **BTW/VAT** — confirm own-account securities/crypto trading is exempt, and re-check if a management fee is later introduced.
|
||||
|
||||
**Regulatory (Wft lawyer, not just belastingadviseur):**
|
||||
14. Does the IBKR order/message rate meet MiFID II's **HFT** technical threshold (removes the own-account exemption)? Needs a real order-flow review.
|
||||
15. **Bybit EU migration — operationally urgent:** the global platform closes EEA/NL access ~1 July 2026; you migrate to the Austria-MiCA "Bybit EU" entity (fresh KYC, Travel Rule proof-of-wallet for transfers >€1,000). **Bybit EU reportedly cannot offer USDT** (Tether has no MiCA EMT authorization), so USDT balances must convert to EUR or a MiCA stablecoin — **is that forced conversion a taxable disposal, and under which box?** Verify directly on Bybit's compliance pages before the deadline.
|
||||
|
||||
## 6. Immediate next build step
|
||||
|
||||
**Build the append-only double-entry ledger core + the IBKR Flex ingestion asset first — one Dagster asset landing IBKR Flex trades/cash/NAV into a `raw` schema, normalizing into `journal_entries`/`journal_postings` with the balanced-entry DB trigger, and joining an ECB `fx_rates` table by value_date to produce EUR-normalized postings.**
|
||||
|
||||
Why this one component first, over everything else:
|
||||
- IBKR is your EUR-native, better-documented venue with a stable Flex Web Service API and an existing importer pattern to model on — lowest integration risk, so you validate the schema and the balanced-entry invariant against real data before adding Bybit's messier surface (funding fees, USDT proxy, migration churn).
|
||||
- It exercises **every** load-bearing design decision at once: append-only immutability, the EUR-sum-to-zero constraint, the strategy_tag dimension, and the ECB-rate join — the FX-revaluation split and cost-basis lot tables extend cleanly on top once this spine is proven.
|
||||
- It's a natural Dagster asset alongside your existing nightly ETL, reusing your monitoring/DB-anchor-SSOT patterns — no new infrastructure, and it turns tax season into a query rather than an annual export scramble regardless of which box you end up in.
|
||||
|
||||
Defer the Bybit importer, the cost-basis lot engine, and any beancount export until this spine is landing correct, reconciled EUR postings from IBKR.
|
||||
29
docs/superpowers/VRP-REMOVAL-TODO.md
Normal file
29
docs/superpowers/VRP-REMOVAL-TODO.md
Normal file
@@ -0,0 +1,29 @@
|
||||
# VRP full removal — DONE (2026-07-20/21)
|
||||
|
||||
**Status: COMPLETE.** VRP code fully removed on branch `chore/remove-vrp-code` (9 commits over base a263b67),
|
||||
full suite green (2018 passed, 2 skipped), whole-branch review clean (no Critical/Important). OPRA data KEPT.
|
||||
|
||||
## What was removed
|
||||
- 7 VRP application modules (`vrp_book/eval/defined_risk_eval/exec_record/marking/pricing/research`) + `black_scholes.py`
|
||||
- The DVOL pipeline (`deribit_dvol.py`, `dvol_ingest.py`, the `VolIndexClient` port) — DVOL was the IV leg of VRP
|
||||
- The `vrp_nav` Dagster asset + its ~14 OPRA-freeze/backfill helper chain
|
||||
- The `XspOptionBarsRepo` **class** + the `_build_vrp` migration builder + the `backfill-xsp-opra`/`ingest-dvol`/
|
||||
`execute-vrp`/`vrp-eval`/`vrp-defined-risk-eval` CLI commands + the `fxhnt-opra-backfill` CronJob manifest
|
||||
- The `vrp`/`vrp_exec` registry entries + `_REF_ALIAS`/`_IBKR_ACCOUNT_SIDS`/`SIM_BOOKS` literals
|
||||
- 13 VRP test files; comment/docstring scrubs across 16 files; test updates (assert vrp-ABSENT / synthetic entries)
|
||||
|
||||
## What was KEPT (Constraint 1 — verified intact)
|
||||
- `xsp_option_bars` **table** + `XspOptionBarRow` model (`cockpit_models.py:436`) + the 13-year OPRA `.dbn` data
|
||||
- `ForwardNavRepo.xsp_freeze_max_date()` (`forward_nav.py:145`) — still reads the kept table
|
||||
- The generic freeze-staleness health axis (`_FREEZE_SOURCES = {"opra-pit"}`) stays
|
||||
- `deribit_funding` (carry edge) untouched
|
||||
|
||||
## CronJob cluster-resource — VERIFIED not present (no action needed)
|
||||
Checked 2026-07-21 on `bizworx-prod-kapsule`: `kubectl get cronjob fxhnt-opra-backfill -n foxhunt` → NotFound,
|
||||
0 hits cluster-wide. The manifest was never applied live (or already gone), so removing it from git was
|
||||
sufficient. The 5 surviving foxhunt CronJobs (ucits/bybit/ibkr rebalancers, ucits-volume, factory) are all
|
||||
non-VRP.
|
||||
|
||||
## Merged + pushed
|
||||
Branch `chore/remove-vrp-code` merged to master (`2f957bb`) and pushed to origin. Live on prod after the next
|
||||
normal git-sync + rollout-restart.
|
||||
549
docs/superpowers/plans/2026-06-21-paper-dynamic-weighting.md
Normal file
549
docs/superpowers/plans/2026-06-21-paper-dynamic-weighting.md
Normal file
@@ -0,0 +1,549 @@
|
||||
# Paper book: ISV-adaptive sleeve weighting — Implementation Plan (Phase 2F-A3c)
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or
|
||||
> superpowers:executing-plans to implement this plan task-by-task. TDD. Steps use checkbox (`- [ ]`) syntax.
|
||||
|
||||
**Goal:** Replace the paper book's equal-weight sleeve sizing with **ISV-adaptive** inverse-vol weights —
|
||||
state-derived lookback (`L_t`), concentration cap (`cap_t`), and a max-with-floor cold-start — driven by
|
||||
each sleeve's OWN causally-accumulated daily returns, applied identically in the backfill and the nightly
|
||||
snapshot.
|
||||
|
||||
**Architecture:** Two new pure functions in `application/paper_book.py` — `sleeve_daily_return` (one shared
|
||||
return formula) and `isv_adaptive_weights` (the A/B/C ISV logic, reusing the inverse-vol math from
|
||||
`book_allocator`). A new `paper_sleeve_ret(run_date, sleeve, ret)` table + `PaperRepo` read/write so the
|
||||
nightly snapshot rebuilds the same trailing window the backfill accumulates. `backfill_paper_book` and
|
||||
`PaperSnapshotService` swap `equal_weights` → `isv_adaptive_weights` and persist each date's per-sleeve
|
||||
return. Read-only/additive; equal-weight remains the cold-start fallback.
|
||||
|
||||
**Tech Stack:** Python 3.12, pytest, SQLAlchemy, Dagster. **Repo:** `~/Work/fxhnt`, branch
|
||||
`feat/paper-dynamic-weighting`. **Spec:** `docs/superpowers/specs/2026-06-21-paper-dynamic-weighting-design.md`.
|
||||
|
||||
**Verified context:**
|
||||
- `book_allocator._inverse_vol_weights(window: {edge: np.ndarray}, max_sleeve_weight) -> {edge: float}` does
|
||||
raw `∝1/std` normalize + an **iterative cap-and-renormalize loop** (mirror it for the two-sided water-fill).
|
||||
- `domain.paper.Position(sleeve, symbol, qty, entry_price, entry_date)`. `target_positions(...)` re-derives
|
||||
positions each run_date at that date's close → `positions_before(d).entry_price` = prior date's close.
|
||||
- `PaperRepo.positions_before(run_date) -> list[Position]` (prior persisted book) already exists; `upsert_nav`
|
||||
/ `migrate` (create_all) / StaticPool-SQLite test pattern exist.
|
||||
- `paper_book.equal_weights(symbol_weights_by_sleeve)` exists (cold-start fallback).
|
||||
- Call sites: `backfill_paper_book` (`application/paper_backfill.py`), `PaperSnapshotService.snapshot`
|
||||
(`application/paper_snapshot.py`), assets driver `build_paper_book_snapshot` (passes `repo`).
|
||||
- Structural constants: `L_min=10`, `L_max=60`, `φ=0.5`, `κ=2.0`. `epoch_day = date.toordinal()`.
|
||||
|
||||
---
|
||||
|
||||
## Task 1: shared `sleeve_daily_return` pure function
|
||||
|
||||
**Files:** Modify `src/fxhnt/application/paper_book.py`; Test `tests/unit/test_sleeve_daily_return.py`
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
```python
|
||||
# tests/unit/test_sleeve_daily_return.py
|
||||
from fxhnt.application.paper_book import sleeve_daily_return
|
||||
from fxhnt.domain.paper import Position
|
||||
|
||||
def _pos(sym, qty, entry): # sleeve fixed; entry_price carries the PRIOR date's close
|
||||
return Position("s", sym, qty, entry, "2026-01-01")
|
||||
|
||||
def test_long_return_is_pnl_over_notional():
|
||||
prev = [_pos("A", 2.0, 100.0)] # notional 200
|
||||
r = sleeve_daily_return(prev, {"A": 110.0}) # pnl = 2*(110-100)=20
|
||||
assert r == 20.0 / 200.0
|
||||
|
||||
def test_short_position_return():
|
||||
prev = [_pos("A", -2.0, 100.0)] # notional |−2*100|=200
|
||||
r = sleeve_daily_return(prev, {"A": 90.0}) # pnl = -2*(90-100)=20
|
||||
assert r == 20.0 / 200.0
|
||||
|
||||
def test_unknown_symbol_contributes_zero():
|
||||
prev = [_pos("A", 2.0, 100.0), _pos("B", 1.0, 50.0)] # notional 200+50=250
|
||||
r = sleeve_daily_return(prev, {"A": 110.0}) # B unknown -> 0 pnl; pnl=20
|
||||
assert r == 20.0 / 250.0
|
||||
|
||||
def test_no_prior_positions_returns_none():
|
||||
assert sleeve_daily_return([], {"A": 110.0}) is None
|
||||
|
||||
def test_zero_notional_returns_none():
|
||||
assert sleeve_daily_return([_pos("A", 0.0, 100.0)], {"A": 110.0}) is None
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run → FAIL.** `uv run pytest tests/unit/test_sleeve_daily_return.py -q` (ImportError).
|
||||
- [ ] **Step 3: Implement** in `paper_book.py` (near `equal_weights`):
|
||||
|
||||
```python
|
||||
from fxhnt.domain.paper import ( # add Position to the existing import block
|
||||
Position, Trade, diff_trades, realized_delta, target_positions,
|
||||
)
|
||||
|
||||
def sleeve_daily_return(prev_positions: list[Position],
|
||||
close_by_symbol: dict[str, float]) -> float | None:
|
||||
"""One date's return for a sleeve from its prior-date positions, marked to today's closes.
|
||||
|
||||
The prior close is carried by `Position.entry_price` (positions are re-derived each date at that
|
||||
date's close). Return = Σ qty·(close − entry_price) / Σ|qty·entry_price|. Unknown symbol → entry
|
||||
(0 pnl). None when there is no prior position or the notional is 0. Pure; no I/O.
|
||||
"""
|
||||
notional = sum(abs(p.qty * p.entry_price) for p in prev_positions)
|
||||
if notional <= 0.0:
|
||||
return None
|
||||
pnl = sum(p.qty * (close_by_symbol.get(p.symbol, p.entry_price) - p.entry_price)
|
||||
for p in prev_positions)
|
||||
return pnl / notional
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run → PASS.**
|
||||
- [ ] **Step 5: Commit** `git add -p` (verify no `*.dbn`) then
|
||||
`git commit -m "feat(paper): shared sleeve_daily_return pure fn"`
|
||||
|
||||
---
|
||||
|
||||
## Task 2: `isv_adaptive_weights` helper (A adaptive lookback, B floor, C cap)
|
||||
|
||||
**Files:** Modify `src/fxhnt/application/paper_book.py`; Test `tests/unit/test_isv_adaptive_weights.py`
|
||||
|
||||
- [ ] **Step 1: Write the failing tests**
|
||||
|
||||
```python
|
||||
# tests/unit/test_isv_adaptive_weights.py
|
||||
import math
|
||||
from fxhnt.application.paper_book import isv_adaptive_weights
|
||||
|
||||
def _series(vals, start=1): # {epoch_day: ret}
|
||||
return {start + i: v for i, v in enumerate(vals)}
|
||||
|
||||
def test_empty_active_returns_empty():
|
||||
assert isv_adaptive_weights({}, []) == {}
|
||||
|
||||
def test_no_history_is_equal_weight():
|
||||
w = isv_adaptive_weights({"a": {}, "b": {}}, ["a", "b"])
|
||||
assert w == {"a": 0.5, "b": 0.5}
|
||||
|
||||
def test_low_vol_sleeve_gets_higher_weight():
|
||||
lo = _series([0.001, -0.001] * 30) # ~60 obs low vol
|
||||
hi = _series([0.05, -0.05] * 30) # ~60 obs high vol
|
||||
w = isv_adaptive_weights({"lo": lo, "hi": hi}, ["lo", "hi"])
|
||||
assert math.isclose(sum(w.values()), 1.0, abs_tol=1e-9)
|
||||
assert w["lo"] > w["hi"]
|
||||
|
||||
def test_rising_vol_regime_shrinks_lookback():
|
||||
# disjoint regime ratio rho>1 (recent block more volatile than the prior block) -> L_t < L_max.
|
||||
# calm must extend through the PRIOR block (indices 40-49); only the last 10 (indices 50-59) are hot,
|
||||
# so std(recent L_min=10) > std(prior L_min=10). Expose L_t via the debug return for determinism.
|
||||
calm = [0.01, -0.01] * 25 # 50 obs low vol (covers the prior block)
|
||||
hot = [0.06, -0.06] * 5 # last 10 obs high vol (the recent block)
|
||||
s = _series(calm + hot)
|
||||
_, dbg = isv_adaptive_weights({"x": s, "y": _series([0.01, -0.01] * 30)},
|
||||
["x", "y"], _debug=True)
|
||||
assert dbg["L_t"] < 60
|
||||
flat = _series([0.01, -0.01] * 40)
|
||||
_, dbg2 = isv_adaptive_weights({"x": flat, "y": flat}, ["x", "y"], _debug=True)
|
||||
assert dbg2["L_t"] == 60
|
||||
|
||||
def test_cap_never_below_one_over_n():
|
||||
s = _series([0.02, -0.02] * 30)
|
||||
_, dbg = isv_adaptive_weights({"a": s, "b": s, "c": s}, ["a", "b", "c"], _debug=True)
|
||||
assert dbg["cap_t"] >= 1.0 / 3 - 1e-9
|
||||
|
||||
def test_floor_holds_after_waterfill():
|
||||
# one very-low-vol sleeve would dominate; floor keeps the other above phi/N.
|
||||
dom = _series([0.0001, -0.0001] * 40)
|
||||
weak = _series([0.2, -0.2] * 40)
|
||||
w = isv_adaptive_weights({"dom": dom, "weak": weak}, ["dom", "weak"])
|
||||
assert math.isclose(sum(w.values()), 1.0, abs_tol=1e-9)
|
||||
assert w["weak"] >= 0.5 / 2 - 1e-9 # w_floor = phi/N = 0.25
|
||||
assert w["dom"] <= 1.0 + 1e-9
|
||||
|
||||
def test_mixed_history_cold_sleeve_gets_floor():
|
||||
has = _series([0.02, -0.02] * 30)
|
||||
w = isv_adaptive_weights({"has": has, "cold": {1: 0.01}}, ["has", "cold"]) # cold has <2 obs
|
||||
assert math.isclose(w["cold"], 0.5 / 2, abs_tol=1e-9) # exactly w_floor
|
||||
assert math.isclose(sum(w.values()), 1.0, abs_tol=1e-9)
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run → FAIL.**
|
||||
- [ ] **Step 3: Implement** in `paper_book.py`. Add `import numpy as np` and
|
||||
`from fxhnt.application.book_allocator import _inverse_vol_weights` is NOT used (we own the clamp); instead:
|
||||
|
||||
```python
|
||||
import numpy as np
|
||||
|
||||
_L_MIN = 10
|
||||
_L_MAX = 60
|
||||
_PHI = 0.5
|
||||
_KAPPA = 2.0
|
||||
|
||||
def _regime_lookback(series_by_sleeve: dict[str, dict[int, float]], active: list[str]) -> int:
|
||||
"""Adaptive L_t from a DISJOINT recent-vs-prior vol ratio on the active sleeves' mean return series.
|
||||
rho = std(recent L_min) / std(prior L_min); rho>1 (vol rising) -> shorten. Hold L_max until 2*L_min
|
||||
obs exist. Floor L_min, anchor L_max."""
|
||||
# union date axis, 0-fill, mean across active = one representative series
|
||||
days = sorted({d for s in active for d in series_by_sleeve.get(s, {})})
|
||||
if len(days) < 2 * _L_MIN:
|
||||
return _L_MAX
|
||||
idx = {d: i for i, d in enumerate(days)}
|
||||
mat = np.zeros((len(active), len(days)))
|
||||
for r, s in enumerate(active):
|
||||
for d, v in series_by_sleeve.get(s, {}).items():
|
||||
mat[r, idx[d]] = v
|
||||
rep = mat.mean(axis=0)
|
||||
recent, prior = rep[-_L_MIN:], rep[-2 * _L_MIN:-_L_MIN]
|
||||
sp = float(np.std(prior))
|
||||
if sp <= 1e-12:
|
||||
return _L_MAX
|
||||
rho = float(np.std(recent)) / sp
|
||||
return int(round(min(_L_MAX, max(_L_MIN, _L_MAX / max(1.0, rho)))))
|
||||
|
||||
def _waterfill(init: dict[str, float], w_floor: float, cap: float) -> dict[str, float]:
|
||||
"""Two-sided iterative clamp-to-[w_floor, cap] + renormalize the free mass, until stable (<=N passes).
|
||||
Feasible because w_floor*N <= 1 <= cap*N. Mirrors the cap loop in book_allocator._inverse_vol_weights."""
|
||||
w = dict(init)
|
||||
for _ in range(len(w) + 1):
|
||||
clamped = {k: v for k, v in w.items() if v <= w_floor + 1e-15 or v >= cap - 1e-15}
|
||||
if clamped:
|
||||
fixed = {k: min(cap, max(w_floor, w[k])) for k in clamped}
|
||||
free = {k: w[k] for k in w if k not in clamped}
|
||||
remaining = 1.0 - sum(fixed.values())
|
||||
ft = sum(free.values())
|
||||
nxt = dict(fixed)
|
||||
for k in free:
|
||||
nxt[k] = (free[k] / ft * remaining) if ft > 1e-12 else (remaining / len(free) if free else 0.0)
|
||||
else:
|
||||
nxt = w
|
||||
if all(abs(nxt[k] - w[k]) < 1e-12 for k in w):
|
||||
w = nxt
|
||||
break
|
||||
w = nxt
|
||||
return {k: min(cap, max(w_floor, v)) for k, v in w.items()}
|
||||
|
||||
def isv_adaptive_weights(returns_by_sleeve: dict[str, dict[int, float]],
|
||||
active_sleeves: list[str], *, _debug: bool = False):
|
||||
"""ISV-adaptive sleeve weights: adaptive lookback L_t (A), N_eff-derived cap_t (C), and a
|
||||
max-with-floor cold-start via two-sided water-fill (B). Cold-start / no-history -> equal-weight.
|
||||
Pure. Returns {sleeve: weight} (sum 1), or ({...}, {"L_t":..,"cap_t":..}) when _debug."""
|
||||
active = [s for s in active_sleeves if s]
|
||||
n = len(active)
|
||||
if n == 0:
|
||||
return ({}, {"L_t": _L_MAX, "cap_t": 0.0}) if _debug else {}
|
||||
has = [s for s in active if len(returns_by_sleeve.get(s, {})) >= 2]
|
||||
if not has:
|
||||
eq = {s: 1.0 / n for s in active}
|
||||
return (eq, {"L_t": _L_MAX, "cap_t": 1.0}) if _debug else eq
|
||||
L_t = _regime_lookback(returns_by_sleeve, has)
|
||||
inv = {}
|
||||
for s in has:
|
||||
vals = [returns_by_sleeve[s][d] for d in sorted(returns_by_sleeve[s])][-L_t:]
|
||||
v = float(np.std(vals)) if len(vals) > 1 else 0.0
|
||||
inv[s] = (1.0 / v) if v > 1e-12 else 0.0
|
||||
tot = sum(inv.values())
|
||||
raw = {s: inv[s] / tot for s in has} if tot > 0 else {s: 1.0 / len(has) for s in has}
|
||||
n_eff = (sum(raw.values()) ** 2) / sum(v * v for v in raw.values()) if raw else 1.0
|
||||
cap_t = min(1.0, max(1.0 / n, _KAPPA / n_eff))
|
||||
w_floor = _PHI / n
|
||||
cold = [s for s in active if s not in has]
|
||||
init = {s: w_floor for s in cold}
|
||||
has_mass = max(0.0, 1.0 - w_floor * len(cold))
|
||||
for s in has:
|
||||
init[s] = raw[s] * has_mass
|
||||
w = _waterfill(init, w_floor, cap_t)
|
||||
return (w, {"L_t": L_t, "cap_t": cap_t}) if _debug else w
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run → PASS.** Adjust tolerances only if a test's expected math is off (not the asserts' intent).
|
||||
- [ ] **Step 5: Commit** `git commit -m "feat(paper): isv_adaptive_weights (adaptive lookback/cap + max-with-floor)"`
|
||||
|
||||
---
|
||||
|
||||
## Task 3: `paper_sleeve_ret` table + PaperRepo read/write
|
||||
|
||||
**Files:** Modify `src/fxhnt/adapters/persistence/cockpit_models.py`,
|
||||
`src/fxhnt/adapters/persistence/paper_repo.py`; Test `tests/integration/test_paper_sleeve_ret.py`
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
```python
|
||||
# tests/integration/test_paper_sleeve_ret.py
|
||||
import datetime as dt
|
||||
from fxhnt.adapters.persistence.paper_repo import PaperRepo
|
||||
|
||||
def _repo():
|
||||
r = PaperRepo("sqlite://"); r.migrate(); return r
|
||||
|
||||
def test_upsert_and_read_trailing_returns():
|
||||
r = _repo(); at = dt.datetime(2026, 1, 2, tzinfo=dt.UTC)
|
||||
r.upsert_sleeve_ret("2026-01-01", "ts", 0.01, at=at)
|
||||
r.upsert_sleeve_ret("2026-01-02", "ts", 0.02, at=at)
|
||||
r.upsert_sleeve_ret("2026-01-02", "sc", -0.01, at=at)
|
||||
out = r.sleeve_returns(before="2026-01-03")
|
||||
ed1 = dt.date(2026, 1, 1).toordinal(); ed2 = dt.date(2026, 1, 2).toordinal()
|
||||
assert out["ts"] == {ed1: 0.01, ed2: 0.02}
|
||||
assert out["sc"] == {ed2: -0.01}
|
||||
|
||||
def test_before_is_strict_and_idempotent():
|
||||
r = _repo(); at = dt.datetime(2026, 1, 2, tzinfo=dt.UTC)
|
||||
r.upsert_sleeve_ret("2026-01-02", "ts", 0.02, at=at)
|
||||
r.upsert_sleeve_ret("2026-01-02", "ts", 0.05, at=at) # overwrite same (date,sleeve)
|
||||
assert r.sleeve_returns(before="2026-01-02") == {} # strict < before
|
||||
out = r.sleeve_returns(before="2026-01-03")
|
||||
assert out["ts"][dt.date(2026, 1, 2).toordinal()] == 0.05
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run → FAIL.**
|
||||
- [ ] **Step 3a: Add the model** to `cockpit_models.py` (after `PaperNavRow`):
|
||||
|
||||
```python
|
||||
class PaperSleeveRetRow(CockpitBase):
|
||||
"""One sleeve's simulated daily return for one run_date (backfill + snapshot both write). Feeds the
|
||||
ISV-adaptive trailing-vol window. (run_date, sleeve) PK — idempotent per date+sleeve."""
|
||||
__tablename__ = "paper_sleeve_ret"
|
||||
run_date: Mapped[dt.date] = mapped_column(Date, primary_key=True)
|
||||
sleeve: Mapped[str] = mapped_column(String(32), primary_key=True)
|
||||
ret: Mapped[float] = mapped_column(Float)
|
||||
src_updated_at: Mapped[dt.datetime] = mapped_column(DateTime)
|
||||
```
|
||||
|
||||
- [ ] **Step 3b: Add repo methods** to `paper_repo.py` (import `PaperSleeveRetRow`; add after the nav block):
|
||||
|
||||
```python
|
||||
def upsert_sleeve_ret(self, run_date: str, sleeve: str, ret: float, *, at: dt.datetime) -> None:
|
||||
"""Idempotent per (run_date, sleeve): write/overwrite that date's per-sleeve simulated return."""
|
||||
rd = dt.date.fromisoformat(run_date)
|
||||
with Session(self._engine) as s:
|
||||
s.merge(PaperSleeveRetRow(run_date=rd, sleeve=sleeve, ret=float(ret), src_updated_at=at))
|
||||
s.commit()
|
||||
|
||||
def sleeve_returns(self, *, before: str) -> dict[str, dict[int, float]]:
|
||||
"""Trailing per-sleeve return series STRICTLY before `before`, as {sleeve: {epoch_day: ret}}
|
||||
(epoch_day = date.toordinal()) — the shape isv_adaptive_weights expects."""
|
||||
cutoff = dt.date.fromisoformat(before)
|
||||
out: dict[str, dict[int, float]] = {}
|
||||
with Session(self._engine) as s:
|
||||
rows = s.scalars(select(PaperSleeveRetRow)
|
||||
.where(PaperSleeveRetRow.run_date < cutoff)
|
||||
.order_by(PaperSleeveRetRow.run_date.asc()))
|
||||
for r in rows:
|
||||
out.setdefault(r.sleeve, {})[r.run_date.toordinal()] = r.ret
|
||||
return out
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run → PASS.**
|
||||
- [ ] **Step 5: Commit** `git commit -m "feat(paper): paper_sleeve_ret table + PaperRepo upsert/sleeve_returns"`
|
||||
|
||||
---
|
||||
|
||||
## Task 4: `backfill_paper_book` — causal returns + isv_adaptive_weights
|
||||
|
||||
**Files:** Modify `src/fxhnt/application/paper_backfill.py`; Test extend `tests/integration/test_paper_backfill.py`
|
||||
|
||||
- [ ] **Step 1: Write the failing test** (append to `test_paper_backfill.py`). Use a fake `PositionSource`
|
||||
with two sleeves of differing realized vol over many dates; assert (a) `paper_sleeve_ret` populated, and
|
||||
(b) on a late date the low-vol sleeve's persisted `weight` exceeds the high-vol sleeve's (moved off 1/N).
|
||||
|
||||
```python
|
||||
def test_backfill_uses_isv_adaptive_weights(tmp_path):
|
||||
import datetime as dt
|
||||
from fxhnt.adapters.persistence.paper_repo import PaperRepo
|
||||
from fxhnt.application.paper_backfill import backfill_paper_book
|
||||
|
||||
class Sleeve:
|
||||
def __init__(self, sym): self.sym = sym
|
||||
def current_weights(self, d): return {self.sym: 1.0} # always active, one name
|
||||
|
||||
repo = PaperRepo("sqlite://"); repo.migrate()
|
||||
dates = [f"2026-01-{i:02d}" for i in range(1, 29)] # 28 days
|
||||
# low-vol symbol LO drifts smoothly; high-vol HI oscillates hard
|
||||
closes_by_date = {}
|
||||
for i, d in enumerate(dates):
|
||||
lo = 100.0 + 0.05 * i
|
||||
hi = 100.0 + (8.0 if i % 2 else -8.0)
|
||||
closes_by_date[d] = {"LO": lo, "HI": hi}
|
||||
n = backfill_paper_book(repo, capital=100_000.0,
|
||||
sleeves={"lo": Sleeve("LO"), "hi": Sleeve("HI")},
|
||||
dates=dates, closes_by_date=closes_by_date,
|
||||
at=dt.datetime(2026, 2, 1, tzinfo=dt.UTC))
|
||||
assert n == len(dates)
|
||||
rets = repo.sleeve_returns(before="2026-02-01")
|
||||
assert "lo" in rets and "hi" in rets and len(rets["lo"]) > 10
|
||||
assert repo.weight_for("lo") > repo.weight_for("hi") # low-vol sleeve weighted up
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run → FAIL.**
|
||||
- [ ] **Step 3: Implement.** Replace the `equal_weights` import + sizing with the causal-returns loop:
|
||||
|
||||
```python
|
||||
from fxhnt.application.paper_book import isv_adaptive_weights, persist_paper_book, sleeve_daily_return
|
||||
# (drop the equal_weights import)
|
||||
```
|
||||
|
||||
Rewrite the loop body of `backfill_paper_book` so that, per date ascending:
|
||||
|
||||
```python
|
||||
persisted = 0
|
||||
for date in sorted(dates):
|
||||
returns_by_sleeve = repo.sleeve_returns(before=date) # history strictly before `date`
|
||||
symbol_weights_by_sleeve: dict[str, dict[str, float]] = {}
|
||||
for name, sleeve in sleeves.items():
|
||||
try:
|
||||
w = sleeve.current_weights(date)
|
||||
except Exception as exc: # noqa: BLE001 — one bad sleeve must not drop the date
|
||||
_log.warning("paper-backfill: sleeve %s current_weights failed on %s, skipping: %s",
|
||||
name, date, exc)
|
||||
continue
|
||||
if not w:
|
||||
_log.info("paper-backfill: sleeve %s has no weights as of %s, skipping", name, date)
|
||||
continue
|
||||
symbol_weights_by_sleeve[name] = w
|
||||
active = list(symbol_weights_by_sleeve)
|
||||
persist_paper_book(
|
||||
repo, capital=capital, run_date=date,
|
||||
sleeve_weights=isv_adaptive_weights(returns_by_sleeve, active),
|
||||
symbol_weights_by_sleeve=symbol_weights_by_sleeve,
|
||||
prices=closes_by_date.get(date, {}), at=at)
|
||||
# AFTER sizing+persisting this date, book its realized return (uses THIS date's close) for the
|
||||
# NEXT date's window — causal. prior = the book persisted for the immediately preceding date.
|
||||
prior = repo.positions_before(date)
|
||||
if prior:
|
||||
by_sleeve: dict[str, list] = {}
|
||||
for p in prior:
|
||||
by_sleeve.setdefault(p.sleeve, []).append(p)
|
||||
for sname, plist in by_sleeve.items():
|
||||
r = sleeve_daily_return(plist, closes_by_date.get(date, {}))
|
||||
if r is not None:
|
||||
repo.upsert_sleeve_ret(date, sname, r, at=at)
|
||||
persisted += 1
|
||||
return persisted
|
||||
```
|
||||
|
||||
Also update the module docstring (equal-weight → ISV-adaptive; mention the causal return window).
|
||||
|
||||
- [ ] **Step 4: Run → PASS** (`uv run pytest tests/integration/test_paper_backfill.py -q`). Keep the existing
|
||||
backfill tests green — if one asserted equal-weight sizing, update it to the ISV expectation.
|
||||
- [ ] **Step 5: Commit** `git commit -m "feat(paper): backfill builds causal per-sleeve returns + ISV-adaptive weights"`
|
||||
|
||||
---
|
||||
|
||||
## Task 5: `PaperSnapshotService` — same ISV path off `paper_sleeve_ret`
|
||||
|
||||
**Files:** Modify `src/fxhnt/application/paper_snapshot.py`; Test extend `tests/integration/test_paper_snapshot.py`
|
||||
|
||||
- [ ] **Step 1: Write the failing test** — seed `paper_sleeve_ret` history (two sleeves, differing vol) +
|
||||
a prior-date book via one `persist_paper_book`, then `PaperSnapshotService.snapshot(as_of=...)` and assert
|
||||
(a) it persists positions, (b) `repo.weight_for(low_vol) > repo.weight_for(high_vol)`, (c) a new
|
||||
`paper_sleeve_ret` row for `as_of` was written.
|
||||
|
||||
```python
|
||||
def test_snapshot_uses_isv_adaptive_weights():
|
||||
import datetime as dt
|
||||
from fxhnt.adapters.persistence.paper_repo import PaperRepo
|
||||
from fxhnt.application.paper_snapshot import PaperSnapshotService
|
||||
|
||||
repo = PaperRepo("sqlite://"); repo.migrate()
|
||||
at = dt.datetime(2026, 2, 1, tzinfo=dt.UTC)
|
||||
base = dt.date(2026, 1, 1).toordinal()
|
||||
for k in range(20): # seed trailing returns: lo low-vol, hi high-vol
|
||||
d = dt.date.fromordinal(base + k).isoformat()
|
||||
repo.upsert_sleeve_ret(d, "lo", 0.001 * (1 if k % 2 else -1), at=at)
|
||||
repo.upsert_sleeve_ret(d, "hi", 0.05 * (1 if k % 2 else -1), at=at)
|
||||
# prior book so snapshot has a positions_before to compute as_of's own return
|
||||
from fxhnt.application.paper_book import persist_paper_book
|
||||
persist_paper_book(repo, capital=100_000.0, run_date="2026-01-24",
|
||||
sleeve_weights={"lo": 0.5, "hi": 0.5},
|
||||
symbol_weights_by_sleeve={"lo": {"LO": 1.0}, "hi": {"HI": 1.0}},
|
||||
prices={"LO": 100.0, "HI": 100.0}, at=at)
|
||||
|
||||
class Sleeve:
|
||||
def __init__(self, sym): self.sym = sym
|
||||
def current_weights(self, d): return {self.sym: 1.0}
|
||||
|
||||
svc = PaperSnapshotService(repo, capital=100_000.0,
|
||||
sleeves={"lo": Sleeve("LO"), "hi": Sleeve("HI")},
|
||||
sleeve_weights={}, prices={"LO": 101.0, "HI": 108.0})
|
||||
n = svc.snapshot(as_of="2026-01-25", at=at)
|
||||
assert n > 0
|
||||
assert repo.weight_for("lo") > repo.weight_for("hi")
|
||||
assert repo.sleeve_returns(before="2026-01-26").get("lo", {})
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run → FAIL.**
|
||||
- [ ] **Step 3: Implement.** In `paper_snapshot.py` swap the import (`equal_weights` →
|
||||
`isv_adaptive_weights, sleeve_daily_return`) and rewrite `snapshot`'s sizing + return-persist:
|
||||
|
||||
```python
|
||||
returns_by_sleeve = self._repo.sleeve_returns(before=as_of)
|
||||
# ... existing per-sleeve current_weights try/except loop builds symbol_weights_by_sleeve ...
|
||||
active = list(symbol_weights_by_sleeve)
|
||||
n = persist_paper_book(
|
||||
self._repo, capital=self._capital, run_date=as_of,
|
||||
sleeve_weights=isv_adaptive_weights(returns_by_sleeve, active),
|
||||
symbol_weights_by_sleeve=symbol_weights_by_sleeve,
|
||||
prices=self._prices, at=at, enabled=self._enabled)
|
||||
if self._enabled:
|
||||
prior = self._repo.positions_before(as_of)
|
||||
by_sleeve: dict[str, list] = {}
|
||||
for p in prior:
|
||||
by_sleeve.setdefault(p.sleeve, []).append(p)
|
||||
for sname, plist in by_sleeve.items():
|
||||
r = sleeve_daily_return(plist, self._prices)
|
||||
if r is not None:
|
||||
self._repo.upsert_sleeve_ret(as_of, sname, r, at=at)
|
||||
return n
|
||||
```
|
||||
|
||||
Update the class/method docstrings (equal-weight → ISV-adaptive; note the shared return fn + table).
|
||||
|
||||
- [ ] **Step 4: Run → PASS** (`uv run pytest tests/integration/test_paper_snapshot.py
|
||||
tests/unit/test_paper_snapshot_asset.py -q`). Update any existing test asserting equal-weight.
|
||||
- [ ] **Step 5: Commit** `git commit -m "feat(paper): nightly snapshot uses ISV-adaptive weights off paper_sleeve_ret"`
|
||||
|
||||
---
|
||||
|
||||
## Task 6: Full suite + finish + deploy + re-backfill + verify
|
||||
|
||||
- [ ] **Step 1:** `cd ~/Work/fxhnt && uv run pytest -q` → all green. Fix any equal-weight-era assertions.
|
||||
- [ ] **Step 2:** Confirm nothing stray staged: `git status --short | grep -i '\.dbn'` must print nothing.
|
||||
- [ ] **Step 3:** **superpowers:finishing-a-development-branch** — merge `feat/paper-dynamic-weighting`
|
||||
→ master, push (auto-deploys via Gitea→Argo). Confirm the cockpit workflow Succeeded + dashboard pod
|
||||
healthy (`kubectl get pods -n foxhunt | grep -E 'dashboard|dagster'`).
|
||||
- [ ] **Step 4:** Re-run the backfill in-cluster (rebuilds `paper_positions`, `paper_nav`, `paper_sleeve_ret`):
|
||||
`kubectl exec deploy/dagster -c daemon -n foxhunt -- fxhnt paper-backfill --days 365`. Confirm it reports
|
||||
positions on many dates.
|
||||
- [ ] **Step 5: VERIFY.** Query that the ISV path engaged and the curve is non-flat + risk-weighted:
|
||||
```bash
|
||||
kubectl exec deploy/dagster -c daemon -n foxhunt -- python -c "
|
||||
from sqlalchemy import create_engine, text
|
||||
from fxhnt.config import get_settings
|
||||
e=create_engine(get_settings().operational_dsn)
|
||||
with e.connect() as c:
|
||||
print('sleeve_ret rows:', list(c.execute(text('select sleeve,count(*) n,round(avg(ret)::numeric,5) from paper_sleeve_ret group by sleeve'))))
|
||||
print('distinct sleeve weights (off 1/N?):', list(c.execute(text('select sleeve,count(distinct round(weight::numeric,4)) from paper_positions group by sleeve'))))
|
||||
print('nav span:', list(c.execute(text('select min(equity),max(equity),count(*) from paper_nav'))))
|
||||
"
|
||||
```
|
||||
Confirm: `paper_sleeve_ret` populated; `paper_positions.weight` takes >1 distinct value per sleeve (proves
|
||||
weights move, not stuck at 1/N); `paper_nav` equity varies (non-flat). Then load
|
||||
`https://dashboard.fxhnt.ai/paper/replay` and `/paper` and report what renders.
|
||||
|
||||
---
|
||||
|
||||
## Self-review notes
|
||||
- `sleeve_daily_return` + `isv_adaptive_weights` are the single shared sources used by BOTH backfill +
|
||||
snapshot (consistency #5). No duplicated return/weight formula.
|
||||
- Causality: each date's return is booked AFTER sizing that date, used only for the next date's window.
|
||||
- Two-sided water-fill (Task 2 `_waterfill`) guarantees the floor holds post-renormalize (#4); cap+floor one
|
||||
joint range (`[w_floor, cap_t]`).
|
||||
- Disjoint regime windows (#2): `_regime_lookback` compares recent vs prior block, holds `L_max` < 2·L_min.
|
||||
- Mixed-history (#6): cold (<2 obs) active sleeves pinned at `w_floor`; all-cold → equal-weight.
|
||||
- Documented inert-at-N≤3 caveats stand (cap rarely binds, lookback often in bootstrap) — degrades to
|
||||
equal-weight, never a flat/degenerate book.
|
||||
- Read-only/additive: no sleeve `run()`/`advance()`, `combine_book`, or gate-record change.
|
||||
- NEVER `git add -A` — `*.dbn` is gitignored; verify nothing stray staged before each commit.
|
||||
|
||||
## Acceptance criteria (from spec)
|
||||
- `isv_adaptive_weights` (active-filter, equal-weight bootstrap, adaptive `L_t`/`cap_t`, mixed-history,
|
||||
two-sided water-fill); unit-tested incl. (i)-(vii). ✅ T2
|
||||
- `sleeve_daily_return` pure fn; both callers import it. ✅ T1, T4, T5
|
||||
- `paper_sleeve_ret` table + `upsert_sleeve_ret`/`sleeve_returns`; idempotent; tested. ✅ T3
|
||||
- backfill builds causal window + ISV weights + persists returns; integration-tested. ✅ T4
|
||||
- snapshot reads trailing returns + ISV weights; tested. ✅ T5
|
||||
- re-backfill → `paper_sleeve_ret` populated, `/paper/replay` non-flat + risk-weighted, `/paper` matches. ✅ T6
|
||||
- full suite green; read-only/additive; no `*.dbn` staged. ✅ T6
|
||||
@@ -0,0 +1,139 @@
|
||||
# Paper book: equal-weight + wire unlock to warehouse — Implementation Plan (Phase 2F-A3b)
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: subagent-driven-development (recommended) or
|
||||
> executing-plans. TDD. Steps use `- [ ]`.
|
||||
|
||||
**Goal:** Make the paper book actually hold positions: size the replayable sleeves by equal-weight (1/N
|
||||
active sleeves) in BOTH the backfill and the nightly snapshot, and build the unlock (and snapshot ts-trend)
|
||||
sleeve from the in-cluster warehouse instead of the absent `crypto_pit_dir`.
|
||||
|
||||
**Architecture:** A shared `equal_weights` helper used by `backfill_paper_book` (per date) and the
|
||||
`PaperSnapshotService`/asset path; unlock's vol panel built from the warehouse `crypto_close_panel`
|
||||
(close + fallback qvol/funding) + the existing `unlock_calendar.json` events. Read-only/additive.
|
||||
|
||||
**Tech Stack:** Python 3.12, pytest, Dagster. **Repo:** `~/Work/fxhnt`, branch
|
||||
`feat/backfill-stablecoin-unlock`. **Spec:** `docs/superpowers/specs/2026-06-21-paper-equal-weight-and-unlock-design.md`.
|
||||
|
||||
**Verified context:** warehouse `crypto_close_panel(suffix="USDT")` → 197 symbols × 365 closes; ts-trend
|
||||
`current_weights` returns 70 names on the latest date; `_paper_backfill_sleeve_weights()` returns `{}` (the
|
||||
allocator covers a different edge set) → 0 positions; `_unlock()` + `_live_sleeves` use the absent
|
||||
`/root/.fxhnt/crypto_pit`. `UnlockShortRunner(panel={sym:{day:(close,qvol,funding)}}, events=[{sym,cliff_day,n,cat}])`.
|
||||
|
||||
---
|
||||
|
||||
## Task 1: shared `equal_weights` helper + `backfill_paper_book` uses per-date 1/N
|
||||
|
||||
**Files:** Modify `src/fxhnt/application/paper_book.py` (add helper) + `src/fxhnt/application/paper_backfill.py`;
|
||||
Test `tests/unit/test_equal_weights.py` + extend `tests/integration/test_paper_backfill.py`
|
||||
|
||||
- [ ] **Step 1: Failing unit test**
|
||||
```python
|
||||
# tests/unit/test_equal_weights.py
|
||||
from fxhnt.application.paper_book import equal_weights
|
||||
def test_equal_weights_over_active_sleeves():
|
||||
swb = {"a": {"X": 1.0}, "b": {"Y": -1.0}, "c": {}} # c is empty/inactive
|
||||
assert equal_weights(swb) == {"a": 0.5, "b": 0.5} # 1/N over the 2 active; empties dropped
|
||||
def test_equal_weights_empty():
|
||||
assert equal_weights({"a": {}}) == {}
|
||||
```
|
||||
- [ ] **Step 2:** run → FAIL.
|
||||
- [ ] **Step 3:** Add to `paper_book.py`:
|
||||
```python
|
||||
def equal_weights(symbol_weights_by_sleeve: dict[str, dict[str, float]]) -> dict[str, float]:
|
||||
"""Equal-weight the sleeves that produced a non-empty book. {sleeve: 1/N} over active sleeves."""
|
||||
active = [s for s, w in symbol_weights_by_sleeve.items() if w]
|
||||
if not active:
|
||||
return {}
|
||||
w = 1.0 / len(active)
|
||||
return {s: w for s in active}
|
||||
```
|
||||
- [ ] **Step 4:** PASS.
|
||||
- [ ] **Step 5: Failing integration test** — extend `test_paper_backfill.py`: with 2 active sleeves on a
|
||||
date, assert each gets notional `0.5*capital` (positions sized, not zero). Then make `backfill_paper_book`
|
||||
compute `sleeve_weights` per date from the day's active sleeves via `equal_weights(...)` (build
|
||||
`symbol_weights_by_sleeve` first, then `equal_weights`, then `persist_paper_book`). Drop the incoming
|
||||
`sleeve_weights` param dependency (keep it for back-compat but override with equal_weights, OR remove it
|
||||
and update callers — your call; keep the test green).
|
||||
- [ ] **Step 6:** PASS. **Step 7:** `git commit -m "feat(paper): equal-weight active replayable sleeves (1/N) in backfill"`
|
||||
|
||||
---
|
||||
|
||||
## Task 2: nightly snapshot path uses equal_weights (so live /paper isn't flat)
|
||||
|
||||
**Files:** Modify `src/fxhnt/application/paper_snapshot.py`; Test extend `tests/integration/test_paper_snapshot.py`
|
||||
|
||||
- [ ] **Step 1: Failing test** — `PaperSnapshotService.snapshot(...)` with 2 active sleeves persists
|
||||
positions sized at `0.5*capital` each (currently it relies on injected `sleeve_weights` which is `{}` in
|
||||
prod → 0 positions). Assert non-empty positions.
|
||||
- [ ] **Step 2:** run → FAIL.
|
||||
- [ ] **Step 3:** In `PaperSnapshotService.snapshot`, after building `symbol_weights_by_sleeve` from each
|
||||
sleeve's `current_weights`, compute `sleeve_weights = equal_weights(symbol_weights_by_sleeve)` (instead of
|
||||
using the injected/allocator weights) before `persist_paper_book`. Keep the per-sleeve skip-on-fail.
|
||||
- [ ] **Step 4:** PASS. **Step 5:** `git commit -m "feat(paper): nightly snapshot equal-weights active sleeves"`
|
||||
|
||||
---
|
||||
|
||||
## Task 3: build unlock (+ snapshot ts-trend) from the warehouse, not crypto_pit_dir
|
||||
|
||||
**Files:** Modify `src/fxhnt/cli.py` (`_paper_backfill_sleeves._unlock`) + `src/fxhnt/adapters/orchestration/assets.py`
|
||||
(`_live_sleeves` `_tstrend` + `_unlock`); Test `tests/integration/test_unlock_warehouse_panel.py`
|
||||
|
||||
- [ ] **Step 1: Failing test** — build a fake warehouse `crypto_close_panel` ({sym:{day:close}}) for a name
|
||||
with a cliff in a historical-cliff calendar; construct the unlock sleeve from a warehouse-derived
|
||||
`(close, qvol, funding)` panel; assert `current_weights(as_of in the pre-window)` returns that name with a
|
||||
negative (short) weight.
|
||||
- [ ] **Step 2:** run → FAIL.
|
||||
- [ ] **Step 3:** Add a helper `warehouse_unlock_panel(close_panel) -> {sym: {day: (close, qvol_fallback,
|
||||
funding_fallback)}}` (qvol_fallback large enough to clear `min_qvol`, funding 0.0 — documented
|
||||
approximation since the warehouse `features` may lack qvol/funding). In `cli.py _unlock()`: replace
|
||||
`CryptoPitPanelSource(s.crypto_pit_dir).panel()` with `warehouse_unlock_panel(<the warehouse close
|
||||
panel already built for ts-trend>)`; keep `events = json.loads(unlock_calendar.json)`;
|
||||
`UnlockShortRunner(panel, events)`. In `assets.py _live_sleeves`: change BOTH `_tstrend` (use
|
||||
`crypto_close_panel` instead of `CryptoPitPanelSource`) and `_unlock` (warehouse panel) the same way, so
|
||||
the nightly snapshot matches the backfill.
|
||||
- [ ] **Step 4:** PASS. **Step 5:** `git commit -m "feat(paper): unlock + snapshot ts-trend read warehouse panel (drop crypto_pit_dir)"`
|
||||
|
||||
---
|
||||
|
||||
## Task 4: drop the dead allocator-weights helper from the CLI
|
||||
|
||||
**Files:** Modify `src/fxhnt/cli.py`
|
||||
|
||||
- [ ] **Step 1:** Remove the now-unused `_paper_backfill_sleeve_weights` call + helper from the
|
||||
`paper-backfill` command (equal-weight in `backfill_paper_book` replaces it). Pass `sleeves` +
|
||||
`dates` + `closes_by_date` to `backfill_paper_book` (which now equal-weights internally). Keep the
|
||||
existing CLI test green (update it for the new signature if needed).
|
||||
- [ ] **Step 2:** `uv run pytest tests/integration/test_paper_backfill_cli.py -q` → PASS.
|
||||
- [ ] **Step 3:** `git commit -m "refactor(paper): drop dead allocator sleeve-weights from paper-backfill"`
|
||||
|
||||
---
|
||||
|
||||
## Task 5: Full suite + finish + deploy + re-backfill + verify NON-flat curve
|
||||
|
||||
- [ ] **Step 1:** `uv run pytest -q` → all green.
|
||||
- [ ] **Step 2:** **superpowers:finishing-a-development-branch** — merge `feat/backfill-stablecoin-unlock`
|
||||
→ master, push (auto-deploys). Confirm the cockpit workflow Succeeded + dashboard pod healthy.
|
||||
- [ ] **Step 3:** Re-run backfill in-cluster:
|
||||
`kubectl exec deploy/dagster -c daemon -n foxhunt -- fxhnt paper-backfill --days 365`. Confirm it reports
|
||||
positions on many dates (not "0 positions").
|
||||
- [ ] **Step 4: VERIFY** — query `paper_positions` per sleeve (non-empty for ts-trend + unlock; stablecoin
|
||||
on depeg days) via:
|
||||
`kubectl exec deploy/dagster -c daemon -n foxhunt -- python -c "from sqlalchemy import create_engine,text; from fxhnt.config import get_settings; e=create_engine(get_settings().operational_dsn); [print(r.sleeve, r.n) for r in e.connect().execute(text('select sleeve,count(*) n from paper_positions group by sleeve'))]"`.
|
||||
Confirm `https://dashboard.fxhnt.ai/paper/replay` equity curve is NON-flat (varies from $100k) and
|
||||
`/paper` shows live positions. Report what renders.
|
||||
|
||||
---
|
||||
|
||||
## Self-review notes
|
||||
- equal_weights is the single sizing source for BOTH backfill + nightly snapshot (consistency).
|
||||
- unlock vol panel from warehouse close + fallback qvol/funding (documented approximation); never crash.
|
||||
- Read-only/additive; no sleeve `run()`/`advance()` or `combine_book` changed.
|
||||
- The full `_live_sleeves`↔backfill composition dedup remains a noted follow-up (not this plan).
|
||||
- Don't `git add -A` (`*.dbn` gitignored — verify).
|
||||
|
||||
## Acceptance criteria (from spec)
|
||||
- shared `equal_weights` (1/N active); unit-tested. ✅ T1
|
||||
- backfill + nightly snapshot both use it; positions persist. ✅ T1, T2
|
||||
- unlock builds from warehouse + calendar (no crypto_pit_dir); produces pre-cliff shorts; tested. ✅ T3
|
||||
- re-backfill → paper_positions non-empty across dates; /paper/replay curve NON-flat. ✅ T5
|
||||
- live /paper book populated by the nightly snapshot. ✅ T2, T5
|
||||
199
docs/superpowers/plans/2026-06-21-paper-replay.md
Normal file
199
docs/superpowers/plans/2026-06-21-paper-replay.md
Normal file
@@ -0,0 +1,199 @@
|
||||
# Paper Replay (history scrubber) — Implementation Plan (Phase 2F-A3)
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: superpowers:subagent-driven-development (recommended) or
|
||||
> executing-plans. TDD throughout. Steps use `- [ ]`.
|
||||
|
||||
**Goal:** Backfill the paper book over warehouse history into a `paper_nav` equity series + per-date
|
||||
positions, and add a `/paper/replay` scrubber to watch the equity curve + book evolve.
|
||||
|
||||
**Architecture:** Reuse `persist_paper_book` (now also writing `paper_nav`); a backfill iterates past dates
|
||||
ascending, recomputing as_of-aware sleeves' `current_weights(as_of)` + warehouse closes per date; a
|
||||
`/paper/replay` HTMX view scrubs `paper_nav` dates. xsfunding excluded (live-only). Read-only.
|
||||
|
||||
**Tech Stack:** Python 3.12, SQLAlchemy, FastAPI+Jinja2+HTMX, Typer CLI, pytest. **Repo:** `~/Work/fxhnt`,
|
||||
branch `feat/paper-replay`. **Spec:** `docs/superpowers/specs/2026-06-21-paper-replay-design.md`.
|
||||
**Run tests:** `uv run pytest <path> -q`.
|
||||
|
||||
**Reuse (exists):** `persist_paper_book(repo, *, capital, run_date, sleeve_weights, symbol_weights_by_sleeve,
|
||||
prices, at, enabled)`; `PaperRepo` with `realized_pnl()`, `latest_positions()`, `recent_trades()`,
|
||||
`replace_positions`; `nav_sparkline(nav, ...)`; each as_of-aware sleeve's `current_weights(as_of)`;
|
||||
`PaperSnapshotService`; `WarehousePriceProvider.fetch(market, start, end) -> PriceSeries`.
|
||||
|
||||
---
|
||||
|
||||
## Task 1: `paper_nav` ORM table
|
||||
|
||||
**Files:** Modify `src/fxhnt/adapters/persistence/cockpit_models.py`; Test `tests/unit/test_paper_nav_model.py`
|
||||
|
||||
- [ ] **Step 1: Failing test**
|
||||
```python
|
||||
# tests/unit/test_paper_nav_model.py
|
||||
from fxhnt.adapters.persistence.cockpit_models import PaperNavRow
|
||||
def test_paper_nav_table():
|
||||
assert PaperNavRow.__tablename__ == "paper_nav"
|
||||
```
|
||||
- [ ] **Step 2:** run → FAIL.
|
||||
- [ ] **Step 3:** Add (same `CockpitBase`/imports as the other rows):
|
||||
```python
|
||||
class PaperNavRow(CockpitBase):
|
||||
__tablename__ = "paper_nav"
|
||||
run_date: Mapped[dt.date] = mapped_column(Date, primary_key=True)
|
||||
capital: Mapped[float] = mapped_column(Float)
|
||||
realized: Mapped[float] = mapped_column(Float)
|
||||
unrealized: Mapped[float] = mapped_column(Float)
|
||||
equity: Mapped[float] = mapped_column(Float)
|
||||
src_updated_at: Mapped[dt.datetime] = mapped_column(DateTime)
|
||||
```
|
||||
- [ ] **Step 4:** PASS. **Step 5:** `git commit -m "feat(replay): paper_nav ORM row"`
|
||||
|
||||
---
|
||||
|
||||
## Task 2: PaperRepo nav + history reads
|
||||
|
||||
**Files:** Modify `src/fxhnt/adapters/persistence/paper_repo.py`; Test `tests/integration/test_paper_repo_nav.py`
|
||||
|
||||
- [ ] **Step 1: Failing test**
|
||||
```python
|
||||
# tests/integration/test_paper_repo_nav.py
|
||||
import datetime as dt
|
||||
from fxhnt.adapters.persistence.paper_repo import PaperRepo
|
||||
def test_nav_history_and_lookups():
|
||||
repo = PaperRepo("sqlite://"); repo.migrate()
|
||||
at = dt.datetime(2026,6,21,tzinfo=dt.UTC)
|
||||
repo.upsert_nav("2026-06-20", capital=100000.0, realized=0.0, unrealized=10.0, equity=100010.0, at=at)
|
||||
repo.upsert_nav("2026-06-21", capital=100000.0, realized=5.0, unrealized=20.0, equity=100025.0, at=at)
|
||||
repo.upsert_nav("2026-06-21", capital=100000.0, realized=5.0, unrealized=25.0, equity=100030.0, at=at) # idempotent overwrite
|
||||
hist = repo.nav_history()
|
||||
assert [h.run_date for h in hist] == ["2026-06-20", "2026-06-21"] # ascending, deduped
|
||||
assert hist[-1].equity == 100030.0
|
||||
```
|
||||
- [ ] **Step 2:** run → FAIL.
|
||||
- [ ] **Step 3:** Add to `PaperRepo`: `upsert_nav(run_date, *, capital, realized, unrealized, equity, at)`
|
||||
(delete-then-insert that run_date, or merge — idempotent); `nav_history() -> list[NavPoint]` ascending by
|
||||
run_date (define a small `NavPoint` dataclass with run_date/equity/realized/unrealized);
|
||||
`positions_at(run_date) -> list[Position]` (rows for that run_date); `trades_on(run_date) -> list[Trade]`
|
||||
(trades whose run_date matches — `paper_trades.run_date` was added in 2F-MVP fix). Mirror existing methods.
|
||||
- [ ] **Step 4:** PASS. **Step 5:** `git commit -m "feat(replay): PaperRepo upsert_nav + nav_history/positions_at/trades_on"`
|
||||
|
||||
---
|
||||
|
||||
## Task 3: write `paper_nav` from `persist_paper_book`
|
||||
|
||||
**Files:** Modify `src/fxhnt/application/paper_book.py`; Test extend `tests/integration/test_paper_book.py`
|
||||
|
||||
- [ ] **Step 1: Failing test** — after `persist_paper_book(... run_date=D, prices=P ...)`, assert
|
||||
`repo.nav_history()[-1]` has `run_date==D` and `equity == capital + realized + Σ qty·(P[sym]-entry)`.
|
||||
- [ ] **Step 2:** run → FAIL.
|
||||
- [ ] **Step 3:** In `persist_paper_book`, after positions/trades/realized are persisted, compute
|
||||
`unrealized = Σ over persisted positions qty*(prices.get(sym, entry) - entry)`,
|
||||
`realized = repo.realized_pnl()` (cumulative ≤ run_date when backfill processes ascending),
|
||||
`equity = capital + realized + unrealized`, and `repo.upsert_nav(run_date, capital=capital,
|
||||
realized=realized, unrealized=unrealized, equity=equity, at=at)`. (Guarded by `enabled`.)
|
||||
- [ ] **Step 4:** PASS. **Step 5:** `git commit -m "feat(replay): persist_paper_book writes paper_nav per run_date"`
|
||||
|
||||
---
|
||||
|
||||
## Task 4: backfill service
|
||||
|
||||
**Files:** Create `src/fxhnt/application/paper_backfill.py`; Test `tests/integration/test_paper_backfill.py`
|
||||
|
||||
- [ ] **Step 1: Failing test**
|
||||
```python
|
||||
# tests/integration/test_paper_backfill.py
|
||||
import datetime as dt
|
||||
from fxhnt.adapters.persistence.paper_repo import PaperRepo
|
||||
from fxhnt.application.paper_backfill import backfill_paper_book
|
||||
|
||||
class _Sleeve: # as_of-aware fake
|
||||
def current_weights(self, as_of): return {"BTCUSDT": 1.0}
|
||||
|
||||
def test_backfill_builds_nav_series():
|
||||
repo = PaperRepo("sqlite://"); repo.migrate()
|
||||
dates = ["2026-06-19","2026-06-20","2026-06-21"]
|
||||
closes_by_date = {"2026-06-19":{"BTCUSDT":100.0}, "2026-06-20":{"BTCUSDT":110.0}, "2026-06-21":{"BTCUSDT":121.0}}
|
||||
n = backfill_paper_book(
|
||||
repo, capital=100_000.0,
|
||||
sleeves={"crypto_tstrend": _Sleeve()}, sleeve_weights={"crypto_tstrend": 1.0},
|
||||
dates=dates, closes_by_date=closes_by_date, at=dt.datetime(2026,6,21,tzinfo=dt.UTC))
|
||||
hist = repo.nav_history()
|
||||
assert [h.run_date for h in hist] == dates # one nav row per date, ascending
|
||||
assert n == len(dates)
|
||||
```
|
||||
- [ ] **Step 2:** run → FAIL.
|
||||
- [ ] **Step 3:** Implement `backfill_paper_book(repo, *, capital, sleeves, sleeve_weights, dates,
|
||||
closes_by_date, at) -> int`: for each date ascending, build `symbol_weights_by_sleeve = {s:
|
||||
src.current_weights(date) for s,src in sleeves.items()}` under per-sleeve try/except (skip on
|
||||
error/empty), then `persist_paper_book(repo, capital=capital, run_date=date,
|
||||
sleeve_weights=<surviving>, symbol_weights_by_sleeve=<surviving>, prices=closes_by_date.get(date,{}),
|
||||
at=at)`. Return count of dates with a persisted nav. (xsfunding is simply not passed in `sleeves` by the
|
||||
caller — see Task 5.)
|
||||
- [ ] **Step 4:** PASS. **Step 5:** `git commit -m "feat(replay): backfill_paper_book over a date range"`
|
||||
|
||||
---
|
||||
|
||||
## Task 5: `fxt paper-backfill` CLI
|
||||
|
||||
**Files:** Modify `src/fxhnt/cli.py`; Test `tests/integration/test_paper_backfill_cli.py`
|
||||
|
||||
- [ ] **Step 1: Failing test** — invoke the Typer command via `CliRunner` with a tmp sqlite DSN + a
|
||||
fake/seeded warehouse (or monkeypatched price provider) and assert it exits 0 + `PaperRepo(dsn).nav_history()`
|
||||
is non-empty.
|
||||
- [ ] **Step 2:** run → FAIL.
|
||||
- [ ] **Step 3:** Add a `paper_backfill` Typer command: assemble the as_of-aware sleeves (stablecoin,
|
||||
unlock, ts-trend — NOT xsfunding; reuse the `_live_sleeves` composition from the snapshot asset, filtered),
|
||||
`current_sleeve_weights(...)`, the date range (`--days N` default e.g. 365, capped to warehouse start),
|
||||
and per-date closes via `WarehousePriceProvider` (fetch each symbol's range once → `{date: close}` →
|
||||
`closes_by_date`). Call `backfill_paper_book(...)`. Print a summary (dates, positions persisted).
|
||||
- [ ] **Step 4:** PASS. **Step 5:** `git commit -m "feat(replay): fxt paper-backfill CLI"`
|
||||
|
||||
---
|
||||
|
||||
## Task 6: `/paper/replay` view (scrubber + curve + book@date)
|
||||
|
||||
**Files:** Modify `src/fxhnt/adapters/web/app.py`; Create `templates/replay.html` + `_replay_at.html`;
|
||||
Test `tests/integration/test_paper_replay_web.py`
|
||||
|
||||
- [ ] **Step 1: Failing test** — seed `paper_nav` + `paper_positions` for 2 dates; build app; TestClient:
|
||||
`GET /paper/replay` 200 + contains a range input + the curve; `GET /paper/replay/at?date=<d>` 200 + shows
|
||||
that date's positions. Empty history → `GET /paper/replay` 200 + "run `fxt paper-backfill`".
|
||||
- [ ] **Step 2:** run → FAIL.
|
||||
- [ ] **Step 3:** Add routes (reuse the `create_app` repo/svc wiring):
|
||||
- `GET /paper/replay` → `replay.html`: equity curve (`nav_sparkline` over `nav_history` equities), a
|
||||
`<input type=range min=0 max=N-1>` over the dates + Play/Pause (small JS or `hx-trigger`), and the
|
||||
`_replay_at.html` partial for the latest date by default. A note: "funding sleeve is live-only — excluded
|
||||
from replay." Empty `nav_history` → the backfill prompt.
|
||||
- `GET /paper/replay/at?date=<d>` → `_replay_at.html`: the book at `d` — `positions_at(d)` marked at d's
|
||||
stored entry/close (PnL vs entry) + `trades_on(d)` + that date's equity from `nav_history`.
|
||||
- Nav links `/paper` ↔ `/paper/replay`. Use `data-label` tables (mobile-friendly, matching the cockpit).
|
||||
- [ ] **Step 4:** PASS. **Step 5:** `git commit -m "feat(replay): /paper/replay scrubber + curve + book@date"`
|
||||
|
||||
---
|
||||
|
||||
## Task 7: Full suite + finish + deploy + backfill run
|
||||
|
||||
- [ ] **Step 1:** `uv run pytest -q` → all green.
|
||||
- [ ] **Step 2:** Use **superpowers:finishing-a-development-branch** — merge `feat/paper-replay` → master,
|
||||
push (auto-deploys via the Gitea webhook). Confirm the cockpit workflow Succeeded + the dashboard pod is
|
||||
healthy (boots clean).
|
||||
- [ ] **Step 3:** Run the backfill once in-cluster: `kubectl exec deploy/dagster -c dagster -n foxhunt --
|
||||
fxt paper-backfill --days 365` (or via a one-shot Job using the cockpit image). Confirm it populates
|
||||
`paper_nav`.
|
||||
- [ ] **Step 4:** Verify `https://dashboard.fxhnt.ai/paper/replay` renders the equity curve + scrubber + a
|
||||
real historical book; `/paper` shows the latest book. Report what renders.
|
||||
|
||||
---
|
||||
|
||||
## Self-review notes
|
||||
- Backfill MUST process dates ascending so `realized_pnl()` accumulates correctly per nav row.
|
||||
- xsfunding excluded from backfill (caller omits it); documented + UI-flagged.
|
||||
- Read-only: recompute from warehouse + domain fns; no broker, no change to the nightly book except the
|
||||
added `paper_nav` write (idempotent).
|
||||
- Don't `git add -A` — `*.dbn` on disk must never be staged (gitignored — verify).
|
||||
|
||||
## Acceptance criteria (from spec)
|
||||
- `paper_nav` table + `upsert_nav`/`nav_history`/`positions_at`/`trades_on`; tested. ✅ T1,T2
|
||||
- `persist_paper_book` writes the equity row per run_date; tested. ✅ T3
|
||||
- `backfill_paper_book` + `fxt paper-backfill` recompute over a range, skip xsfunding/missing; tested. ✅ T4,T5
|
||||
- `/paper/replay` curve + scrubber + book@date; empty + xsfunding notes; tested. ✅ T6
|
||||
- idempotent re-run; nightly snapshot unchanged except paper_nav. ✅ T3 (upsert), T4
|
||||
- post-deploy `/paper/replay` shows real history. ✅ T7
|
||||
230
docs/superpowers/plans/2026-06-21-paper-snapshot.md
Normal file
230
docs/superpowers/plans/2026-06-21-paper-snapshot.md
Normal file
@@ -0,0 +1,230 @@
|
||||
# Paper-Snapshot Job — Implementation Plan (Phase 2F-2)
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: superpowers:subagent-driven-development (recommended) or
|
||||
> executing-plans. TDD throughout. Steps use `- [ ]`.
|
||||
|
||||
**Goal:** Populate the `/paper` cockpit with the live sleeves' real positions: each sleeve exposes
|
||||
`current_weights(as_of)`, a `PaperSnapshotService` combines them with allocator sleeve-weights + warehouse
|
||||
closes → `persist_paper_book`, wired as a nightly Dagster asset.
|
||||
|
||||
**Architecture:** Read-only/additive — a parallel `current_weights()` accessor per sleeve (reusing its
|
||||
existing domain weight logic) + a snapshot service + a Dagster asset after `combined_forward_nav`. The
|
||||
return-producing hot path + gate record are untouched.
|
||||
|
||||
**Tech Stack:** Python 3.12, Dagster, SQLAlchemy, pytest. **Repo:** `~/Work/fxhnt`, branch
|
||||
`feat/paper-snapshot`. **Spec:** `docs/superpowers/specs/2026-06-21-paper-snapshot-design.md`.
|
||||
**Run tests:** `uv run pytest <path> -q`.
|
||||
|
||||
**Backend already available (2F MVP, merged):** `persist_paper_book(repo, *, capital, run_date,
|
||||
sleeve_weights, symbol_weights_by_sleeve, prices, at, enabled) -> int` in `application/paper_book.py`;
|
||||
`PaperRepo`; domain `paper.py`.
|
||||
|
||||
**Live sleeves (registry):** crypto-momentum (vestigial trend), 60/40 (beta), xsfunding (crypto-funding),
|
||||
unlock (crypto-event), crypto ts-trend (crypto-trend), stablecoin (crypto-stablecoin).
|
||||
|
||||
---
|
||||
|
||||
## Task 1: `PositionSource` protocol
|
||||
|
||||
**Files:** Create `src/fxhnt/ports/position_source.py`; Test `tests/unit/test_position_source.py`
|
||||
|
||||
- [ ] **Step 1: Failing test**
|
||||
```python
|
||||
# tests/unit/test_position_source.py
|
||||
from fxhnt.ports.position_source import PositionSource
|
||||
class _Stub:
|
||||
def current_weights(self, as_of: str) -> dict[str, float]:
|
||||
return {"USDTUSD": -1.0}
|
||||
def test_protocol_structural():
|
||||
s: PositionSource = _Stub()
|
||||
assert s.current_weights("2026-06-21") == {"USDTUSD": -1.0}
|
||||
```
|
||||
- [ ] **Step 2:** run → FAIL.
|
||||
- [ ] **Step 3:** `ports/position_source.py`:
|
||||
```python
|
||||
from __future__ import annotations
|
||||
from typing import Protocol
|
||||
class PositionSource(Protocol):
|
||||
"""A live sleeve's current per-symbol target weights (signed) as of a rebalance date.
|
||||
Returns {} when the sleeve has no data / can't surface weights (caller skips + logs)."""
|
||||
def current_weights(self, as_of: str) -> dict[str, float]: ...
|
||||
```
|
||||
- [ ] **Step 4:** PASS. **Step 5:** `git commit -m "feat(paper-snap): PositionSource protocol"`
|
||||
|
||||
---
|
||||
|
||||
## Task 2: stablecoin `current_weights()` (worked example — follow this pattern for the others)
|
||||
|
||||
**Files:** Modify `src/fxhnt/application/stablecoin_runner.py` (or the stablecoin sleeve service);
|
||||
Test `tests/unit/test_stablecoin_current_weights.py`
|
||||
|
||||
- [ ] **Step 1: Failing test**
|
||||
```python
|
||||
# tests/unit/test_stablecoin_current_weights.py
|
||||
from fxhnt.application.stablecoin_runner import StableReversionRunner
|
||||
def test_current_weights_shorts_rich_names():
|
||||
# panel = {symbol: {epoch_day: close}}; day 20000 has USDT rich (>1+thresh), DAI at peg
|
||||
panel = {"USDTUSD": {20000: 1.01}, "DAIUSD": {20000: 1.0001}}
|
||||
r = StableReversionRunner(panel, thresh_bp=50.0)
|
||||
w = r.current_weights("epoch:20000") # or the runner's date convention
|
||||
assert w["USDTUSD"] < 0 # rich -> short
|
||||
assert w.get("DAIUSD", 0.0) == 0.0
|
||||
```
|
||||
- [ ] **Step 2:** run → FAIL.
|
||||
- [ ] **Step 3:** Add `current_weights(self, as_of)` to `StableReversionRunner` reusing its existing logic:
|
||||
compute `dev = {s: series[day] - 1.0 for s, series in self._p.items() if day in series}` for the latest
|
||||
available day ≤ as_of, then `return short_rich_weights(dev, self._th)`. (Mirror exactly how `run()`'s loop
|
||||
builds `dev` + calls `short_rich_weights`; reuse `self._th`.) Resolve `as_of`→epoch-day with the runner's
|
||||
existing date helper.
|
||||
- [ ] **Step 4:** PASS. **Step 5:** `git commit -m "feat(paper-snap): stablecoin current_weights"`
|
||||
|
||||
---
|
||||
|
||||
## Task 3: `current_weights()` for the other crypto sleeves (unlock, xsfunding, ts-trend)
|
||||
|
||||
**Files:** Modify the unlock / xsfunding / ts-trend sleeve runners/services; Tests per sleeve under
|
||||
`tests/unit/test_<sleeve>_current_weights.py`
|
||||
|
||||
For EACH of unlock, xsfunding, crypto ts-trend: add `current_weights(as_of)` reusing that sleeve's existing
|
||||
domain weight fn for the latest day, with a focused unit test (reuse the sleeve's existing domain test
|
||||
fixtures). Specifics (study each runner first):
|
||||
- **unlock shorts** → `inverse_vol_capped_weights(vols, max_name_weight)` on the names with an active
|
||||
unlock window as of `as_of`, **negated** (shorts). Test: an in-window name gets a negative weight.
|
||||
- **xsfunding** → `construction_weights(scores, mode, quantile=…)` on the latest funding scores. Test: top/
|
||||
bottom quantile get +/− weights, middle 0.
|
||||
- **crypto ts-trend** → per symbol `ts_trend_signal(closes[sym], lookbacks)`; long/flat sign → weight
|
||||
(equal-weight across symbols with a positive signal, scaled to the sleeve's gross). Test: an up-trending
|
||||
series → positive weight, a down/flat one → 0.
|
||||
|
||||
- [ ] **Step per sleeve:** failing test → implement `current_weights` (reuse domain fn) → PASS → commit
|
||||
`feat(paper-snap): <sleeve> current_weights`.
|
||||
|
||||
---
|
||||
|
||||
## Task 4: `current_weights()` for the equity/beta sleeves (60/40, crypto-momentum)
|
||||
|
||||
**Files:** Modify the sixtyforty + crypto-momentum sleeve services; Tests under `tests/unit/`
|
||||
|
||||
- **60/40** → constant weights `{"SPY": 0.6, "IEF": 0.4}` (or the sleeve's `_SIXTYFORTY_WEIGHTS`). Test:
|
||||
returns those two weights summing to 1.0.
|
||||
- **crypto-momentum (vestigial trend overlay)** → its current long/flat weight from its existing signal; if
|
||||
the sleeve is purely a vestigial overlay with no clean per-symbol weight, `current_weights` returns `{}`
|
||||
(documented — it'll be skipped). Test: returns `{}` or the documented weight.
|
||||
|
||||
- [ ] **Step per sleeve:** failing test → implement → PASS → commit `feat(paper-snap): <sleeve> current_weights`.
|
||||
|
||||
---
|
||||
|
||||
## Task 5: allocator `current_sleeve_weights()` accessor
|
||||
|
||||
**Files:** Modify `src/fxhnt/application/book_allocator.py`; Test `tests/unit/test_allocator_sleeve_weights.py`
|
||||
|
||||
- [ ] **Step 1: Failing test**
|
||||
```python
|
||||
# tests/unit/test_allocator_sleeve_weights.py
|
||||
import numpy as np
|
||||
from fxhnt.application.book_allocator import current_sleeve_weights
|
||||
def test_current_sleeve_weights_inverse_vol():
|
||||
# two edges, edge B 2x the vol of A -> A gets more weight; weights sum ~1
|
||||
series = {"A": {1:0.001,2:-0.001,3:0.001}, "B": {1:0.01,2:-0.01,3:0.01}}
|
||||
w = current_sleeve_weights(series, vol_lookback=3, max_sleeve_weight=0.5)
|
||||
assert abs(sum(w.values()) - 1.0) < 1e-6
|
||||
assert w["A"] > w["B"]
|
||||
```
|
||||
- [ ] **Step 2:** run → FAIL.
|
||||
- [ ] **Step 3:** Add a public `current_sleeve_weights(series_by_edge, *, vol_lookback=60,
|
||||
max_sleeve_weight=...) -> dict[str, float]` thin wrapper that runs the SAME trailing-window inverse-vol
|
||||
computation `combine_book` uses for the latest window (reuse `align_edges` + `_inverse_vol_weights` over
|
||||
the last `vol_lookback` rows). Do not change `combine_book`.
|
||||
- [ ] **Step 4:** PASS. **Step 5:** `git commit -m "feat(paper-snap): allocator current_sleeve_weights"`
|
||||
|
||||
---
|
||||
|
||||
## Task 6: `PaperSnapshotService`
|
||||
|
||||
**Files:** Create `src/fxhnt/application/paper_snapshot.py`; Test `tests/integration/test_paper_snapshot.py`
|
||||
|
||||
- [ ] **Step 1: Failing test**
|
||||
```python
|
||||
# tests/integration/test_paper_snapshot.py
|
||||
import datetime as dt
|
||||
from fxhnt.adapters.persistence.paper_repo import PaperRepo
|
||||
from fxhnt.application.paper_snapshot import PaperSnapshotService
|
||||
|
||||
class _Sleeve:
|
||||
def __init__(self, w): self._w = w
|
||||
def current_weights(self, as_of): return self._w
|
||||
|
||||
def test_snapshot_persists_multi_sleeve_and_skips_failures():
|
||||
repo = PaperRepo("sqlite://"); repo.migrate()
|
||||
class _Boom:
|
||||
def current_weights(self, as_of): raise RuntimeError("no data")
|
||||
svc = PaperSnapshotService(
|
||||
repo, capital=100_000.0,
|
||||
sleeves={"stablecoin_rotation": _Sleeve({"USDTUSD": -1.0}), "broken": _Boom()},
|
||||
sleeve_weights={"stablecoin_rotation": 0.6, "broken": 0.4},
|
||||
prices={"USDTUSD": 2.0})
|
||||
n = svc.snapshot(as_of="2026-06-21", at=dt.datetime(2026,6,21,tzinfo=dt.UTC))
|
||||
pos = repo.latest_positions()
|
||||
assert any(p.symbol == "USDTUSD" for p in pos) # good sleeve persisted
|
||||
assert all(p.sleeve != "broken" for p in pos) # failing sleeve skipped, not fatal
|
||||
```
|
||||
- [ ] **Step 2:** run → FAIL.
|
||||
- [ ] **Step 3:** Implement `PaperSnapshotService(repo, *, capital, sleeves: dict[str, PositionSource],
|
||||
sleeve_weights: dict[str,float], prices: dict[str,float])`. `snapshot(as_of, at)`: build
|
||||
`symbol_weights_by_sleeve` by calling each sleeve's `current_weights(as_of)` inside try/except (log +
|
||||
skip on error, also drop from sleeve_weights), then call `persist_paper_book(repo, capital=capital,
|
||||
run_date=as_of, sleeve_weights=…, symbol_weights_by_sleeve=…, prices=prices, at=at)` and return its count.
|
||||
(In production, `prices` + `sleeves` + `sleeve_weights` are assembled by the asset in Task 7; the service
|
||||
stays pure-ish + injectable for tests.)
|
||||
- [ ] **Step 4:** PASS. **Step 5:** `git commit -m "feat(paper-snap): PaperSnapshotService (skip-on-fail)"`
|
||||
|
||||
---
|
||||
|
||||
## Task 7: Dagster asset `paper_book_snapshot` wired into the daily schedule
|
||||
|
||||
**Files:** Modify `src/fxhnt/adapters/orchestration/assets.py` + `definitions.py`;
|
||||
Test `tests/integration/test_paper_snapshot_asset.py`
|
||||
|
||||
- [ ] **Step 1: Failing test** — materialize/call the plain builder with a fake warehouse + fake sleeves and
|
||||
assert it persists positions (mirror the existing `build_combined_forward_nav` test pattern in the repo).
|
||||
- [ ] **Step 2:** run → FAIL.
|
||||
- [ ] **Step 3:** Add `build_paper_book_snapshot(...)` (plain fn, unit-testable) + `@asset
|
||||
paper_book_snapshot` that `deps=[combined_forward_nav]`. It assembles inputs for `PaperSnapshotService`:
|
||||
the live sleeves (from the registry/composition), `current_sleeve_weights(...)` from the book's edge
|
||||
series, and latest closes per referenced symbol via `WarehousePriceProvider`. Construct `PaperRepo` from
|
||||
the same cockpit DSN as `ForwardNavRepo`. Guard the whole asset body in try/except so a snapshot failure
|
||||
logs but does NOT fail the run (it must not break `combined_forward_nav`). Add the asset to the
|
||||
`Definitions` in `definitions.py` so the daily 23:30 schedule materializes it after the book.
|
||||
- [ ] **Step 4:** PASS; also `uv run pytest tests/integration -q -k "forward or book or dagster or definition"`
|
||||
to confirm the nightly graph still loads + no regression.
|
||||
- [ ] **Step 5:** `git commit -m "feat(paper-snap): paper_book_snapshot Dagster asset (daily, after book)"`
|
||||
|
||||
---
|
||||
|
||||
## Task 8: Full suite + finish + deploy
|
||||
|
||||
- [ ] **Step 1:** `uv run pytest -q` → all green.
|
||||
- [ ] **Step 2:** Use **superpowers:finishing-a-development-branch** — merge `feat/paper-snapshot` → master,
|
||||
push to Gitea.
|
||||
- [ ] **Step 3:** Deploy: `./scripts/argo-deploy-cockpit.sh` (or via the Gitea webhook auto-deploy from the
|
||||
push). Verify the workflow Succeeded.
|
||||
- [ ] **Step 4:** Trigger one snapshot (materialize the `paper_book_snapshot` asset once via Dagster, or wait
|
||||
for 23:30) and confirm `https://dashboard.fxhnt.ai/paper` (or the cockpit `/paper` route) shows real
|
||||
positions + live MTM. Report what renders.
|
||||
|
||||
---
|
||||
|
||||
## Self-review notes
|
||||
- Per-sleeve `current_weights` MUST reuse the sleeve's existing domain fn (no re-derivation) — fidelity.
|
||||
- A failing/no-data sleeve returns `{}` / is caught + skipped; never breaks the nightly book.
|
||||
- Read-only/additive: `combined_forward_nav` + gate record unchanged.
|
||||
- `do NOT git add -A` — `*.dbn` data on disk must not be staged (gitignored; verify).
|
||||
|
||||
## Acceptance criteria (from spec)
|
||||
- each live sleeve has `current_weights(as_of)` reusing domain logic, unit-tested. ✅ T2,T3,T4
|
||||
- allocator exposes `current_sleeve_weights`, unit-tested. ✅ T5
|
||||
- `PaperSnapshotService.snapshot()` persists multi-sleeve + skips failures. ✅ T6
|
||||
- `paper_book_snapshot` asset wired into daily schedule, deps combined_forward_nav, wiring-tested. ✅ T7
|
||||
- post-deploy `/paper` shows real positions/trades/PnL. ✅ T8
|
||||
- nightly book + gate record unchanged. ✅ (additive; asset try/except)
|
||||
375
docs/superpowers/plans/2026-06-21-paper-trading-cockpit.md
Normal file
375
docs/superpowers/plans/2026-06-21-paper-trading-cockpit.md
Normal file
@@ -0,0 +1,375 @@
|
||||
# Paper-Trading Cockpit (live MTM) — Implementation Plan (Phase 2F MVP)
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended)
|
||||
> or superpowers:executing-plans. TDD throughout (this is real code). Steps use `- [ ]` checkboxes.
|
||||
|
||||
**Goal:** Persist the positions + trades the book already computes, and add a `/paper` cockpit view that
|
||||
marks them to live Binance prices (HTMX poll) → combined paper equity + per-sleeve PnL + trade log.
|
||||
|
||||
**Architecture:** Hexagonal, matching the repo. Pure domain math (`domain/paper.py`); persistence
|
||||
(`paper_repo.py` + ORM rows in `cockpit_models.py`); a `LivePrice` port + ccxt adapter; a
|
||||
`PaperBookService` that derives/persists on rebalance and builds the read model; web routes + templates.
|
||||
|
||||
**Tech Stack:** Python 3.12, SQLAlchemy, FastAPI + Jinja2 + HTMX, ccxt, pytest.
|
||||
|
||||
**Spec:** `docs/superpowers/specs/2026-06-21-paper-trading-cockpit-design.md`. **Repo:** `~/Work/fxhnt`,
|
||||
branch `feat/paper-trading-cockpit`.
|
||||
|
||||
**Run tests:** `cd ~/Work/fxhnt && uv run pytest <path> -q`
|
||||
|
||||
---
|
||||
|
||||
## Task 1: `paper_capital` config
|
||||
|
||||
**Files:** Modify `src/fxhnt/config.py`; Test `tests/unit/test_config_paper.py`
|
||||
|
||||
- [ ] **Step 1: Failing test**
|
||||
```python
|
||||
# tests/unit/test_config_paper.py
|
||||
from fxhnt.config import Settings
|
||||
def test_paper_capital_default():
|
||||
s = Settings()
|
||||
assert s.paper_capital == 100_000.0
|
||||
```
|
||||
- [ ] **Step 2:** `uv run pytest tests/unit/test_config_paper.py -q` → FAIL (no attribute `paper_capital`).
|
||||
- [ ] **Step 3:** In `src/fxhnt/config.py` add to `Settings` (pydantic):
|
||||
```python
|
||||
paper_capital: float = 100_000.0 # starting paper book capital for the cockpit paper view
|
||||
```
|
||||
- [ ] **Step 4:** rerun → PASS.
|
||||
- [ ] **Step 5:** `git add -A && git commit -m "feat(paper): paper_capital config (default 100k)"`
|
||||
|
||||
---
|
||||
|
||||
## Task 2: Pure domain — target positions, trade diff, mark-to-market
|
||||
|
||||
**Files:** Create `src/fxhnt/domain/paper.py`; Test `tests/unit/test_paper_domain.py`
|
||||
|
||||
- [ ] **Step 1: Failing test**
|
||||
```python
|
||||
# tests/unit/test_paper_domain.py
|
||||
from fxhnt.domain.paper import Position, target_positions, diff_trades, mark_to_market
|
||||
|
||||
def test_target_positions_notional_split():
|
||||
# sleeve weight 0.5 of 100k = 50k; symbol weight -1.0 (short) at price 2.0 -> qty -25000
|
||||
pos = target_positions(sleeve="stablecoin_rotation", sleeve_weight=0.5, capital=100_000.0,
|
||||
symbol_weights={"USDTUSD": -1.0}, prices={"USDTUSD": 2.0}, date="2026-06-21")
|
||||
assert pos[0].symbol == "USDTUSD"
|
||||
assert pos[0].qty == -25_000.0
|
||||
assert pos[0].entry_price == 2.0
|
||||
|
||||
def test_diff_trades_entry_exit_resize():
|
||||
prior = [Position("s","BTC", 1.0, 100.0, "2026-06-20")]
|
||||
target = [Position("s","BTC", 1.5, 110.0, "2026-06-21"), Position("s","ETH", 2.0, 50.0, "2026-06-21")]
|
||||
trades = diff_trades(prior, target, ts="2026-06-21T00:00:00Z")
|
||||
kinds = {(t.symbol, t.side, t.qty) for t in trades}
|
||||
assert ("BTC","BUY",0.5) in kinds # resize up
|
||||
assert ("ETH","BUY",2.0) in kinds # new entry
|
||||
# a full exit of a prior symbol yields a SELL of its qty
|
||||
trades2 = diff_trades([Position("s","BTC",1.0,100.0,"d")], [], ts="t")
|
||||
assert (trades2[0].symbol, trades2[0].side, trades2[0].qty) == ("BTC","SELL",1.0)
|
||||
|
||||
def test_mark_to_market():
|
||||
pos = [Position("s","BTC", 2.0, 100.0, "d")]
|
||||
mtm = mark_to_market(pos, prices={"BTC": 110.0})
|
||||
assert mtm.unrealized == 20.0 # 2*(110-100)
|
||||
assert mtm.per_symbol["BTC"] == 20.0
|
||||
```
|
||||
- [ ] **Step 2:** run → FAIL (module missing).
|
||||
- [ ] **Step 3:** Implement `src/fxhnt/domain/paper.py` (pure, no I/O):
|
||||
```python
|
||||
"""Pure paper-book math: target positions from weights+capital, trade diff, mark-to-market. No I/O."""
|
||||
from __future__ import annotations
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Position:
|
||||
sleeve: str
|
||||
symbol: str
|
||||
qty: float
|
||||
entry_price: float
|
||||
entry_date: str
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Trade:
|
||||
ts: str
|
||||
sleeve: str
|
||||
symbol: str
|
||||
side: str # BUY | SELL
|
||||
qty: float
|
||||
price: float
|
||||
reason: str
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MTM:
|
||||
unrealized: float
|
||||
per_symbol: dict[str, float] = field(default_factory=dict)
|
||||
|
||||
def target_positions(sleeve: str, sleeve_weight: float, capital: float,
|
||||
symbol_weights: dict[str, float], prices: dict[str, float], date: str) -> list[Position]:
|
||||
notional = sleeve_weight * capital
|
||||
out: list[Position] = []
|
||||
for sym, w in symbol_weights.items():
|
||||
px = prices.get(sym)
|
||||
if not px or px <= 0:
|
||||
continue
|
||||
qty = (w * notional) / px
|
||||
if qty == 0.0:
|
||||
continue
|
||||
out.append(Position(sleeve, sym, qty, px, date))
|
||||
return out
|
||||
|
||||
def diff_trades(prior: list[Position], target: list[Position], ts: str, reason: str = "rebalance") -> list[Trade]:
|
||||
pri = {(p.sleeve, p.symbol): p for p in prior}
|
||||
tgt = {(p.sleeve, p.symbol): p for p in target}
|
||||
trades: list[Trade] = []
|
||||
for key in sorted(set(pri) | set(tgt)):
|
||||
sleeve, sym = key
|
||||
pq = pri[key].qty if key in pri else 0.0
|
||||
tq = tgt[key].qty if key in tgt else 0.0
|
||||
d = tq - pq
|
||||
if abs(d) < 1e-9:
|
||||
continue
|
||||
px = (tgt[key].entry_price if key in tgt else pri[key].entry_price)
|
||||
trades.append(Trade(ts, sleeve, sym, "BUY" if d > 0 else "SELL", abs(d), px, reason))
|
||||
return trades
|
||||
|
||||
def mark_to_market(positions: list[Position], prices: dict[str, float]) -> MTM:
|
||||
per: dict[str, float] = {}
|
||||
total = 0.0
|
||||
for p in positions:
|
||||
px = prices.get(p.symbol, p.entry_price) # unknown -> entry (0 pnl)
|
||||
pnl = p.qty * (px - p.entry_price)
|
||||
per[p.symbol] = per.get(p.symbol, 0.0) + pnl
|
||||
total += pnl
|
||||
return MTM(unrealized=total, per_symbol=per)
|
||||
```
|
||||
- [ ] **Step 4:** run → PASS.
|
||||
- [ ] **Step 5:** `git commit -m "feat(paper): pure domain — target positions, trade diff, MTM"`
|
||||
|
||||
---
|
||||
|
||||
## Task 3: ORM rows for positions + trades
|
||||
|
||||
**Files:** Modify `src/fxhnt/adapters/persistence/cockpit_models.py`; Test `tests/unit/test_paper_models.py`
|
||||
|
||||
- [ ] **Step 1: Failing test**
|
||||
```python
|
||||
# tests/unit/test_paper_models.py
|
||||
from fxhnt.adapters.persistence.cockpit_models import PaperPositionRow, PaperTradeRow
|
||||
def test_paper_tables_exist():
|
||||
assert PaperPositionRow.__tablename__ == "paper_positions"
|
||||
assert PaperTradeRow.__tablename__ == "paper_trades"
|
||||
```
|
||||
- [ ] **Step 2:** run → FAIL (import error).
|
||||
- [ ] **Step 3:** Add to `cockpit_models.py` (same `CockpitBase`, imports already present — add `Date`/`DateTime`/`Float`/`String`/`Integer` are imported):
|
||||
```python
|
||||
class PaperPositionRow(CockpitBase):
|
||||
__tablename__ = "paper_positions"
|
||||
run_date: Mapped[dt.date] = mapped_column(Date, primary_key=True)
|
||||
sleeve: Mapped[str] = mapped_column(String(32), primary_key=True)
|
||||
symbol: Mapped[str] = mapped_column(String(32), primary_key=True)
|
||||
qty: Mapped[float] = mapped_column(Float)
|
||||
entry_price: Mapped[float] = mapped_column(Float)
|
||||
entry_date: Mapped[str] = mapped_column(String(10))
|
||||
weight: Mapped[float] = mapped_column(Float)
|
||||
src_updated_at: Mapped[dt.datetime] = mapped_column(DateTime)
|
||||
|
||||
class PaperTradeRow(CockpitBase):
|
||||
__tablename__ = "paper_trades"
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
|
||||
ts: Mapped[dt.datetime] = mapped_column(DateTime)
|
||||
sleeve: Mapped[str] = mapped_column(String(32))
|
||||
symbol: Mapped[str] = mapped_column(String(32))
|
||||
side: Mapped[str] = mapped_column(String(4))
|
||||
qty: Mapped[float] = mapped_column(Float)
|
||||
price: Mapped[float] = mapped_column(Float)
|
||||
reason: Mapped[str] = mapped_column(String(64))
|
||||
```
|
||||
- [ ] **Step 4:** run → PASS.
|
||||
- [ ] **Step 5:** `git commit -m "feat(paper): paper_positions + paper_trades ORM rows"`
|
||||
|
||||
---
|
||||
|
||||
## Task 4: PaperRepo (persistence)
|
||||
|
||||
**Files:** Create `src/fxhnt/adapters/persistence/paper_repo.py`; Test `tests/integration/test_paper_repo.py`
|
||||
|
||||
- [ ] **Step 1: Failing test**
|
||||
```python
|
||||
# tests/integration/test_paper_repo.py
|
||||
import datetime as dt
|
||||
from fxhnt.adapters.persistence.paper_repo import PaperRepo
|
||||
from fxhnt.domain.paper import Position, Trade
|
||||
|
||||
def test_paper_repo_roundtrip():
|
||||
repo = PaperRepo("sqlite://"); repo.migrate()
|
||||
at = dt.datetime(2026,6,21,23,30, tzinfo=dt.UTC)
|
||||
repo.replace_positions("2026-06-21",
|
||||
[Position("stablecoin_rotation","USDTUSD",-25000.0,2.0,"2026-06-21")], weight_by_sleeve={"stablecoin_rotation":0.5}, at=at)
|
||||
repo.record_trades([Trade("2026-06-21T23:30:00Z","stablecoin_rotation","USDTUSD","SELL",25000.0,2.0,"rebalance")], at=at)
|
||||
pos = repo.latest_positions()
|
||||
assert pos[0].symbol == "USDTUSD" and pos[0].qty == -25000.0
|
||||
assert repo.weight_for("stablecoin_rotation") == 0.5
|
||||
assert repo.recent_trades(10)[0].side == "SELL"
|
||||
```
|
||||
- [ ] **Step 2:** run → FAIL.
|
||||
- [ ] **Step 3:** Implement `paper_repo.py` mirroring `ForwardNavRepo` (engine/StaticPool for sqlite, `CockpitBase.metadata.create_all` in `migrate()`; `replace_positions(run_date, positions, weight_by_sleeve, at)` deletes that run_date's rows then inserts; `latest_positions()` returns the max run_date's rows as `Position`; `weight_for(sleeve)` from latest; `record_trades(trades, at)` inserts; `recent_trades(n)` newest-first). Use the same engine-construction block as `ForwardNavRepo.__init__`.
|
||||
- [ ] **Step 4:** run → PASS.
|
||||
- [ ] **Step 5:** `git commit -m "feat(paper): PaperRepo persistence (positions + trades)"`
|
||||
|
||||
---
|
||||
|
||||
## Task 5: LivePrice port + ccxt adapter + fake
|
||||
|
||||
**Files:** Create `src/fxhnt/ports/live_price.py`, `src/fxhnt/adapters/exchange/ccxt_live_price.py`;
|
||||
Test `tests/unit/test_live_price_fake.py`
|
||||
|
||||
- [ ] **Step 1: Failing test (the fake used by later tests)**
|
||||
```python
|
||||
# tests/unit/test_live_price_fake.py
|
||||
from fxhnt.ports.live_price import LivePrice, FakeLivePrice
|
||||
def test_fake_live_price():
|
||||
lp: LivePrice = FakeLivePrice({"BTCUSDT": 110.0})
|
||||
assert lp.get_prices(["BTCUSDT","UNKNOWN"]) == {"BTCUSDT": 110.0}
|
||||
```
|
||||
- [ ] **Step 2:** run → FAIL.
|
||||
- [ ] **Step 3:** `ports/live_price.py`:
|
||||
```python
|
||||
from __future__ import annotations
|
||||
from typing import Protocol
|
||||
|
||||
class LivePrice(Protocol):
|
||||
def get_prices(self, symbols: list[str]) -> dict[str, float]: ...
|
||||
|
||||
class FakeLivePrice:
|
||||
def __init__(self, prices: dict[str, float]) -> None:
|
||||
self._p = prices
|
||||
def get_prices(self, symbols: list[str]) -> dict[str, float]:
|
||||
return {s: self._p[s] for s in symbols if s in self._p}
|
||||
```
|
||||
Then `adapters/exchange/ccxt_live_price.py` — a `CcxtLivePrice` using `ccxt.binance().fetch_tickers()`,
|
||||
with an in-memory last-good cache so a failed fetch returns the previous prices (never raises to the web
|
||||
layer); maps a sleeve symbol → binance pair where needed. (Covered by the integration test via the fake;
|
||||
the real adapter is thin.)
|
||||
- [ ] **Step 4:** run → PASS.
|
||||
- [ ] **Step 5:** `git commit -m "feat(paper): LivePrice port + fake + ccxt adapter"`
|
||||
|
||||
---
|
||||
|
||||
## Task 6: PaperBookService — derive/persist + read model
|
||||
|
||||
**Files:** Create `src/fxhnt/application/paper_book.py`; Test `tests/integration/test_paper_book.py`
|
||||
|
||||
- [ ] **Step 1: Failing test**
|
||||
```python
|
||||
# tests/integration/test_paper_book.py
|
||||
import datetime as dt
|
||||
from fxhnt.adapters.persistence.paper_repo import PaperRepo
|
||||
from fxhnt.ports.live_price import FakeLivePrice
|
||||
from fxhnt.application.paper_book import PaperBookService
|
||||
|
||||
def test_derive_persist_and_view():
|
||||
repo = PaperRepo("sqlite://"); repo.migrate()
|
||||
svc = PaperBookService(repo, capital=100_000.0)
|
||||
# one sleeve, weight 0.5, short USDTUSD weight -1.0 at price 2.0
|
||||
svc.derive_and_persist(run_date="2026-06-21",
|
||||
sleeve_weights={"stablecoin_rotation": 0.5},
|
||||
symbol_weights_by_sleeve={"stablecoin_rotation": {"USDTUSD": -1.0}},
|
||||
prices={"USDTUSD": 2.0}, at=dt.datetime(2026,6,21,tzinfo=dt.UTC))
|
||||
view = svc.view(live=FakeLivePrice({"USDTUSD": 1.9})) # price dropped -> short profits
|
||||
assert view.capital == 100_000.0
|
||||
assert view.equity > 100_000.0 # short gained as price fell
|
||||
assert any(p.symbol == "USDTUSD" for p in view.positions)
|
||||
assert view.sleeves["stablecoin_rotation"].pnl > 0
|
||||
```
|
||||
- [ ] **Step 2:** run → FAIL.
|
||||
- [ ] **Step 3:** Implement `paper_book.py`:
|
||||
- `derive_and_persist(run_date, sleeve_weights, symbol_weights_by_sleeve, prices, at)`: for each sleeve,
|
||||
`target_positions(...)`; gather all → `diff_trades(prior=repo.latest_positions(), target)` →
|
||||
`repo.replace_positions(...)` + `repo.record_trades(...)`.
|
||||
- `view(live: LivePrice) -> PaperView`: load `repo.latest_positions()`, `live.get_prices([syms])`,
|
||||
`mark_to_market(...)`, build `PaperView(capital, equity=capital+unrealized, positions=[...with last+pnl],
|
||||
sleeves={sleeve: SleeveView(weight, pnl, n)}, trades=repo.recent_trades(25))`.
|
||||
- Define `PaperView`, `PositionView`, `SleeveView` dataclasses here.
|
||||
- [ ] **Step 4:** run → PASS.
|
||||
- [ ] **Step 5:** `git commit -m "feat(paper): PaperBookService derive/persist + live view model"`
|
||||
|
||||
---
|
||||
|
||||
## Task 7: Hook derive_and_persist into the daily rebalance
|
||||
|
||||
**Files:** Modify the combined-book/forward advance path (`src/fxhnt/application/forward_tracker.py` or the
|
||||
cron entrypoint that runs the daily book); Test `tests/integration/test_paper_hook.py`
|
||||
|
||||
- [ ] **Step 1: Failing test** — assert that after the book advance runs, `PaperRepo.latest_positions()` is
|
||||
non-empty (seed a 1-sleeve book with a known weight + price via the existing test fixtures).
|
||||
```python
|
||||
# tests/integration/test_paper_hook.py (skeleton — wire to the actual advance fixture)
|
||||
def test_advance_persists_paper_positions(seeded_book_repo):
|
||||
# run the daily advance with paper persistence enabled; then:
|
||||
assert seeded_book_repo.paper.latest_positions() # non-empty
|
||||
```
|
||||
- [ ] **Step 2:** run → FAIL.
|
||||
- [ ] **Step 3:** In the daily advance path, after the book computes `book_allocator` sleeve weights +
|
||||
each sleeve's symbol weights + the day's closes, call `PaperBookService(paper_repo,
|
||||
settings.paper_capital).derive_and_persist(...)`. Inject `paper_repo` where `ForwardNavRepo` is built
|
||||
(same DSN). Guard with a setting `paper_enabled: bool = True` so it can be turned off.
|
||||
- [ ] **Step 4:** run → PASS; also run the existing forward/book tests to confirm no regression:
|
||||
`uv run pytest tests/integration -q -k "forward or book"`.
|
||||
- [ ] **Step 5:** `git commit -m "feat(paper): persist positions/trades on daily book rebalance"`
|
||||
|
||||
---
|
||||
|
||||
## Task 8: `/paper` + `/paper/live` routes + templates
|
||||
|
||||
**Files:** Modify `src/fxhnt/adapters/web/app.py`; Create
|
||||
`src/fxhnt/adapters/web/templates/paper.html` + `_paper_live.html`; Test `tests/integration/test_paper_web.py`
|
||||
|
||||
- [ ] **Step 1: Failing test**
|
||||
```python
|
||||
# tests/integration/test_paper_web.py
|
||||
from starlette.testclient import TestClient
|
||||
from fxhnt.adapters.persistence.paper_repo import PaperRepo
|
||||
from fxhnt.ports.live_price import FakeLivePrice
|
||||
from fxhnt.adapters.web.app import create_app
|
||||
# build app with a seeded PaperRepo + FakeLivePrice (extend create_app to accept paper_repo + live)
|
||||
|
||||
def test_paper_page_and_live(seeded_paper_app):
|
||||
c = TestClient(seeded_paper_app)
|
||||
r = c.get("/paper"); assert r.status_code == 200 and "Paper" in r.text
|
||||
r2 = c.get("/paper/live"); assert r2.status_code == 200 and "equity" in r2.text.lower()
|
||||
```
|
||||
- [ ] **Step 2:** run → FAIL.
|
||||
- [ ] **Step 3:** Extend `create_app(...)` to accept `paper_repo` + `live: LivePrice` (default real ones in
|
||||
`create_app()` from settings; tests inject fakes). Add routes:
|
||||
- `GET /paper` → render `paper.html` with `PaperBookService(paper_repo, capital).view(live)`.
|
||||
- `GET /paper/live` → render `_paper_live.html` (the equity + positions partial), the HTMX poll target.
|
||||
`paper.html`: page shell + combined equity (reuse `nav_sparkline`) + sleeve cards + positions table +
|
||||
trades log; the live numbers wrapped in a div `hx-get="/paper/live" hx-trigger="every 45s"`.
|
||||
`_paper_live.html`: just the equity + positions table partial. Add a nav link to `/paper` from the fleet page.
|
||||
- [ ] **Step 4:** run → PASS.
|
||||
- [ ] **Step 5:** `git commit -m "feat(paper): /paper + /paper/live cockpit view (HTMX live MTM)"`
|
||||
|
||||
---
|
||||
|
||||
## Task 9: Full suite + finish
|
||||
|
||||
- [ ] **Step 1:** `uv run pytest -q` → all green (no regressions).
|
||||
- [ ] **Step 2:** Use **superpowers:finishing-a-development-branch** — merge `feat/paper-trading-cockpit`
|
||||
→ master, push to Gitea, then deploy via `./scripts/argo-deploy-cockpit.sh` and verify
|
||||
`dashboard.fxhnt.ai/paper` renders (note: the cockpit is served behind the dashboard; confirm route).
|
||||
|
||||
---
|
||||
|
||||
## Self-review notes
|
||||
- Live equity feed for *equities* is out of MVP scope (marked at last close) — documented in spec.
|
||||
- Symbol→price-source mapping (crypto vs equity) lives in `CcxtLivePrice` / the view; unknown→entry (0 PnL).
|
||||
- A3 (replay) reuses `paper_positions` history; B (alerts) is separate. Not in this plan.
|
||||
|
||||
## Acceptance criteria (from spec)
|
||||
- paper_positions + paper_trades populated by a rebalance; target-qty + diff math unit-tested. ✅ T2,T4,T7
|
||||
- `/paper` shows equity + sleeves + positions + trades from seeded data (mocked prices). ✅ T8
|
||||
- `/paper/live` returns updated MTM; crypto live via ccxt (equities last-close). ✅ T6,T8
|
||||
- price-fetch failure degrades gracefully (last-good cache). ✅ T5
|
||||
- read-only, no broker side effects. ✅ (derivation only; no execute path)
|
||||
190
docs/superpowers/plans/2026-06-22-faithful-deltaneutral-carry.md
Normal file
190
docs/superpowers/plans/2026-06-22-faithful-deltaneutral-carry.md
Normal file
@@ -0,0 +1,190 @@
|
||||
# Faithful delta-neutral carry — Implementation Plan (Phase 2F-A5)
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development. TDD. `- [ ]` steps.
|
||||
|
||||
**Goal:** Fix the xsfunding carry artifact (Sharpe 30.9) by modeling the REAL delta-neutral return
|
||||
(`basis + funding`) — ingest spot closes, build a delta-neutral return panel, feed it to the carry sleeve.
|
||||
Sizing stays the existing ISV-adaptive overlay ("match the system").
|
||||
|
||||
**Architecture:** Spot stored as `feature='spot_close'` in the warehouse (mirror funding). A
|
||||
`delta_neutral_return_panel(spot, perp, funding)` = `(spot_ret − perp_ret) + funding` (funding-only fallback).
|
||||
The backfill/snapshot feed that to the existing `sleeve_daily_carry` (unchanged: `Σ w·x / gross`) in place of
|
||||
raw funding. Read-only/additive; overlay sizing untouched.
|
||||
|
||||
**Tech Stack:** Python 3.12, pytest, DuckDB, httpx. **Repo:** `~/Work/fxhnt`, branch `feat/faithful-deltaneutral-carry`.
|
||||
**Spec:** `docs/superpowers/specs/2026-06-22-faithful-deltaneutral-carry-design.md`.
|
||||
|
||||
**Verified context:**
|
||||
- `BinanceSpotHistory().daily_close(symbol) -> {epoch_day: close}` (`adapters/data/binance_spot_history.py`),
|
||||
per-symbol retry/never-raise. Reuse for spot ingest.
|
||||
- Feature store: `write_features_bulk(items)` idempotent per (symbol, ts-range, feature); `crypto_funding_panel`
|
||||
(feature='funding') at duckdb_feature_store.py:123 — MIRROR for spot. `crypto_close_panel` = perp closes.
|
||||
- A4 wiring: `_paper_backfill_sleeves` builds `XsFundingReplay` (carry, return_mode="carry"); the
|
||||
`paper_backfill` command builds `funding_by_date` and passes it to `backfill_paper_book(..., funding_by_date=)`;
|
||||
carry sleeves book `sleeve_daily_carry(prev_shadow, funding_by_date[date])`. We REPLACE that input with the
|
||||
delta-neutral return.
|
||||
- `funding_ingest.py`/`crypto-funding-ingest` CLI/`crypto_funding` asset (A4) — MIRROR for spot.
|
||||
|
||||
---
|
||||
|
||||
## Task 1: spot ingest (adapter reuse) + `crypto_spot_panel` + CLI + asset
|
||||
|
||||
**Files:** Create `src/fxhnt/application/spot_ingest.py`; Modify `duckdb_feature_store.py`, `cli.py`,
|
||||
`adapters/orchestration/assets.py` (+ definitions); Test `tests/integration/test_spot_ingest.py` +
|
||||
`tests/integration/test_crypto_spot_panel.py`
|
||||
|
||||
- [ ] **Step 1: Failing tests**
|
||||
```python
|
||||
# tests/integration/test_crypto_spot_panel.py
|
||||
from fxhnt.adapters.warehouse.duckdb_feature_store import DuckDbFeatureStore
|
||||
def test_crypto_spot_panel_reads_spot_close(tmp_path):
|
||||
st = DuckDbFeatureStore(str(tmp_path/"w.duckdb"))
|
||||
st.write_features("BTCUSDT", [(86400, {"spot_close": 100.0}), (172800, {"spot_close": 110.0})])
|
||||
st.write_features("BTCUSDT", [(86400, {"close": 99.0})]) # perp close must NOT leak
|
||||
assert st.crypto_spot_panel()["BTCUSDT"] == {1: 100.0, 2: 110.0}
|
||||
```
|
||||
```python
|
||||
# tests/integration/test_spot_ingest.py
|
||||
from fxhnt.adapters.warehouse.duckdb_feature_store import DuckDbFeatureStore
|
||||
from fxhnt.application.spot_ingest import ingest_spot
|
||||
class FakeFetcher:
|
||||
def daily_close(self, sym, start_ms=0): return {1: 100.0, 2: 110.0}
|
||||
def test_ingest_spot_writes_spot_close_idempotent(tmp_path):
|
||||
st = DuckDbFeatureStore(str(tmp_path/"w.duckdb"))
|
||||
assert ingest_spot(st, FakeFetcher(), ["BTCUSDT"]) == 1
|
||||
assert st.crypto_spot_panel()["BTCUSDT"] == {1: 100.0, 2: 110.0}
|
||||
ingest_spot(st, FakeFetcher(), ["BTCUSDT"]) # re-run
|
||||
assert st.crypto_spot_panel()["BTCUSDT"] == {1: 100.0, 2: 110.0}
|
||||
def test_ingest_spot_skips_empty(tmp_path):
|
||||
st = DuckDbFeatureStore(str(tmp_path/"w.duckdb"))
|
||||
class E:
|
||||
def daily_close(self, s, start_ms=0): return {}
|
||||
assert ingest_spot(st, E(), ["XUSDT"]) == 0
|
||||
```
|
||||
- [ ] **Step 2:** Run → FAIL.
|
||||
- [ ] **Step 3a:** `crypto_spot_panel(*, suffix='USDT')` in duckdb_feature_store.py — exact mirror of
|
||||
`crypto_funding_panel` but `feature='spot_close'`.
|
||||
- [ ] **Step 3b:** `src/fxhnt/application/spot_ingest.py`:
|
||||
```python
|
||||
"""Ingest Binance SPOT daily-close history into the warehouse feature store as feature='spot_close'
|
||||
(the long leg of the delta-neutral carry). Idempotent per (symbol, day). One-shot / single-writer."""
|
||||
from __future__ import annotations
|
||||
from typing import Any, Protocol
|
||||
class _SpotFetcher(Protocol):
|
||||
def daily_close(self, symbol: str, start_ms: int = 0) -> dict[int, float]: ...
|
||||
def ingest_spot(store: Any, fetcher: _SpotFetcher, symbols: list[str]) -> int:
|
||||
items: list[tuple[str, list[tuple[int, dict[str, float]]]]] = []
|
||||
for sym in symbols:
|
||||
series = fetcher.daily_close(sym)
|
||||
if not series:
|
||||
continue
|
||||
rows = [(int(day) * 86_400, {"spot_close": float(c)}) for day, c in sorted(series.items())]
|
||||
if rows:
|
||||
items.append((sym, rows))
|
||||
if not items:
|
||||
return 0
|
||||
store.write_features_bulk(items)
|
||||
return len(items)
|
||||
```
|
||||
- [ ] **Step 3c:** CLI `crypto-spot-ingest [--days 400]` in cli.py (mirror `crypto-funding-ingest`: open
|
||||
`DuckDbFeatureStore(s.warehouse_path)`, `symbols = sorted(store.crypto_close_panel())`, `BinanceSpotHistory()`,
|
||||
`ingest_spot(store, ...)`, echo count). Dagster asset `crypto_spot` (mirror `crypto_funding`, deps=[crypto_bars],
|
||||
daily/single-writer) + register in definitions.py (update the asset-count test).
|
||||
- [ ] **Step 4:** PASS: `uv run pytest tests/integration/test_spot_ingest.py tests/integration/test_crypto_spot_panel.py -q`.
|
||||
- [ ] **Step 5:** Regression `uv run pytest -k "warehouse or ingest or asset or funding or spot" -q`.
|
||||
- [ ] **Step 6:** Commit `feat(carry): ingest spot closes (feature=spot_close) + crypto_spot_panel + CLI/asset`.
|
||||
|
||||
---
|
||||
|
||||
## Task 2: `delta_neutral_return_panel` + feed it to the carry sleeve
|
||||
|
||||
**Files:** Modify `src/fxhnt/application/funding_panel.py`, `src/fxhnt/cli.py`,
|
||||
`src/fxhnt/adapters/orchestration/assets.py`; Test `tests/integration/test_delta_neutral_return.py`
|
||||
|
||||
- [ ] **Step 1: Failing test**
|
||||
```python
|
||||
# tests/integration/test_delta_neutral_return.py
|
||||
from fxhnt.application.funding_panel import delta_neutral_return_panel
|
||||
def test_dn_is_basis_plus_funding():
|
||||
spot = {"A": {1: 100.0, 2: 110.0}}; perp = {"A": {1: 100.0, 2: 121.0}}; funding = {"A": {2: 0.001}}
|
||||
dn = delta_neutral_return_panel(spot, perp, funding)
|
||||
# day2: spot_ret 0.10, perp_ret 0.21 -> basis -0.11 + funding 0.001 = -0.109
|
||||
assert abs(dn["A"][2] - (-0.109)) < 1e-9
|
||||
assert 1 not in dn["A"] # no prior close for day 1 -> no dn
|
||||
def test_dn_funding_only_fallback_when_no_spot():
|
||||
perp = {"A": {1: 100.0, 2: 110.0}}; funding = {"A": {2: 0.02}}
|
||||
dn = delta_neutral_return_panel({}, perp, funding)
|
||||
assert abs(dn["A"][2] - 0.02) < 1e-9 # no spot -> funding-only
|
||||
```
|
||||
- [ ] **Step 2:** Run → FAIL.
|
||||
- [ ] **Step 3a:** Add to `funding_panel.py`:
|
||||
```python
|
||||
def delta_neutral_return_panel(spot_panel, perp_panel, funding_panel):
|
||||
"""{sym: {day: dn_ret}} where dn_ret(d) = (spot_ret - perp_ret) + funding(d). Per coin/day:
|
||||
requires a perp close at d AND d-1; spot leg used when spot has d AND d-1, else funding-only
|
||||
(the live strategy's documented fallback). Pure."""
|
||||
out: dict[str, dict[int, float]] = {}
|
||||
for sym, pdays in perp_panel.items():
|
||||
sp = spot_panel.get(sym, {}); fnd = funding_panel.get(sym, {}); row: dict[int, float] = {}
|
||||
for d in pdays:
|
||||
if (d - 1) not in pdays or pdays[d - 1] <= 0:
|
||||
continue
|
||||
perp_ret = pdays[d] / pdays[d - 1] - 1.0
|
||||
f = fnd.get(d, 0.0)
|
||||
if (d in sp) and ((d - 1) in sp) and sp[d - 1] > 0:
|
||||
spot_ret = sp[d] / sp[d - 1] - 1.0
|
||||
row[d] = (spot_ret - perp_ret) + f
|
||||
else:
|
||||
row[d] = f # funding-only fallback
|
||||
if row:
|
||||
out[sym] = row
|
||||
return out
|
||||
```
|
||||
- [ ] **Step 3b:** In `cli.py` `paper_backfill`: build the spot panel
|
||||
(`DuckDbFeatureStore(s.warehouse_path).crypto_spot_panel()`) and replace the `funding_by_date` construction
|
||||
with a delta-neutral one: `dn = delta_neutral_return_panel(spot_panel, panel, funding_panel)` then invert to
|
||||
`dn_by_date = {iso:{sym: dn[sym][day]}}`; pass `funding_by_date=dn_by_date` to `backfill_paper_book`. (`panel`
|
||||
is the perp close panel already built; `funding_panel = crypto_funding_panel()`.)
|
||||
- [ ] **Step 3c:** Mirror in the snapshot assembly (`assets.py _assemble_paper_snapshot_inputs` / wherever the
|
||||
carry funding-for-as_of would be provided) so the live snapshot books the delta-neutral return for as_of —
|
||||
live==replay. If the snapshot doesn't yet pass a carry input, wire it to pass the as_of delta-neutral return
|
||||
per symbol (latest dn from the panels). REPORT exactly what you wired.
|
||||
- [ ] **Step 4:** PASS: `uv run pytest tests/integration/test_delta_neutral_return.py tests/integration/test_paper_backfill_cli.py -q`.
|
||||
- [ ] **Step 5:** Regression `uv run pytest -k "paper or backfill or carry or funding or snapshot" -q`.
|
||||
- [ ] **Step 6:** Commit `feat(carry): delta_neutral_return_panel (basis+funding) feeds the xsfunding carry sleeve`.
|
||||
|
||||
---
|
||||
|
||||
## Task 3: deploy + ingest spot + re-backfill + HONEST 3-edge verify
|
||||
|
||||
- [ ] **Step 1:** `uv run pytest -q` → green. **Step 2:** no `*.dbn` staged.
|
||||
- [ ] **Step 3:** **superpowers:finishing-a-development-branch** — merge → master, push (auto-deploy). Confirm
|
||||
workflow Succeeded + pods rolled. (If git.fxhnt.ai:22 is down again: `kubectl rollout restart
|
||||
deploy/tailscale-gitlab-proxy -n foxhunt`, wait ready, retry push.)
|
||||
- [ ] **Step 4:** Ingest spot (funding already ingested in A4; re-run if stale):
|
||||
`kubectl exec deploy/dagster -c daemon -n foxhunt -- fxhnt crypto-spot-ingest --days 400`.
|
||||
- [ ] **Step 5:** Re-backfill: `kubectl exec deploy/dagster -c daemon -n foxhunt -- fxhnt paper-backfill --days 365`.
|
||||
- [ ] **Step 6: VERIFY HONESTLY** — the success bar is "plausible + non-dominating", NOT a target number:
|
||||
- xsfunding **standalone Sharpe** — must drop from 30.9 to a plausible band (~1–4). Report it.
|
||||
- 3-edge nav return/Sharpe/maxDD + per-sleeve standalone + the OOS H1/H2 split + cost-drag.
|
||||
- **If xsfunding Sharpe is STILL absurd (>~6) after the basis term**, that proves the daily book can't
|
||||
represent carry → recommend the live-only fallback (revert xsfunding from the replay). Say so plainly.
|
||||
- `/paper` + `/paper/replay` render. Report all numbers honestly (incl. whether the basis fix worked).
|
||||
|
||||
---
|
||||
|
||||
## Self-review notes
|
||||
- The fix is the RETURN (basis+funding), not the sizing — sizing stays the existing ISV-adaptive overlay
|
||||
("match the system").
|
||||
- `sleeve_daily_carry` unchanged; it's fed the delta-neutral return instead of raw funding.
|
||||
- Funding-only fallback (no spot) = the live strategy's documented behavior.
|
||||
- Gross in the book (post-hoc cost lens), matching the directional sleeves.
|
||||
- Honest exit: if Sharpe still absurd, carry → live-only (pre-agreed).
|
||||
- NEVER `git add -A`; verify no `*.dbn` staged.
|
||||
|
||||
## Acceptance criteria (from spec)
|
||||
- spot ingest + crypto_spot_panel; idempotent; tested. ✅ T1
|
||||
- delta_neutral_return_panel (basis+funding, funding-only fallback); tested. ✅ T2
|
||||
- carry sleeve fed the delta-neutral return (basis injects variance); backfill+snapshot; tested. ✅ T2
|
||||
- in-cluster ingest spot → re-backfill → xsfunding Sharpe plausible OR honest live-only verdict; reported. ✅ T3
|
||||
- read-only/additive; sizing unchanged; no `*.dbn`. ✅ T3
|
||||
@@ -0,0 +1,237 @@
|
||||
# Funding-history ingest + replayable xsfunding (3rd edge) — Implementation Plan (Phase 2F-A4)
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development. TDD. `- [ ]` steps.
|
||||
|
||||
**Goal:** Ingest Binance perp funding-rate history into the warehouse, expose it, and add a faithful
|
||||
replayable xsfunding sleeve so the paper book becomes a **3-edge** book.
|
||||
|
||||
**Architecture:** Funding stored as `feature='funding'` in the existing DuckDB feature store (no new table).
|
||||
A never-raise httpx adapter fetches Binance `fundingRate` history; an ingest fn aggregates 8h→daily and bulk-
|
||||
writes; a `crypto_funding_panel` accessor + `warehouse_funding_panel` assemble the `(close,qvol,funding)` panel
|
||||
xsfunding/unlock need; a replayable xsfunding `PositionSource` runs the pure domain pipeline over it; the
|
||||
backfill adds it. Read-only/additive to the A3f accounting + A3h overlay.
|
||||
|
||||
**Tech Stack:** Python 3.12, pytest, httpx, DuckDB. **Repo:** `~/Work/fxhnt`, branch `feat/funding-ingest-xsfunding`.
|
||||
**Spec:** `docs/superpowers/specs/2026-06-22-funding-ingest-xsfunding-replay-design.md`.
|
||||
|
||||
**Verified context:**
|
||||
- Feature store: `features(symbol, ts, feature, value)`, `ts=epoch_day·86400`; `write_features_bulk(items:
|
||||
list[(symbol, list[(ts, {feature:value})])])` idempotent per (symbol, ts-range); `crypto_close_panel(suffix)`
|
||||
reads `feature='close'`; `read_panel(symbols, feature)`.
|
||||
- xsfunding domain (`domain/cross_sectional_funding.py`, pure): `Panel={sym:{day:(close,qvol,funding)}}`;
|
||||
`eligible_asof(panel, asof_day, *, min_qvol, min_history, vol_window=30)`; `funding_score(panel, eligible,
|
||||
asof_day, lookback_days=7)`; `construction_weights(scores, mode, *, quantile=0.2, borrowable=None)`.
|
||||
- Live/backtest params: `min_qvol=1e6, min_history=30, lookback_days=7, quantile=0.2`. Replay mode =
|
||||
`"market_neutral"` (backtest-validated; `executable` needs a live borrowable set we can't reconstruct
|
||||
historically). qvol fallback `1e7` (≥ min_qvol → liquidity gate effectively off in replay, as with unlock).
|
||||
- `PositionSource` = `Protocol{ current_weights(as_of: str) -> dict[str,float] }`. Backfill sleeves built in
|
||||
`cli.py:_paper_backfill_sleeves(s, panel)` (xsfunding currently excluded ~line 903).
|
||||
- httpx is in the cockpit image. Binance public: `https://fapi.binance.com/fapi/v1/fundingRate`.
|
||||
|
||||
---
|
||||
|
||||
## Task 1: `BinanceFundingHistory` adapter
|
||||
|
||||
**Files:** Create `src/fxhnt/adapters/exchange/binance_funding_history.py`; Test `tests/unit/test_binance_funding_history.py`
|
||||
|
||||
- [ ] **Step 1: Failing test** (mock httpx — no network):
|
||||
```python
|
||||
# tests/unit/test_binance_funding_history.py
|
||||
from fxhnt.adapters.exchange.binance_funding_history import BinanceFundingHistory
|
||||
|
||||
class FakeResp:
|
||||
def __init__(self, data): self._d = data
|
||||
def raise_for_status(self): pass
|
||||
def json(self): return self._d
|
||||
|
||||
class FakeClient:
|
||||
def __init__(self, pages): self.pages = pages; self.calls = []
|
||||
def get(self, url, params=None, timeout=None):
|
||||
self.calls.append(params);
|
||||
return FakeResp(self.pages.pop(0) if self.pages else [])
|
||||
|
||||
def test_fetch_paginates_and_parses():
|
||||
p1 = [{"symbol": "BTCUSDT", "fundingTime": 1000, "fundingRate": "0.0001"}] * 1000
|
||||
p2 = [{"symbol": "BTCUSDT", "fundingTime": 2000, "fundingRate": "-0.0002"}]
|
||||
fc = FakeClient([p1, p2, []])
|
||||
out = BinanceFundingHistory(client=fc).fetch("BTCUSDT", 0, 9999)
|
||||
assert len(out) == 1001 and out[-1] == (2000, -0.0002) # (fundingTime ms, rate float)
|
||||
|
||||
def test_fetch_error_returns_empty_not_raise():
|
||||
class BoomClient:
|
||||
def get(self, *a, **k): raise RuntimeError("429")
|
||||
assert BinanceFundingHistory(client=BoomClient()).fetch("X", 0, 1) == []
|
||||
```
|
||||
- [ ] **Step 2:** Run → FAIL.
|
||||
- [ ] **Step 3:** Implement. `BinanceFundingHistory(client=None, base="https://fapi.binance.com")` (default
|
||||
`client=httpx.Client()`). `fetch(symbol, start_ms, end_ms)`: loop `GET /fapi/v1/fundingRate` with
|
||||
`{symbol, startTime, endTime, limit:1000}`; parse `(int(fundingTime), float(fundingRate))`; advance
|
||||
`startTime = last fundingTime + 1`; stop when a page has `<1000` rows or is empty or exceeds `end_ms`;
|
||||
wrap the whole loop in `try/except Exception → return what's collected (or [])`. Cap iterations (e.g. ≤200)
|
||||
as a runaway guard.
|
||||
- [ ] **Step 4:** Run → PASS. **Step 5:** Commit `feat(funding): BinanceFundingHistory adapter (paginated, never-raise)`.
|
||||
|
||||
---
|
||||
|
||||
## Task 2: `crypto_funding_panel` accessor
|
||||
|
||||
**Files:** Modify `src/fxhnt/adapters/warehouse/duckdb_feature_store.py`; Test `tests/integration/test_crypto_funding_panel.py`
|
||||
|
||||
- [ ] **Step 1: Failing test**
|
||||
```python
|
||||
# tests/integration/test_crypto_funding_panel.py
|
||||
from fxhnt.adapters.warehouse.duckdb_feature_store import DuckDbFeatureStore
|
||||
|
||||
def test_crypto_funding_panel_reads_funding_feature(tmp_path):
|
||||
st = DuckDbFeatureStore(str(tmp_path / "w.duckdb"))
|
||||
st.write_features("BTCUSDT", [(86400, {"funding": 0.0003}), (172800, {"funding": -0.0001})])
|
||||
st.write_features("ETHUSDT", [(86400, {"funding": 0.0005})])
|
||||
st.write_features("BTCUSDT", [(86400, {"close": 100.0})]) # close must NOT leak into funding panel
|
||||
p = st.crypto_funding_panel()
|
||||
assert p["BTCUSDT"] == {1: 0.0003, 2: -0.0001} # ts//86400 = epoch_day
|
||||
assert p["ETHUSDT"] == {1: 0.0005}
|
||||
```
|
||||
- [ ] **Step 2:** Run → FAIL.
|
||||
- [ ] **Step 3:** Add `crypto_funding_panel(*, suffix='USDT')` mirroring `crypto_close_panel` but
|
||||
`feature='funding'`.
|
||||
- [ ] **Step 4:** PASS. **Step 5:** Commit `feat(funding): crypto_funding_panel accessor`.
|
||||
|
||||
---
|
||||
|
||||
## Task 3: `ingest_funding` + CLI + Dagster asset
|
||||
|
||||
**Files:** Create `src/fxhnt/application/funding_ingest.py`; Modify `src/fxhnt/cli.py` +
|
||||
`src/fxhnt/adapters/orchestration/assets.py` (+ definitions); Test `tests/integration/test_funding_ingest.py`
|
||||
|
||||
- [ ] **Step 1: Failing test** (fake fetcher + real in-memory store):
|
||||
```python
|
||||
# tests/integration/test_funding_ingest.py
|
||||
from fxhnt.adapters.warehouse.duckdb_feature_store import DuckDbFeatureStore
|
||||
from fxhnt.application.funding_ingest import ingest_funding
|
||||
|
||||
class FakeFetcher:
|
||||
def fetch(self, sym, s, e):
|
||||
# two 8h stamps on epoch_day 1 (86400..) + one on day 2 -> daily SUM
|
||||
return [(86400_000, 0.0001), (86400_000 + 8*3600_000, 0.0002), (172800_000, -0.0003)]
|
||||
|
||||
def test_ingest_aggregates_8h_to_daily_sum_idempotent(tmp_path):
|
||||
st = DuckDbFeatureStore(str(tmp_path / "w.duckdb"))
|
||||
n = ingest_funding(st, FakeFetcher(), ["BTCUSDT"], start_ms=0, end_ms=10**13)
|
||||
assert n == 1
|
||||
p = st.crypto_funding_panel()
|
||||
assert abs(p["BTCUSDT"][1] - 0.0003) < 1e-12 # day1 = 0.0001+0.0002 (summed)
|
||||
assert abs(p["BTCUSDT"][2] - (-0.0003)) < 1e-12
|
||||
ingest_funding(st, FakeFetcher(), ["BTCUSDT"], start_ms=0, end_ms=10**13) # re-run
|
||||
assert st.crypto_funding_panel()["BTCUSDT"] == p["BTCUSDT"] # idempotent
|
||||
```
|
||||
- [ ] **Step 2:** Run → FAIL.
|
||||
- [ ] **Step 3:** `ingest_funding(store, fetcher, symbols, *, start_ms, end_ms) -> int`: per symbol
|
||||
`fetcher.fetch(sym, start_ms, end_ms)`; aggregate by `epoch_day = ms // 86_400_000` summing rates;
|
||||
build `rows=[(day*86400, {"funding": daily})...]`; collect `(sym, rows)`; `store.write_features_bulk(items)`;
|
||||
return count of symbols with ≥1 row. Skip empty fetches.
|
||||
- [ ] **Step 4:** PASS.
|
||||
- [ ] **Step 5:** Wire a CLI `crypto-funding-ingest [--days 400]` in `cli.py`: open the live warehouse store
|
||||
(same path the paper-backfill uses via `get_settings()`), `symbols = list(store.crypto_close_panel())`,
|
||||
`BinanceFundingHistory()`, `start_ms = (today-days)·86_400_000`, call `ingest_funding`, echo count. Add a
|
||||
Dagster asset `crypto_funding` (mirror an existing crypto ingest asset; daily) in `assets.py` + register in
|
||||
`definitions.py`. Keep it a one-shot single-writer (note in the docstring).
|
||||
- [ ] **Step 6:** `uv run pytest tests/integration/test_funding_ingest.py -q` + a CLI smoke test if one exists.
|
||||
Commit `feat(funding): ingest_funding (8h->daily) + CLI + Dagster asset`.
|
||||
|
||||
---
|
||||
|
||||
## Task 4: `warehouse_funding_panel` + replayable xsfunding sleeve
|
||||
|
||||
**Files:** Create `src/fxhnt/application/funding_panel.py` + `src/fxhnt/application/xsfunding_replay.py`;
|
||||
Test `tests/integration/test_xsfunding_replay.py`
|
||||
|
||||
- [ ] **Step 1: Failing test**
|
||||
```python
|
||||
# tests/integration/test_xsfunding_replay.py
|
||||
from fxhnt.application.funding_panel import warehouse_funding_panel
|
||||
from fxhnt.application.xsfunding_replay import XsFundingReplay
|
||||
|
||||
def test_warehouse_funding_panel_assembles_tuple():
|
||||
closes = {"A": {1: 10.0, 2: 11.0}, "B": {1: 5.0, 2: 5.0}}
|
||||
funding = {"A": {1: 0.01, 2: 0.01}, "B": {1: -0.01, 2: -0.01}}
|
||||
panel = warehouse_funding_panel(closes, funding, qvol_fallback=1e7)
|
||||
assert panel["A"][1] == (10.0, 1e7, 0.01)
|
||||
|
||||
def test_xsfunding_replay_longs_low_funding_shorts_high():
|
||||
# 40 days history so eligibility/lookback clear; A high funding, B low funding
|
||||
closes = {s: {d: 10.0 for d in range(1, 41)} for s in ("A", "B")}
|
||||
funding = {"A": {d: 0.02 for d in range(1, 41)}, "B": {d: -0.02 for d in range(1, 41)}}
|
||||
panel = warehouse_funding_panel(closes, funding, qvol_fallback=1e7)
|
||||
w = XsFundingReplay(panel).current_weights("1970-02-10") # epoch_day 40
|
||||
assert w.get("A", 0) < 0 and w.get("B", 0) > 0 # short high-funding A, long low-funding B
|
||||
assert abs(sum(w.values())) < 1e-9 # market-neutral (dollar-neutral)
|
||||
```
|
||||
- [ ] **Step 2:** Run → FAIL.
|
||||
- [ ] **Step 3:**
|
||||
- `funding_panel.py`: `warehouse_funding_panel(close_panel, funding_panel, *, qvol_fallback=1e7) ->
|
||||
{sym:{day:(close, qvol_fallback, funding_panel.get(sym,{}).get(day, 0.0))}}` over the close panel's
|
||||
(sym,day) keys. (Funding 0.0 only where genuinely missing.)
|
||||
- `xsfunding_replay.py`: `class XsFundingReplay` (implements `PositionSource`):
|
||||
`__init__(self, panel, *, min_qvol=1e6, min_history=30, lookback_days=7, quantile=0.2)`;
|
||||
`current_weights(as_of)`: `day = (date.fromisoformat(as_of) - date(1970,1,1)).days`;
|
||||
`elig = eligible_asof(panel, day, min_qvol=…, min_history=…)`; if not elig → `{}`;
|
||||
`scores = funding_score(panel, elig, day, lookback_days=…)`;
|
||||
`return construction_weights(scores, "market_neutral", quantile=…)`.
|
||||
- [ ] **Step 4:** PASS. **Step 5:** Commit `feat(funding): warehouse_funding_panel + XsFundingReplay sleeve`.
|
||||
|
||||
---
|
||||
|
||||
## Task 5: wire xsfunding into the backfill (3-edge) + fix unlock funding
|
||||
|
||||
**Files:** Modify `src/fxhnt/cli.py` (`_paper_backfill_sleeves`); Test extend `tests/integration/test_paper_backfill*.py`
|
||||
|
||||
- [ ] **Step 1: Failing/asserting test** — build a fake warehouse (closes + funding) with a clear
|
||||
high/low-funding split; assert `_paper_backfill_sleeves` returns a sleeve set INCLUDING `"xsfunding"`, and
|
||||
that its `current_weights(<a date with history>)` is non-empty + market-neutral.
|
||||
- [ ] **Step 2:** Run → FAIL.
|
||||
- [ ] **Step 3:** In `_paper_backfill_sleeves`: build `funding_panel = store.crypto_funding_panel()`,
|
||||
`fp = warehouse_funding_panel(close_panel, funding_panel)`, add `sleeves["xsfunding"] =
|
||||
XsFundingReplay(fp)`. ALSO switch the unlock sleeve's panel to use `fp` (real funding) instead of the
|
||||
`funding=0` fallback (bonus from the spec). Update the "xsfunding excluded" comment → now included (3 edges).
|
||||
- [ ] **Step 4:** PASS. **Step 5:** Commit `feat(funding): add xsfunding (3rd edge) to the paper backfill`.
|
||||
|
||||
---
|
||||
|
||||
## Task 6: Full suite + finish + deploy + ingest + re-backfill + 3-edge verify
|
||||
|
||||
- [ ] **Step 1:** `uv run pytest -q` → green. **Step 2:** no `*.dbn` staged.
|
||||
- [ ] **Step 3:** **superpowers:finishing-a-development-branch** — merge → master, push (auto-deploy). Confirm
|
||||
workflow Succeeded + pods rolled.
|
||||
- [ ] **Step 4:** Ingest funding in-cluster (one-shot, single-writer — ensure no other warehouse write runs):
|
||||
`kubectl exec deploy/dagster -c daemon -n foxhunt -- fxhnt crypto-funding-ingest --days 400`. Confirm it
|
||||
reports N symbols written; spot-check `crypto_funding_panel` is populated.
|
||||
- [ ] **Step 5:** Re-backfill (now 3-edge):
|
||||
`kubectl exec deploy/dagster -c daemon -n foxhunt -- fxhnt paper-backfill --days 365`. Confirm sleeves now
|
||||
list `xsfunding` and it has positions.
|
||||
- [ ] **Step 6: VERIFY (the real test)** — does the 3rd uncorrelated edge de-concentrate the regime risk?
|
||||
- per-sleeve standalone + in-book contribution (xsfunding should appear, ~uncorrelated to the other two);
|
||||
- the deployed nav curve (return / Sharpe / maxDD) + the **OOS H1/H2 split** — is H1 less weak and the
|
||||
blended Sharpe higher/steadier than the 2-edge book (which was H1 Sh ~0.9 / H2 ~5.6)?
|
||||
- the cost-drag rerun on the 3-edge turnover.
|
||||
- `/paper` + `/paper/replay` render with 3 sleeves. Report all numbers honestly (incl. whether xsfunding
|
||||
actually helped — if it didn't de-concentrate, say so).
|
||||
|
||||
---
|
||||
|
||||
## Self-review notes
|
||||
- Funding = a new `feature='funding'`; no schema change. Ingest idempotent (per symbol,ts-range).
|
||||
- Replay xsfunding uses the SAME pure domain pipeline + live params (market_neutral mode); causal.
|
||||
- `warehouse_funding_panel` serves xsfunding AND fixes unlock's funding=0 fallback.
|
||||
- Read-only/additive: A3f accounting, A3h overlay, shadow signal, other 2 sleeves untouched; live xsfunding
|
||||
path untouched (replay is additive).
|
||||
- External API: per-symbol never-raise + idempotent; one-shot single-writer ingest.
|
||||
- NEVER `git add -A`; verify no `*.dbn` staged before each commit.
|
||||
|
||||
## Acceptance criteria (from spec)
|
||||
- BinanceFundingHistory paginates + never-raises; tested. ✅ T1
|
||||
- crypto_funding_panel reads funding; tested. ✅ T2
|
||||
- ingest_funding 8h→daily sum + idempotent + CLI + asset; tested. ✅ T3
|
||||
- warehouse_funding_panel + XsFundingReplay (market-neutral, causal); tested. ✅ T4
|
||||
- backfill includes xsfunding (3 edges) + unlock funding fixed; tested. ✅ T5
|
||||
- in-cluster ingest → 3-edge re-backfill → OOS/cost verify (did the 3rd edge de-concentrate?); rendered. ✅ T6
|
||||
- read-only/additive; no `*.dbn`. ✅ T6
|
||||
196
docs/superpowers/plans/2026-06-22-paper-adaptive-vol-budget.md
Normal file
196
docs/superpowers/plans/2026-06-22-paper-adaptive-vol-budget.md
Normal file
@@ -0,0 +1,196 @@
|
||||
# Paper book: ISV-adaptive vol budget — Implementation Plan (Phase 2F-A3h)
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or
|
||||
> superpowers:executing-plans. TDD. Steps use checkbox (`- [ ]`).
|
||||
|
||||
**Goal:** Make the vol BUDGET state-adaptive — scaled by trailing book Sharpe, bounded [25%,35%], bootstrapped
|
||||
to 30% — applied identically in both sizing paths (live == replay). Beats fixed 30/35% on Sharpe per the A/B.
|
||||
|
||||
**Architecture:** A pure `adaptive_target_vol(book_returns)` helper feeds the per-date target-vol into
|
||||
`vol_target_leverage` inside `paper_book_returns` (per historical date, from the running book series it builds)
|
||||
and `paper_risk_overlay` (latest date, from the prior book series `bret`). `vol_target_leverage` primitive
|
||||
unchanged. Only the budget becomes adaptive; A3f accounting, ISV weights, killswitch, shadow signal untouched.
|
||||
|
||||
**Tech Stack:** Python 3.12, pytest. **Repo:** `~/Work/fxhnt`, branch `feat/paper-adaptive-vol-budget`.
|
||||
**Spec:** `docs/superpowers/specs/2026-06-22-paper-adaptive-vol-budget-design.md`.
|
||||
|
||||
**Verified context (current code, paper_book.py):**
|
||||
- Constants: `_TARGET_VOL=0.30, _KELLY_FRACTION=1.0, _MAX_LEVERAGE=3.0, _VOL_LOOKBACK=60,
|
||||
_PERIODS_PER_YEAR=365, _KILL_DD=0.25, _REENTER_DD=0.12`. `import math`, `import numpy as np`, `statistics`
|
||||
NOT imported (use `np`/`math` or add `statistics`).
|
||||
- `paper_book_returns(...)` loop: sets `out[d]` (when `w_held` not None) at top, then computes
|
||||
`before_d`/`active`/`w_held`/`lev_held = vol_target_leverage(w_held, before_d, target_vol=target_vol, ...)`.
|
||||
- `paper_risk_overlay(...)`: computes `weights`, then `leverage = vol_target_leverage(weights, returns,
|
||||
target_vol=target_vol, ...)`, then `bret = paper_book_returns(returns, target_vol=target_vol, ...)`, then
|
||||
replays `DrawdownKillSwitch` over `bret`. (Leverage is computed BEFORE bret today — Task 2 reorders.)
|
||||
|
||||
---
|
||||
|
||||
## Task 1: `adaptive_target_vol` helper + constants
|
||||
|
||||
**Files:** Modify `src/fxhnt/application/paper_book.py`; Test `tests/unit/test_adaptive_target_vol.py`
|
||||
|
||||
- [ ] **Step 1: Failing tests**
|
||||
```python
|
||||
# tests/unit/test_adaptive_target_vol.py
|
||||
from fxhnt.application.paper_book import adaptive_target_vol
|
||||
|
||||
def test_bootstrap_below_min_obs_returns_base():
|
||||
assert adaptive_target_vol([0.01] * 5) == 0.30 # <20 obs -> base
|
||||
|
||||
def test_strong_sharpe_hits_vol_max():
|
||||
strong = [0.012, 0.010] * 40 # positive mean, tiny vol -> huge Sharpe
|
||||
assert adaptive_target_vol(strong) == 0.35 # capped at vol_max
|
||||
|
||||
def test_negative_mean_floors_vol_min():
|
||||
weak = [0.010, -0.012] * 40 # negative-ish mean -> Sharpe<=0 -> floor
|
||||
assert adaptive_target_vol(weak) == 0.25 # vol_min
|
||||
|
||||
def test_zero_vol_window_no_nan_floors_min():
|
||||
assert adaptive_target_vol([0.0] * 60) == 0.25 # sd=0 -> ts 0 -> frac 0 -> vol_min, no NaN
|
||||
|
||||
def test_monotone_and_bounded():
|
||||
lo = adaptive_target_vol([0.001, -0.0005] * 40) # weak positive
|
||||
hi = adaptive_target_vol([0.012, 0.010] * 40) # strong
|
||||
assert 0.25 <= lo <= hi <= 0.35
|
||||
```
|
||||
- [ ] **Step 2:** Run → FAIL (ImportError).
|
||||
- [ ] **Step 3:** Add constants (near the A3g overlay constants) + the helper to `paper_book.py`. Add
|
||||
`import statistics as st` at the top if not present (else use numpy). Constants:
|
||||
```python
|
||||
_VOL_MIN = 0.25 # adaptive vol-budget floor (user ceiling concern: never "massive")
|
||||
_VOL_MAX = 0.35 # adaptive vol-budget ceiling
|
||||
_VOL_SH_REF = 2.0 # trailing book Sharpe at which the budget reaches vol_max
|
||||
_VOL_MIN_OBS = 20 # min trailing book returns before adapting (else bootstrap to base)
|
||||
```
|
||||
Helper (place after `vol_target_leverage`):
|
||||
```python
|
||||
def adaptive_target_vol(book_returns: list[float], *, vol_min: float = _VOL_MIN, vol_max: float = _VOL_MAX,
|
||||
sh_ref: float = _VOL_SH_REF, base: float = _TARGET_VOL,
|
||||
win: int = _VOL_LOOKBACK, min_obs: int = _VOL_MIN_OBS,
|
||||
periods_per_year: int = _PERIODS_PER_YEAR) -> float:
|
||||
"""ISV-adaptive vol budget: scale the target vol by the trailing book Sharpe, bounded [vol_min, vol_max].
|
||||
Eases toward vol_min in weak patches, rides vol_max when the edge is firing. Bootstrap to `base` until
|
||||
`min_obs` trailing book returns exist. Pure; causal (caller passes the strictly-prior book series)."""
|
||||
w = book_returns[-win:]
|
||||
if len(w) < min_obs:
|
||||
return base
|
||||
sd = st.pstdev(w)
|
||||
ts = (st.mean(w) / sd * math.sqrt(periods_per_year)) if sd > 0 else 0.0
|
||||
frac = max(0.0, min(1.0, ts / sh_ref))
|
||||
return vol_min + (vol_max - vol_min) * frac
|
||||
```
|
||||
- [ ] **Step 4:** Run → PASS (`uv run pytest tests/unit/test_adaptive_target_vol.py -q`).
|
||||
- [ ] **Step 5:** Commit `git commit -m "feat(paper): adaptive_target_vol (trailing-Sharpe vol budget, bounded [25,35])"`
|
||||
|
||||
---
|
||||
|
||||
## Task 2: wire the adaptive budget into both sizing paths
|
||||
|
||||
**Files:** Modify `src/fxhnt/application/paper_book.py`; Test extend `tests/unit/test_paper_risk_overlay.py`
|
||||
|
||||
- [ ] **Step 1: Failing test** (append):
|
||||
```python
|
||||
def test_overlay_uses_adaptive_budget():
|
||||
from fxhnt.application.paper_book import (
|
||||
paper_risk_overlay, paper_book_returns, adaptive_target_vol, vol_target_leverage, isv_adaptive_weights)
|
||||
R = {"a": {i + 1: (0.012 if i % 2 else 0.010) for i in range(80)}} # strong positive Sharpe
|
||||
w, lev, _killed = paper_risk_overlay(R, ["a"])
|
||||
bret = paper_book_returns(R)
|
||||
tv = adaptive_target_vol(list(bret.values()))
|
||||
expected = vol_target_leverage(isv_adaptive_weights(R, ["a"]), R, target_vol=tv)
|
||||
assert tv == 0.35 # strong trailing Sharpe -> rides the 35% ceiling
|
||||
assert abs(lev - expected) < 1e-9 # overlay leverage uses the adaptive budget
|
||||
|
||||
def test_overlay_weak_history_eases_budget():
|
||||
from fxhnt.application.paper_book import paper_book_returns, adaptive_target_vol
|
||||
R = {"a": {i + 1: (0.010 if i % 2 else -0.012) for i in range(80)}} # weak/negative
|
||||
tv = adaptive_target_vol(list(paper_book_returns(R).values()))
|
||||
assert tv == 0.25 # eases to the 25% floor
|
||||
```
|
||||
- [ ] **Step 2:** Run → FAIL.
|
||||
- [ ] **Step 3a:** In `paper_book_returns`, make the per-date budget adaptive. Replace the `lev_held = ...`
|
||||
line so the target vol comes from the running book series `out` (strictly prior):
|
||||
```python
|
||||
w_held = isv_adaptive_weights(before_d, active)
|
||||
tv = adaptive_target_vol(list(out.values()), base=target_vol)
|
||||
lev_held = vol_target_leverage(w_held, before_d, target_vol=tv, kelly_fraction=kelly_fraction,
|
||||
vol_lookback=vol_lookback, periods_per_year=periods_per_year,
|
||||
max_leverage=max_leverage)
|
||||
```
|
||||
(`out` holds book returns through the prior date at this point — causal. `target_vol` param is now the
|
||||
bootstrap base.) Update the docstring: budget is adaptive (trailing-Sharpe, [25,35]); `target_vol` = base.
|
||||
- [ ] **Step 3b:** In `paper_risk_overlay`, REORDER so leverage uses the adaptive budget derived from `bret`:
|
||||
```python
|
||||
weights = isv_adaptive_weights(returns_by_sleeve, active_sleeves)
|
||||
bret = paper_book_returns(returns_by_sleeve, target_vol=target_vol, kelly_fraction=kelly_fraction,
|
||||
vol_lookback=vol_lookback, periods_per_year=periods_per_year,
|
||||
max_leverage=max_leverage)
|
||||
tv = adaptive_target_vol(list(bret.values()), base=target_vol)
|
||||
leverage = vol_target_leverage(weights, returns_by_sleeve, target_vol=tv, kelly_fraction=kelly_fraction,
|
||||
vol_lookback=vol_lookback, periods_per_year=periods_per_year,
|
||||
max_leverage=max_leverage)
|
||||
ks = DrawdownKillSwitch(kill_dd=kill_dd, reenter_dd=reenter_dd)
|
||||
for d in sorted(bret):
|
||||
ks.observe(bret[d])
|
||||
return weights, leverage, ks.killed
|
||||
```
|
||||
Update the overlay docstring: vol budget is ISV-adaptive (trailing-Sharpe, bounded [25%,35%], bootstrap 30%)
|
||||
rather than a fixed 30%.
|
||||
- [ ] **Step 4:** Run → PASS (`uv run pytest tests/unit/test_paper_risk_overlay.py -q`).
|
||||
- [ ] **Step 5: Regression** — all green (these assert shape/sign/consistency, not a specific fixed budget;
|
||||
update any that hard-coded fixed-budget leverage, preserving intent, REPORT):
|
||||
`uv run pytest tests/unit/test_vol_target_leverage.py tests/unit/test_drawdown_killswitch.py
|
||||
tests/unit/test_isv_adaptive_weights.py tests/integration/test_paper_backfill.py
|
||||
tests/integration/test_paper_snapshot.py tests/integration/test_paper_compounding.py -q`.
|
||||
- [ ] **Step 6:** Commit `git commit -m "feat(paper): apply adaptive vol budget in overlay + book-return replay"`
|
||||
|
||||
---
|
||||
|
||||
## Task 3: Full suite + finish + deploy + re-backfill + verify
|
||||
|
||||
- [ ] **Step 1:** `cd ~/Work/fxhnt && uv run pytest -q` → all green (update any remaining fixed-budget
|
||||
assertion, preserving intent; report it).
|
||||
- [ ] **Step 2:** `git status --short | grep -i '\.dbn'` must print nothing.
|
||||
- [ ] **Step 3:** **superpowers:finishing-a-development-branch** — merge `feat/paper-adaptive-vol-budget` →
|
||||
master, push (auto-deploys via Gitea→Argo). Confirm cockpit workflow Succeeded + pods rolled.
|
||||
- [ ] **Step 4:** Re-run backfill in-cluster:
|
||||
`kubectl exec deploy/dagster -c daemon -n foxhunt -- fxhnt paper-backfill --days 365`.
|
||||
- [ ] **Step 5: VERIFY** — curve ≈ the sweep's ADAPT[25→35] row (~+220% gross, Sharpe ~3.4, DD ~−14.5%),
|
||||
budget capped at 35%:
|
||||
```bash
|
||||
kubectl exec deploy/dagster -c daemon -n foxhunt -- python -c "
|
||||
from sqlalchemy import create_engine, text
|
||||
import statistics as st, math
|
||||
from fxhnt.config import get_settings
|
||||
e=create_engine(get_settings().operational_dsn)
|
||||
with e.connect() as c:
|
||||
nav=[float(x[0]) for x in c.execute(text('select equity from paper_nav order by run_date'))]
|
||||
rets=[nav[i]/nav[i-1]-1 for i in range(1,len(nav)) if nav[i-1]]
|
||||
peak=0.0; mdd=0.0
|
||||
for v in nav:
|
||||
peak=max(peak,v); mdd=min(mdd, v/peak-1)
|
||||
sh=(st.mean(rets)/st.pstdev(rets)*math.sqrt(365)) if st.pstdev(rets)>0 else 0
|
||||
print(f'final/start {nav[-1]:.0f}/{nav[0]:.0f} = {(nav[-1]/nav[0]-1)*100:+.1f}% Sharpe {sh:.2f} maxDD {mdd*100:.1f}%')
|
||||
"
|
||||
```
|
||||
Confirm a strongly profitable curve (~Sharpe 3.4, ~+220%, ~−14.5% DD). Load `https://dashboard.fxhnt.ai/paper`
|
||||
+ `/paper/replay`; report final equity, Sharpe, max-DD, and that `/paper` (live) agrees.
|
||||
|
||||
---
|
||||
|
||||
## Self-review notes
|
||||
- Adaptive budget = pure fn of the book's own trailing returns; bounded [25%,35%] (never exceeds the user's
|
||||
ceiling); bootstrap 30%; eases in weak patches (the Sharpe-lifting half), rides 35% when strong.
|
||||
- Same `adaptive_target_vol` call in `paper_book_returns` (per date, from `out`) AND `paper_risk_overlay`
|
||||
(latest, from `bret`) → live == replay. Causal (strictly-prior book series).
|
||||
- A3f equity accounting, ISV weights, killswitch, shadow signal, vol_target_leverage primitive untouched.
|
||||
- Procyclicality bounded by the narrow band + dormant kill; honest 1-sample caveat documented in the spec.
|
||||
- NEVER `git add -A`; verify no `*.dbn` staged.
|
||||
|
||||
## Acceptance criteria (from spec)
|
||||
- `adaptive_target_vol`: bootstrap/<min_obs→base, strong→vol_max, weak→vol_min, monotone, bounded, no NaN. ✅ T1
|
||||
- both paths use it on the strictly-prior book series; consistent (live==replay); causal. ✅ T2
|
||||
- full suite green. ✅ T2, T3
|
||||
- re-backfill → ~+220% / Sharpe ~3.4 / ~−14.5% DD, capped at 35%; /paper agrees; verified. ✅ T3
|
||||
- accounting/strategy mechanics read-only; no `*.dbn`. ✅ T3
|
||||
254
docs/superpowers/plans/2026-06-22-paper-equity-accounting-fix.md
Normal file
254
docs/superpowers/plans/2026-06-22-paper-equity-accounting-fix.md
Normal file
@@ -0,0 +1,254 @@
|
||||
# Paper book: compounding equity-accounting fix — Implementation Plan (Phase 2F-A3f)
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or
|
||||
> superpowers:executing-plans. TDD. Steps use checkbox (`- [ ]`).
|
||||
|
||||
**Goal:** Make the paper-book equity curve reflect the true compounding mark-to-market of the held book
|
||||
(fixing the bug where `unrealized≡0` discarded all held-position appreciation), so the cockpit shows the real
|
||||
profitable curve.
|
||||
|
||||
**Architecture:** `equity_d = equity_{d-1} + Σ positions_before(d).qty·(close_d − entry)`; size each date's
|
||||
throttled book off the running equity (compounding). Internal to `persist_paper_book` + `PaperBookService`
|
||||
(+ two new repo reads); backfill/snapshot are unchanged callers. The shadow signal, `paper_risk_overlay`, and
|
||||
sizing weights/leverage are untouched — only the throttled-book sizing base (capital→equity) and the nav
|
||||
equity computation change.
|
||||
|
||||
**Tech Stack:** Python 3.12, pytest. **Repo:** `~/Work/fxhnt`, branch `feat/paper-equity-accounting`.
|
||||
**Spec:** `docs/superpowers/specs/2026-06-22-paper-equity-accounting-fix-design.md`.
|
||||
|
||||
**Verified context (current code):**
|
||||
- `PaperBookService.derive_and_persist(run_date, sleeve_weights, symbol_weights_by_sleeve, prices, at,
|
||||
leverage=1.0)` sizes `target_positions(..., capital=self._capital * leverage ...)` then diff_trades +
|
||||
realized_delta + replace_positions/replace_trades/set_realized_for_run_date.
|
||||
- `persist_paper_book(repo, *, capital, run_date, sleeve_weights, symbol_weights_by_sleeve, prices, at,
|
||||
enabled=True, leverage=1.0)`: calls derive_and_persist, then `positions=positions_at(run_date)`,
|
||||
`unrealized=Σ qty·(price−entry)` (≡0, the bug), `realized=realized_pnl_through(run_date)`,
|
||||
`equity=capital+realized+unrealized`, `upsert_nav(...)`.
|
||||
- `PaperBookService.view`: `equity=self._capital + self._repo.realized_pnl() + unrealized` (line 388).
|
||||
- `PaperRepo`: has `positions_before(run_date)`, `positions_at`, `nav_history() -> [NavPoint(run_date,equity,
|
||||
realized,unrealized)]`, `upsert_nav`. Engine pattern: `Session(self._engine)`, `select`, `PaperNavRow`
|
||||
imported.
|
||||
- `realized_delta`/`set_realized_for_run_date`/`realized_pnl*` stay in place (unused for equity after this
|
||||
change — out-of-scope cleanup).
|
||||
|
||||
---
|
||||
|
||||
## Task 1: `nav_equity_before` + `latest_nav_equity` repo reads
|
||||
|
||||
**Files:** Modify `src/fxhnt/adapters/persistence/paper_repo.py`; Test `tests/integration/test_paper_nav_equity_reads.py`
|
||||
|
||||
- [ ] **Step 1: Failing test**
|
||||
```python
|
||||
# tests/integration/test_paper_nav_equity_reads.py
|
||||
import datetime as dt
|
||||
from fxhnt.adapters.persistence.paper_repo import PaperRepo
|
||||
|
||||
def _repo():
|
||||
r = PaperRepo("sqlite://"); r.migrate(); return r
|
||||
|
||||
def test_nav_equity_before_and_latest():
|
||||
r = _repo(); at = dt.datetime(2026, 1, 2, tzinfo=dt.UTC)
|
||||
assert r.nav_equity_before("2026-01-01") is None # nothing yet
|
||||
assert r.latest_nav_equity() is None
|
||||
r.upsert_nav("2026-01-01", capital=100000.0, realized=0.0, unrealized=0.0, equity=100000.0, at=at)
|
||||
r.upsert_nav("2026-01-02", capital=100000.0, realized=0.0, unrealized=5000.0, equity=105000.0, at=at)
|
||||
assert r.nav_equity_before("2026-01-02") == 100000.0 # strictly-prior
|
||||
assert r.nav_equity_before("2026-01-01") is None
|
||||
assert r.nav_equity_before("2026-01-03") == 105000.0
|
||||
assert r.latest_nav_equity() == 105000.0
|
||||
```
|
||||
- [ ] **Step 2:** Run → FAIL.
|
||||
- [ ] **Step 3:** Add to `PaperRepo` (after `nav_history`):
|
||||
```python
|
||||
def nav_equity_before(self, run_date: str) -> float | None:
|
||||
"""The most recent paper_nav equity STRICTLY BEFORE `run_date` (the prior equity to compound from)."""
|
||||
rd = dt.date.fromisoformat(run_date)
|
||||
with Session(self._engine) as s:
|
||||
return s.scalar(select(PaperNavRow.equity)
|
||||
.where(PaperNavRow.run_date < rd)
|
||||
.order_by(PaperNavRow.run_date.desc()).limit(1))
|
||||
|
||||
def latest_nav_equity(self) -> float | None:
|
||||
"""The most recent paper_nav equity (for the live view's curve baseline)."""
|
||||
with Session(self._engine) as s:
|
||||
return s.scalar(select(PaperNavRow.equity)
|
||||
.order_by(PaperNavRow.run_date.desc()).limit(1))
|
||||
```
|
||||
- [ ] **Step 4:** Run → PASS. **Step 5:** Regression `uv run pytest tests/integration/test_paper_repo_nav.py -q`.
|
||||
- [ ] **Step 6:** Commit `git commit -m "feat(paper): PaperRepo nav_equity_before + latest_nav_equity"`
|
||||
|
||||
---
|
||||
|
||||
## Task 2: compounding equity in `persist_paper_book` + `derive_and_persist(capital_base)`
|
||||
|
||||
**Files:** Modify `src/fxhnt/application/paper_book.py`; Test `tests/integration/test_paper_compounding.py`
|
||||
|
||||
- [ ] **Step 1: Failing test**
|
||||
```python
|
||||
# tests/integration/test_paper_compounding.py
|
||||
import datetime as dt
|
||||
from fxhnt.adapters.persistence.paper_repo import PaperRepo
|
||||
from fxhnt.application.paper_book import persist_paper_book
|
||||
|
||||
def _p(repo, d, px, lev=1.0):
|
||||
persist_paper_book(repo, capital=100000.0, run_date=d, sleeve_weights={"s": 1.0},
|
||||
symbol_weights_by_sleeve={"s": {"X": 1.0}}, prices={"X": px}, at=dt.datetime(2026,1,1,tzinfo=dt.UTC),
|
||||
leverage=lev)
|
||||
|
||||
def test_equity_compounds_off_prior_equity():
|
||||
r = PaperRepo("sqlite://"); r.migrate()
|
||||
_p(r, "2026-01-01", 100.0) # day0: no prior -> equity = capital
|
||||
assert r.nav_history()[-1].equity == 100000.0
|
||||
_p(r, "2026-01-02", 110.0) # prior qty 1000 *(110-100)=+10000 -> 110000
|
||||
assert abs(r.nav_history()[-1].equity - 110000.0) < 1e-6
|
||||
pos = r.positions_at("2026-01-02") # sized off 110000 -> qty 110000/110=1000
|
||||
assert abs(pos[0].qty - 110000.0 / 110.0) < 1e-6
|
||||
_p(r, "2026-01-03", 121.0) # +10% again, compounded -> 121000
|
||||
assert abs(r.nav_history()[-1].equity - 121000.0) < 1e-6
|
||||
|
||||
def test_killed_flat_book_keeps_equity_flat():
|
||||
r = PaperRepo("sqlite://"); r.migrate()
|
||||
_p(r, "2026-01-01", 100.0) # equity 100000, book qty 1000
|
||||
_p(r, "2026-01-02", 110.0) # equity 110000
|
||||
_p(r, "2026-01-03", 130.0, lev=0.0) # kill: prior(qty 1000)*(130-110)=+20000 -> 130000, book FLAT
|
||||
assert abs(r.nav_history()[-1].equity - 130000.0) < 1e-6
|
||||
assert r.positions_at("2026-01-03") == []
|
||||
_p(r, "2026-01-04", 200.0) # prior book empty -> gain 0 -> equity flat 130000
|
||||
assert abs(r.nav_history()[-1].equity - 130000.0) < 1e-6
|
||||
|
||||
def test_rerun_is_idempotent():
|
||||
r = PaperRepo("sqlite://"); r.migrate()
|
||||
for d, px in [("2026-01-01",100.0),("2026-01-02",110.0),("2026-01-03",121.0)]:
|
||||
_p(r, d, px)
|
||||
first = [(n.run_date, n.equity) for n in r.nav_history()]
|
||||
for d, px in [("2026-01-01",100.0),("2026-01-02",110.0),("2026-01-03",121.0)]:
|
||||
_p(r, d, px) # re-run ascending
|
||||
assert [(n.run_date, n.equity) for n in r.nav_history()] == first
|
||||
```
|
||||
- [ ] **Step 2:** Run → FAIL (current code: day2 equity stays ~100000 because unrealized≡0).
|
||||
- [ ] **Step 3a:** In `PaperBookService.derive_and_persist`, add a `capital_base` param and size off it:
|
||||
signature → `def derive_and_persist(self, run_date, sleeve_weights, symbol_weights_by_sleeve, prices, at,
|
||||
leverage: float = 1.0, capital_base: float | None = None) -> None:`; at the top
|
||||
`base = self._capital if capital_base is None else capital_base`; change the `target_positions(...)` call to
|
||||
`capital=base * leverage`. Leave diff_trades / realized_delta / replace_* unchanged (trades + realized
|
||||
recording stay; realized no longer drives equity).
|
||||
- [ ] **Step 3b:** Rewrite the equity block of `persist_paper_book` (replace the `positions=positions_at(...)`
|
||||
→ `upsert_nav(...)` block, and move the sizing AFTER the equity computation):
|
||||
```python
|
||||
if not enabled:
|
||||
return 0
|
||||
# Compounding mark-to-market: today's equity = prior equity + the PRIOR book marked to today's close.
|
||||
# (positions_before / nav_equity_before are strictly-prior, so this is causal + idempotent.)
|
||||
prior = repo.positions_before(run_date)
|
||||
gain = sum(p.qty * (prices.get(p.symbol, p.entry_price) - p.entry_price) for p in prior)
|
||||
e_prev = repo.nav_equity_before(run_date)
|
||||
if e_prev is None:
|
||||
e_prev = capital # day 0 seed
|
||||
equity = e_prev + gain
|
||||
# Size today's throttled book off the COMPOUNDED equity (gains reinvest).
|
||||
PaperBookService(repo, capital).derive_and_persist(
|
||||
run_date=run_date, sleeve_weights=sleeve_weights,
|
||||
symbol_weights_by_sleeve=symbol_weights_by_sleeve, prices=prices, at=at,
|
||||
leverage=leverage, capital_base=equity)
|
||||
# realized = cumulative through prior day; unrealized = today's MTM gain; equity = capital+realized+unrealized.
|
||||
repo.upsert_nav(run_date, capital=capital, realized=e_prev - capital,
|
||||
unrealized=gain, equity=equity, at=at)
|
||||
return len(repo.positions_at(run_date))
|
||||
```
|
||||
Update the docstring (compounding MTM equity; sizes off running equity). Remove the now-stale comment block
|
||||
about `realized_pnl_through`/`unrealized≡0`.
|
||||
- [ ] **Step 4:** Run → PASS (`uv run pytest tests/integration/test_paper_compounding.py -q`).
|
||||
- [ ] **Step 5:** Regression — these will likely need assertion updates to compounding semantics (update
|
||||
preserving intent, REPORT each): `uv run pytest tests/integration/test_paper_hook.py
|
||||
tests/integration/test_paper_backfill.py tests/integration/test_paper_snapshot.py
|
||||
tests/integration/test_paper_repo_nav.py -q`. (`test_persist_paper_book_leverage_scales_notional` should
|
||||
still pass: day-0 gain 0 → equity = capital = 100k, qty = leverage·100k/px.) The backfill
|
||||
`*_realized_is_monotone` test asserts the OLD realized model — update it to assert idempotent equity
|
||||
reproduction (re-run gives identical nav) instead.
|
||||
- [ ] **Step 6:** Commit `git commit -m "fix(paper): compounding mark-to-market equity (size book off running equity)"`
|
||||
|
||||
---
|
||||
|
||||
## Task 3: `PaperBookService.view` equity = latest nav + live intraday MTM
|
||||
|
||||
**Files:** Modify `src/fxhnt/application/paper_book.py`; Test extend `tests/integration/test_paper_web.py` or `test_paper_hook.py`
|
||||
|
||||
- [ ] **Step 1: Failing test** (add where PaperBookService.view is exercised; if a NullLivePrice/fake live
|
||||
fixture exists, reuse it — otherwise a tiny stub):
|
||||
```python
|
||||
def test_view_equity_is_nav_curve_plus_live_intraday():
|
||||
import datetime as dt
|
||||
from fxhnt.adapters.persistence.paper_repo import PaperRepo
|
||||
from fxhnt.application.paper_book import PaperBookService, persist_paper_book
|
||||
|
||||
class FakeLive:
|
||||
def __init__(self, px): self.px = px
|
||||
def get_prices(self, syms): return {s: self.px for s in syms}
|
||||
|
||||
r = PaperRepo("sqlite://"); r.migrate(); at = dt.datetime(2026, 1, 1, tzinfo=dt.UTC)
|
||||
persist_paper_book(r, capital=100000.0, run_date="2026-01-01", sleeve_weights={"s": 1.0},
|
||||
symbol_weights_by_sleeve={"s": {"X": 1.0}}, prices={"X": 100.0}, at=at)
|
||||
persist_paper_book(r, capital=100000.0, run_date="2026-01-02", sleeve_weights={"s": 1.0},
|
||||
symbol_weights_by_sleeve={"s": {"X": 1.0}}, prices={"X": 110.0}, at=at)
|
||||
# nav curve through last close = 110000; current book (qty 110000/110=1000, entry 110) marked live @ 120
|
||||
view = PaperBookService(r, 100000.0).view(FakeLive(120.0))
|
||||
# equity = latest_nav_equity (110000) + 1000*(120-110)=10000 -> 120000
|
||||
assert abs(view.equity - 120000.0) < 1e-6
|
||||
```
|
||||
- [ ] **Step 2:** Run → FAIL (current: `capital + realized_pnl() + unrealized`).
|
||||
- [ ] **Step 3:** Change the `view` return equity (line ~388) to:
|
||||
```python
|
||||
equity=(self._repo.latest_nav_equity() or self._capital) + unrealized,
|
||||
```
|
||||
(`unrealized` is already the current book marked to live prices in the existing loop. Drop the
|
||||
`realized_pnl()` term — superseded by the nav curve baseline.)
|
||||
- [ ] **Step 4:** Run → PASS. **Step 5:** Regression `uv run pytest tests/integration/test_paper_web.py tests/integration/test_paper_hook.py -q` (update any view-equity assertion to the new basis).
|
||||
- [ ] **Step 6:** Commit `git commit -m "fix(paper): live view equity = nav curve + live intraday MTM"`
|
||||
|
||||
---
|
||||
|
||||
## Task 4: Full suite + finish + deploy + re-backfill + verify the real curve
|
||||
|
||||
- [ ] **Step 1:** `cd ~/Work/fxhnt && uv run pytest -q` → all green (update any remaining old-equity-model
|
||||
assertions to compounding semantics, intent preserved).
|
||||
- [ ] **Step 2:** `git status --short | grep -i '\.dbn'` must print nothing.
|
||||
- [ ] **Step 3:** **superpowers:finishing-a-development-branch** — merge `feat/paper-equity-accounting` →
|
||||
master, push (auto-deploys via Gitea→Argo). Confirm cockpit workflow Succeeded + pods rolled.
|
||||
- [ ] **Step 4:** Re-run backfill in-cluster:
|
||||
`kubectl exec deploy/dagster -c daemon -n foxhunt -- fxhnt paper-backfill --days 365`.
|
||||
- [ ] **Step 5: VERIFY** — the corrected curve is now profitable:
|
||||
```bash
|
||||
kubectl exec deploy/dagster -c daemon -n foxhunt -- python -c "
|
||||
from sqlalchemy import create_engine, text
|
||||
from fxhnt.config import get_settings
|
||||
e=create_engine(get_settings().operational_dsn)
|
||||
with e.connect() as c:
|
||||
print('nav: start/min/max/final/n:', list(c.execute(text('select round((select equity from paper_nav order by run_date asc limit 1)::numeric,0) start, round(min(equity)::numeric,0) lo, round(max(equity)::numeric,0) hi, round((select equity from paper_nav order by run_date desc limit 1)::numeric,0) final, count(*) n from paper_nav'))))
|
||||
print('max drawdown:', list(c.execute(text('with e as (select run_date,equity,max(equity) over (order by run_date) pk from paper_nav) select round(min((equity-pk)/pk)::numeric,3) maxdd from e'))))
|
||||
"
|
||||
```
|
||||
Confirm `final` ≫ `start` (the curve reflects the +92%/+195% edges, compounded, after the risk overlay —
|
||||
profitable, not −8%). Load `https://dashboard.fxhnt.ai/paper` + `/paper/replay` and report the new final
|
||||
equity, max drawdown, and whether `/paper` (live) agrees with the curve.
|
||||
|
||||
---
|
||||
|
||||
## Self-review notes
|
||||
- equity = prior equity + prior-book-marked-to-today → compounding falls out because the book is sized off the
|
||||
running equity. Killed/flat days → empty prior book → 0 gain → flat equity. Day 0 → equity = capital.
|
||||
- Causal: `positions_before` + `nav_equity_before` are strictly-prior. Idempotent: ascending recompute /
|
||||
per-run_date upsert.
|
||||
- Live view = nav curve baseline + live intraday MTM (no double count: curve is through last close, intraday
|
||||
is since last close).
|
||||
- Strategy untouched: shadow signal, paper_risk_overlay, weights/leverage/kill all unchanged; only the
|
||||
throttled-book sizing base + nav equity change.
|
||||
- realized_delta/realized_pnl* left in place (recorded but unused for equity) — bounded blast radius.
|
||||
- NEVER `git add -A`; `*.dbn` gitignored — verify nothing stray staged before each commit.
|
||||
|
||||
## Acceptance criteria (from spec)
|
||||
- `nav_equity_before`/`latest_nav_equity` strictly-prior/latest, None-when-empty. ✅ T1
|
||||
- compounding equity (rising sleeve grows multiplicatively; killed→flat; day0=capital; idempotent). ✅ T2
|
||||
- `derive_and_persist` sizes off `capital_base`. ✅ T2
|
||||
- live view equity = nav curve + live intraday MTM. ✅ T3
|
||||
- full suite green; re-backfill → profitable curve; /paper agrees; verified. ✅ T4
|
||||
- strategy read-only/additive; no `*.dbn`. ✅ T4
|
||||
236
docs/superpowers/plans/2026-06-22-paper-marginal-gate-cleanup.md
Normal file
236
docs/superpowers/plans/2026-06-22-paper-marginal-gate-cleanup.md
Normal file
@@ -0,0 +1,236 @@
|
||||
# Paper book: marginal-gate + cleanup — Implementation Plan (Phase 2F-A3e)
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or
|
||||
> superpowers:executing-plans. TDD. Steps use checkbox (`- [ ]`).
|
||||
|
||||
**Goal:** Add leave-one-out marginal-Sharpe edge pruning (`marginal_gate`) to the paper book's sizing (both
|
||||
backfill + snapshot, live==replay), and clear two logged follow-ups: dead `equal_weights` and the
|
||||
`_regime_lookback` zero-fill bias.
|
||||
|
||||
**Architecture:** A pure `marginal_gate_active(returns, active)` helper in `application/paper_book.py` that
|
||||
reuses `book_allocator.align_edges` + `_marginal_prune`; it prunes the active set before
|
||||
`isv_adaptive_weights` in both `paper_risk_overlay` and `paper_book_returns`. `_regime_lookback` becomes
|
||||
NaN-aware. `equal_weights` + its test removed. Read-only/additive; no new tables.
|
||||
|
||||
**Tech Stack:** Python 3.12, pytest. **Repo:** `~/Work/fxhnt`, branch `feat/paper-marginal-gate`.
|
||||
**Spec:** `docs/superpowers/specs/2026-06-22-paper-marginal-gate-cleanup-design.md`.
|
||||
|
||||
**Verified context:**
|
||||
- `book_allocator.align_edges(series_by_edge) -> (dates, {edge: np.ndarray})` (union axis, 0-fill flat days).
|
||||
- `book_allocator._marginal_prune(window: {edge: np.ndarray}, max_sleeve_weight) -> {edge: np.ndarray}`
|
||||
(leave-one-out: bench a sleeve only when removing it raises trailing book Sharpe; shrinks to ≥1 sleeve).
|
||||
- `paper_book.py` has: `isv_adaptive_weights(returns_by_sleeve, active)`, `_regime_lookback(series_by_sleeve,
|
||||
active)` (currently `np.zeros` fill + mean), `paper_risk_overlay` (weights=isv over active),
|
||||
`paper_book_returns` (per-date `active=[s for s,r in before_d.items() if r]`; `w_held=isv(before_d,active)`),
|
||||
constants `_VOL_LOOKBACK=60`, `_PHI=0.5`, `_L_MIN=10`, `_L_MAX=60`.
|
||||
- `equal_weights` has NO production caller (only `tests/unit/test_equal_weights.py`). The
|
||||
`test_*_equal_weights_active_sleeves_size_positions` tests in backfill/snapshot assert cold-start sizing and
|
||||
do NOT call the function — leave them.
|
||||
|
||||
---
|
||||
|
||||
## Task 1: `marginal_gate_active` helper
|
||||
|
||||
**Files:** Modify `src/fxhnt/application/paper_book.py`; Test `tests/unit/test_marginal_gate_active.py`
|
||||
|
||||
- [ ] **Step 1: Failing tests**
|
||||
```python
|
||||
# tests/unit/test_marginal_gate_active.py
|
||||
from fxhnt.application.paper_book import marginal_gate_active
|
||||
|
||||
def _series(vals, start=1):
|
||||
return {start + i: v for i, v in enumerate(vals)}
|
||||
|
||||
def test_benches_dead_non_diversifying_sleeve():
|
||||
good = _series([0.02, 0.01, 0.03, 0.00] * 15) # positive-mean, real Sharpe
|
||||
dead = _series([-0.02, -0.01, -0.03, 0.00] * 15) # negative-mean, drags the book
|
||||
kept = marginal_gate_active({"good": good, "dead": dead}, ["good", "dead"])
|
||||
assert "good" in kept and "dead" not in kept
|
||||
|
||||
def test_keeps_both_diversifying_sleeves():
|
||||
a = _series([0.03, 0.01] * 30) # both positive-mean, anti-phase -> diversify
|
||||
b = _series([0.01, 0.03] * 30)
|
||||
kept = marginal_gate_active({"a": a, "b": b}, ["a", "b"])
|
||||
assert set(kept) == {"a", "b"}
|
||||
|
||||
def test_cold_sleeve_passes_through_and_no_pruning_under_two_history():
|
||||
good = _series([0.02, 0.01] * 30)
|
||||
kept = marginal_gate_active({"good": good, "new": {1: 0.01}}, ["good", "new"]) # only 1 has-history
|
||||
assert kept == ["good", "new"] # <2 has-history -> unchanged, order preserved
|
||||
|
||||
def test_never_empty_and_preserves_input_order():
|
||||
a = _series([0.02, 0.01] * 30); b = _series([0.015, 0.02] * 30)
|
||||
kept = marginal_gate_active({"a": a, "b": b}, ["a", "b"])
|
||||
assert kept and kept == [s for s in ["a", "b"] if s in kept]
|
||||
```
|
||||
- [ ] **Step 2:** Run → FAIL. (If the exact Sharpe ordering in test 1/2 differs from intent, adjust the
|
||||
numeric series to make `dead` clearly book-Sharpe-lowering / `a`,`b` clearly diversifying — preserve the
|
||||
assertion intent, report any change.)
|
||||
- [ ] **Step 3:** Implement in `paper_book.py` (after `isv_adaptive_weights`; add the import at top with the
|
||||
other imports):
|
||||
```python
|
||||
from fxhnt.application.book_allocator import align_edges as _align_edges, _marginal_prune
|
||||
```
|
||||
```python
|
||||
def marginal_gate_active(returns_by_sleeve: dict[str, dict[int, float]], active_sleeves: list[str], *,
|
||||
vol_lookback: int = _VOL_LOOKBACK, max_sleeve_weight: float = _PHI) -> list[str]:
|
||||
"""Leave-one-out marginal-Sharpe edge gate (reuses book_allocator._marginal_prune): bench a has-history
|
||||
sleeve only when REMOVING it raises the trailing book Sharpe (keeps anticorrelated hedges, benches dead /
|
||||
non-diversifying sleeves). Cold sleeves (<2 returns) pass through untouched; <2 has-history → unchanged.
|
||||
Causal over the trailing `vol_lookback`. Returns the kept active sleeves in input order. Pure."""
|
||||
has = [s for s in active_sleeves if len(returns_by_sleeve.get(s, {})) >= 2]
|
||||
cold = [s for s in active_sleeves if s not in has]
|
||||
if len(has) < 2:
|
||||
return list(active_sleeves)
|
||||
_, arrs = _align_edges({s: returns_by_sleeve[s] for s in has})
|
||||
window = {s: a[max(0, a.size - vol_lookback):] for s, a in arrs.items()}
|
||||
kept_has = set(_marginal_prune(window, max_sleeve_weight))
|
||||
return [s for s in active_sleeves if s in cold or s in kept_has]
|
||||
```
|
||||
- [ ] **Step 4:** Run → PASS (`uv run pytest tests/unit/test_marginal_gate_active.py -q`).
|
||||
- [ ] **Step 5:** Commit `git commit -m "feat(paper): marginal_gate_active (leave-one-out edge pruning)"`
|
||||
|
||||
---
|
||||
|
||||
## Task 2: wire marginal_gate into `paper_risk_overlay` + `paper_book_returns`
|
||||
|
||||
**Files:** Modify `src/fxhnt/application/paper_book.py`; Test extend `tests/unit/test_paper_risk_overlay.py`
|
||||
|
||||
- [ ] **Step 1: Failing test** (append):
|
||||
```python
|
||||
def test_overlay_drops_dead_sleeve_from_weights():
|
||||
from fxhnt.application.paper_book import paper_risk_overlay
|
||||
good = {i + 1: v for i, v in enumerate([0.02, 0.01, 0.03, 0.00] * 15)}
|
||||
dead = {i + 1: v for i, v in enumerate([-0.02, -0.01, -0.03, 0.00] * 15)}
|
||||
w, _lev, _killed = paper_risk_overlay({"good": good, "dead": dead}, ["good", "dead"])
|
||||
assert "good" in w and w.get("dead", 0.0) == 0.0 # dead benched -> no weight
|
||||
```
|
||||
- [ ] **Step 2:** Run → FAIL (dead currently gets inverse-vol weight).
|
||||
- [ ] **Step 3:** In `paper_risk_overlay`, prune before weighting:
|
||||
```python
|
||||
active = marginal_gate_active(returns_by_sleeve, active_sleeves)
|
||||
weights = isv_adaptive_weights(returns_by_sleeve, active)
|
||||
```
|
||||
(replace the existing `weights = isv_adaptive_weights(returns_by_sleeve, active_sleeves)` line; everything
|
||||
else in the fn unchanged.)
|
||||
In `paper_book_returns`, inside the loop, prune the per-date active set before `isv_adaptive_weights`:
|
||||
```python
|
||||
active = [s for s, r in before_d.items() if r]
|
||||
active = marginal_gate_active(before_d, active)
|
||||
w_held = isv_adaptive_weights(before_d, active)
|
||||
```
|
||||
- [ ] **Step 4:** Run → PASS (`uv run pytest tests/unit/test_paper_risk_overlay.py -q`).
|
||||
- [ ] **Step 5:** Regression `uv run pytest tests/unit/test_isv_adaptive_weights.py tests/unit/test_vol_target_leverage.py tests/unit/test_drawdown_killswitch.py -q`.
|
||||
- [ ] **Step 6:** Commit `git commit -m "feat(paper): apply marginal_gate in risk overlay + book-return replay"`
|
||||
|
||||
---
|
||||
|
||||
## Task 3: `_regime_lookback` NaN-aware
|
||||
|
||||
**Files:** Modify `src/fxhnt/application/paper_book.py`; Test `tests/unit/test_regime_lookback.py`
|
||||
|
||||
- [ ] **Step 1: Failing/guard test**
|
||||
```python
|
||||
# tests/unit/test_regime_lookback.py
|
||||
from fxhnt.application.paper_book import _regime_lookback
|
||||
|
||||
def test_late_joining_sleeve_does_not_zero_bias_regime():
|
||||
# 'a' alone has a rising-vol regime over 60 days -> L_t < 60.
|
||||
calm = [0.005, -0.005] * 25; hot = [0.05, -0.05] * 5
|
||||
a = {i + 1: v for i, v in enumerate(calm + hot)}
|
||||
assert _regime_lookback({"a": a}, ["a"]) < 60
|
||||
# 'b' joins only for the last 10 days. Zero-filling its first 50 days would distort the pooled
|
||||
# mean's vol; NaN-aware ignores the missing cells, so the regime is still detected (L_t < 60).
|
||||
b = {i + 1: (0.005 if i % 2 else -0.005) for i in range(50, 60)}
|
||||
assert _regime_lookback({"a": a, "b": b}, ["a", "b"]) < 60
|
||||
|
||||
def test_all_calm_holds_l_max():
|
||||
flat = {i + 1: (0.01 if i % 2 else -0.01) for i in range(60)}
|
||||
assert _regime_lookback({"a": flat}, ["a"]) == 60
|
||||
```
|
||||
- [ ] **Step 2:** Run → the late-joining test may already pass or fail depending on the zero-fill distortion;
|
||||
run to see. Either way implement Step 3 (the NaN-aware version is the correct intent).
|
||||
- [ ] **Step 3:** Replace the matrix build + mean in `_regime_lookback` (keep the rest — the `< 2*_L_MIN`
|
||||
early return, the `sp <= 1e-12` guard, the final clamp):
|
||||
```python
|
||||
idx = {d: i for i, d in enumerate(days)}
|
||||
mat = np.full((len(active), len(days)), np.nan)
|
||||
for r, s in enumerate(active):
|
||||
for d, v in series_by_sleeve.get(s, {}).items():
|
||||
mat[r, idx[d]] = v
|
||||
# NaN-aware mean across active sleeves so a sleeve missing a day does NOT get a spurious 0
|
||||
# (which would depress the pooled vol and bias the regime ratio toward L_max).
|
||||
with np.errstate(invalid="ignore"):
|
||||
rep = np.nanmean(mat, axis=0)
|
||||
rep = np.nan_to_num(rep, nan=0.0) # a day with NO active sleeve -> neutral 0 (rare; union axis)
|
||||
recent, prior = rep[-_L_MIN:], rep[-2 * _L_MIN:-_L_MIN]
|
||||
sp = float(np.std(prior))
|
||||
if sp <= 1e-12:
|
||||
return _L_MAX
|
||||
rho = float(np.std(recent)) / sp
|
||||
return int(round(min(_L_MAX, max(_L_MIN, _L_MAX / max(1.0, rho)))))
|
||||
```
|
||||
- [ ] **Step 4:** Run → PASS (`uv run pytest tests/unit/test_regime_lookback.py tests/unit/test_isv_adaptive_weights.py -q`; the existing A3c regime test inside the isv suite must stay green).
|
||||
- [ ] **Step 5:** Commit `git commit -m "fix(paper): _regime_lookback NaN-aware (drop zero-fill bias)"`
|
||||
|
||||
---
|
||||
|
||||
## Task 4: remove dead `equal_weights`
|
||||
|
||||
**Files:** Modify `src/fxhnt/application/paper_book.py`; Delete `tests/unit/test_equal_weights.py`
|
||||
|
||||
- [ ] **Step 1:** Confirm no production caller: `grep -rn "equal_weights" src/` shows only the `def
|
||||
equal_weights` line (and `marginal_gate_active`/others unrelated). If any `src/` file calls it, STOP and
|
||||
report (do not remove).
|
||||
- [ ] **Step 2:** Remove the `equal_weights` function from `src/fxhnt/application/paper_book.py` and
|
||||
`git rm tests/unit/test_equal_weights.py`.
|
||||
- [ ] **Step 3:** Run the paper suite: `uv run pytest tests/unit/ tests/integration/ -k paper -q` → all green
|
||||
(the backfill/snapshot `*_equal_weights_*` cold-start tests still pass; they don't call the function).
|
||||
- [ ] **Step 4:** Commit `git commit -m "refactor(paper): remove dead equal_weights helper + its test"`
|
||||
|
||||
---
|
||||
|
||||
## Task 5: Full suite + finish + deploy + re-backfill + verify
|
||||
|
||||
- [ ] **Step 1:** `cd ~/Work/fxhnt && uv run pytest -q` → all green.
|
||||
- [ ] **Step 2:** `git status --short | grep -i '\.dbn'` must print nothing.
|
||||
- [ ] **Step 3:** **superpowers:finishing-a-development-branch** — merge `feat/paper-marginal-gate` → master,
|
||||
push (auto-deploys via Gitea→Argo). Confirm cockpit workflow Succeeded + dashboard/dagster pods rolled.
|
||||
- [ ] **Step 4:** Re-run backfill in-cluster:
|
||||
`kubectl exec deploy/dagster -c daemon -n foxhunt -- fxhnt paper-backfill --days 365`.
|
||||
- [ ] **Step 5: VERIFY** — book still populated (gate didn't empty it) + whether any sleeve gets benched:
|
||||
```bash
|
||||
kubectl exec deploy/dagster -c daemon -n foxhunt -- python -c "
|
||||
from sqlalchemy import create_engine, text
|
||||
from fxhnt.config import get_settings
|
||||
e=create_engine(get_settings().operational_dsn)
|
||||
with e.connect() as c:
|
||||
print('positions per sleeve:', list(c.execute(text('select sleeve,count(*) n,count(distinct run_date) d from paper_positions group by sleeve order by sleeve'))))
|
||||
print('shadow sleeves (composition always present):', list(c.execute(text('select sleeve,count(distinct run_date) d from paper_shadow_positions group by sleeve order by sleeve'))))
|
||||
print('dates where a shadow sleeve had NO throttled position (benched-or-killed):', list(c.execute(text('select s.sleeve, count(*) from (select distinct run_date,sleeve from paper_shadow_positions) s left join (select distinct run_date,sleeve from paper_positions) p on s.run_date=p.run_date and s.sleeve=p.sleeve where p.sleeve is null group by s.sleeve order by s.sleeve'))))
|
||||
print('nav span:', list(c.execute(text('select round(min(equity)::numeric,0),round(max(equity)::numeric,0),count(*) from paper_nav'))))
|
||||
"
|
||||
```
|
||||
Confirm the book is non-empty; compare benched/killed dates per sleeve (the marginal gate may now bench a
|
||||
sleeve on some dates where the throttled book is otherwise non-flat). Load `https://dashboard.fxhnt.ai/paper`
|
||||
+ `/paper/replay` and report what renders + whether any sleeve gets benched.
|
||||
|
||||
---
|
||||
|
||||
## Self-review notes
|
||||
- `marginal_gate_active` reuses the trusted `_marginal_prune` (leave-one-out, keeps hedges) — not a Sharpe floor.
|
||||
- Same helper in `paper_risk_overlay` + `paper_book_returns` → live `/paper` == `/paper/replay`.
|
||||
- Cold sleeves pass through; `<2` has-history → unchanged; never empties the book (`_marginal_prune` ≥1).
|
||||
- Causal: prunes over strictly-prior `returns_by_sleeve` / `before_d`.
|
||||
- `_regime_lookback` NaN-aware: missing cell ≠ spurious 0; `L_t` still bounded `[L_min, L_max]`.
|
||||
- `equal_weights` removal verified to have no production caller first.
|
||||
- NEVER `git add -A`; `*.dbn` gitignored — verify nothing stray staged before each commit.
|
||||
|
||||
## Acceptance criteria (from spec)
|
||||
- `marginal_gate_active`: benches dead/non-diversifying, keeps diversifying, cold pass-through, <2-history
|
||||
unchanged, never empty. ✅ T1
|
||||
- applied in overlay + book-return replay; dead sleeve dropped from overlay weights. ✅ T2
|
||||
- `_regime_lookback` NaN-aware; A3c regime tests green. ✅ T3
|
||||
- `equal_weights` + its test removed; suite green. ✅ T4
|
||||
- re-backfill → book non-empty; /paper + /paper/replay render; benched sleeves reported. ✅ T5
|
||||
- read-only/additive; no `*.dbn`. ✅ T5
|
||||
124
docs/superpowers/plans/2026-06-22-paper-overlay-retune.md
Normal file
124
docs/superpowers/plans/2026-06-22-paper-overlay-retune.md
Normal file
@@ -0,0 +1,124 @@
|
||||
# Paper book: risk-overlay re-tune — Implementation Plan (Phase 2F-A3g)
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or
|
||||
> superpowers:executing-plans. TDD. Steps use checkbox (`- [ ]`).
|
||||
|
||||
**Goal:** Re-tune the live paper-book risk overlay to fit the validated edges (per the A/B sweep on the
|
||||
corrected curve): 30% full-Kelly vol-target, a dormant wide (−25/−12) killswitch, marginal-gate OFF.
|
||||
|
||||
**Architecture:** Constants + two call-site removals in `application/paper_book.py`. The overlay functions
|
||||
(`vol_target_leverage`, `DrawdownKillSwitch`, `paper_risk_overlay`, `paper_book_returns`) are unchanged in
|
||||
shape — only the parameter constants and the (removed) marginal-gate step change. Both sizing paths stay
|
||||
consistent (live == replay).
|
||||
|
||||
**Tech Stack:** Python 3.12, pytest. **Repo:** `~/Work/fxhnt`, branch `feat/paper-overlay-retune`.
|
||||
**Spec:** `docs/superpowers/specs/2026-06-22-paper-overlay-retune-design.md`.
|
||||
|
||||
**Verified context:**
|
||||
- Constants in `paper_book.py`: `_TARGET_VOL=0.12, _KELLY_FRACTION=0.5, _MAX_LEVERAGE=3.0, _VOL_LOOKBACK=60,
|
||||
_PERIODS_PER_YEAR=365, _KILL_DD=0.15, _REENTER_DD=0.07`.
|
||||
- `paper_risk_overlay` has `active = marginal_gate_active(returns_by_sleeve, active_sleeves)` before
|
||||
`weights = isv_adaptive_weights(returns_by_sleeve, active)`.
|
||||
- `paper_book_returns` per-date loop has `active = [s for s,r in before_d.items() if r]` then
|
||||
`active = marginal_gate_active(before_d, active)` then `w_held = isv_adaptive_weights(before_d, active)`.
|
||||
- Only one test asserts the gate-via-overlay behavior: `tests/unit/test_paper_risk_overlay.py::
|
||||
test_overlay_drops_dead_sleeve_from_weights`. `vol_target_leverage`/`DrawdownKillSwitch`/`marginal_gate_active`
|
||||
unit tests pass explicit params (not the module defaults), so the constant change does not affect them.
|
||||
|
||||
---
|
||||
|
||||
## Task 1: re-tune constants + disable marginal-gate in both paths
|
||||
|
||||
**Files:** Modify `src/fxhnt/application/paper_book.py`; Test update `tests/unit/test_paper_risk_overlay.py`
|
||||
|
||||
- [ ] **Step 1: Update the affected test FIRST** in `tests/unit/test_paper_risk_overlay.py`. Replace
|
||||
`test_overlay_drops_dead_sleeve_from_weights` (which asserted the gate benched a dead sleeve via the
|
||||
overlay) with a test asserting the gate is now OFF — the dead sleeve receives a (small, inverse-vol) weight:
|
||||
```python
|
||||
def test_overlay_no_longer_gates_dead_sleeve():
|
||||
# marginal-gate is OFF in the overlay (A3g): a dead sleeve is NOT benched here; it still receives an
|
||||
# inverse-vol weight. (The gate fn keeps its own direct unit tests in test_marginal_gate_active.)
|
||||
from fxhnt.application.paper_book import paper_risk_overlay
|
||||
good = {i + 1: v for i, v in enumerate([0.02, 0.01, 0.03, 0.00] * 15)}
|
||||
dead = {i + 1: v for i, v in enumerate([-0.02, -0.01, -0.03, 0.00] * 15)}
|
||||
w, _lev, _killed = paper_risk_overlay({"good": good, "dead": dead}, ["good", "dead"])
|
||||
assert w.get("dead", 0.0) > 0.0 # not benched anymore
|
||||
assert abs(sum(w.values()) - 1.0) < 1e-9
|
||||
```
|
||||
- [ ] **Step 2:** Run → FAIL (current overlay still gates the dead sleeve to 0).
|
||||
- [ ] **Step 3a: Update the constants** in `paper_book.py`:
|
||||
`_TARGET_VOL = 0.30`, `_KELLY_FRACTION = 1.0`, `_KILL_DD = 0.25`, `_REENTER_DD = 0.12`.
|
||||
(Leave `_MAX_LEVERAGE=3.0`, `_VOL_LOOKBACK=60`, `_PERIODS_PER_YEAR=365`, `_L_MIN/_L_MAX/_PHI/_KAPPA`.)
|
||||
- [ ] **Step 3b: Remove the marginal-gate call from `paper_risk_overlay`:** delete the
|
||||
`active = marginal_gate_active(returns_by_sleeve, active_sleeves)` line and weight directly over
|
||||
`active_sleeves`:
|
||||
```python
|
||||
weights = isv_adaptive_weights(returns_by_sleeve, active_sleeves)
|
||||
```
|
||||
- [ ] **Step 3c: Remove the marginal-gate call from `paper_book_returns`:** in the per-date loop, delete the
|
||||
`active = marginal_gate_active(before_d, active)` line, keeping:
|
||||
```python
|
||||
active = [s for s, r in before_d.items() if r]
|
||||
w_held = isv_adaptive_weights(before_d, active)
|
||||
```
|
||||
- [ ] **Step 3d:** Add a one-line note to the `paper_risk_overlay` docstring: overlay is 30% full-Kelly
|
||||
vol-target + a dormant wide (−25/−12) killswitch; marginal-gate intentionally OFF for the current small
|
||||
book (the `marginal_gate_active` fn is retained + unit-tested, re-enable when the book grows).
|
||||
- [ ] **Step 4:** Run → PASS: `uv run pytest tests/unit/test_paper_risk_overlay.py -q`.
|
||||
- [ ] **Step 5: Regression:** `uv run pytest tests/unit/test_marginal_gate_active.py tests/unit/test_vol_target_leverage.py tests/unit/test_drawdown_killswitch.py tests/unit/test_isv_adaptive_weights.py tests/integration/test_paper_backfill.py tests/integration/test_paper_snapshot.py tests/integration/test_paper_compounding.py -q` → all green (these use explicit params or assert shape, so the constant change is safe; report any that need a value update).
|
||||
- [ ] **Step 6: Commit** (NEVER `git add -A`; never stage `*.dbn`):
|
||||
```
|
||||
git add src/fxhnt/application/paper_book.py tests/unit/test_paper_risk_overlay.py
|
||||
git status --short | grep -i '\.dbn' && echo "ABORT: dbn staged" || echo "ok"
|
||||
git commit -m "feat(paper): re-tune overlay (30% full-Kelly vol-target, wide dormant kill, gate off)"
|
||||
```
|
||||
End commit body with: `Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>`
|
||||
|
||||
---
|
||||
|
||||
## Task 2: Full suite + finish + deploy + re-backfill + verify the re-tuned curve
|
||||
|
||||
- [ ] **Step 1:** `cd ~/Work/fxhnt && uv run pytest -q` → all green (update any remaining test that hard-coded
|
||||
the old overlay constants/gate behavior, preserving intent; report it).
|
||||
- [ ] **Step 2:** `git status --short | grep -i '\.dbn'` must print nothing.
|
||||
- [ ] **Step 3:** **superpowers:finishing-a-development-branch** — merge `feat/paper-overlay-retune` → master,
|
||||
push (auto-deploys via Gitea→Argo). Confirm cockpit workflow Succeeded + pods rolled.
|
||||
- [ ] **Step 4:** Re-run backfill in-cluster:
|
||||
`kubectl exec deploy/dagster -c daemon -n foxhunt -- fxhnt paper-backfill --days 365`.
|
||||
- [ ] **Step 5: VERIFY** — the re-tuned curve matches the sweep's 30%-full-Kelly row (~+150–200% gross, Sharpe
|
||||
~3, DD ~−14%):
|
||||
```bash
|
||||
kubectl exec deploy/dagster -c daemon -n foxhunt -- python -c "
|
||||
from sqlalchemy import create_engine, text
|
||||
import statistics as st, math
|
||||
from fxhnt.config import get_settings
|
||||
e=create_engine(get_settings().operational_dsn)
|
||||
with e.connect() as c:
|
||||
nav=[float(x[0]) for x in c.execute(text('select equity from paper_nav order by run_date'))]
|
||||
rets=[nav[i]/nav[i-1]-1 for i in range(1,len(nav)) if nav[i-1]]
|
||||
peak=0.0; mdd=0.0
|
||||
for v in nav:
|
||||
peak=max(peak,v); mdd=min(mdd, v/peak-1)
|
||||
sh=(st.mean(rets)/st.pstdev(rets)*math.sqrt(365)) if st.pstdev(rets)>0 else 0
|
||||
print(f'final/start: {nav[-1]:.0f}/{nav[0]:.0f} = {(nav[-1]/nav[0]-1)*100:.1f}% Sharpe {sh:.2f} maxDD {mdd*100:.1f}%')
|
||||
"
|
||||
```
|
||||
Confirm a strongly profitable curve (~Sharpe 3, ~+150–200%, DD ~−14%). Load `https://dashboard.fxhnt.ai/paper`
|
||||
+ `/paper/replay`; report final equity, Sharpe, max-DD, and that `/paper` (live) agrees with the curve.
|
||||
|
||||
---
|
||||
|
||||
## Self-review notes
|
||||
- Pure config change: 4 constants + 2 call-site removals; overlay fn shapes unchanged.
|
||||
- Gate removed from BOTH `paper_risk_overlay` and `paper_book_returns` → live == replay preserved.
|
||||
- Wide kill (−25/−12) is dormant at 30% vol (vol-target caps DD below threshold) = free tail-insurance; kept.
|
||||
- `marginal_gate_active` fn + its tests retained (re-enablable); only the call sites removed.
|
||||
- A3f equity accounting + shadow signal + sizing mechanics untouched.
|
||||
- NEVER `git add -A`; verify no `*.dbn` staged before each commit.
|
||||
|
||||
## Acceptance criteria (from spec)
|
||||
- constants 0.30/1.0/0.25/0.12; overlay tests green. ✅ T1
|
||||
- marginal-gate not called by overlay/book-returns; the gate-via-overlay test updated. ✅ T1
|
||||
- full suite green. ✅ T1, T2
|
||||
- re-backfill → re-tuned profitable curve (~Sharpe 3, ~+150–200%, ~−14% DD); /paper agrees; verified. ✅ T2
|
||||
- accounting/strategy mechanics read-only; no `*.dbn`. ✅ T2
|
||||
599
docs/superpowers/plans/2026-06-22-paper-vol-target-killswitch.md
Normal file
599
docs/superpowers/plans/2026-06-22-paper-vol-target-killswitch.md
Normal file
@@ -0,0 +1,599 @@
|
||||
# Paper book: vol-target leverage + drawdown killswitch — Implementation Plan (Phase 2F-A3d)
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or
|
||||
> superpowers:executing-plans. TDD. Steps use checkbox (`- [ ]`).
|
||||
|
||||
**Goal:** Layer the live `book_allocator` risk overlay onto A3c's sleeve weights: vol-target + fractional-Kelly
|
||||
leverage and a drawdown killswitch, driven off a kill-proof per-sleeve return signal, applied identically in
|
||||
the backfill and the nightly snapshot.
|
||||
|
||||
**Architecture:** New pure fns in `application/paper_book.py` — `vol_target_leverage`, `DrawdownKillSwitch`,
|
||||
`paper_book_returns`, and a single `paper_risk_overlay(returns, active) -> (weights, leverage, killed)` both
|
||||
paths call. A `leverage` arg on `persist_paper_book` scales notional only. A new `paper_shadow_positions` unit
|
||||
book (never flattened) makes `sleeve_daily_return` survive kill periods so re-entry + vol estimates stay
|
||||
correct. Read-only/additive; default `leverage=1.0` leaves every existing test unchanged.
|
||||
|
||||
**Tech Stack:** Python 3.12, pytest, SQLAlchemy, Dagster. **Repo:** `~/Work/fxhnt`, branch
|
||||
`feat/paper-vol-target`. **Spec:** `docs/superpowers/specs/2026-06-22-paper-vol-target-killswitch-design.md`.
|
||||
|
||||
**Verified context:**
|
||||
- `book_allocator.combine_book` step 4: `target_vol_daily = target_vol/√ppy`; `lev = min(MAX_LEVERAGE,
|
||||
kelly·target_vol_daily/pv)` where `pv = std` of the trailing weighted-portfolio return; `_MAX_LEVERAGE=3.0`.
|
||||
`apply_drawdown_killswitch`: shadow_eq cumulative `∏(1+r)`, kill when `dd>kill_dd`, re-enter when
|
||||
`dd<reenter_dd`. Defaults: `target_vol=0.12, kelly_fraction=0.5, vol_lookback=60, periods_per_year=365,
|
||||
kill_dd=0.15, reenter_dd=0.07`.
|
||||
- `domain.paper.target_positions(sleeve, sleeve_weight, capital, symbol_weights, prices, date)` → notional =
|
||||
`sleeve_weight·capital`; `Position(sleeve, symbol, qty, entry_price, entry_date)`. `sleeve_daily_return`
|
||||
(A3c) is leverage/weight-invariant.
|
||||
- `persist_paper_book(repo, *, capital, run_date, sleeve_weights, symbol_weights_by_sleeve, prices, at,
|
||||
enabled=True)` → `PaperBookService(repo, capital).derive_and_persist(...)` (uses
|
||||
`target_positions(..., capital=self._capital ...)`) + writes `paper_nav`. `equal_weights` is now dead (A3c).
|
||||
- A3c helpers present: `isv_adaptive_weights(returns_by_sleeve, active)`, `sleeve_daily_return(prev, closes)`;
|
||||
`PaperRepo.sleeve_returns(before=)`/`upsert_sleeve_ret`, `positions_before(run_date)`, `replace_positions`.
|
||||
- Backfill/snapshot bodies (post-A3c) compute the clean return from `positions_before` — A3d switches that to
|
||||
`shadow_positions_before` and adds the overlay + shadow persistence.
|
||||
|
||||
---
|
||||
|
||||
## Task 1: `vol_target_leverage` pure fn
|
||||
|
||||
**Files:** Modify `src/fxhnt/application/paper_book.py`; Test `tests/unit/test_vol_target_leverage.py`
|
||||
|
||||
- [ ] **Step 1: Failing test**
|
||||
```python
|
||||
# tests/unit/test_vol_target_leverage.py
|
||||
import math
|
||||
from fxhnt.application.paper_book import vol_target_leverage
|
||||
|
||||
def _series(vals, start=1):
|
||||
return {start + i: v for i, v in enumerate(vals)}
|
||||
|
||||
def test_no_history_cold_start_is_unlevered():
|
||||
assert vol_target_leverage({"a": {}}, {"a": {}}) == 1.0 # weights, returns both empty -> 1.0
|
||||
|
||||
def test_lower_vol_gives_more_leverage():
|
||||
lo = vol_target_leverage({"a": 1.0}, {"a": _series([0.002, -0.002] * 40)})
|
||||
hi = vol_target_leverage({"a": 1.0}, {"a": _series([0.02, -0.02] * 40)})
|
||||
assert lo > hi
|
||||
|
||||
def test_capped_at_max_leverage():
|
||||
tiny = vol_target_leverage({"a": 1.0}, {"a": _series([1e-6, -1e-6] * 40)}, max_leverage=3.0)
|
||||
assert tiny == 3.0
|
||||
|
||||
def test_formula_matches_book_allocator():
|
||||
# single sleeve weight 1.0, known std -> lev = min(MAX, kelly*(target_vol/sqrt(ppy))/pv)
|
||||
import numpy as np
|
||||
r = _series([0.01, -0.01, 0.005, -0.005] * 20)
|
||||
w = {"a": 1.0}
|
||||
lev = vol_target_leverage(w, {"a": r}, target_vol=0.12, kelly_fraction=0.5,
|
||||
periods_per_year=365, vol_lookback=60, max_leverage=3.0)
|
||||
vals = [r[d] for d in sorted(r)][-60:]
|
||||
pv = float(np.std(vals)); tvd = 0.12 / math.sqrt(365)
|
||||
assert math.isclose(lev, min(3.0, 0.5 * tvd / pv), rel_tol=1e-9)
|
||||
```
|
||||
- [ ] **Step 2:** Run → FAIL.
|
||||
- [ ] **Step 3:** Implement in `paper_book.py` (after `isv_adaptive_weights`; add module constants near the
|
||||
A3c constants `_L_MIN…`):
|
||||
```python
|
||||
_TARGET_VOL = 0.12
|
||||
_KELLY_FRACTION = 0.5
|
||||
_MAX_LEVERAGE = 3.0
|
||||
_VOL_LOOKBACK = 60
|
||||
_PERIODS_PER_YEAR = 365
|
||||
_KILL_DD = 0.15
|
||||
_REENTER_DD = 0.07
|
||||
|
||||
|
||||
def vol_target_leverage(weights: dict[str, float], returns_by_sleeve: dict[str, dict[int, float]], *,
|
||||
target_vol: float = _TARGET_VOL, kelly_fraction: float = _KELLY_FRACTION,
|
||||
vol_lookback: int = _VOL_LOOKBACK, periods_per_year: int = _PERIODS_PER_YEAR,
|
||||
max_leverage: float = _MAX_LEVERAGE) -> float:
|
||||
"""Fractional-Kelly vol-target leverage over the trailing weighted-portfolio return (mirrors
|
||||
book_allocator.combine_book step 4). lev = min(max_leverage, kelly·(target_vol/√ppy)/pv) where pv is the
|
||||
std of Σ wᵢ·rᵢ over the last `vol_lookback` of the union date axis. Cold-start (pv≤1e-12 or <2 obs) → 1.0
|
||||
(unlevered = A3c behavior; never flat from lack of data). Pure."""
|
||||
days = sorted({d for s in weights for d in returns_by_sleeve.get(s, {})})
|
||||
if len(days) < 2:
|
||||
return 1.0
|
||||
win = days[max(0, len(days) - vol_lookback):]
|
||||
port = [sum(weights.get(s, 0.0) * returns_by_sleeve.get(s, {}).get(d, 0.0) for s in weights) for d in win]
|
||||
pv = float(np.std(port)) if len(port) > 1 else 0.0
|
||||
if pv <= 1e-12:
|
||||
return 1.0
|
||||
return min(max_leverage, kelly_fraction * (target_vol / math.sqrt(periods_per_year)) / pv)
|
||||
```
|
||||
(Add `import math` if not present — check the top of the file; numpy `np` already imported by A3c.)
|
||||
- [ ] **Step 4:** Run → PASS.
|
||||
- [ ] **Step 5:** Commit `git commit -m "feat(paper): vol_target_leverage (fractional-Kelly vol-target)"`
|
||||
|
||||
---
|
||||
|
||||
## Task 2: `DrawdownKillSwitch` class
|
||||
|
||||
**Files:** Modify `src/fxhnt/application/paper_book.py`; Test `tests/unit/test_drawdown_killswitch.py`
|
||||
|
||||
- [ ] **Step 1: Failing test**
|
||||
```python
|
||||
# tests/unit/test_drawdown_killswitch.py
|
||||
from fxhnt.application.paper_book import DrawdownKillSwitch
|
||||
|
||||
def test_not_killed_before_threshold():
|
||||
ks = DrawdownKillSwitch(kill_dd=0.15, reenter_dd=0.07)
|
||||
ks.observe(-0.10) # 10% dd < 15%
|
||||
assert ks.killed is False
|
||||
|
||||
def test_kills_past_threshold():
|
||||
ks = DrawdownKillSwitch(kill_dd=0.15, reenter_dd=0.07)
|
||||
ks.observe(-0.20) # 20% dd > 15%
|
||||
assert ks.killed is True
|
||||
|
||||
def test_reenters_after_recovery_with_hysteresis():
|
||||
ks = DrawdownKillSwitch(kill_dd=0.15, reenter_dd=0.07)
|
||||
ks.observe(-0.20); assert ks.killed is True # killed
|
||||
ks.observe(0.05); assert ks.killed is True # dd still ~16% > 7% -> stay killed (hysteresis)
|
||||
ks.observe(0.12); assert ks.killed is False # recovered inside 7% -> re-enter
|
||||
|
||||
def test_decision_uses_prior_state_only():
|
||||
# .killed read BEFORE observe(next) is the state applying to the next date
|
||||
ks = DrawdownKillSwitch(kill_dd=0.15, reenter_dd=0.07)
|
||||
assert ks.killed is False # initial: not killed
|
||||
ks.observe(-0.20)
|
||||
assert ks.killed is True
|
||||
```
|
||||
- [ ] **Step 2:** Run → FAIL.
|
||||
- [ ] **Step 3:** Implement in `paper_book.py` (mirrors `apply_drawdown_killswitch`):
|
||||
```python
|
||||
class DrawdownKillSwitch:
|
||||
"""Drawdown kill-switch with hysteresis (mirrors book_allocator.apply_drawdown_killswitch). Tracks the
|
||||
UN-throttled shadow equity; `.killed` (read before `observe`-ing the next return) is the state to apply to
|
||||
the next date. kill when running dd > kill_dd; re-enter when dd < reenter_dd. Causal; pure (no I/O)."""
|
||||
|
||||
def __init__(self, kill_dd: float = _KILL_DD, reenter_dd: float = _REENTER_DD) -> None:
|
||||
self.kill_dd = kill_dd
|
||||
self.reenter_dd = reenter_dd
|
||||
self.shadow_eq = 1.0
|
||||
self.peak = 1.0
|
||||
self.killed = False
|
||||
|
||||
def observe(self, book_return: float) -> None:
|
||||
self.shadow_eq *= (1.0 + book_return)
|
||||
self.peak = max(self.peak, self.shadow_eq)
|
||||
dd = (self.peak - self.shadow_eq) / self.peak if self.peak > 0 else 0.0
|
||||
if not self.killed and dd > self.kill_dd:
|
||||
self.killed = True
|
||||
elif self.killed and dd < self.reenter_dd:
|
||||
self.killed = False
|
||||
```
|
||||
- [ ] **Step 4:** Run → PASS.
|
||||
- [ ] **Step 5:** Commit `git commit -m "feat(paper): DrawdownKillSwitch (hysteresis, mirrors allocator)"`
|
||||
|
||||
---
|
||||
|
||||
## Task 3: `paper_book_returns` + `paper_risk_overlay` pure fns
|
||||
|
||||
**Files:** Modify `src/fxhnt/application/paper_book.py`; Test `tests/unit/test_paper_risk_overlay.py`
|
||||
|
||||
- [ ] **Step 1: Failing test**
|
||||
```python
|
||||
# tests/unit/test_paper_risk_overlay.py
|
||||
from fxhnt.application.paper_book import paper_book_returns, paper_risk_overlay
|
||||
|
||||
def _series(vals, start=1):
|
||||
return {start + i: v for i, v in enumerate(vals)}
|
||||
|
||||
def test_book_returns_are_causal_and_keyed_by_day():
|
||||
# one sleeve, weight ~1; book return on day d uses weights/leverage held from before d
|
||||
r = _series([0.01, -0.01] * 30)
|
||||
out = paper_book_returns({"a": r})
|
||||
assert isinstance(out, dict) and len(out) >= 1
|
||||
# first day has no prior-held weights -> not in output (no book return yet)
|
||||
assert min(r) not in out
|
||||
|
||||
def test_overlay_returns_weights_leverage_killed():
|
||||
r = _series([0.01, -0.01] * 30)
|
||||
w, lev, killed = paper_risk_overlay({"a": r}, ["a"])
|
||||
assert set(w) == {"a"} and w["a"] == 1.0
|
||||
assert lev > 0.0
|
||||
assert killed in (True, False)
|
||||
|
||||
def test_overlay_kills_after_sustained_drawdown():
|
||||
# a long losing streak in the (single-sleeve) book -> killed True
|
||||
losing = _series([-0.03] * 40)
|
||||
_, _, killed = paper_risk_overlay({"a": losing}, ["a"])
|
||||
assert killed is True
|
||||
|
||||
def test_overlay_not_killed_on_calm_history():
|
||||
calm = _series([0.001, -0.001] * 40)
|
||||
_, _, killed = paper_risk_overlay({"a": calm}, ["a"])
|
||||
assert killed is False
|
||||
```
|
||||
- [ ] **Step 2:** Run → FAIL.
|
||||
- [ ] **Step 3:** Implement in `paper_book.py` (after `vol_target_leverage` + `DrawdownKillSwitch`):
|
||||
```python
|
||||
def paper_book_returns(returns_by_sleeve: dict[str, dict[int, float]], *,
|
||||
target_vol: float = _TARGET_VOL, kelly_fraction: float = _KELLY_FRACTION,
|
||||
vol_lookback: int = _VOL_LOOKBACK, periods_per_year: int = _PERIODS_PER_YEAR,
|
||||
max_leverage: float = _MAX_LEVERAGE) -> dict[int, float]:
|
||||
"""UN-throttled paper book daily return series (the killswitch's regime signal). Ascending over the union
|
||||
date axis; weights/leverage are recomputed from returns STRICTLY BEFORE each date and held into it, so
|
||||
bret(d) = lev_held·Σ w_held·r_d. Causal — no look-ahead. The historical 'active' set is every sleeve with
|
||||
prior return history (a faithful proxy; identical in both call paths). Pure."""
|
||||
all_days = sorted({d for r in returns_by_sleeve.values() for d in r})
|
||||
out: dict[int, float] = {}
|
||||
w_held: dict[str, float] | None = None
|
||||
lev_held = 1.0
|
||||
for d in all_days:
|
||||
if w_held is not None:
|
||||
out[d] = lev_held * sum(w_held.get(s, 0.0) * returns_by_sleeve.get(s, {}).get(d, 0.0)
|
||||
for s in w_held)
|
||||
before_d = {s: {k: v for k, v in r.items() if k < d} for s, r in returns_by_sleeve.items()}
|
||||
active = [s for s, r in before_d.items() if r]
|
||||
w_held = isv_adaptive_weights(before_d, active)
|
||||
lev_held = vol_target_leverage(w_held, before_d, target_vol=target_vol,
|
||||
kelly_fraction=kelly_fraction, vol_lookback=vol_lookback,
|
||||
periods_per_year=periods_per_year, max_leverage=max_leverage)
|
||||
return out
|
||||
|
||||
|
||||
def paper_risk_overlay(returns_by_sleeve: dict[str, dict[int, float]], active_sleeves: list[str], *,
|
||||
target_vol: float = _TARGET_VOL, kelly_fraction: float = _KELLY_FRACTION,
|
||||
vol_lookback: int = _VOL_LOOKBACK, periods_per_year: int = _PERIODS_PER_YEAR,
|
||||
max_leverage: float = _MAX_LEVERAGE, kill_dd: float = _KILL_DD,
|
||||
reenter_dd: float = _REENTER_DD) -> tuple[dict[str, float], float, bool]:
|
||||
"""Single entry point for one date's risk overlay given its trailing (strictly-prior) returns. Returns
|
||||
(ISV weights over active, vol-target leverage, killed). `killed` replays a DrawdownKillSwitch over the
|
||||
un-throttled book-return history → the state applying to THIS date. Both backfill and snapshot call this
|
||||
on their trailing window so live /paper == /paper/replay. Pure."""
|
||||
weights = isv_adaptive_weights(returns_by_sleeve, active_sleeves)
|
||||
leverage = vol_target_leverage(weights, returns_by_sleeve, target_vol=target_vol,
|
||||
kelly_fraction=kelly_fraction, vol_lookback=vol_lookback,
|
||||
periods_per_year=periods_per_year, max_leverage=max_leverage)
|
||||
ks = DrawdownKillSwitch(kill_dd=kill_dd, reenter_dd=reenter_dd)
|
||||
bret = paper_book_returns(returns_by_sleeve, target_vol=target_vol, kelly_fraction=kelly_fraction,
|
||||
vol_lookback=vol_lookback, periods_per_year=periods_per_year,
|
||||
max_leverage=max_leverage)
|
||||
for d in sorted(bret):
|
||||
ks.observe(bret[d])
|
||||
return weights, leverage, ks.killed
|
||||
```
|
||||
- [ ] **Step 4:** Run → PASS.
|
||||
- [ ] **Step 5:** Commit `git commit -m "feat(paper): paper_book_returns + paper_risk_overlay (causal risk overlay)"`
|
||||
|
||||
---
|
||||
|
||||
## Task 4: `paper_shadow_positions` table + repo methods
|
||||
|
||||
**Files:** Modify `src/fxhnt/adapters/persistence/cockpit_models.py`,
|
||||
`src/fxhnt/adapters/persistence/paper_repo.py`; Test `tests/integration/test_paper_shadow_positions.py`
|
||||
|
||||
- [ ] **Step 1: Failing test**
|
||||
```python
|
||||
# tests/integration/test_paper_shadow_positions.py
|
||||
import datetime as dt
|
||||
from fxhnt.adapters.persistence.paper_repo import PaperRepo
|
||||
from fxhnt.domain.paper import Position
|
||||
|
||||
def _repo():
|
||||
r = PaperRepo("sqlite://"); r.migrate(); return r
|
||||
|
||||
def test_replace_and_read_shadow_before():
|
||||
r = _repo(); at = dt.datetime(2026, 1, 2, tzinfo=dt.UTC)
|
||||
r.replace_shadow_positions("2026-01-01",
|
||||
[Position("ts", "BTCUSDT", 0.01, 100.0, "2026-01-01")], at=at)
|
||||
r.replace_shadow_positions("2026-01-02",
|
||||
[Position("ts", "BTCUSDT", 0.02, 110.0, "2026-01-02")], at=at)
|
||||
before2 = r.shadow_positions_before("2026-01-02")
|
||||
assert len(before2) == 1 and before2[0].entry_price == 100.0 # the prior date's shadow book
|
||||
assert r.shadow_positions_before("2026-01-01") == [] # nothing strictly before
|
||||
|
||||
def test_replace_is_idempotent_per_run_date():
|
||||
r = _repo(); at = dt.datetime(2026, 1, 2, tzinfo=dt.UTC)
|
||||
r.replace_shadow_positions("2026-01-01", [Position("ts", "BTCUSDT", 0.01, 100.0, "2026-01-01")], at=at)
|
||||
r.replace_shadow_positions("2026-01-01", [Position("ts", "ETHUSDT", 0.5, 50.0, "2026-01-01")], at=at)
|
||||
before = r.shadow_positions_before("2026-01-02")
|
||||
assert len(before) == 1 and before[0].symbol == "ETHUSDT" # replaced, not appended
|
||||
```
|
||||
- [ ] **Step 2:** Run → FAIL.
|
||||
- [ ] **Step 3a:** Add model to `cockpit_models.py` (after `PaperSleeveRetRow`):
|
||||
```python
|
||||
class PaperShadowPositionRow(CockpitBase):
|
||||
"""Unit (sleeve_weight=1, capital=1) sleeve composition per run_date — the kill-PROOF return signal.
|
||||
Persisted every date and NEVER flattened by the killswitch, so per-sleeve returns survive kill periods
|
||||
(re-entry detection + ISV vol estimate stay correct). (run_date, sleeve, symbol) PK — idempotent."""
|
||||
__tablename__ = "paper_shadow_positions"
|
||||
run_date: Mapped[dt.date] = mapped_column(Date, primary_key=True)
|
||||
sleeve: Mapped[str] = mapped_column(String(32), primary_key=True)
|
||||
symbol: Mapped[str] = mapped_column(String(32), primary_key=True)
|
||||
qty: Mapped[float] = mapped_column(Float)
|
||||
entry_price: Mapped[float] = mapped_column(Float)
|
||||
entry_date: Mapped[str] = mapped_column(String(10))
|
||||
src_updated_at: Mapped[dt.datetime] = mapped_column(DateTime)
|
||||
```
|
||||
- [ ] **Step 3b:** In `paper_repo.py` add `PaperShadowPositionRow` to the cockpit_models import, and add (after
|
||||
`shadow`-free position methods, e.g. near `positions_before`):
|
||||
```python
|
||||
def replace_shadow_positions(self, run_date: str, positions: list[Position], *, at: dt.datetime) -> None:
|
||||
"""Idempotent per run_date: rewrite that date's UNIT shadow book (never throttled)."""
|
||||
rd = dt.date.fromisoformat(run_date)
|
||||
with Session(self._engine) as s:
|
||||
s.execute(delete(PaperShadowPositionRow).where(PaperShadowPositionRow.run_date == rd))
|
||||
for p in positions:
|
||||
s.add(PaperShadowPositionRow(
|
||||
run_date=rd, sleeve=p.sleeve, symbol=p.symbol, qty=p.qty,
|
||||
entry_price=p.entry_price, entry_date=p.entry_date, src_updated_at=at))
|
||||
s.commit()
|
||||
|
||||
def shadow_positions_before(self, run_date: str) -> list[Position]:
|
||||
"""The UNIT shadow book as of the most recent run_date STRICTLY BEFORE `run_date`."""
|
||||
rd = dt.date.fromisoformat(run_date)
|
||||
with Session(self._engine) as s:
|
||||
prev = s.scalar(select(PaperShadowPositionRow.run_date)
|
||||
.where(PaperShadowPositionRow.run_date < rd)
|
||||
.order_by(PaperShadowPositionRow.run_date.desc()).limit(1))
|
||||
if prev is None:
|
||||
return []
|
||||
rows = s.scalars(select(PaperShadowPositionRow)
|
||||
.where(PaperShadowPositionRow.run_date == prev)
|
||||
.order_by(PaperShadowPositionRow.sleeve, PaperShadowPositionRow.symbol))
|
||||
return [Position(r.sleeve, r.symbol, r.qty, r.entry_price, r.entry_date) for r in rows]
|
||||
```
|
||||
(`delete`, `select`, `Session`, `Position` already imported in paper_repo.py — confirm.)
|
||||
- [ ] **Step 4:** Run → PASS. **Step 5:** Regression `uv run pytest tests/integration/test_paper_repo.py -q`.
|
||||
- [ ] **Step 6:** Commit `git commit -m "feat(paper): paper_shadow_positions unit book (kill-proof return signal)"`
|
||||
|
||||
---
|
||||
|
||||
## Task 5: `persist_paper_book(leverage=…)` + `unit_shadow_positions` helper
|
||||
|
||||
**Files:** Modify `src/fxhnt/application/paper_book.py`; Test extend `tests/integration/test_paper_hook.py`
|
||||
(or `tests/unit/`), add `tests/unit/test_unit_shadow_positions.py`
|
||||
|
||||
- [ ] **Step 1: Failing tests**
|
||||
```python
|
||||
# tests/unit/test_unit_shadow_positions.py
|
||||
from fxhnt.application.paper_book import unit_shadow_positions
|
||||
|
||||
def test_builds_unit_notional_composition():
|
||||
swb = {"ts": {"BTCUSDT": 1.0}, "sc": {"USDTUSD": -1.0}}
|
||||
prices = {"BTCUSDT": 100.0, "USDTUSD": 1.0}
|
||||
out = unit_shadow_positions(swb, prices, "2026-01-01")
|
||||
by = {(p.sleeve, p.symbol): p for p in out}
|
||||
assert by[("ts", "BTCUSDT")].qty == 1.0 / 100.0 # w*1*1 / px
|
||||
assert by[("sc", "USDTUSD")].qty == -1.0 / 1.0
|
||||
assert all(p.entry_date == "2026-01-01" for p in out)
|
||||
```
|
||||
```python
|
||||
# add to tests/integration/test_paper_hook.py
|
||||
def test_persist_paper_book_leverage_scales_notional():
|
||||
import datetime as dt
|
||||
from fxhnt.adapters.persistence.paper_repo import PaperRepo
|
||||
from fxhnt.application.paper_book import persist_paper_book
|
||||
repo = PaperRepo("sqlite://"); repo.migrate()
|
||||
at = dt.datetime(2026, 1, 1, tzinfo=dt.UTC)
|
||||
kw = dict(capital=100_000.0, run_date="2026-01-01",
|
||||
sleeve_weights={"ts": 1.0}, symbol_weights_by_sleeve={"ts": {"BTCUSDT": 1.0}},
|
||||
prices={"BTCUSDT": 100.0}, at=at)
|
||||
persist_paper_book(repo, **kw, leverage=2.0)
|
||||
pos = repo.positions_at("2026-01-01")
|
||||
assert len(pos) == 1 and pos[0].qty == 2.0 * 100_000.0 / 100.0 # notional doubled
|
||||
nav = repo.nav_history()[-1]
|
||||
assert nav.equity == 100_000.0 # equity base unscaled (no pnl yet)
|
||||
```
|
||||
- [ ] **Step 2:** Run → FAIL.
|
||||
- [ ] **Step 3a:** Add `unit_shadow_positions` to `paper_book.py` (near `equal_weights`):
|
||||
```python
|
||||
def unit_shadow_positions(symbol_weights_by_sleeve: dict[str, dict[str, float]],
|
||||
prices: dict[str, float], date: str) -> list[Position]:
|
||||
"""The UNIT (sleeve_weight=1, capital=1) composition for every active sleeve — the kill-proof signal
|
||||
book. Reuses target_positions so qty/entry semantics match the throttled book. Pure."""
|
||||
out: list[Position] = []
|
||||
for sleeve, sw in symbol_weights_by_sleeve.items():
|
||||
out.extend(target_positions(sleeve=sleeve, sleeve_weight=1.0, capital=1.0,
|
||||
symbol_weights=sw, prices=prices, date=date))
|
||||
return out
|
||||
```
|
||||
- [ ] **Step 3b:** Add a `leverage` param to `persist_paper_book` and `PaperBookService.derive_and_persist`:
|
||||
- `persist_paper_book(..., enabled: bool = True, leverage: float = 1.0)` → pass `leverage` into
|
||||
`PaperBookService(repo, capital).derive_and_persist(..., leverage=leverage)`.
|
||||
- In `derive_and_persist(self, run_date, sleeve_weights, symbol_weights_by_sleeve, prices, at, leverage: float = 1.0)`:
|
||||
change the `target_positions(...)` call to use `capital=self._capital * leverage` (leave the
|
||||
`paper_nav` equity computation using `self._capital` unchanged — base stays unscaled):
|
||||
```python
|
||||
target.extend(target_positions(
|
||||
sleeve=sleeve, sleeve_weight=sw, capital=self._capital * leverage,
|
||||
symbol_weights=symbol_weights, prices=prices, date=run_date))
|
||||
```
|
||||
- [ ] **Step 4:** Run → PASS. **Step 5:** Regression — `uv run pytest tests/integration/test_paper_hook.py
|
||||
tests/integration/test_paper_backfill.py tests/integration/test_paper_snapshot.py -q` (default `leverage=1.0`
|
||||
must leave all green).
|
||||
- [ ] **Step 6:** Commit `git commit -m "feat(paper): persist_paper_book leverage (notional only) + unit_shadow_positions"`
|
||||
|
||||
---
|
||||
|
||||
## Task 6: backfill — risk overlay + shadow book + clean signal from shadow
|
||||
|
||||
**Files:** Modify `src/fxhnt/application/paper_backfill.py`; Test extend `tests/integration/test_paper_backfill.py`
|
||||
|
||||
- [ ] **Step 1: Failing test** — a kill period flattens the throttled book while the shadow book + signal
|
||||
continue, and a later recovery restores positions:
|
||||
```python
|
||||
def test_backfill_killswitch_flattens_then_reenters():
|
||||
import datetime as dt
|
||||
from fxhnt.adapters.persistence.paper_repo import PaperRepo
|
||||
from fxhnt.application.paper_backfill import backfill_paper_book
|
||||
|
||||
class Sleeve:
|
||||
def __init__(self, sym): self.sym = sym
|
||||
def current_weights(self, d): return {self.sym: 1.0}
|
||||
|
||||
repo = PaperRepo("sqlite://"); repo.migrate()
|
||||
dates = [f"2026-{(1+(i//28)):02d}-{(1+(i%28)):02d}" for i in range(112)] # ~4 months
|
||||
closes = {}
|
||||
px = 100.0
|
||||
for i, d in enumerate(dates):
|
||||
# calm up, then a deep crash (drives dd>15%), then recovery
|
||||
if i < 40: px *= 1.001
|
||||
elif i < 70: px *= 0.985 # ~36% drawdown over 30 days -> kill fires
|
||||
else: px *= 1.01 # recovery -> eventual re-enter
|
||||
closes[d] = {"X": px}
|
||||
n = backfill_paper_book(repo, capital=100_000.0, sleeves={"s": Sleeve("X")},
|
||||
dates=dates, closes_by_date=closes, at=dt.datetime(2026, 6, 1, tzinfo=dt.UTC))
|
||||
assert n == len(dates)
|
||||
# shadow signal is continuous (no gaps) across the whole window
|
||||
assert len(repo.sleeve_returns(before=dates[-1]).get("s", {})) > 80
|
||||
# at least one date during the crash has a FLAT throttled book (killed) ...
|
||||
flat = [d for d in dates[55:75] if repo.positions_at(d) == []]
|
||||
assert flat, "expected a killed/flat throttled book during the crash"
|
||||
# ... while the shadow book for those same dates is non-empty (signal survives)
|
||||
assert all(repo.shadow_positions_before(d) for d in flat)
|
||||
```
|
||||
- [ ] **Step 2:** Run → FAIL.
|
||||
- [ ] **Step 3:** Edit `backfill_paper_book`:
|
||||
- Import: `from fxhnt.application.paper_book import (paper_risk_overlay, persist_paper_book,
|
||||
sleeve_daily_return, unit_shadow_positions)` (drop `isv_adaptive_weights`).
|
||||
- Replace the sizing + return-booking block with:
|
||||
```python
|
||||
active = list(symbol_weights_by_sleeve)
|
||||
weights, lev_raw, killed = paper_risk_overlay(returns_by_sleeve, active)
|
||||
eff_lev = 0.0 if killed else lev_raw
|
||||
persist_paper_book(
|
||||
repo, capital=capital, run_date=date,
|
||||
sleeve_weights=weights, symbol_weights_by_sleeve=symbol_weights_by_sleeve,
|
||||
prices=closes_by_date.get(date, {}), at=at, leverage=eff_lev)
|
||||
# Persist the UNIT shadow book for this date (never flattened) so the per-sleeve return signal
|
||||
# survives kill periods. Then book the clean per-sleeve return from the PRIOR shadow book marked to
|
||||
# this date's close — feeds the next date's window. Causal.
|
||||
prices_d = closes_by_date.get(date, {})
|
||||
repo.replace_shadow_positions(date, unit_shadow_positions(symbol_weights_by_sleeve, prices_d, date), at=at)
|
||||
if prices_d:
|
||||
prior = repo.shadow_positions_before(date)
|
||||
by_sleeve: dict[str, list] = {}
|
||||
for p in prior:
|
||||
by_sleeve.setdefault(p.sleeve, []).append(p)
|
||||
for sname, plist in by_sleeve.items():
|
||||
r = sleeve_daily_return(plist, prices_d)
|
||||
if r is not None:
|
||||
repo.upsert_sleeve_ret(date, sname, r, at=at)
|
||||
persisted += 1
|
||||
return persisted
|
||||
```
|
||||
- Update the module/function docstring: ISV weights + vol-target leverage + killswitch; clean signal from
|
||||
the unit shadow book.
|
||||
- [ ] **Step 4:** Run → PASS (`uv run pytest tests/integration/test_paper_backfill.py -q`). Update any
|
||||
pre-existing test that asserted the A3c return-source (positions_before) if it now differs; report it.
|
||||
- [ ] **Step 5:** Commit `git commit -m "feat(paper): backfill applies risk overlay + unit shadow signal"`
|
||||
|
||||
---
|
||||
|
||||
## Task 7: snapshot — same overlay + shadow + clean signal
|
||||
|
||||
**Files:** Modify `src/fxhnt/application/paper_snapshot.py`; Test extend `tests/integration/test_paper_snapshot.py`
|
||||
|
||||
- [ ] **Step 1: Failing test** — seed a losing trailing signal so `paper_risk_overlay` reports `killed`, then
|
||||
assert the snapshot persists a FLAT throttled book (kill applied) while still persisting the unit shadow
|
||||
book + a new `paper_sleeve_ret` row:
|
||||
```python
|
||||
def test_snapshot_applies_killswitch_when_drawdown():
|
||||
import datetime as dt
|
||||
from fxhnt.adapters.persistence.paper_repo import PaperRepo
|
||||
from fxhnt.application.paper_snapshot import PaperSnapshotService
|
||||
repo = PaperRepo("sqlite://"); repo.migrate()
|
||||
at = dt.datetime(2026, 2, 1, tzinfo=dt.UTC)
|
||||
base = dt.date(2026, 1, 1).toordinal()
|
||||
for k in range(40): # sustained losses -> killed
|
||||
d = dt.date.fromordinal(base + k).isoformat()
|
||||
repo.upsert_sleeve_ret(d, "s", -0.03, at=at)
|
||||
class Sleeve:
|
||||
def current_weights(self, d): return {"X": 1.0}
|
||||
svc = PaperSnapshotService(repo, capital=100_000.0, sleeves={"s": Sleeve()},
|
||||
sleeve_weights={}, prices={"X": 100.0})
|
||||
as_of = dt.date.fromordinal(base + 40).isoformat()
|
||||
n = svc.snapshot(as_of=as_of, at=at)
|
||||
assert n == 0 # killed -> flat throttled book
|
||||
assert repo.positions_at(as_of) == []
|
||||
# shadow book + signal still recorded for as_of
|
||||
assert any(p.sleeve == "s" for p in repo.shadow_positions_before(dt.date.fromordinal(base + 41).isoformat()))
|
||||
```
|
||||
- [ ] **Step 2:** Run → FAIL.
|
||||
- [ ] **Step 3:** Edit `PaperSnapshotService.snapshot`:
|
||||
- Import: `from fxhnt.application.paper_book import (paper_risk_overlay, persist_paper_book,
|
||||
sleeve_daily_return, unit_shadow_positions)` (drop `isv_adaptive_weights`).
|
||||
- Replace the sizing + return-booking block with:
|
||||
```python
|
||||
returns_by_sleeve = self._repo.sleeve_returns(before=as_of)
|
||||
active = list(symbol_weights_by_sleeve)
|
||||
weights, lev_raw, killed = paper_risk_overlay(returns_by_sleeve, active)
|
||||
eff_lev = 0.0 if killed else lev_raw
|
||||
n = persist_paper_book(
|
||||
self._repo, capital=self._capital, run_date=as_of,
|
||||
sleeve_weights=weights, symbol_weights_by_sleeve=symbol_weights_by_sleeve,
|
||||
prices=self._prices, at=at, enabled=self._enabled, leverage=eff_lev)
|
||||
if self._enabled:
|
||||
self._repo.replace_shadow_positions(
|
||||
as_of, unit_shadow_positions(symbol_weights_by_sleeve, self._prices, as_of), at=at)
|
||||
if self._prices:
|
||||
prior = self._repo.shadow_positions_before(as_of)
|
||||
by_sleeve: dict[str, list] = {}
|
||||
for p in prior:
|
||||
by_sleeve.setdefault(p.sleeve, []).append(p)
|
||||
for sname, plist in by_sleeve.items():
|
||||
r = sleeve_daily_return(plist, self._prices)
|
||||
if r is not None:
|
||||
self._repo.upsert_sleeve_ret(as_of, sname, r, at=at)
|
||||
return n
|
||||
```
|
||||
- Update the method docstring (overlay + shadow signal; live == replay).
|
||||
- [ ] **Step 4:** Run → PASS (`uv run pytest tests/integration/test_paper_snapshot.py
|
||||
tests/integration/test_paper_snapshot_asset.py -q`). Update pre-existing tests if the return-source change
|
||||
shifts an assertion; report it.
|
||||
- [ ] **Step 5:** Commit `git commit -m "feat(paper): snapshot applies risk overlay + unit shadow signal"`
|
||||
|
||||
---
|
||||
|
||||
## Task 8: Full suite + finish + deploy + re-backfill + verify
|
||||
|
||||
- [ ] **Step 1:** `cd ~/Work/fxhnt && uv run pytest -q` → all green. Fix any A3c-era assertion the
|
||||
return-source/leverage change shifts.
|
||||
- [ ] **Step 2:** `git status --short | grep -i '\.dbn'` must print nothing.
|
||||
- [ ] **Step 3:** **superpowers:finishing-a-development-branch** — merge `feat/paper-vol-target` → master,
|
||||
push (auto-deploys via Gitea→Argo). Confirm the cockpit workflow Succeeded + dashboard/dagster pods rolled.
|
||||
- [ ] **Step 4:** Re-run backfill in-cluster:
|
||||
`kubectl exec deploy/dagster -c daemon -n foxhunt -- fxhnt paper-backfill --days 365`.
|
||||
- [ ] **Step 5: VERIFY** — leverage + killswitch visible and drawdown capped:
|
||||
```bash
|
||||
kubectl exec deploy/dagster -c daemon -n foxhunt -- python -c "
|
||||
from sqlalchemy import create_engine, text
|
||||
from fxhnt.config import get_settings
|
||||
e=create_engine(get_settings().operational_dsn)
|
||||
with e.connect() as c:
|
||||
print('shadow rows:', list(c.execute(text('select count(*) from paper_shadow_positions'))))
|
||||
print('flat (killed) throttled dates:', list(c.execute(text('select count(*) from paper_nav n where not exists (select 1 from paper_positions p where p.run_date=n.run_date)'))))
|
||||
print('nav span + final:', list(c.execute(text('select round(min(equity)::numeric,0) lo, round(max(equity)::numeric,0) hi, count(*) n from paper_nav'))))
|
||||
print('max drawdown vs A3c -29%:', list(c.execute(text('with e as (select run_date, equity, max(equity) over (order by run_date) pk from paper_nav) select round(min((equity-pk)/pk)::numeric,3) maxdd from e'))))
|
||||
"
|
||||
```
|
||||
Confirm: `paper_shadow_positions` populated; some killed/flat dates exist during drawdowns; the
|
||||
`maxdd` is shallower than A3c's −0.29. Load `https://dashboard.fxhnt.ai/paper` + `/paper/replay` and report
|
||||
the levered, kill-aware curve.
|
||||
|
||||
---
|
||||
|
||||
## Self-review notes
|
||||
- `paper_risk_overlay` is the ONE entry point both paths call → live `/paper` == `/paper/replay`.
|
||||
- Clean signal from the UNIT shadow book (never flattened) → re-entry detection + ISV vol estimate survive
|
||||
kills. `sleeve_daily_return` is leverage/weight-invariant so the shadow value matches the throttled value on
|
||||
non-killed days.
|
||||
- Causality: weights/leverage/kill all from strictly-prior returns; `bret(d)` uses prior-held weights.
|
||||
- `leverage` scales notional only; equity base stays `capital`; default 1.0 keeps existing callers/tests green.
|
||||
- Read-only/additive; no change to `book_allocator`, sleeve run()/advance(), combine_book, gate record, brokers.
|
||||
- NEVER `git add -A`; `*.dbn` gitignored — verify nothing stray staged before each commit.
|
||||
|
||||
## Acceptance criteria (from spec)
|
||||
- `vol_target_leverage` formula/cap/cold-start. ✅ T1
|
||||
- `DrawdownKillSwitch` hysteresis + causal. ✅ T2
|
||||
- `paper_book_returns`/`paper_risk_overlay` causal `(weights, leverage, killed)`. ✅ T3
|
||||
- `paper_shadow_positions` table + repo; idempotent. ✅ T4
|
||||
- `persist_paper_book(leverage=)` scales notional not base; default 1.0 green. ✅ T5
|
||||
- backfill + snapshot apply overlay + unit shadow signal; kill flattens actual book, shadow continues. ✅ T6, T7
|
||||
- re-backfill → leverage + capped DD; shadow populated; /paper + /paper/replay render. ✅ T8
|
||||
- full suite green; read-only/additive; no `*.dbn`. ✅ T8
|
||||
@@ -0,0 +1,151 @@
|
||||
# Bybit Testnet Real-Paper Execution Leg — Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: superpowers:subagent-driven-development (recommended) or superpowers:executing-plans, task-by-task. Steps use `- [ ]` checkboxes.
|
||||
|
||||
**Goal:** Run the `bybit_4edge` book as a real-paper track on Bybit testnet — real orders/fills, the risk overlay re-applied on the testnet book's own state, and decomposed, tested slippage — to validate execution *mechanics* before mainnet micro-live.
|
||||
|
||||
**Architecture:** A SEPARATE Dagster job (`bybit_testnet_execution_job`, no CronJob) reads the latest raw `bybit_4edge` sleeve weights, re-applies the overlay on testnet equity/drawdown, diffs vs testnet positions, places ccxt market orders on Bybit testnet, and records fills with decomposed slippage + a post-rebalance NAV. Pure logic (slippage, order planning, overlay-reapply, execution-gap verdict) is unit-tested; the adapter is tested against mocked ccxt.
|
||||
|
||||
**Tech stack:** Python 3.12, ccxt (bybit sandbox), Dagster, SQLAlchemy/Postgres (bybit_features warehouse = TimescaleFeatureStore), pytest. Spec: `docs/superpowers/specs/2026-06-28-bybit-testnet-real-paper-execution-design.md`.
|
||||
|
||||
**Sources of truth to mirror:** `src/fxhnt/adapters/broker/ibkr.py` (broker shape), `src/fxhnt/application/execution.py` (exec protocol), the existing risk-overlay code (single source — do NOT reimplement), `config.py:59` crypto-exec config (triple-gate pattern).
|
||||
|
||||
---
|
||||
|
||||
## PHASE 1 — execution loop (dry-runnable + testnet), no cockpit
|
||||
|
||||
### Task 1: Pin the raw pre-overlay sleeve-weights source (LOAD-BEARING — do first)
|
||||
|
||||
**Files:** read `src/fxhnt/adapters/orchestration/assets.py` (the `bybit_4edge` / `bybit_paper_book` assets), `src/fxhnt/application/paper_book.py`, the sleeve strategies' `current_weights(as_of)`.
|
||||
|
||||
- [ ] **Step 1:** Trace how `bybit_4edge` builds its book. Determine whether the RAW per-sleeve target weights (pre-overlay) for the latest `as_of` are persisted/exposed anywhere, or only post-overlay sim positions.
|
||||
- [ ] **Step 2:** If a clean raw-weights read-model exists, document the exact function/table to call. If NOT, add a minimal read-model: a function `latest_raw_sleeve_weights(repo) -> dict[str, float]` that returns the strategies' `current_weights(latest_as_of)` for the bybit_4edge sleeves (the same inputs the book uses pre-overlay). Add a test that it returns the expected weights for a seeded as_of.
|
||||
- [ ] **Step 3 (GATE):** If the raw weights cannot be obtained cleanly (only sim-marked positions available), STOP and escalate — the leg's correctness depends on this. Do not proceed to Task 4 until resolved.
|
||||
|
||||
### Task 2: `slippage.py` — pure slippage model (TDD)
|
||||
|
||||
**Files:** Create `src/fxhnt/application/slippage.py`; Test `tests/unit/test_slippage.py`.
|
||||
|
||||
- [ ] **Step 1: Failing tests**
|
||||
```python
|
||||
import math
|
||||
from fxhnt.application.slippage import (
|
||||
execution_slippage_bps, timing_drift_bps, slippage_vs_assumed_bps)
|
||||
|
||||
def test_buy_pays_above_mid_is_positive():
|
||||
# bought at 101 vs mid 100 → +100 bps cost
|
||||
assert execution_slippage_bps(101.0, 100.0, "BUY") == 100.0
|
||||
def test_sell_below_mid_is_positive_cost():
|
||||
# sold at 99 vs mid 100 → +100 bps cost
|
||||
assert execution_slippage_bps(99.0, 100.0, "SELL") == 100.0
|
||||
def test_timing_drift_sign():
|
||||
# BUY, mid moved 100→101 from the book mark → +100 bps adverse drift
|
||||
assert timing_drift_bps(101.0, 100.0, "BUY") == 100.0
|
||||
def test_decomposition_sums_to_total():
|
||||
fill, mid, mark = 102.0, 101.0, 100.0
|
||||
total = execution_slippage_bps(fill, mid, "BUY") + timing_drift_bps(mid, mark, "BUY")
|
||||
# fill-vs-mark for a BUY = (102-100)/100*1e4 = 200
|
||||
assert abs(total - 200.0) < 1e-6
|
||||
def test_assumed_delta():
|
||||
assert slippage_vs_assumed_bps(40.0, 35.0) == 5.0 # 5bp worse than the 35bp tier
|
||||
def test_guards_none_on_bad_inputs():
|
||||
assert execution_slippage_bps(101.0, 0.0, "BUY") is None
|
||||
assert timing_drift_bps(101.0, -1.0, "SELL") is None
|
||||
```
|
||||
- [ ] **Step 2:** Run → FAIL (module missing).
|
||||
- [ ] **Step 3: Implement**
|
||||
```python
|
||||
from __future__ import annotations
|
||||
def _sign(side: str) -> int:
|
||||
return 1 if side.upper() == "BUY" else -1
|
||||
def execution_slippage_bps(fill: float, mid_at_order: float, side: str) -> float | None:
|
||||
if mid_at_order <= 0.0: return None
|
||||
return _sign(side) * (fill - mid_at_order) / mid_at_order * 1e4
|
||||
def timing_drift_bps(mid_at_order: float, book_mark: float, side: str) -> float | None:
|
||||
if book_mark <= 0.0: return None
|
||||
return _sign(side) * (mid_at_order - book_mark) / book_mark * 1e4
|
||||
def slippage_vs_assumed_bps(execution_slippage: float, assumed_tier_bps: float) -> float:
|
||||
return execution_slippage - assumed_tier_bps
|
||||
```
|
||||
- [ ] **Step 4:** Run → PASS. **Step 5:** Commit `feat(exec): pure decomposed slippage model`.
|
||||
|
||||
### Task 3: `execution_gap.py` — mechanics verdict + indicative slippage (TDD)
|
||||
|
||||
**Files:** Create `src/fxhnt/application/execution_gap.py`; Test `tests/unit/test_execution_gap.py`.
|
||||
|
||||
- [ ] **Step 1: Failing tests** — a `mechanics_verdict(rows, target_positions, tolerance) -> Verdict` that is PASS when every recorded post-rebalance position tracks its effective target within `tolerance` and there are no error/gap rows; FAIL otherwise with the offending symbol(s). And `indicative_slippage(fills) -> dict[tier, {mean_delta_bps, coverage}]` aggregating `slippage_vs_assumed` per cost-tier, EXCLUDING `low_confidence` fills, reporting coverage (n real-liquidity fills / n).
|
||||
- [ ] **Step 2:** Run → FAIL. **Step 3:** Implement pure functions over the recorded-row DTOs. **Step 4:** Run → PASS. **Step 5:** Commit `feat(exec): execution-gap mechanics verdict + indicative slippage`.
|
||||
|
||||
### Task 4: `bybit_testnet_execution.py` — overlay-reapply + order planning (TDD)
|
||||
|
||||
**Files:** Create `src/fxhnt/application/bybit_testnet_execution.py`; Test `tests/unit/test_bybit_testnet_execution.py`. Reuse the EXISTING overlay (import it; do not reimplement).
|
||||
|
||||
- [ ] **Step 1: Failing tests for `effective_weights`** — given raw sleeve weights + a testnet nav history that is in deep drawdown, the re-applied overlay killswitch de-risks (weights → 0/reduced); given a calm testnet history, weights pass through vol-targeted. Assert it calls the SAME overlay used by the book (parametrize/inject it).
|
||||
- [ ] **Step 2: Failing tests for `plan_orders`**:
|
||||
```python
|
||||
def test_buy_when_target_above_current():
|
||||
intents = plan_orders({"BTCUSDT": 0.5}, {"BTCUSDT": 0.0}, equity=10_000,
|
||||
marks={"BTCUSDT": 100.0}, limits=_lim())
|
||||
assert intents == [OrderIntent("BTCUSDT", qty=50.0, reduce_only=False)] # 0.5*10000/100
|
||||
def test_reduce_only_on_close():
|
||||
intents = plan_orders({"BTCUSDT": 0.0}, {"BTCUSDT": 50.0}, 10_000, {"BTCUSDT":100.0}, _lim())
|
||||
assert intents[0].reduce_only is True and intents[0].qty == -50.0
|
||||
def test_min_notional_filters_dust():
|
||||
intents = plan_orders({"X": 0.0001}, {"X": 0.0}, 10_000, {"X": 100.0}, _lim(min_notional=50))
|
||||
assert intents == [] # 0.0001*10000=1 USD < 50 minNotional
|
||||
def test_qty_step_snap_and_gross_cap():
|
||||
# Σ|weight·equity| > equity*max_gross(1.0) → clipped
|
||||
...
|
||||
```
|
||||
- [ ] **Step 3:** Implement `effective_weights` (overlay re-apply on testnet nav) + `plan_orders` (qtyStep snap, minNotional/minOrderQty filter, reduce_only, gross-cap assert+clip). **Step 4:** PASS. **Step 5:** Commit `feat(exec): bybit testnet overlay-reapply + diff order planning`.
|
||||
|
||||
### Task 5: `BybitExecution` ccxt adapter (TDD, mocked ccxt)
|
||||
|
||||
**Files:** Create `src/fxhnt/adapters/broker/bybit.py`; Test `tests/unit/test_bybit_execution_adapter.py`.
|
||||
|
||||
- [ ] **Step 1: Failing tests** with a fake ccxt exchange: `set_sandbox_mode(True)` called when testnet; `place_market` issues `create_order(symbol,'market',side,abs(qty),params={'reduceOnly':...})`; `mid` returns bid/ask midpoint and `None` on an empty/one-sided book; `instrument_limits` parses minOrderQty/qtyStep/minNotional from `load_markets`; api_secret never appears in `repr`/logs.
|
||||
- [ ] **Step 2:** FAIL. **Step 3:** Implement (ccxt.bybit; inject the exchange factory for tests). **Step 4:** PASS. **Step 5:** Commit `feat(broker): ccxt Bybit testnet execution adapter`.
|
||||
|
||||
### Task 6: Persistence — tables + repo (TDD)
|
||||
|
||||
**Files:** Modify `src/fxhnt/adapters/persistence/cockpit_models.py` (or the relevant models) + the repo; migration in the repo `migrate()`; Test `tests/integration/test_bybit_testnet_repo.py`.
|
||||
|
||||
- [ ] **Step 1: Failing tests** — `upsert_testnet_nav`, `read_testnet_nav`, `insert_testnet_fills`, `read_testnet_fills`, and a per-day `executed_marker` check (idempotency).
|
||||
- [ ] **Step 2:** FAIL. **Step 3:** Add tables `bybit_testnet_nav` + `bybit_testnet_fills` (schema per spec §4.4) `CREATE TABLE IF NOT EXISTS` in migrate(); repo methods. **Step 4:** PASS. **Step 5:** Commit `feat(persist): bybit testnet nav + fills tables`.
|
||||
|
||||
### Task 7: Config — `BybitExecutionConfig` (TDD)
|
||||
|
||||
**Files:** Modify `src/fxhnt/config.py`; Test `tests/unit/test_config.py`.
|
||||
|
||||
- [ ] **Step 1: Failing test** — env `FXHNT_BYBIT_EXEC_*` populates the config; defaults `testnet=True, allow_live=False, kill_switch=False, max_gross=1.0`; a `can_trade_live` property is True ONLY when `testnet=False AND allow_live=True AND kill_switch=False`.
|
||||
- [ ] **Step 2:** FAIL. **Step 3:** Implement. **Step 4:** PASS. **Step 5:** Commit `feat(config): BybitExecutionConfig with triple-gate`.
|
||||
|
||||
### Task 8: Dagster job — reconcile + record assets + wiring (TDD)
|
||||
|
||||
**Files:** Modify `src/fxhnt/adapters/orchestration/assets.py` + `definitions.py`; Test `tests/integration/test_bybit_testnet_job_wiring.py`.
|
||||
|
||||
- [ ] **Step 1: Failing tests** — `bybit_testnet_execution_job` exists; it contains `bybit_testnet_reconcile` + `bybit_testnet_record`; it is NOT in `combined_book_forward_job`; schedule `bybit_testnet_daily` registered; `kill_switch=True` → reconcile places zero orders (asset-level test with a fake adapter); a per-day marker → a second run is a no-op.
|
||||
- [ ] **Step 2:** FAIL. **Step 3:** Implement the two assets (reconcile: build adapter from settings, raw weights → effective_weights → plan_orders → per-order mid+place+fill, best-effort BrokerError → structured result, idempotency guard; record: write fills+nav). Register the job + schedule (default_status per ops; start STOPPED until creds present). **Step 4:** PASS. **Step 5:** Commit `feat(orch): bybit testnet execution Dagster job + schedule`.
|
||||
|
||||
### Task 9: Infra manifests (no CronJob)
|
||||
|
||||
**Files:** Create `infra/k8s/secrets/bybit-testnet-credentials.yaml` (template, NO real keys committed), `infra/k8s/network-policies/bybit-testnet-egress.yaml` (dagster pod → api-testnet.bybit.com:443).
|
||||
|
||||
- [ ] **Step 1:** Add the secret template (documented: populate via `kubectl create secret`, never commit keys) + the NetworkPolicy. **Step 2:** Commit `chore(infra): bybit testnet secret template + egress netpol`.
|
||||
|
||||
### Phase-1 acceptance
|
||||
A testnet run (creds present, schedule enabled) reconciles the book, places real testnet orders, records fills with decomposed slippage + a NAV row, and `mechanics_verdict` computes — all without touching `combined_book_forward_job` or real money. Full suite green.
|
||||
|
||||
---
|
||||
|
||||
## PHASE 2 — surfacing (outline, separate plan when P1 lands)
|
||||
- Cockpit "real-paper" card reading `bybit_testnet_*`: equity curve, mechanics verdict (PASS/FAIL + offending symbols), indicative slippage per tier vs assumed (with coverage + the "magnitude pending mainnet" label), side-by-side vs the sim `paper_nav`.
|
||||
- Wire into the dashboard service + a template; verify locally per the cockpit dev-loop (real browser, read the numbers).
|
||||
|
||||
---
|
||||
|
||||
## Self-review notes
|
||||
- Spec coverage: every spec §4 component → a task; slippage (§3) → Task 2; verdict (§5) → Task 3; load-bearing dep (§6) → Task 1 gate.
|
||||
- No placeholders in pure-logic tasks (code given). Integration tasks specify files + the exact assertions.
|
||||
- Type consistency: `OrderIntent(symbol, qty, reduce_only)`, `Fill(symbol, side, qty, avg_price, fee, ts, order_id)`, slippage fns return `float|None` — used consistently across Tasks 2/4/5/8.
|
||||
- Deferred honestly: slippage MAGNITUDE validation (mainnet micro-live), cockpit (Phase 2), real money (separate spec).
|
||||
@@ -0,0 +1,480 @@
|
||||
# Bybit 4-edge Levered Shadow Track (Phase 1) Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Record an observe-only levered shadow of the `bybit_4edge` book (static differentiated leverage, book-gross ≤ 1.5x) alongside the honest naive track, so we learn live whether the levered book reconciles OOS — no capital, no adaptive re-tune, no killswitch.
|
||||
|
||||
**Architecture:** Reuse every existing part. The levered book is the SAME per-sleeve funding-net returns combined with a static per-sleeve leverage shape and a hard gross cap, instead of equal weight. One new pure combine function, a leverage-shape parameter threaded through the existing `BybitFourEdgeStrategy` / backtest-summary / forward-nav asset, a registry row, and one persist call. The nightly `ForwardTracker` + `cockpit_forward` + reconciliation gate all work unchanged.
|
||||
|
||||
**Tech Stack:** Python 3.13, SQLAlchemy (cockpit repos), Dagster assets, pytest, in-memory sqlite `TimescaleFeatureStore` for tests.
|
||||
|
||||
## Global Constraints
|
||||
|
||||
- Observe-only: NO capital deployment, NO change to the naive `bybit_4edge` track.
|
||||
- Book-gross cap: `MAX_BOOK_GROSS = 1.5` (hard). Leverage shape (relative): `crypto_tstrend 1.0, unlock 1.5, xsfunding 5.0, positioning 2.0` — carry-weighted, scaled down to hit the 1.5x gross cap.
|
||||
- The levered gate MUST get its own `backtest_summary` row (else the recon gate silently degrades — see `reference_fxhnt_forward_gate_two_store_sync`).
|
||||
- The heavy full-history compute lives ONLY in the `bybit-persist-sleeve-ret` own-memory Job, never the OOM-prone dagster daemon.
|
||||
- Never commit secrets; commit messages end `Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>`.
|
||||
|
||||
---
|
||||
|
||||
### Task 1: `levered_eqwt_daily_returns` — the gross-capped levered combine
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/application/bybit_forward_track.py` (add function + two module constants near `naive_eqwt_daily_returns`)
|
||||
- Test: `tests/integration/test_bybit_forward_track.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: `levered_eqwt_daily_returns(returns_by_sleeve: dict[str, dict[int, float]], sleeves: list[str], shape: dict[str, float], *, max_gross: float = 1.5, cost_bps: float = 0.0) -> list[tuple[int, float]]`; constants `_LEVERED_SLEEVE_SHAPE: dict[str,float]`, `MAX_BOOK_GROSS: float`.
|
||||
|
||||
- [ ] **Step 1: Write the failing tests**
|
||||
|
||||
```python
|
||||
# in tests/integration/test_bybit_forward_track.py
|
||||
def test_levered_eqwt_uniform_shape_equals_naive() -> None:
|
||||
"""With a uniform shape (all 1.0) and max_gross >= 1, the levered combine reduces EXACTLY to the naive
|
||||
equal-weight book — same weights, same gross 1.0."""
|
||||
from fxhnt.application.bybit_forward_track import (
|
||||
levered_eqwt_daily_returns, naive_eqwt_daily_returns)
|
||||
rbs = {"a": {1: 0.01, 2: -0.02, 3: 0.03}, "b": {1: 0.00, 2: 0.04, 3: -0.01}}
|
||||
naive = naive_eqwt_daily_returns(rbs, ["a", "b"], cost_bps=0.0)
|
||||
lev = levered_eqwt_daily_returns(rbs, ["a", "b"], {"a": 1.0, "b": 1.0}, max_gross=1.5, cost_bps=0.0)
|
||||
assert lev == naive
|
||||
|
||||
def test_levered_eqwt_carry_weighted_and_gross_capped() -> None:
|
||||
"""A carry-heavy shape puts more weight on the high-shape sleeve, and the book gross is capped at
|
||||
max_gross (the summed absolute weight never exceeds it)."""
|
||||
from fxhnt.application.bybit_forward_track import levered_eqwt_daily_returns
|
||||
rbs = {"trend": {1: 0.02, 2: 0.02}, "carry": {1: 0.02, 2: 0.02}}
|
||||
lev = levered_eqwt_daily_returns(rbs, ["trend", "carry"], {"trend": 1.0, "carry": 5.0},
|
||||
max_gross=1.5, cost_bps=0.0)
|
||||
# both sleeves move +2%; shape 1:5 -> carry weight 5x trend; scaled so gross = 1.5.
|
||||
# weights: trend=1/2, carry=5/2 -> gross=3.0 -> scale 0.5 -> trend=0.25, carry=1.25 (gross 1.5)
|
||||
# day return = 0.25*0.02 + 1.25*0.02 = 0.03
|
||||
assert abs(lev[0][1] - 0.03) < 1e-12
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run to verify they fail**
|
||||
|
||||
Run: `uv run pytest tests/integration/test_bybit_forward_track.py -k "levered_eqwt" -q`
|
||||
Expected: FAIL with `ImportError: cannot import name 'levered_eqwt_daily_returns'`.
|
||||
|
||||
- [ ] **Step 3: Implement the function + constants**
|
||||
|
||||
Add near the top of `bybit_forward_track.py` (after `_EPOCH`) the constants, and after `naive_eqwt_daily_returns` the function:
|
||||
|
||||
```python
|
||||
# Phase-1 static leverage shape (relative per-sleeve leverage), scaled down to the book-gross cap. Carry is
|
||||
# the low-vol workhorse; trend stays ~1x (its liquidation vector). See the 2026-07-03 leverage spec.
|
||||
_LEVERED_SLEEVE_SHAPE: dict[str, float] = {
|
||||
"crypto_tstrend": 1.0, "unlock": 1.5, "xsfunding": 5.0, "positioning": 2.0}
|
||||
MAX_BOOK_GROSS = 1.5
|
||||
|
||||
|
||||
def levered_eqwt_daily_returns(returns_by_sleeve: dict[str, dict[int, float]], sleeves: list[str],
|
||||
shape: dict[str, float], *, max_gross: float = 1.5,
|
||||
cost_bps: float = 0.0) -> list[tuple[int, float]]:
|
||||
"""The book's per-day return series `[(epoch_day, ret), ...]` under a STATIC per-sleeve leverage `shape`
|
||||
(relative leverage), scaled so the summed absolute weight (book gross) never exceeds `max_gross`. Mirrors
|
||||
`naive_eqwt_daily_returns` exactly (static weight over the present set, absent-today sleeve idles at 0,
|
||||
same turnover charge) — it IS the naive book when `shape` is uniform and `max_gross >= 1`."""
|
||||
present = [s for s in sleeves if returns_by_sleeve.get(s)]
|
||||
if not present:
|
||||
return []
|
||||
n = len(present)
|
||||
w = {s: shape.get(s, 1.0) / n for s in present}
|
||||
gross = sum(abs(v) for v in w.values())
|
||||
if gross > max_gross and gross > 0:
|
||||
scale = max_gross / gross
|
||||
w = {s: v * scale for s, v in w.items()}
|
||||
all_days = sorted({d for s in present for d in returns_by_sleeve[s]})
|
||||
daily: list[tuple[int, float]] = []
|
||||
prev_w = {s: 0.0 for s in present}
|
||||
for d in all_days:
|
||||
today = [s for s in present if d in returns_by_sleeve[s]]
|
||||
if not today:
|
||||
continue
|
||||
g = sum(w[s] * returns_by_sleeve[s][d] for s in today)
|
||||
if cost_bps:
|
||||
cur_w = {s: (w[s] if d in returns_by_sleeve[s] else 0.0) for s in present}
|
||||
turnover = sum(abs(cur_w[s] - prev_w[s]) for s in present)
|
||||
g -= turnover * cost_bps / 1e4
|
||||
prev_w = cur_w
|
||||
daily.append((d, g))
|
||||
return daily
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run to verify they pass**
|
||||
|
||||
Run: `uv run pytest tests/integration/test_bybit_forward_track.py -k "levered_eqwt" -q`
|
||||
Expected: PASS (2 passed).
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/application/bybit_forward_track.py tests/integration/test_bybit_forward_track.py
|
||||
git commit -m "feat(bybit-levered): gross-capped levered eq-wt combine (Phase 1)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 2: Thread `leverage_shape` through `BybitFourEdgeStrategy`
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/application/bybit_forward_track.py` (`BybitFourEdgeStrategy.__init__` + `.advance`)
|
||||
- Test: `tests/integration/test_bybit_forward_track.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `levered_eqwt_daily_returns`, `_LEVERED_SLEEVE_SHAPE`, `MAX_BOOK_GROSS` (Task 1).
|
||||
- Produces: `BybitFourEdgeStrategy(store, *, universe=None, sleeves=None, cost_bps=5.5, unlock_events=None, stablecoin_spot_panel=None, leverage_shape: dict[str,float] | None = None, max_gross: float = MAX_BOOK_GROSS)`. `leverage_shape=None` → naive (unchanged); set → levered.
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
```python
|
||||
def test_bybit_four_edge_strategy_levered_differs_from_naive() -> None:
|
||||
"""With a leverage_shape the strategy books the levered book (carry-weighted, gross-capped); without it,
|
||||
the naive book — same days, different returns."""
|
||||
from fxhnt.application.bybit_forward_track import BybitFourEdgeStrategy, _LEVERED_SLEEVE_SHAPE
|
||||
store = TimescaleFeatureStore("sqlite://", table="bybit_features")
|
||||
_seed_momentum(store, n_symbols=8, days=260)
|
||||
_seed_carry(store, days=260)
|
||||
naive, _ = BybitFourEdgeStrategy(store, cost_bps=0.0).advance(None, {})
|
||||
lev, _ = BybitFourEdgeStrategy(store, cost_bps=0.0, leverage_shape=_LEVERED_SLEEVE_SHAPE).advance(None, {})
|
||||
store.close()
|
||||
assert [d for d, _ in lev] == [d for d, _ in naive] # same day spine
|
||||
assert [r for _, r in lev] != [r for _, r in naive] # different (levered) returns
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run to verify it fails**
|
||||
|
||||
Run: `uv run pytest tests/integration/test_bybit_forward_track.py -k "strategy_levered_differs" -q`
|
||||
Expected: FAIL with `TypeError: __init__() got an unexpected keyword argument 'leverage_shape'`.
|
||||
|
||||
- [ ] **Step 3: Implement — add the params + branch**
|
||||
|
||||
In `BybitFourEdgeStrategy.__init__`, add after `self._stablecoin_spot_panel = ...` (keep existing lines):
|
||||
|
||||
```python
|
||||
self._leverage_shape = leverage_shape
|
||||
self._max_gross = max_gross
|
||||
```
|
||||
|
||||
and extend the signature with `leverage_shape: dict[str, float] | None = None, max_gross: float = MAX_BOOK_GROSS`.
|
||||
|
||||
In `.advance`, replace the `daily = naive_eqwt_daily_returns(...)` line with:
|
||||
|
||||
```python
|
||||
if self._leverage_shape is None:
|
||||
daily = naive_eqwt_daily_returns(returns_by_sleeve, active, cost_bps=self._cost_bps)
|
||||
else:
|
||||
daily = levered_eqwt_daily_returns(returns_by_sleeve, active, self._leverage_shape,
|
||||
max_gross=self._max_gross, cost_bps=self._cost_bps)
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run to verify it passes**
|
||||
|
||||
Run: `uv run pytest tests/integration/test_bybit_forward_track.py -k "strategy_levered_differs or backtest_summary or naive_eqwt" -q`
|
||||
Expected: PASS (naive path unchanged; levered path differs).
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/application/bybit_forward_track.py tests/integration/test_bybit_forward_track.py
|
||||
git commit -m "feat(bybit-levered): thread leverage_shape through BybitFourEdgeStrategy"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 3: Parametrize `bybit_4edge_backtest_summary` for the levered strategy_id
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/application/bybit_forward_track.py` (`bybit_4edge_backtest_summary`)
|
||||
- Test: `tests/integration/test_bybit_forward_track.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: Task 2's `leverage_shape`.
|
||||
- Produces: `bybit_4edge_backtest_summary(store, *, universe=None, sleeves=None, cost_bps=5.5, unlock_events=None, stablecoin_spot_panel=None, leverage_shape=None, strategy_id="bybit_4edge")` — same return type `BacktestSummary`, `strategy_id` overridable.
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
```python
|
||||
def test_levered_backtest_summary_uses_levered_strategy_id_and_differs() -> None:
|
||||
"""The levered backtest reference carries strategy_id 'bybit_4edge_levered' and a DIFFERENT cagr from the
|
||||
naive one (leverage scales the book), so the levered recon gate reconciles against the levered backtest."""
|
||||
from fxhnt.application.bybit_forward_track import bybit_4edge_backtest_summary, _LEVERED_SLEEVE_SHAPE
|
||||
store = TimescaleFeatureStore("sqlite://", table="bybit_features")
|
||||
_seed_momentum(store, n_symbols=8, days=260)
|
||||
_seed_carry(store, days=260)
|
||||
naive = bybit_4edge_backtest_summary(store, cost_bps=0.0)
|
||||
lev = bybit_4edge_backtest_summary(store, cost_bps=0.0, leverage_shape=_LEVERED_SLEEVE_SHAPE,
|
||||
strategy_id="bybit_4edge_levered")
|
||||
store.close()
|
||||
assert lev.strategy_id == "bybit_4edge_levered"
|
||||
assert lev.cagr != naive.cagr
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run to verify it fails**
|
||||
|
||||
Run: `uv run pytest tests/integration/test_bybit_forward_track.py -k "levered_backtest_summary_uses" -q`
|
||||
Expected: FAIL with `TypeError: ... unexpected keyword argument 'leverage_shape'`.
|
||||
|
||||
- [ ] **Step 3: Implement — add params + thread them**
|
||||
|
||||
Extend `bybit_4edge_backtest_summary`'s signature with `leverage_shape: dict[str, float] | None = None, strategy_id: str = "bybit_4edge"`. Pass `leverage_shape=leverage_shape` into the `BybitFourEdgeStrategy(...)` construction inside it, and replace both literal `strategy_id="bybit_4edge"` occurrences (the empty-rows early return AND the final return) with `strategy_id=strategy_id`.
|
||||
|
||||
- [ ] **Step 4: Run to verify it passes**
|
||||
|
||||
Run: `uv run pytest tests/integration/test_bybit_forward_track.py -k "backtest_summary" -q`
|
||||
Expected: PASS (naive summary unchanged; levered summary distinct).
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/application/bybit_forward_track.py tests/integration/test_bybit_forward_track.py
|
||||
git commit -m "feat(bybit-levered): parametrize backtest_summary for the levered strategy_id"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 4: Registry row `bybit_4edge_levered`
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/registry.py` (add entry after `bybit_4edge`)
|
||||
- Test: `tests/integration/test_bybit_forward_track.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: `STRATEGY_REGISTRY["bybit_4edge_levered"]` with `state_file="bybit_4edge_levered_state"`, reconciliation gate `min_forward_days=21`.
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
```python
|
||||
def test_registry_has_bybit_4edge_levered_shadow() -> None:
|
||||
from fxhnt.registry import STRATEGY_REGISTRY
|
||||
e = STRATEGY_REGISTRY["bybit_4edge_levered"]
|
||||
assert e["state_file"] == "bybit_4edge_levered_state"
|
||||
assert e["venue"] == "bybit-perp"
|
||||
assert e["gate_spec"] == {"gate_type": "reconciliation", "min_forward_days": 21}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run to verify it fails**
|
||||
|
||||
Run: `uv run pytest tests/integration/test_bybit_forward_track.py -k "registry_has_bybit_4edge_levered" -q`
|
||||
Expected: FAIL with `KeyError: 'bybit_4edge_levered'`.
|
||||
|
||||
- [ ] **Step 3: Implement — add the registry entry**
|
||||
|
||||
Immediately after the `"bybit_4edge": { ... },` block in `registry.py`, add:
|
||||
|
||||
```python
|
||||
# OBSERVE-ONLY shadow of bybit_4edge under STATIC differentiated leverage (carry-weighted, book-gross
|
||||
# <= 1.5x). Records what the levered book WOULD do so we can validate it OOS before any capital rides it
|
||||
# (Phase 1 of the 2026-07-03 leverage spec). Same reconciliation gate as the naive book.
|
||||
"bybit_4edge_levered": {
|
||||
"display_name": "Bybit 4-edge book (levered shadow, <=1.5x)", "sleeve": "crypto-bybit",
|
||||
"venue": "bybit-perp", "state_file": "bybit_4edge_levered_state",
|
||||
"tier": "research", "execution": "paper",
|
||||
"gate_spec": {"gate_type": "reconciliation", "min_forward_days": 21},
|
||||
},
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run to verify it passes**
|
||||
|
||||
Run: `uv run pytest tests/integration/test_bybit_forward_track.py -k "registry_has_bybit_4edge_levered" -q`
|
||||
Expected: PASS.
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/registry.py tests/integration/test_bybit_forward_track.py
|
||||
git commit -m "feat(bybit-levered): register bybit_4edge_levered shadow track"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 5: Nightly asset `bybit_4edge_levered_nav` + wire into the job
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/adapters/orchestration/assets.py` (add `build_bybit_4edge_levered_forward_nav` next to `build_bybit_4edge_forward_nav`; add `bybit_4edge_levered_nav` asset next to `bybit_4edge_nav`; add it to `combined_book_forward_job` and `cockpit_forward`'s `deps`)
|
||||
- Test: `tests/integration/test_bybit_forward_track.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: Task 2's `leverage_shape` param, `_LEVERED_SLEEVE_SHAPE`.
|
||||
- Produces: `build_bybit_4edge_levered_forward_nav(store, state_path, *, universe=None, cost_bps=5.5, unlock_events=None, stablecoin_spot_panel=None) -> ForwardStatus`; Dagster asset `bybit_4edge_levered_nav`.
|
||||
|
||||
- [ ] **Step 1: Write the failing test (the plain builder, no Dagster)**
|
||||
|
||||
```python
|
||||
def test_build_bybit_4edge_levered_forward_nav_books_the_levered_book(tmp_path) -> None:
|
||||
"""The levered forward builder freezes T0 and books the levered (not naive) book to its OWN state file."""
|
||||
from fxhnt.adapters.orchestration.assets import build_bybit_4edge_levered_forward_nav
|
||||
store = TimescaleFeatureStore("sqlite://", table="bybit_features")
|
||||
_seed_momentum(store, n_symbols=8, days=130)
|
||||
_seed_carry(store, days=130)
|
||||
state = str(tmp_path / "bybit_4edge_levered_state.json")
|
||||
st = build_bybit_4edge_levered_forward_nav(store, state, cost_bps=0.0)
|
||||
store.close()
|
||||
assert st.inception is not None and (tmp_path / "bybit_4edge_levered_state.json").exists()
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run to verify it fails**
|
||||
|
||||
Run: `uv run pytest tests/integration/test_bybit_forward_track.py -k "build_bybit_4edge_levered" -q`
|
||||
Expected: FAIL with `ImportError: cannot import name 'build_bybit_4edge_levered_forward_nav'`.
|
||||
|
||||
- [ ] **Step 3: Implement the plain builder**
|
||||
|
||||
Directly after `build_bybit_4edge_forward_nav` in `assets.py`, add:
|
||||
|
||||
```python
|
||||
def build_bybit_4edge_levered_forward_nav(store, state_path, *, universe=None, cost_bps=5.5,
|
||||
unlock_events=None, stablecoin_spot_panel=None): # type: ignore[no-untyped-def]
|
||||
"""Observe-only LEVERED shadow of the 4-edge book: the SAME sleeves under the static `_LEVERED_SLEEVE_SHAPE`
|
||||
(carry-weighted, book-gross <= MAX_BOOK_GROSS), stepped into its OWN state file. No capital; records what
|
||||
the levered book would do so it can be validated OOS before deployment."""
|
||||
from fxhnt.application.bybit_forward_track import (
|
||||
MAX_BOOK_GROSS, _LEVERED_SLEEVE_SHAPE, BybitFourEdgeStrategy)
|
||||
from fxhnt.application.forward_tracker import ForwardTracker
|
||||
|
||||
strategy = BybitFourEdgeStrategy(
|
||||
store, universe=universe, cost_bps=cost_bps, unlock_events=unlock_events,
|
||||
stablecoin_spot_panel=stablecoin_spot_panel,
|
||||
leverage_shape=_LEVERED_SLEEVE_SHAPE, max_gross=MAX_BOOK_GROSS)
|
||||
return ForwardTracker(strategy, state_path).step()
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run to verify the builder test passes**
|
||||
|
||||
Run: `uv run pytest tests/integration/test_bybit_forward_track.py -k "build_bybit_4edge_levered" -q`
|
||||
Expected: PASS.
|
||||
|
||||
- [ ] **Step 5: Add the Dagster asset + wire it in**
|
||||
|
||||
After the `bybit_4edge_nav` asset, add an asset that mirrors it exactly but calls the levered builder and writes `bybit_4edge_levered_state.json`:
|
||||
|
||||
```python
|
||||
@asset(deps=[refresh_bybit_warehouse])
|
||||
def bybit_4edge_levered_nav(context: AssetExecutionContext) -> dict: # type: ignore[type-arg]
|
||||
"""OBSERVE-ONLY levered shadow of bybit_4edge (static leverage, book-gross <= 1.5x). Steps its own
|
||||
ForwardTracker; forward-nav-ONLY (light, OOM-proof), same as bybit_4edge_nav."""
|
||||
import urllib.error
|
||||
|
||||
from fxhnt.adapters.warehouse.timescale_feature_store import TimescaleFeatureStore
|
||||
from fxhnt.application.bybit_liquidity import liquid_universe
|
||||
from fxhnt.application.unlock_calendar_loader import load_unlock_events, operational_unlock_repo
|
||||
from fxhnt.config import get_settings
|
||||
|
||||
s = get_settings()
|
||||
store = TimescaleFeatureStore(s.operational_dsn, table="bybit_features")
|
||||
try:
|
||||
universe = liquid_universe(store) or None
|
||||
unlock_events = load_unlock_events(operational_unlock_repo(s), f"{_data_dir()}/unlock_calendar.json")
|
||||
st = build_bybit_4edge_levered_forward_nav(
|
||||
store, f"{_data_dir()}/bybit_4edge_levered_state.json",
|
||||
universe=universe, cost_bps=s.cost_bps_per_turnover, unlock_events=unlock_events)
|
||||
context.log.info(
|
||||
f"bybit_4edge_levered_nav: T0={st.inception}, {st.forward_days}d through {st.last_date} "
|
||||
f"(+{st.forward_return_pct:.2f}%, Sharpe {st.forward_sharpe:.2f})")
|
||||
return {"inception": st.inception, "last_date": st.last_date, "forward_days": st.forward_days,
|
||||
"forward_return_pct": st.forward_return_pct, "forward_sharpe": st.forward_sharpe}
|
||||
except (urllib.error.URLError, TimeoutError, ConnectionError, OSError) as e:
|
||||
context.log.warning(f"bybit_4edge_levered_nav step failed (state unchanged): {e}")
|
||||
return {"inception": None, "last_date": None, "forward_days": 0}
|
||||
finally:
|
||||
store.close()
|
||||
```
|
||||
|
||||
Then add `bybit_4edge_levered_nav` to BOTH the `combined_book_forward_job` asset selection and the `cockpit_forward` `@asset(deps=[...])` list (the line beginning `@asset(deps=[combined_forward_nav, ...`), alongside the existing `bybit_4edge_nav`.
|
||||
|
||||
- [ ] **Step 6: Run the whole forward-track + orchestration test surface**
|
||||
|
||||
Run: `uv run pytest tests/integration/test_bybit_forward_track.py tests/integration/test_bybit_book_eval.py -q`
|
||||
Expected: PASS (all).
|
||||
|
||||
- [ ] **Step 7: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/adapters/orchestration/assets.py tests/integration/test_bybit_forward_track.py
|
||||
git commit -m "feat(bybit-levered): nightly bybit_4edge_levered_nav shadow asset + job wiring"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 6: Persist the levered backtest reference in the persist Job
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/cli.py` (`bybit_persist_sleeve_ret`, right after the existing naive `bybit_4edge_backtest_summary` upsert)
|
||||
- Test: `tests/integration/test_bybit_forward_track.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: Task 3's `bybit_4edge_backtest_summary(..., leverage_shape=, strategy_id=)`.
|
||||
- Produces: a `backtest_summary` row `strategy_id='bybit_4edge_levered'` so the levered recon gate reconciles (does not silently degrade).
|
||||
|
||||
- [ ] **Step 1: Write the failing test (the provider gets a levered reference)**
|
||||
|
||||
```python
|
||||
def test_levered_backtest_ref_activates_the_levered_gate() -> None:
|
||||
"""Persisting the levered summary gives the cockpit ref provider a real reference for
|
||||
'bybit_4edge_levered' — the levered gate's band is active, not the degraded no-reference timer."""
|
||||
from fxhnt.application.bybit_forward_track import bybit_4edge_backtest_summary, _LEVERED_SLEEVE_SHAPE
|
||||
from fxhnt.application.forward_ingest import CockpitBacktestRefProvider
|
||||
store = TimescaleFeatureStore("sqlite://", table="bybit_features")
|
||||
_seed_momentum(store, n_symbols=8, days=260)
|
||||
_seed_carry(store, days=260)
|
||||
sm = bybit_4edge_backtest_summary(store, cost_bps=0.0, leverage_shape=_LEVERED_SLEEVE_SHAPE,
|
||||
strategy_id="bybit_4edge_levered")
|
||||
store.close()
|
||||
repo = ForwardNavRepo("sqlite://")
|
||||
repo.migrate()
|
||||
repo.upsert_backtest_summary(sm, at=dt.datetime(2026, 7, 3))
|
||||
ref = CockpitBacktestRefProvider(repo).reference("bybit_4edge_levered", 21)
|
||||
assert ref is not None and ref.expected_window_return != 0.0
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run to verify it fails**
|
||||
|
||||
Run: `uv run pytest tests/integration/test_bybit_forward_track.py -k "levered_backtest_ref_activates" -q`
|
||||
Expected: PASS already at the unit level (it exercises the pure function + repo directly). This test guards the MECHANISM; the CLI wiring in Step 3 is what makes it happen in prod. Run it now to confirm green, then add the CLI wiring so the prod Job actually writes the row.
|
||||
|
||||
- [ ] **Step 3: Implement — persist the levered summary in the CLI**
|
||||
|
||||
In `bybit_persist_sleeve_ret`, immediately after the existing `fwd_repo.upsert_backtest_summary(bt, ...)` line, add:
|
||||
|
||||
```python
|
||||
from fxhnt.application.bybit_forward_track import _LEVERED_SLEEVE_SHAPE
|
||||
bt_lev = bybit_4edge_backtest_summary(
|
||||
store, universe=universe, cost_bps=cost_bps, unlock_events=unlock_events,
|
||||
leverage_shape=_LEVERED_SLEEVE_SHAPE, strategy_id="bybit_4edge_levered")
|
||||
fwd_repo.upsert_backtest_summary(bt_lev, at=dt.datetime.now(dt.UTC).replace(tzinfo=None))
|
||||
log.info("bybit-persist-sleeve-ret: bybit_4edge_levered backtest ref cagr=%.1f%% -> backtest_summary",
|
||||
bt_lev.cagr * 100)
|
||||
```
|
||||
|
||||
(The `bybit_4edge_backtest_summary` import already exists a few lines above from the naive persist.)
|
||||
|
||||
- [ ] **Step 4: Run the full affected surface**
|
||||
|
||||
Run: `uv run pytest tests/integration/test_bybit_forward_track.py -q && uv run pytest tests/ -k "forward_ingest or reconciliation_gate" -q`
|
||||
Expected: PASS (all).
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/cli.py tests/integration/test_bybit_forward_track.py
|
||||
git commit -m "feat(bybit-levered): persist bybit_4edge_levered backtest reference in the persist Job"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Post-implementation (not tasks — deploy/verify, done together after review)
|
||||
|
||||
- Full suite: `uv run pytest tests/ -q` (expect all green).
|
||||
- Deploy: commit → push master → build → dagster rollout (nightly runs `bybit_4edge_levered_nav`).
|
||||
- Activate the reference now: run `bybit-persist-sleeve-ret` locally against prod (writes both naive + levered `backtest_summary` rows).
|
||||
- Trigger `bybit_4edge_levered_nav` + `cockpit_forward` once to freeze T0 and populate the shadow row; verify `forward_summary` shows `bybit_4edge_levered` building, and `CockpitBacktestRefProvider(...).reference("bybit_4edge_levered", 21)` is non-None.
|
||||
- Then: observe. The learning phase — compare the levered shadow vs the naive track live before any Phase 2 work.
|
||||
|
||||
## Self-Review
|
||||
|
||||
- **Spec coverage:** Phase 1 = Component 3 with a static leverage vector (spec's phasing) — Tasks 1–2 (static levered combine + strategy), Task 3 (levered backtest ref), Task 4 (registry/gate), Task 5 (nightly shadow track), Task 6 (reference persisted so the gate doesn't silently degrade). Components 1–2 + killswitch are explicitly Phase 2, not here. Observe-only / no-capital honored (no execution code). ✓
|
||||
- **Placeholder scan:** none — every step has real code/commands.
|
||||
- **Type consistency:** `leverage_shape: dict[str,float] | None` and `strategy_id: str` are used identically across Tasks 2, 3, 5, 6; `_LEVERED_SLEEVE_SHAPE` / `MAX_BOOK_GROSS` defined in Task 1 and imported everywhere after; `build_bybit_4edge_levered_forward_nav` signature matches the naive builder. ✓
|
||||
116
docs/superpowers/plans/2026-07-05-cockpit-levered-track-views.md
Normal file
116
docs/superpowers/plans/2026-07-05-cockpit-levered-track-views.md
Normal file
@@ -0,0 +1,116 @@
|
||||
# Cockpit Levered Track Views Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: superpowers:subagent-driven-development or executing-plans. Steps use `- [ ]`.
|
||||
|
||||
**Goal:** Surface the levered shadow book in the cockpit Backtest (a measured levered curve, 4th book pill) and Replay (naive↔levered track toggle), so the two tracks can be compared visually. Observe-only.
|
||||
|
||||
**Architecture:** Thread a `leverage_shape` through the per-coin measured-cost combine (`per_coin_book_returns` → `_bybit_measured_net` → `precompute_single_book_measured`) so the levered book gets a MEASURED curve (never flat); add it to the sim book switch. Add a `track` param to the Replay reading the levered `forward_nav` position-less. Reuse `_LEVERED_SLEEVE_SHAPE`/`MAX_BOOK_GROSS`.
|
||||
|
||||
**Tech Stack:** Python 3.13, FastAPI + Jinja2 + htmx, pytest, playwright (local browser verify).
|
||||
|
||||
## Global Constraints
|
||||
- Naive paths BYTE-IDENTICAL: `leverage_shape=None` and `track=naive` produce exactly the prior output.
|
||||
- Backtest is MEASURED cost only — never a flat/cost-blind levered curve (mirage foot-gun).
|
||||
- Shape `{crypto_tstrend 1.0, unlock 1.5, xsfunding 5.0, positioning 2.0}`, book-gross ≤ 1.5 (`_LEVERED_SLEEVE_SHAPE`/`MAX_BOOK_GROSS`).
|
||||
- MANDATORY local browser verification before "done" (feedback_verify_cockpit_locally_not_prod); read the rendered numbers, screenshot naive vs levered.
|
||||
- Never `git add -A`; commit end `Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>`.
|
||||
|
||||
---
|
||||
|
||||
### Task 1: `per_coin_book_returns` accepts a gross-capped `leverage_shape`
|
||||
|
||||
**Files:** Modify `src/fxhnt/application/bybit_liquidity_sweep.py`; Test `tests/integration/test_bybit_liquidity_sweep.py` (or the file that tests `per_coin_book_returns`).
|
||||
|
||||
**Interfaces:** Produces `per_coin_book_returns(store, *, universe, sleeves=None, capital=100000.0, ..., leverage_shape: dict[str,float] | None = None, max_gross: float = 1.5)`; `leverage_shape=None` → naive (unchanged).
|
||||
|
||||
- [ ] **Step 1: failing test** — with a uniform shape the per-coin book equals naive; with a carry-weighted shape the summed sleeve factor is gross-capped ≤ max_gross and carry's contribution scales up.
|
||||
|
||||
```python
|
||||
def test_per_coin_book_returns_uniform_shape_equals_naive():
|
||||
# seed a store with >=2 sleeves; naive vs leverage_shape={all:1.0} must give identical net_series
|
||||
...
|
||||
naive = per_coin_book_returns(store, universe=None, capital=100000.0, taker_fee_bps=5.5)
|
||||
lev1 = per_coin_book_returns(store, universe=None, capital=100000.0, taker_fee_bps=5.5,
|
||||
leverage_shape={s: 1.0 for s in _DEFAULT_BYBIT_SLEEVES})
|
||||
assert lev1["net_series"] == naive["net_series"]
|
||||
```
|
||||
|
||||
- [ ] **Step 2: run → fail** (`unexpected keyword argument 'leverage_shape'`).
|
||||
|
||||
- [ ] **Step 3: implement.** Add the two params. Just before the "2) combine eq-wt" loop compute the per-sleeve factor (gross-capped), then use it:
|
||||
|
||||
```python
|
||||
if leverage_shape is None:
|
||||
factor = {s: 1.0 / n_sleeves for s in per_sleeve}
|
||||
else:
|
||||
factor = {s: leverage_shape.get(s, 1.0) / n_sleeves for s in per_sleeve}
|
||||
gross = sum(abs(v) for v in factor.values())
|
||||
if gross > max_gross and gross > 0:
|
||||
scale = max_gross / gross
|
||||
factor = {s: v * scale for s, v in factor.items()}
|
||||
```
|
||||
|
||||
and in the combine loop change `for by_day in per_sleeve.values():` to `for sleeve, by_day in per_sleeve.items():` and replace `w / n_sleeves` with `factor[sleeve] * w` in BOTH the `wrow[coin]` and `crow[coin]` updates (`crow[coin] += factor[sleeve] * w * r`).
|
||||
|
||||
- [ ] **Step 4: run → pass** (uniform-shape==naive; carry-weighted differs + gross-capped).
|
||||
- [ ] **Step 5: commit** `feat(measured): per_coin_book_returns gross-capped leverage_shape`.
|
||||
|
||||
---
|
||||
|
||||
### Task 2: `_bybit_measured_net` + `precompute_single_book_measured` compute the levered measured curve
|
||||
|
||||
**Files:** Modify `src/fxhnt/application/compare_measured_cost.py`; Test its test file.
|
||||
|
||||
**Interfaces:** `_bybit_measured_net(..., leverage_shape=None, max_gross=1.5)` passes through to `per_coin_book_returns`. `precompute_single_book_measured` additionally caches `bybit_4edge_levered` using `_LEVERED_SLEEVE_SHAPE`.
|
||||
|
||||
- [ ] **Step 1: failing test** — `_bybit_measured_net` with `_LEVERED_SLEEVE_SHAPE` yields a curve whose metrics differ from naive (higher Sharpe, carry-weighted) at the SAME measured cost model; and `precompute_single_book_measured` writes a `bybit_4edge_levered` cache entry readable by `read_cached_single_measured`.
|
||||
|
||||
- [ ] **Step 2: run → fail.**
|
||||
- [ ] **Step 3: implement.** Thread `leverage_shape`/`max_gross` from `_bybit_measured_net` into `per_coin_book_returns`. In `precompute_single_book_measured`, after the naive `bybit_4edge` single-book precompute, compute the same for the levered book (`leverage_shape=_LEVERED_SLEEVE_SHAPE` imported from `bybit_forward_track`) and persist under cache key `bybit_4edge_levered`.
|
||||
- [ ] **Step 4: run → pass.**
|
||||
- [ ] **Step 5: commit** `feat(measured): precompute the bybit_4edge_levered measured curve`.
|
||||
|
||||
---
|
||||
|
||||
### Task 3: Backtest sim — add the levered book pill
|
||||
|
||||
**Files:** Modify `src/fxhnt/adapters/web/app.py` (`_SIM_BOOKS`, `_sim_returns`) + `templates/sim.html` (pill label); Test `tests/integration/test_bybit_sim_web.py` / `test_paper_sim_web.py`.
|
||||
|
||||
- [ ] **Step 1: failing test** — `_SIM_BOOKS` includes `bybit_4edge_levered`; `/paper/sim?book=bybit_4edge_levered` renders 200 with the levered pill active; the lazy `/paper/sim/run?book=bybit_4edge_levered` renders the measured curve (or the graceful pending note if the cache is absent) — never a 500, never a flat slider.
|
||||
|
||||
- [ ] **Step 2: run → fail.**
|
||||
- [ ] **Step 3: implement.** Add `"bybit_4edge_levered"` to `_SIM_BOOKS`; make `_sim_returns("bybit_4edge_levered")` read `bybit_sleeve_returns()` (same sleeves; the levered curve comes from the measured cache, not this raw table). Add the 4th pill to `sim.html`'s book switch with label "Bybit 4-edge levered · shadow". `read_cached_single_measured` already resolves any book key.
|
||||
- [ ] **Step 4: run → pass.**
|
||||
- [ ] **Step 5: commit** `feat(cockpit): levered book pill in the Backtest`.
|
||||
|
||||
---
|
||||
|
||||
### Task 4: Replay — naive↔levered track toggle
|
||||
|
||||
**Files:** Modify `src/fxhnt/adapters/web/app.py` (`paper_replay`, `paper_replay_at`) + `templates/replay.html`; Test `tests/integration/test_bybit_sim_web.py` or the replay test.
|
||||
|
||||
**Interfaces:** `paper_replay(request, venue="bybit", track="naive")`; `track="levered"` reads `bybit_4edge_levered` forward_nav position-less.
|
||||
|
||||
- [ ] **Step 1: failing test** — `/paper/replay?track=levered` renders 200 showing the levered forward_nav curve and the "return-defined — no scrubbable positions" note (no positions); `/paper/replay` (default) and `?track=naive` are byte-identical to before; the toggle appears in the HTML; leak-clean.
|
||||
|
||||
- [ ] **Step 2: run → fail.**
|
||||
- [ ] **Step 3: implement.** Add `track` param. `track=="levered"` → read the `bybit_4edge_levered` forward track via the DashboardService `detail("bybit_4edge_levered")` (its `history` NAV points) → build the sparkline + a position-less context (reuse the carry note). `track=="naive"` → the current venue nav_history path unchanged. Add a naive↔levered toggle to `replay.html` (relative-URL pills, like the venue toggle). `paper_replay_at` ignores positions when `track=levered`.
|
||||
- [ ] **Step 4: run → pass.**
|
||||
- [ ] **Step 5: commit** `feat(cockpit): naive<->levered track toggle in Replay`.
|
||||
|
||||
---
|
||||
|
||||
### Task 5: Local browser verification + deploy
|
||||
|
||||
**Files:** none (verification + deploy).
|
||||
|
||||
- [ ] **Step 1** Run the cockpit locally against a seeded/prod-mirrored DB per reference_fxhnt_local_cockpit_dev_loop.
|
||||
- [ ] **Step 2** Open `/paper/sim?book=bybit_4edge_levered` in playwright; confirm the levered pill is active and a curve (or the pending note) renders; screenshot naive vs levered; READ the rendered Sharpe/CAGR numbers and confirm they match the levered backtest_summary (Sh~3.25, cagr~46%) when the measured cache is present.
|
||||
- [ ] **Step 3** Open `/paper/replay?track=levered`; confirm the levered forward_nav curve renders position-less with the note; toggle back to naive; screenshot both.
|
||||
- [ ] **Step 4** Full suite green: `uv run pytest tests/ -q`.
|
||||
- [ ] **Step 5** Deploy: commit → push master → build → dagster rollout; re-run the `compare-measured-precompute` in-cluster Job so the levered measured cache populates. Verify on prod the levered pill renders a measured curve.
|
||||
|
||||
## Self-Review
|
||||
- Spec coverage: Component 1 (measured levered curve) = Tasks 1-3; Component 2 (replay toggle) = Task 4; browser guardrail + deploy = Task 5. Naive-byte-identical + measured-not-flat constraints are the Task 1/3 tests. ✓
|
||||
- Placeholders: none — combine-code change is spelled out; test intents concrete.
|
||||
- Type consistency: `leverage_shape: dict[str,float] | None` + `max_gross: float` threaded identically through per_coin_book_returns → _bybit_measured_net → precompute; `track: str` in the two replay routes. ✓
|
||||
1346
docs/superpowers/plans/2026-07-08-deterministic-forward-state.md
Normal file
1346
docs/superpowers/plans/2026-07-08-deterministic-forward-state.md
Normal file
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,670 @@
|
||||
# Declarative Strategy-Level Allocation Policy + Capital Ceiling (Sub-project B2a) Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** A versioned in-code `AllocationPolicy` + a pure `compute_allocation`/`target_capital` that turn the deploy-tier strategies into ex-ante-sized per-strategy target weights (equal-weight the marginal-Sharpe-gate survivors; ex-ante vol-target leverage from the FULL-history vol, never trailing; a rare latching aggregate killswitch) capped by a hard human `capital_ceiling`, recorded + policy-version-stamped.
|
||||
|
||||
**Architecture:** `allocation_policy.py` holds the versioned policy + content-hash (mirrors B1's gate_policy). `allocation_engine.py` composes pure helpers (full-history vol, equal-weight marginal-Sharpe prune, killswitch latch check, reusing B1's `promotion_corr.deploy_book_returns` for the equal-weight aggregate) into `compute_allocation` → per-strategy weights and `target_capital` → per-strategy dollars capped at the ceiling. A nightly step records the snapshot to `strategy_allocation`. No capital is moved (that is sub-project C).
|
||||
|
||||
**Tech Stack:** Python 3.12, numpy, dataclasses, SQLAlchemy 2.0 ORM (sqlite tests / Postgres prod), pytest (`uv run pytest`), ruff (E,F,I,UP,B,SIM; line-length 120).
|
||||
|
||||
## Global Constraints
|
||||
|
||||
- Spec: `docs/superpowers/specs/2026-07-09-declarative-allocation-policy-b2a-design.md` (authoritative).
|
||||
- Builds on A (merged: `forward_nav` records via `ForwardNavRepo.nav_history`) + B1 (merged: deploy-tier set = registry `tier=="deploy"` ∪ `promoted_strategy_ids()`; `promotion_corr.deploy_book_returns` for the equal-weight per-date aggregate).
|
||||
- **NO reactive de-sizing** (the load-bearing rule): the ex-ante leverage `K` derives from the FULL-history vol (stable), NEVER a trailing window. The only drawdown-reactive element is the rare, latching aggregate killswitch. Do NOT reuse `book_allocator.combine_book` (its `kelly · target_vol / trailing_port_vol` is the rejected reactive path).
|
||||
- REAL MONEY: `capital_ceiling` is a HARD cap — total deployed gross never exceeds it; err safe (exclude a short-history strategy, deploy nothing when no eligible strategies, flatten on killswitch). No capital is MOVED by B2a (targets only).
|
||||
- Policy lives in git-committed code, never runtime/DB. A change = commit + `ALLOCATION_POLICY_VERSION` bump; the content-derived `allocation_policy_hash` self-check forces the bump.
|
||||
- Determinism: `compute_allocation`/`target_capital` are pure — no wall-clock, no I/O; recomputable from (policy + records).
|
||||
- Idempotency: DB writes are `Session.merge` upserts. Never save tests to the repo root; tests under `tests/unit/` or `tests/integration/`. Commit after every task; messages end with the `Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>` trailer.
|
||||
|
||||
---
|
||||
|
||||
### Task 1: `allocation_policy.py` — the versioned allocation policy + hash
|
||||
|
||||
**Files:**
|
||||
- Create: `src/fxhnt/application/allocation_policy.py`
|
||||
- Test: `tests/unit/test_allocation_policy.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: `AllocationPolicy` (frozen dataclass, fields below), `ALLOCATION_POLICY`, `ALLOCATION_POLICY_VERSION: int = 1`, `allocation_policy_hash(policy, version) -> str`.
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
```python
|
||||
# tests/unit/test_allocation_policy.py
|
||||
from fxhnt.application.allocation_policy import (
|
||||
ALLOCATION_POLICY, ALLOCATION_POLICY_VERSION, AllocationPolicy, allocation_policy_hash)
|
||||
|
||||
|
||||
def test_policy_defaults():
|
||||
p = ALLOCATION_POLICY
|
||||
assert (p.target_vol, p.kelly_fraction, p.max_leverage, p.max_strategy_weight) == (0.12, 0.5, 1.5, 0.5)
|
||||
assert (p.marginal_gate, p.min_history_days) == (True, 60)
|
||||
assert (p.kill_dd, p.reenter_dd) == (0.25, 0.10)
|
||||
assert p.capital_ceiling == 35_000.0
|
||||
|
||||
|
||||
def test_hash_content_derived_and_pinned():
|
||||
assert allocation_policy_hash(ALLOCATION_POLICY, ALLOCATION_POLICY_VERSION) == \
|
||||
allocation_policy_hash(ALLOCATION_POLICY, ALLOCATION_POLICY_VERSION) # stable
|
||||
assert allocation_policy_hash(AllocationPolicy(target_vol=0.10), ALLOCATION_POLICY_VERSION) != \
|
||||
allocation_policy_hash(ALLOCATION_POLICY, ALLOCATION_POLICY_VERSION) # field change
|
||||
assert allocation_policy_hash(ALLOCATION_POLICY, 2) != allocation_policy_hash(ALLOCATION_POLICY, 1)
|
||||
assert allocation_policy_hash(ALLOCATION_POLICY, ALLOCATION_POLICY_VERSION) == _EXPECTED_HASH # noqa: F821
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `uv run pytest tests/unit/test_allocation_policy.py -q`
|
||||
Expected: FAIL with `ModuleNotFoundError: No module named 'fxhnt.application.allocation_policy'`.
|
||||
|
||||
- [ ] **Step 3: Implement `allocation_policy.py`**
|
||||
|
||||
```python
|
||||
# src/fxhnt/application/allocation_policy.py
|
||||
"""The declarative, versioned strategy-level ALLOCATION policy (sub-project B2a).
|
||||
|
||||
Turns the deploy-tier strategies into ex-ante-sized target weights. Single source for the allocation knobs
|
||||
(previously scattered as book_allocator function defaults + config settings). A change = a code commit + an
|
||||
ALLOCATION_POLICY_VERSION bump; the content-derived allocation_policy_hash self-check forces the bump (same
|
||||
discipline as A's definition_hash and B1's policy_hash). The capital_ceiling — the HARD human gate on total
|
||||
deployed gross — lives in this versioned policy, so a ceiling change is audited. NO reactive de-sizing: the
|
||||
sizing derives from full-history vol, and the only drawdown-reactive element is the rare aggregate killswitch."""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from dataclasses import asdict, dataclass
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class AllocationPolicy:
|
||||
target_vol: float = 0.12 # ex-ante annualised vol target for the book
|
||||
kelly_fraction: float = 0.5 # fractional-Kelly scalar
|
||||
max_leverage: float = 1.5 # hard cap on the ex-ante leverage K
|
||||
max_strategy_weight: float = 0.5 # per-strategy weight cap (before leverage)
|
||||
marginal_gate: bool = True # leave-one-out equal-weight marginal-Sharpe quality filter
|
||||
min_history_days: int = 60 # a strategy needs >= this many return obs for a stable ex-ante vol
|
||||
kill_dd: float = 0.25 # RARE aggregate killswitch: flatten beyond this drawdown (latching)
|
||||
reenter_dd: float = 0.10 # re-enter once the drawdown recovers inside this
|
||||
capital_ceiling: float = 35_000.0 # HARD fund-level dollar cap on total deployed gross (the human gate)
|
||||
|
||||
|
||||
ALLOCATION_POLICY = AllocationPolicy()
|
||||
ALLOCATION_POLICY_VERSION = 1
|
||||
|
||||
|
||||
def allocation_policy_hash(policy: AllocationPolicy, version: int) -> str:
|
||||
canonical = json.dumps(asdict(policy), sort_keys=True, separators=(",", ":"))
|
||||
return hashlib.sha256(f"{canonical}|v{version}".encode()).hexdigest()
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Pin the hash + run**
|
||||
|
||||
Run: `uv run python -c "from fxhnt.application.allocation_policy import ALLOCATION_POLICY, ALLOCATION_POLICY_VERSION, allocation_policy_hash; print(allocation_policy_hash(ALLOCATION_POLICY, ALLOCATION_POLICY_VERSION))"`
|
||||
Paste the value as `_EXPECTED_HASH = "<value>"` near the top of the test file.
|
||||
Run: `uv run pytest tests/unit/test_allocation_policy.py -q` → Expected: PASS (2 passed).
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/application/allocation_policy.py tests/unit/test_allocation_policy.py
|
||||
git commit -F - <<'EOF'
|
||||
feat(alloc): versioned AllocationPolicy + content-derived hash + capital_ceiling (B2a)
|
||||
|
||||
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
|
||||
EOF
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 2: allocation engine — pure helpers (full-history vol, equal-weight marginal prune, killswitch latch)
|
||||
|
||||
**Files:**
|
||||
- Create: `src/fxhnt/application/allocation_engine.py`
|
||||
- Test: `tests/unit/test_allocation_engine_helpers.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `promotion_corr.deploy_book_returns` (B1, the equal-weight per-date aggregate), `per_period_sharpe` (`fxhnt.domain.gauntlet`).
|
||||
- Produces:
|
||||
- `full_history_vol(returns: dict[str, float]) -> float` — annualised (√365) std of the return values over the WHOLE series; `0.0` if `< 2` obs.
|
||||
- `equal_weight_marginal_prune(returns_by_strategy: dict[str, dict[str, float]]) -> set[str]` — leave-one-out: bench a strategy only when removing it raises the EQUAL-WEIGHT book's per-period Sharpe; shrinks to ≥1.
|
||||
- `killswitch_active(book_returns: dict[str, float], kill_dd: float, reenter_dd: float) -> bool` — the final latch state after folding the drawdown killswitch over the sorted series.
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
```python
|
||||
# tests/unit/test_allocation_engine_helpers.py
|
||||
import numpy as np
|
||||
|
||||
from fxhnt.application.allocation_engine import (
|
||||
equal_weight_marginal_prune, full_history_vol, killswitch_active)
|
||||
|
||||
|
||||
def test_full_history_vol_is_annualised_std():
|
||||
r = {f"2026-06-{d:02d}": v for d, v in zip(range(1, 6), [0.01, -0.01, 0.02, -0.02, 0.0])}
|
||||
assert abs(full_history_vol(r) - float(np.std([0.01, -0.01, 0.02, -0.02, 0.0]) * np.sqrt(365.0))) < 1e-12
|
||||
assert full_history_vol({"2026-06-01": 0.01}) == 0.0 # < 2 obs → 0
|
||||
|
||||
|
||||
def test_marginal_prune_benches_non_additive_strategy():
|
||||
dates = [f"2026-06-{d:02d}" for d in range(1, 41)]
|
||||
good = {d: (0.01 if i % 2 else -0.008) for i, d in enumerate(dates)} # positive-Sharpe
|
||||
noise = {d: (0.03 if i % 3 == 0 else -0.02) for i, d in enumerate(dates)} # drags the book
|
||||
kept = equal_weight_marginal_prune({"good": good, "noise": noise})
|
||||
assert "good" in kept and "noise" not in kept
|
||||
|
||||
|
||||
def test_killswitch_latches_on_deep_dd_and_reenters():
|
||||
# a −30% run then recovery; kill at 0.25, reenter at 0.10
|
||||
down = {f"2026-06-{d:02d}": -0.04 for d in range(1, 11)} # ~ -33% cumulative → breach 0.25
|
||||
assert killswitch_active(down, kill_dd=0.25, reenter_dd=0.10) is True
|
||||
calm = {f"2026-06-{d:02d}": 0.001 for d in range(1, 11)}
|
||||
assert killswitch_active(calm, kill_dd=0.25, reenter_dd=0.10) is False
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `uv run pytest tests/unit/test_allocation_engine_helpers.py -q`
|
||||
Expected: FAIL with `ModuleNotFoundError: No module named 'fxhnt.application.allocation_engine'`.
|
||||
|
||||
- [ ] **Step 3: Implement the helpers**
|
||||
|
||||
```python
|
||||
# src/fxhnt/application/allocation_engine.py
|
||||
"""The pure, ex-ante strategy-level allocation engine (sub-project B2a).
|
||||
|
||||
compute_allocation turns each live strategy's return series (from A's record) into ex-ante-sized target
|
||||
weights: equal-weight the marginal-Sharpe-gate survivors, size to a target vol using the FULL-history vol
|
||||
(stable — NOT trailing, so a recent vol spike does not change the leverage), flatten on a rare latching
|
||||
aggregate drawdown killswitch. target_capital caps total deployed gross at the policy's hard capital_ceiling.
|
||||
No reactive de-sizing; the killswitch is the only drawdown-reactive element."""
|
||||
from __future__ import annotations
|
||||
|
||||
import numpy as np
|
||||
|
||||
from fxhnt.application.allocation_policy import AllocationPolicy
|
||||
from fxhnt.application.promotion_corr import deploy_book_returns
|
||||
from fxhnt.domain.gauntlet import per_period_sharpe
|
||||
|
||||
_ANN = np.sqrt(365.0)
|
||||
|
||||
|
||||
def full_history_vol(returns: dict[str, float]) -> float:
|
||||
vals = list(returns.values())
|
||||
if len(vals) < 2:
|
||||
return 0.0
|
||||
return float(np.std(np.array(vals)) * _ANN)
|
||||
|
||||
|
||||
def _book_sharpe(returns_by_strategy: dict[str, dict[str, float]]) -> float:
|
||||
book = deploy_book_returns(returns_by_strategy) # equal-weight per-date average
|
||||
if len(book) < 2:
|
||||
return 0.0
|
||||
return float(per_period_sharpe(np.array([book[d] for d in sorted(book)])))
|
||||
|
||||
|
||||
def equal_weight_marginal_prune(returns_by_strategy: dict[str, dict[str, float]]) -> set[str]:
|
||||
kept = set(returns_by_strategy)
|
||||
while len(kept) > 1:
|
||||
base = _book_sharpe({e: returns_by_strategy[e] for e in kept})
|
||||
drop = next((e for e in kept
|
||||
if _book_sharpe({x: returns_by_strategy[x] for x in kept - {e}}) > base + 1e-9), None)
|
||||
if drop is None:
|
||||
break
|
||||
kept.discard(drop)
|
||||
return kept
|
||||
|
||||
|
||||
def killswitch_active(book_returns: dict[str, float], kill_dd: float, reenter_dd: float) -> bool:
|
||||
killed = False
|
||||
shadow_eq = 1.0
|
||||
peak = 1.0
|
||||
for d in sorted(book_returns):
|
||||
shadow_eq *= (1.0 + book_returns[d])
|
||||
peak = max(peak, shadow_eq)
|
||||
dd = (peak - shadow_eq) / peak if peak > 0 else 0.0
|
||||
if not killed and dd > kill_dd:
|
||||
killed = True
|
||||
elif killed and dd < reenter_dd:
|
||||
killed = False
|
||||
return killed
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run tests → PASS. Ruff.**
|
||||
|
||||
Run: `uv run pytest tests/unit/test_allocation_engine_helpers.py -q` → PASS (3 passed).
|
||||
Run: `uv run ruff check src/fxhnt/application/allocation_engine.py tests/unit/test_allocation_engine_helpers.py`
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/application/allocation_engine.py tests/unit/test_allocation_engine_helpers.py
|
||||
git commit -F - <<'EOF'
|
||||
feat(alloc): pure allocation helpers — full-history vol, equal-weight marginal prune, killswitch latch (B2a)
|
||||
|
||||
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
|
||||
EOF
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 3: `compute_allocation` — the ex-ante composition (+ the anti-reactive invariant)
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/application/allocation_engine.py` (add `compute_allocation`)
|
||||
- Test: `tests/unit/test_compute_allocation.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: the Task-2 helpers, `AllocationPolicy` (Task 1), `deploy_book_returns`.
|
||||
- Produces: `compute_allocation(returns_by_strategy: dict[str, dict[str, float]], policy: AllocationPolicy) -> dict[str, float]` — per-strategy target weights (ex-ante-sized, gated, killswitch-aware).
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
```python
|
||||
# tests/unit/test_compute_allocation.py
|
||||
from fxhnt.application.allocation_policy import AllocationPolicy
|
||||
from fxhnt.application.allocation_engine import compute_allocation
|
||||
|
||||
|
||||
def _series(n, ret): # n days of a constant-ish return
|
||||
return {f"2026-{(6 + d // 30):02d}-{(d % 30) + 1:02d}": ret for d in range(n)}
|
||||
|
||||
|
||||
def test_excludes_short_history_and_empty_returns_empty():
|
||||
pol = AllocationPolicy(min_history_days=60, marginal_gate=False)
|
||||
assert compute_allocation({"short": _series(10, 0.001)}, pol) == {} # < 60 obs → excluded → empty
|
||||
|
||||
|
||||
def test_equal_weight_and_ex_ante_leverage():
|
||||
pol = AllocationPolicy(min_history_days=5, marginal_gate=False, max_leverage=1.5,
|
||||
kelly_fraction=0.5, target_vol=0.12, kill_dd=0.99, reenter_dd=0.98)
|
||||
import numpy as np
|
||||
a = {f"2026-06-{d:02d}": (0.01 if d % 2 else -0.005) for d in range(1, 21)}
|
||||
b = {f"2026-06-{d:02d}": (-0.005 if d % 2 else 0.01) for d in range(1, 21)}
|
||||
w = compute_allocation({"a": a, "b": b}, pol)
|
||||
assert set(w) == {"a", "b"}
|
||||
assert abs(w["a"] - w["b"]) < 1e-9 # equal base weights
|
||||
# K applied: weights sum to K (the leverage), each = K/2
|
||||
k = sum(w.values())
|
||||
assert 0 < k <= 1.5
|
||||
|
||||
|
||||
def test_recent_vol_spike_does_not_change_K_antireactive():
|
||||
# THE load-bearing invariant: ex-ante vol uses the FULL history, so appending a recent spike must not
|
||||
# change the leverage on the earlier window.
|
||||
pol = AllocationPolicy(min_history_days=5, marginal_gate=False, kill_dd=0.99, reenter_dd=0.98)
|
||||
base = {f"2026-06-{d:02d}": (0.01 if d % 2 else -0.01) for d in range(1, 61)}
|
||||
w_before = compute_allocation({"a": dict(base), "b": dict(base)}, pol)
|
||||
spiked = dict(base); spiked["2026-06-30"] = 0.5 # a single huge recent day
|
||||
w_after = compute_allocation({"a": spiked, "b": dict(base)}, pol)
|
||||
# the full-history vol of "a" changes (it must — vol IS a full-history stat), so K adapts SLOWLY over the
|
||||
# whole record, NOT reactively to the last window. Assert K moved only via the full-history vol, i.e. the
|
||||
# book vol recomputed over ALL 60 days (not a trailing subset):
|
||||
from fxhnt.application.allocation_engine import full_history_vol
|
||||
from fxhnt.application.promotion_corr import deploy_book_returns
|
||||
vol_all = full_history_vol(deploy_book_returns({"a": spiked, "b": dict(base)}))
|
||||
k_after = sum(w_after.values())
|
||||
assert abs(k_after - min(pol.max_leverage, pol.kelly_fraction * pol.target_vol / vol_all)) < 1e-9
|
||||
|
||||
|
||||
def test_killswitch_flattens():
|
||||
pol = AllocationPolicy(min_history_days=5, marginal_gate=False, kill_dd=0.20, reenter_dd=0.10)
|
||||
crash = {f"2026-06-{d:02d}": -0.04 for d in range(1, 11)} # deep DD → killswitch
|
||||
w = compute_allocation({"a": crash, "b": crash}, pol)
|
||||
assert all(v == 0.0 for v in w.values()) # flattened
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `uv run pytest tests/unit/test_compute_allocation.py -q`
|
||||
Expected: FAIL with `ImportError: cannot import name 'compute_allocation'`.
|
||||
|
||||
- [ ] **Step 3: Implement `compute_allocation`**
|
||||
|
||||
Append to `src/fxhnt/application/allocation_engine.py`:
|
||||
|
||||
```python
|
||||
def compute_allocation(returns_by_strategy: dict[str, dict[str, float]],
|
||||
policy: AllocationPolicy) -> dict[str, float]:
|
||||
# 1. eligibility: a stable ex-ante vol needs enough history
|
||||
eligible = {s: r for s, r in returns_by_strategy.items() if len(r) >= policy.min_history_days}
|
||||
if not eligible:
|
||||
return {}
|
||||
# 2. marginal-Sharpe quality gate (equal-weight book), 3. equal-weight survivors
|
||||
survivors = equal_weight_marginal_prune(eligible) if policy.marginal_gate else set(eligible)
|
||||
n = len(survivors)
|
||||
base = {s: 1.0 / n for s in survivors}
|
||||
# per-strategy cap + renormalise (inert for pure equal weight, honours the policy guardrail)
|
||||
if any(w > policy.max_strategy_weight + 1e-12 for w in base.values()):
|
||||
capped = {s: min(w, policy.max_strategy_weight) for s, w in base.items()}
|
||||
tot = sum(capped.values())
|
||||
base = {s: w / tot for s, w in capped.items()} if tot > 0 else base
|
||||
# 5. rare latching aggregate killswitch → flatten
|
||||
book = deploy_book_returns({s: eligible[s] for s in survivors})
|
||||
if killswitch_active(book, policy.kill_dd, policy.reenter_dd):
|
||||
return {s: 0.0 for s in survivors}
|
||||
# 4. ex-ante vol-target leverage from the FULL-history book vol (never trailing)
|
||||
vol = full_history_vol(book)
|
||||
k = min(policy.max_leverage, policy.kelly_fraction * policy.target_vol / vol) if vol > 1e-12 else 0.0
|
||||
return {s: k * w for s, w in base.items()}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run tests → PASS. Ruff.**
|
||||
|
||||
Run: `uv run pytest tests/unit/test_compute_allocation.py -q` → PASS (4 passed).
|
||||
Run: `uv run ruff check src/fxhnt/application/allocation_engine.py tests/unit/test_compute_allocation.py`
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/application/allocation_engine.py tests/unit/test_compute_allocation.py
|
||||
git commit -F - <<'EOF'
|
||||
feat(alloc): compute_allocation — ex-ante vol-target on full-history vol, marginal gate, killswitch (B2a)
|
||||
|
||||
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
|
||||
EOF
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 4: `target_capital` — the hard capital-ceiling scaling
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/application/allocation_engine.py` (add `target_capital`)
|
||||
- Test: `tests/unit/test_target_capital.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: `target_capital(target_weights: dict[str, float], equity: float, policy: AllocationPolicy) -> dict[str, float]` — per-strategy signed dollars; `sum(abs(dollars)) <= policy.capital_ceiling` always.
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
```python
|
||||
# tests/unit/test_target_capital.py
|
||||
from fxhnt.application.allocation_policy import AllocationPolicy
|
||||
from fxhnt.application.allocation_engine import target_capital
|
||||
|
||||
|
||||
def test_ceiling_binds_and_caps_gross():
|
||||
pol = AllocationPolicy(capital_ceiling=10_000.0)
|
||||
# weights sum to 1.0 (gross), equity 50k → implied gross 50k > ceiling 10k → scaled to 10k
|
||||
w = {"a": 0.5, "b": 0.5}
|
||||
d = target_capital(w, equity=50_000.0, policy=pol)
|
||||
assert abs(sum(abs(x) for x in d.values()) - 10_000.0) < 1e-6 # gross capped at ceiling
|
||||
assert abs(d["a"] - d["b"]) < 1e-6 # equal split preserved
|
||||
|
||||
|
||||
def test_ceiling_not_binding_uses_equity():
|
||||
pol = AllocationPolicy(capital_ceiling=100_000.0)
|
||||
d = target_capital({"a": 0.5, "b": 0.5}, equity=20_000.0, policy=pol)
|
||||
assert abs(sum(abs(x) for x in d.values()) - 20_000.0) < 1e-6 # gross = equity·1.0 < ceiling
|
||||
|
||||
|
||||
def test_empty_or_zero_weights():
|
||||
pol = AllocationPolicy(capital_ceiling=10_000.0)
|
||||
assert target_capital({}, 50_000.0, pol) == {}
|
||||
assert target_capital({"a": 0.0, "b": 0.0}, 50_000.0, pol) == {"a": 0.0, "b": 0.0}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `uv run pytest tests/unit/test_target_capital.py -q`
|
||||
Expected: FAIL with `ImportError: cannot import name 'target_capital'`.
|
||||
|
||||
- [ ] **Step 3: Implement `target_capital`**
|
||||
|
||||
Append to `src/fxhnt/application/allocation_engine.py`:
|
||||
|
||||
```python
|
||||
def target_capital(target_weights: dict[str, float], equity: float,
|
||||
policy: AllocationPolicy) -> dict[str, float]:
|
||||
"""Per-strategy signed dollars. Total deployed gross = min(equity·Σ|w|, capital_ceiling) — the HARD
|
||||
human gate: never deploy more than the ceiling (or more than equity·leverage). Signs preserved."""
|
||||
if not target_weights:
|
||||
return {}
|
||||
gross = sum(abs(w) for w in target_weights.values()) * equity
|
||||
if gross <= 0.0:
|
||||
return {s: 0.0 for s in target_weights}
|
||||
deployed = min(gross, policy.capital_ceiling)
|
||||
scale = deployed / gross
|
||||
return {s: w * equity * scale for s, w in target_weights.items()}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run tests → PASS. Ruff.**
|
||||
|
||||
Run: `uv run pytest tests/unit/test_target_capital.py -q` → PASS (3 passed).
|
||||
Run: `uv run ruff check src/fxhnt/application/allocation_engine.py tests/unit/test_target_capital.py`
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/application/allocation_engine.py tests/unit/test_target_capital.py
|
||||
git commit -F - <<'EOF'
|
||||
feat(alloc): target_capital — hard capital_ceiling caps total deployed gross (B2a)
|
||||
|
||||
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
|
||||
EOF
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 5: `strategy_allocation` table + repo write/read
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/adapters/persistence/cockpit_models.py` (add `StrategyAllocationRow`)
|
||||
- Modify: `src/fxhnt/adapters/persistence/forward_nav.py` (add `upsert_allocation`, `current_allocation`)
|
||||
- Test: `tests/integration/test_strategy_allocation.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Produces:
|
||||
- `StrategyAllocationRow(strategy_id, target_weight, target_dollars, policy_version, policy_hash, as_of, computed_at)` — `strategy_id` PK (one current row per strategy, replaced each run).
|
||||
- `ForwardNavRepo.upsert_allocation(strategy_id, target_weight, target_dollars, policy_version, policy_hash, as_of, at)`
|
||||
- `ForwardNavRepo.current_allocation() -> dict[str, float]` — `{strategy_id: target_dollars}` for the latest snapshot.
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
```python
|
||||
# tests/integration/test_strategy_allocation.py
|
||||
import datetime as dt
|
||||
|
||||
from fxhnt.adapters.persistence.forward_nav import ForwardNavRepo
|
||||
|
||||
|
||||
def test_allocation_round_trips(tmp_path):
|
||||
repo = ForwardNavRepo(f"sqlite:///{tmp_path}/a.db"); repo.migrate()
|
||||
at = dt.datetime(2026, 7, 9, 23, 30)
|
||||
repo.upsert_allocation("multistrat", target_weight=0.6, target_dollars=6000.0,
|
||||
policy_version=1, policy_hash="h", as_of="2026-07-09", at=at)
|
||||
repo.upsert_allocation("bybit_4edge", target_weight=0.4, target_dollars=4000.0,
|
||||
policy_version=1, policy_hash="h", as_of="2026-07-09", at=at)
|
||||
assert repo.current_allocation() == {"multistrat": 6000.0, "bybit_4edge": 4000.0}
|
||||
# re-run replaces (idempotent on strategy_id)
|
||||
repo.upsert_allocation("multistrat", target_weight=0.5, target_dollars=5000.0,
|
||||
policy_version=1, policy_hash="h", as_of="2026-07-10", at=at)
|
||||
assert repo.current_allocation()["multistrat"] == 5000.0
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `uv run pytest tests/integration/test_strategy_allocation.py -q`
|
||||
Expected: FAIL with `AttributeError: 'ForwardNavRepo' object has no attribute 'upsert_allocation'`.
|
||||
|
||||
- [ ] **Step 3: Add the ORM row**
|
||||
|
||||
In `cockpit_models.py`, append:
|
||||
|
||||
```python
|
||||
class StrategyAllocationRow(CockpitBase):
|
||||
"""The latest B2a strategy-level allocation snapshot — one row per strategy, replaced each run.
|
||||
target_weight = the ex-ante-sized weight; target_dollars = capital-ceiling-bounded dollars. Stamped with
|
||||
the allocation policy that produced it (audit). A pure derived cache of (policy + A's records + B1 status)."""
|
||||
__tablename__ = "strategy_allocation"
|
||||
strategy_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
target_weight: Mapped[float] = mapped_column(Float)
|
||||
target_dollars: Mapped[float] = mapped_column(Float)
|
||||
policy_version: Mapped[int] = mapped_column(Integer)
|
||||
policy_hash: Mapped[str] = mapped_column(String(64))
|
||||
as_of: Mapped[str] = mapped_column(String(10))
|
||||
computed_at: Mapped[dt.datetime] = mapped_column(DateTime)
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Add the repo methods**
|
||||
|
||||
Add `StrategyAllocationRow` to the `cockpit_models` import in `forward_nav.py`, then add to `ForwardNavRepo`:
|
||||
|
||||
```python
|
||||
def upsert_allocation(self, strategy_id: str, target_weight: float, target_dollars: float,
|
||||
policy_version: int, policy_hash: str, as_of: str, at: dt.datetime) -> None:
|
||||
with Session(self._engine) as s:
|
||||
s.merge(StrategyAllocationRow(
|
||||
strategy_id=strategy_id, target_weight=target_weight, target_dollars=target_dollars,
|
||||
policy_version=policy_version, policy_hash=policy_hash, as_of=as_of, computed_at=at))
|
||||
s.commit()
|
||||
|
||||
def current_allocation(self) -> dict[str, float]:
|
||||
with Session(self._engine) as s:
|
||||
return {r.strategy_id: r.target_dollars for r in s.scalars(select(StrategyAllocationRow))}
|
||||
```
|
||||
|
||||
- [ ] **Step 5: Run tests → PASS + ruff, then commit**
|
||||
|
||||
Run: `uv run pytest tests/integration/test_strategy_allocation.py -q` → PASS.
|
||||
Run: `uv run ruff check src/fxhnt/adapters/persistence/cockpit_models.py src/fxhnt/adapters/persistence/forward_nav.py tests/integration/test_strategy_allocation.py`
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/adapters/persistence/cockpit_models.py src/fxhnt/adapters/persistence/forward_nav.py tests/integration/test_strategy_allocation.py
|
||||
git commit -F - <<'EOF'
|
||||
feat(alloc): strategy_allocation table + repo (record the B2a allocation snapshot) (B2a)
|
||||
|
||||
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
|
||||
EOF
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 6: nightly step — compute + record the allocation from A's records + B1's deploy set
|
||||
|
||||
**Files:**
|
||||
- Create: `src/fxhnt/application/allocation_ingest.py` (`record_allocation(dsn, equity)`)
|
||||
- Modify: `src/fxhnt/adapters/orchestration/assets.py` (`cockpit_forward` calls it after `evaluate_promotions`)
|
||||
- Test: `tests/integration/test_allocation_ingest.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `ForwardNavRepo.nav_history`, `promoted_strategy_ids`, `STRATEGY_REGISTRY` (`tier`), `compute_allocation`/`target_capital` (Tasks 3-4), `ALLOCATION_POLICY`/`ALLOCATION_POLICY_VERSION`/`allocation_policy_hash` (Task 1), `upsert_allocation` (Task 5).
|
||||
- Produces: `record_allocation(dsn: str, equity: float) -> dict[str, float]` — computes the deploy-set allocation, records each strategy's snapshot, returns `{strategy_id: target_dollars}`.
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
```python
|
||||
# tests/integration/test_allocation_ingest.py
|
||||
import datetime as dt
|
||||
|
||||
from fxhnt.adapters.persistence.forward_nav import ForwardNavRepo
|
||||
from fxhnt.application.allocation_ingest import record_allocation
|
||||
from fxhnt.application.forward_models import ForwardNavRow as NavRowDTO
|
||||
from fxhnt import registry
|
||||
|
||||
|
||||
def test_records_allocation_for_deploy_set(tmp_path, monkeypatch):
|
||||
dsn = f"sqlite:///{tmp_path}/ai.db"
|
||||
repo = ForwardNavRepo(dsn); repo.migrate()
|
||||
at = dt.datetime(2026, 7, 9, 23, 30)
|
||||
monkeypatch.setitem(registry.STRATEGY_REGISTRY, "d1",
|
||||
{"display_name": "D1", "sleeve": "x", "gate_spec": {}, "tier": "deploy"})
|
||||
monkeypatch.setitem(registry.STRATEGY_REGISTRY, "d2",
|
||||
{"display_name": "D2", "sleeve": "x", "gate_spec": {}, "tier": "deploy"})
|
||||
for sid, flip in (("d1", 1), ("d2", 0)):
|
||||
for i, dd in enumerate([f"2026-06-{x:02d}" for x in range(1, 21)] +
|
||||
[f"2026-07-{x:02d}" for x in range(1, 41)]):
|
||||
repo.upsert_rows([NavRowDTO(strategy_id=sid, date=dd,
|
||||
ret=(0.01 if (i % 2) == flip else -0.008), nav=1.0)], at=at)
|
||||
out = record_allocation(dsn, equity=20_000.0)
|
||||
assert set(out) <= {"d1", "d2"} # only deploy-set considered
|
||||
assert sum(abs(v) for v in out.values()) <= 35_000.0 + 1e-6 # capped by the default ceiling
|
||||
assert repo.current_allocation() == out # recorded
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `uv run pytest tests/integration/test_allocation_ingest.py -q`
|
||||
Expected: FAIL with `ModuleNotFoundError: No module named 'fxhnt.application.allocation_ingest'`.
|
||||
|
||||
- [ ] **Step 3: Implement `allocation_ingest.py`**
|
||||
|
||||
```python
|
||||
# src/fxhnt/application/allocation_ingest.py
|
||||
"""Compute + record the B2a strategy-level allocation from A's records + B1's deploy set. Runs in the nightly
|
||||
AFTER B1's promotion (so the deploy-tier set is fresh). Pure allocation math (allocation_engine) + a recorded
|
||||
snapshot; no capital is moved (that is sub-project C)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
import logging
|
||||
|
||||
from fxhnt.adapters.persistence.forward_nav import ForwardNavRepo
|
||||
from fxhnt.application.allocation_engine import compute_allocation, target_capital
|
||||
from fxhnt.application.allocation_policy import (
|
||||
ALLOCATION_POLICY, ALLOCATION_POLICY_VERSION, allocation_policy_hash)
|
||||
from fxhnt.registry import STRATEGY_REGISTRY
|
||||
|
||||
_log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def record_allocation(dsn: str, equity: float) -> dict[str, float]:
|
||||
repo = ForwardNavRepo(dsn)
|
||||
repo.migrate()
|
||||
deploy_sids = {sid for sid, m in STRATEGY_REGISTRY.items() if m.get("tier") == "deploy"} \
|
||||
| repo.promoted_strategy_ids()
|
||||
returns = {sid: {r.date: r.ret for r in repo.nav_history(sid)} for sid in deploy_sids}
|
||||
weights = compute_allocation(returns, ALLOCATION_POLICY)
|
||||
dollars = target_capital(weights, equity, ALLOCATION_POLICY)
|
||||
at = dt.datetime.now(dt.timezone.utc).replace(tzinfo=None)
|
||||
phash = allocation_policy_hash(ALLOCATION_POLICY, ALLOCATION_POLICY_VERSION)
|
||||
as_of = at.strftime("%Y-%m-%d")
|
||||
for sid in weights:
|
||||
repo.upsert_allocation(sid, target_weight=weights[sid], target_dollars=dollars.get(sid, 0.0),
|
||||
policy_version=ALLOCATION_POLICY_VERSION, policy_hash=phash, as_of=as_of, at=at)
|
||||
_log.info("record_allocation: %d strategies allocated (gross $%.0f, ceiling $%.0f)",
|
||||
len(weights), sum(abs(v) for v in dollars.values()), ALLOCATION_POLICY.capital_ceiling)
|
||||
return dollars
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Wire it into `cockpit_forward`**
|
||||
|
||||
In `src/fxhnt/adapters/orchestration/assets.py`, the `cockpit_forward` asset currently calls
|
||||
`ingest_forward_state(dsn)` then `evaluate_promotions(dsn)`. Add the allocation record AFTER promotions (so
|
||||
the deploy set is fresh), using the paper capital as the equity input:
|
||||
|
||||
```python
|
||||
from fxhnt.application.allocation_ingest import record_allocation
|
||||
# ... after: promoted = evaluate_promotions(dsn)
|
||||
allocated = record_allocation(dsn, equity=get_settings().paper_capital)
|
||||
context.log.info(f"cockpit_forward: allocated {len(allocated)} strategies (promoted: {promoted or 'none'})")
|
||||
```
|
||||
(Read the current `cockpit_forward` body first; keep the existing ingest + promotion calls + log line, add the allocation record after them.)
|
||||
|
||||
- [ ] **Step 5: Run tests + full suite + ruff, then commit**
|
||||
|
||||
Run: `uv run pytest tests/integration/test_allocation_ingest.py -q` → PASS.
|
||||
Run the FULL suite ONCE (this touches the nightly `cockpit_forward` path): `uv run pytest -q` → PASS.
|
||||
Run: `uv run ruff check src/fxhnt/application/allocation_ingest.py src/fxhnt/adapters/orchestration/assets.py tests/integration/test_allocation_ingest.py`
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/application/allocation_ingest.py src/fxhnt/adapters/orchestration/assets.py tests/integration/test_allocation_ingest.py
|
||||
git commit -F - <<'EOF'
|
||||
feat(alloc): nightly records the B2a allocation from A's records + B1 deploy set (B2a)
|
||||
|
||||
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
|
||||
EOF
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Self-Review notes
|
||||
|
||||
- **Spec coverage:** policy object (§1)→T1; engine helpers (§2)→T2; compute_allocation ex-ante composition (§2)→T3; capital ceiling (§3)→T4; recorded snapshot + audit stamp (§4)→T5+T6; where-human-stays (§5)→T6 (records targets, no capital moved); anti-reactive invariant (error-handling)→T3 test; killswitch→T2/T3; min_history exclusion→T3.
|
||||
- **Anti-reactive invariant:** T3's `test_recent_vol_spike_does_not_change_K_antireactive` pins that `K` comes from the FULL-history book vol (not a trailing window) — the load-bearing guarantee.
|
||||
- **No reactive path:** the plan reuses `promotion_corr.deploy_book_returns` + `per_period_sharpe` and reimplements the killswitch fold; it does NOT touch `book_allocator.combine_book` (the rejected trailing-vol leverage).
|
||||
- **Type consistency:** `compute_allocation(returns_by_strategy: dict[str, dict[str,float]], policy)` → weights; `target_capital(weights, equity, policy)` → dollars; both consumed by `record_allocation` (T6). `deploy_book_returns` (B1) is the equal-weight aggregate used by both the vol + the marginal gate. `full_history_vol`/`equal_weight_marginal_prune`/`killswitch_active` signatures match T2 defs.
|
||||
- **No capital moved:** T6 records dollar TARGETS only; execution (moving capital) is sub-project C.
|
||||
@@ -0,0 +1,720 @@
|
||||
# Declarative Gate + Promotion Policy (Sub-project B1) Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Consolidate the scattered gate + promotion defaults into one in-code, versioned `GatePolicy`; resolve the effective per-track policy at a single point; stamp every gate verdict with the `policy_version` + content-derived `policy_hash` that produced it; and add an automatic correlation backstop to promotion.
|
||||
|
||||
**Architecture:** A new `gate_policy.py` holds the policy defaults (values byte-identical to today's module constants) + a `POLICY_VERSION` + a `policy_hash`. `resolve_policy(POLICY, gate_spec) -> ResolvedPolicy` layers per-track registry overrides on the defaults at one point; `evaluate_reconciliation_gate`/`evaluate_gate` take the `ResolvedPolicy` (their module constants / inline defaults are removed). `forward_ingest` resolves once per track, stamps the verdict onto `forward_summary`, and adds a deploy-book correlation backstop to `evaluate_promotions`. Behavior is byte-identical under the current thresholds.
|
||||
|
||||
**Tech Stack:** Python 3.12, dataclasses, SQLAlchemy 2.0 ORM (sqlite for tests, Postgres prod), pytest (`uv run pytest`), ruff (E,F,I,UP,B,SIM; line-length 120).
|
||||
|
||||
## Global Constraints
|
||||
|
||||
- Spec: `docs/superpowers/specs/2026-07-09-declarative-gate-promotion-policy-b1-design.md` (authoritative).
|
||||
- Builds on sub-project A (merged to master): reads `forward_summary` + `forward_nav` (A's record) and extends `forward_ingest` / the gate functions.
|
||||
- REAL MONEY: the gate must stay WAIT-safe — never a false PASS. The consolidation must produce **byte-identical verdicts** under the current thresholds (a regression anchor test proves it). Gate verdict LOGIC (the comparisons) is unchanged; only where the thresholds come from changes.
|
||||
- Policy lives in **git-committed code**, never runtime-editable in the DB. A policy change = a code commit + a `POLICY_VERSION` bump; the content-derived `policy_hash` self-check forces the bump.
|
||||
- No capital: B1 touches only gate STATUS + promotion (a tracking tier). Capital is B2.
|
||||
- Idempotency: DB writes are `Session.merge` upserts. New `forward_summary` columns are nullable (migration-tolerant). Determinism: no wall-clock inside pure gate/policy functions.
|
||||
- Never save tests to the repo root; tests under `tests/unit/` or `tests/integration/` per the existing layout. Commit after every task; messages end with the `Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>` trailer.
|
||||
- Promotion is ONE-WAY + persistent (unchanged): once promoted, never reverts.
|
||||
|
||||
---
|
||||
|
||||
### Task 1: `gate_policy.py` — the versioned policy object + hash
|
||||
|
||||
**Files:**
|
||||
- Create: `src/fxhnt/application/gate_policy.py`
|
||||
- Test: `tests/unit/test_gate_policy.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Produces:
|
||||
- `GatePolicy` frozen dataclass with fields (defaults = today's constants): `min_forward_days: int = 14`, `recon_tolerance_frac: float = 0.5`, `recon_min_band: float = 0.05`, `recon_min_corr: float = 0.0`, `max_gap_days: int = 1`, `min_days: int = 20`, `min_total_return: float = 0.0`, `min_sharpe: float = 0.0`, `promotion_max_corr: float = 0.5`, `promotion_min_overlap_days: int = 10`.
|
||||
- `POLICY: GatePolicy` — the single current instance (all defaults).
|
||||
- `POLICY_VERSION: int = 1`.
|
||||
- `policy_hash(policy: GatePolicy, version: int) -> str` — sha256 of the canonical field values + version.
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
```python
|
||||
# tests/unit/test_gate_policy.py
|
||||
from fxhnt.application.gate_policy import POLICY, POLICY_VERSION, GatePolicy, policy_hash
|
||||
|
||||
|
||||
def test_policy_defaults_match_todays_constants():
|
||||
# these values must equal the pre-B1 reconciliation_gate / gate.py constants (byte-identical verdicts)
|
||||
assert (POLICY.min_forward_days, POLICY.recon_tolerance_frac, POLICY.recon_min_band,
|
||||
POLICY.recon_min_corr, POLICY.max_gap_days) == (14, 0.5, 0.05, 0.0, 1)
|
||||
assert (POLICY.min_days, POLICY.min_total_return, POLICY.min_sharpe) == (20, 0.0, 0.0)
|
||||
|
||||
|
||||
def test_policy_hash_is_stable_and_content_derived():
|
||||
assert policy_hash(POLICY, POLICY_VERSION) == policy_hash(POLICY, POLICY_VERSION) # stable
|
||||
changed = GatePolicy(min_forward_days=21) # a field changed
|
||||
assert policy_hash(changed, POLICY_VERSION) != policy_hash(POLICY, POLICY_VERSION) # → different hash
|
||||
assert policy_hash(POLICY, 2) != policy_hash(POLICY, 1) # version bump → different
|
||||
|
||||
|
||||
def test_policy_hash_selfcheck_forces_version_bump():
|
||||
# PIN the current hash. If a policy field changes without this literal being updated (and the version
|
||||
# bumped), this test fails — forcing a deliberate, audited change. Update BOTH on any real policy change.
|
||||
assert policy_hash(POLICY, POLICY_VERSION) == _EXPECTED_HASH # noqa: F821 — defined below in the test file
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `uv run pytest tests/unit/test_gate_policy.py -q`
|
||||
Expected: FAIL with `ModuleNotFoundError: No module named 'fxhnt.application.gate_policy'`.
|
||||
|
||||
- [ ] **Step 3: Implement `gate_policy.py`**
|
||||
|
||||
```python
|
||||
# src/fxhnt/application/gate_policy.py
|
||||
"""The declarative, versioned gate + promotion POLICY (sub-project B1).
|
||||
|
||||
Single source for the gate/promotion DEFAULTS that used to be scattered as module constants in
|
||||
reconciliation_gate.py + inline literals in gate.py. Per-track overrides still live in the registry
|
||||
gate_spec; `resolve_policy` (gate_policy_resolve.py) layers them on top. A policy change = a code commit +
|
||||
a POLICY_VERSION bump; the content-derived `policy_hash` self-check test forces the bump (mirrors sub-project
|
||||
A's definition_version discipline). Every gate verdict is stamped with (POLICY_VERSION, policy_hash) for
|
||||
audit — 'PASSed under policy v3' is queryable."""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from dataclasses import asdict, dataclass
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class GatePolicy:
|
||||
# reconciliation gate
|
||||
min_forward_days: int = 14
|
||||
recon_tolerance_frac: float = 0.5
|
||||
recon_min_band: float = 0.05
|
||||
recon_min_corr: float = 0.0
|
||||
max_gap_days: int = 1
|
||||
# absolute gate
|
||||
min_days: int = 20
|
||||
min_total_return: float = 0.0
|
||||
min_sharpe: float = 0.0
|
||||
# promotion (research -> deploy) correlation backstop
|
||||
promotion_max_corr: float = 0.5
|
||||
promotion_min_overlap_days: int = 10
|
||||
|
||||
|
||||
POLICY = GatePolicy()
|
||||
POLICY_VERSION = 1
|
||||
|
||||
|
||||
def policy_hash(policy: GatePolicy, version: int) -> str:
|
||||
canonical = json.dumps(asdict(policy), sort_keys=True, separators=(",", ":"))
|
||||
return hashlib.sha256(f"{canonical}|v{version}".encode()).hexdigest()
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Pin the expected hash in the test, run to verify it passes**
|
||||
|
||||
Compute the hash once and paste it into the test file:
|
||||
Run: `uv run python -c "from fxhnt.application.gate_policy import POLICY, POLICY_VERSION, policy_hash; print(policy_hash(POLICY, POLICY_VERSION))"`
|
||||
Add `_EXPECTED_HASH = "<the printed value>"` near the top of `tests/unit/test_gate_policy.py`.
|
||||
Run: `uv run pytest tests/unit/test_gate_policy.py -q`
|
||||
Expected: PASS (3 passed).
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/application/gate_policy.py tests/unit/test_gate_policy.py
|
||||
git commit -F - <<'EOF'
|
||||
feat(policy): versioned GatePolicy + content-derived policy_hash (B1)
|
||||
|
||||
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
|
||||
EOF
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 2: `resolve_policy` — layer per-track overrides on the policy defaults
|
||||
|
||||
**Files:**
|
||||
- Create: `src/fxhnt/application/gate_policy_resolve.py`
|
||||
- Test: `tests/unit/test_gate_policy_resolve.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `GatePolicy` (Task 1).
|
||||
- Produces:
|
||||
- `ResolvedPolicy` frozen dataclass with the SAME threshold fields the two gates need: `min_forward_days`, `recon_tolerance_frac`, `recon_min_band`, `recon_min_corr`, `max_gap_days`, `min_days`, `min_total_return`, `min_sharpe`.
|
||||
- `resolve_policy(policy: GatePolicy, gate_spec: dict) -> ResolvedPolicy` — each field = `gate_spec.get(<key>, getattr(policy, <field>))`.
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
```python
|
||||
# tests/unit/test_gate_policy_resolve.py
|
||||
from fxhnt.application.gate_policy import POLICY
|
||||
from fxhnt.application.gate_policy_resolve import resolve_policy
|
||||
|
||||
|
||||
def test_unset_falls_back_to_policy_default():
|
||||
r = resolve_policy(POLICY, {})
|
||||
assert r.min_forward_days == 14 and r.recon_tolerance_frac == 0.5 and r.max_gap_days == 1
|
||||
assert r.min_days == 20 and r.min_sharpe == 0.0
|
||||
|
||||
|
||||
def test_gate_spec_override_wins():
|
||||
r = resolve_policy(POLICY, {"min_forward_days": 21, "max_gap_days": 4, "min_sharpe": 0.5})
|
||||
assert r.min_forward_days == 21 and r.max_gap_days == 4 and r.min_sharpe == 0.5
|
||||
assert r.recon_tolerance_frac == 0.5 # unset field still the default
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `uv run pytest tests/unit/test_gate_policy_resolve.py -q`
|
||||
Expected: FAIL with `ModuleNotFoundError: No module named 'fxhnt.application.gate_policy_resolve'`.
|
||||
|
||||
- [ ] **Step 3: Implement `gate_policy_resolve.py`**
|
||||
|
||||
```python
|
||||
# src/fxhnt/application/gate_policy_resolve.py
|
||||
"""Resolve the effective per-track gate policy: registry gate_spec overrides layered on the GatePolicy
|
||||
defaults at ONE point (no `.get(..., CONSTANT)` scattered across the gate functions). The gate evaluators
|
||||
take the ResolvedPolicy — the single place defaults and per-track overrides combine."""
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from fxhnt.application.gate_policy import GatePolicy
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ResolvedPolicy:
|
||||
min_forward_days: int
|
||||
recon_tolerance_frac: float
|
||||
recon_min_band: float
|
||||
recon_min_corr: float
|
||||
max_gap_days: int
|
||||
min_days: int
|
||||
min_total_return: float
|
||||
min_sharpe: float
|
||||
|
||||
|
||||
def resolve_policy(policy: GatePolicy, gate_spec: dict[str, Any]) -> ResolvedPolicy:
|
||||
return ResolvedPolicy(
|
||||
min_forward_days=int(gate_spec.get("min_forward_days", policy.min_forward_days)),
|
||||
recon_tolerance_frac=float(gate_spec.get("recon_tolerance_frac", policy.recon_tolerance_frac)),
|
||||
recon_min_band=float(gate_spec.get("recon_min_band", policy.recon_min_band)),
|
||||
recon_min_corr=float(gate_spec.get("recon_min_corr", policy.recon_min_corr)),
|
||||
max_gap_days=int(gate_spec.get("max_gap_days", policy.max_gap_days)),
|
||||
min_days=int(gate_spec.get("min_days", policy.min_days)),
|
||||
min_total_return=float(gate_spec.get("min_total_return", policy.min_total_return)),
|
||||
min_sharpe=float(gate_spec.get("min_sharpe", policy.min_sharpe)),
|
||||
)
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run test to verify it passes**
|
||||
|
||||
Run: `uv run pytest tests/unit/test_gate_policy_resolve.py -q`
|
||||
Expected: PASS (2 passed).
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/application/gate_policy_resolve.py tests/unit/test_gate_policy_resolve.py
|
||||
git commit -F - <<'EOF'
|
||||
feat(policy): resolve_policy — per-track overrides on GatePolicy defaults, one resolution point (B1)
|
||||
|
||||
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
|
||||
EOF
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 3: Wire the gate functions to `ResolvedPolicy` (remove scattered constants)
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/application/reconciliation_gate.py` (signature + remove module constants)
|
||||
- Modify: `src/fxhnt/application/gate.py` (signature)
|
||||
- Modify: `src/fxhnt/application/forward_ingest.py:55-61` (`_gate_verdict` resolves once)
|
||||
- Modify: existing gate tests that call these functions (re-point to pass a `ResolvedPolicy`)
|
||||
- Test: `tests/unit/test_gate_policy_regression.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `resolve_policy` (Task 2), `ResolvedPolicy`.
|
||||
- Produces:
|
||||
- `evaluate_reconciliation_gate(summary, rows, backtest, policy: ResolvedPolicy) -> GateVerdict` (was `gate_spec: dict`).
|
||||
- `evaluate_gate(summary, policy: ResolvedPolicy) -> GateVerdict` (was `gate_spec: dict`).
|
||||
|
||||
- [ ] **Step 1: Write the failing regression-anchor test**
|
||||
|
||||
```python
|
||||
# tests/unit/test_gate_policy_regression.py
|
||||
from fxhnt.application.forward_models import ForwardNavRow, ForwardSummary
|
||||
from fxhnt.application.gate_policy import POLICY
|
||||
from fxhnt.application.gate_policy_resolve import resolve_policy
|
||||
from fxhnt.application.reconciliation_gate import BacktestReference, evaluate_reconciliation_gate
|
||||
|
||||
|
||||
def _summary(days, total_return, sid="x"):
|
||||
return ForwardSummary(strategy_id=sid, as_of="2026-07-01", days=days, nav=1.0 + total_return,
|
||||
total_return=total_return, sharpe=1.0, maxdd=-0.01)
|
||||
|
||||
|
||||
def test_recon_gate_takes_resolved_policy_and_verdicts_are_unchanged():
|
||||
pol = resolve_policy(POLICY, {})
|
||||
rows = [ForwardNavRow(strategy_id="x", date=f"2026-06-{d:02d}", ret=0.001, nav=1.0 + 0.001 * i)
|
||||
for i, d in enumerate(range(1, 16), 0)]
|
||||
# 15 days >= 14, forward +1.5% vs expected +1.0% (outperforms → in band), clean → PASS
|
||||
v = evaluate_reconciliation_gate(_summary(15, 0.015), rows, BacktestReference(expected_window_return=0.01), pol)
|
||||
assert v.status == "PASS"
|
||||
# short window → WAIT building
|
||||
v2 = evaluate_reconciliation_gate(_summary(5, 0.01), rows[:5], BacktestReference(expected_window_return=0.01), pol)
|
||||
assert v2.status == "WAIT" and "building" in v2.reason
|
||||
# no backtest ref → WAIT (PASS withheld), unchanged
|
||||
v3 = evaluate_reconciliation_gate(_summary(15, 0.015), rows, None, pol)
|
||||
assert v3.status == "WAIT"
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `uv run pytest tests/unit/test_gate_policy_regression.py -q`
|
||||
Expected: FAIL — `evaluate_reconciliation_gate` still takes `gate_spec` (a `ResolvedPolicy` has no `.get`), so it raises `AttributeError`.
|
||||
|
||||
- [ ] **Step 3: Rewire `reconciliation_gate.py`**
|
||||
|
||||
Remove the module constants `MIN_FORWARD_DAYS`, `RECON_TOLERANCE_FRAC`, `RECON_MIN_BAND`, `RECON_MIN_CORR` (lines 36, 43, 47, 58) and change the function to read from the resolved policy. Replace the signature + the four `gate_spec.get(...)` lines (114-117) and the `max_gap_days` read (126):
|
||||
|
||||
```python
|
||||
from fxhnt.application.gate_policy_resolve import ResolvedPolicy # add to imports
|
||||
|
||||
def evaluate_reconciliation_gate(summary: ForwardSummary, rows: list[ForwardNavRow],
|
||||
backtest: BacktestReference | None,
|
||||
policy: ResolvedPolicy) -> GateVerdict:
|
||||
"""... (keep the docstring; note thresholds now come from the resolved policy) ..."""
|
||||
if summary.days < policy.min_forward_days:
|
||||
return GateVerdict("WAIT", f"building ({summary.days}/{policy.min_forward_days} days)")
|
||||
if _has_gaps(rows, policy.max_gap_days):
|
||||
return GateVerdict("WAIT", "execution gap (missing/duplicate forward day in the record)")
|
||||
if backtest is None:
|
||||
return GateVerdict("WAIT", f"{summary.days}d: no backtest reference to reconcile against — PASS withheld "
|
||||
f"(persist a backtest_summary row for this strategy)")
|
||||
expected = backtest.expected_window_return
|
||||
band = policy.recon_tolerance_frac * max(abs(expected), policy.recon_min_band)
|
||||
shortfall = expected - summary.total_return
|
||||
if shortfall > band:
|
||||
return GateVerdict("WAIT", f"forward diverging from backtest (Δ {100.0 * shortfall:+.2f}% below "
|
||||
f"expected, band ±{100.0 * band:.2f}%)")
|
||||
if backtest.daily_returns is not None:
|
||||
corr = _corr([r.ret for r in rows], backtest.daily_returns)
|
||||
if corr is not None and corr < policy.recon_min_corr:
|
||||
return GateVerdict("WAIT", f"forward diverging from backtest (daily corr {corr:+.2f} < "
|
||||
f"{policy.recon_min_corr:.2f})")
|
||||
return GateVerdict("PASS", f"{summary.days}d: +{summary.total_return_pct:.2f}%, reconciles with backtest, "
|
||||
f"clean execution")
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Rewire `gate.py`**
|
||||
|
||||
Change `evaluate_gate(summary, gate_spec)` to take the resolved policy:
|
||||
|
||||
```python
|
||||
from fxhnt.application.gate_policy_resolve import ResolvedPolicy
|
||||
|
||||
def evaluate_gate(summary: ForwardSummary, policy: ResolvedPolicy) -> GateVerdict:
|
||||
if summary.days < policy.min_days:
|
||||
return GateVerdict("WAIT", f"{summary.days}/{policy.min_days} forward days")
|
||||
if summary.total_return <= policy.min_total_return:
|
||||
return GateVerdict("NO_GO", f"forward return {summary.total_return_pct:.2f}% <= "
|
||||
f"{policy.min_total_return * 100:.2f}%")
|
||||
if summary.sharpe < policy.min_sharpe:
|
||||
return GateVerdict("NO_GO", f"forward Sharpe {summary.sharpe:.2f} < {policy.min_sharpe}")
|
||||
return GateVerdict("GO", f"forward Sharpe {summary.sharpe:.2f}, +{summary.total_return_pct:.2f}%")
|
||||
```
|
||||
|
||||
(Preserve the EXACT reason strings that the current `gate.py` emits — read the current file lines 11-21 and keep them byte-identical, only swapping the threshold source. If the current NO_GO/GO reason wording differs from above, keep the current wording.)
|
||||
|
||||
- [ ] **Step 5: Rewire `_gate_verdict` in `forward_ingest.py`**
|
||||
|
||||
Replace the `MIN_FORWARD_DAYS` import and resolve the policy once per track:
|
||||
|
||||
```python
|
||||
# imports: drop MIN_FORWARD_DAYS; add:
|
||||
from fxhnt.application.gate_policy import POLICY
|
||||
from fxhnt.application.gate_policy_resolve import resolve_policy
|
||||
|
||||
def _gate_verdict(summary, rows, gate_spec, backtest_ref, sid):
|
||||
policy = resolve_policy(POLICY, gate_spec)
|
||||
if gate_spec.get("gate_type") == "reconciliation":
|
||||
ref = backtest_ref.reference(sid, max(summary.days, policy.min_forward_days)) if backtest_ref else None
|
||||
return evaluate_reconciliation_gate(summary, rows, ref, policy)
|
||||
return evaluate_gate(summary, policy)
|
||||
```
|
||||
|
||||
- [ ] **Step 6: Fix the existing gate tests + run**
|
||||
|
||||
Update any existing test that calls `evaluate_reconciliation_gate(..., gate_spec)` or `evaluate_gate(summary, gate_spec)` to pass `resolve_policy(POLICY, gate_spec)` instead (grep: `grep -rn "evaluate_reconciliation_gate\|evaluate_gate\|MIN_FORWARD_DAYS" tests/ src/`). Then run the full suite ONCE:
|
||||
Run: `uv run pytest -q`
|
||||
Expected: PASS (all — the verdicts are byte-identical because `POLICY` == the old constants).
|
||||
|
||||
- [ ] **Step 7: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/application/reconciliation_gate.py src/fxhnt/application/gate.py src/fxhnt/application/forward_ingest.py tests/
|
||||
git commit -F - <<'EOF'
|
||||
refactor(policy): gate functions take ResolvedPolicy; remove scattered gate constants (B1)
|
||||
|
||||
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
|
||||
EOF
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 4: Stamp the verdict — `forward_summary` gains policy_version + policy_hash
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/adapters/persistence/cockpit_models.py` (`ForwardSummaryRow` columns)
|
||||
- Modify: `src/fxhnt/adapters/persistence/forward_nav.py` (`upsert_summary` signature)
|
||||
- Test: `tests/integration/test_forward_summary_policy_stamp.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Produces:
|
||||
- `ForwardSummaryRow.policy_version: Mapped[int | None]`, `ForwardSummaryRow.policy_hash: Mapped[str | None]`.
|
||||
- `ForwardNavRepo.upsert_summary(sm, gate_status, gate_reason, at, *, policy_version=None, policy_hash=None)`.
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
```python
|
||||
# tests/integration/test_forward_summary_policy_stamp.py
|
||||
import datetime as dt
|
||||
|
||||
from fxhnt.adapters.persistence.forward_nav import ForwardNavRepo
|
||||
from fxhnt.application.forward_models import ForwardSummary
|
||||
|
||||
|
||||
def test_upsert_summary_stamps_policy_version_and_hash(tmp_path):
|
||||
repo = ForwardNavRepo(f"sqlite:///{tmp_path}/s.db"); repo.migrate()
|
||||
at = dt.datetime(2026, 7, 9, 23, 30)
|
||||
sm = ForwardSummary(strategy_id="x", as_of="2026-07-01", days=15, nav=1.02, total_return=0.02,
|
||||
sharpe=1.0, maxdd=-0.01)
|
||||
repo.upsert_summary(sm, "PASS", "ok", at, policy_version=3, policy_hash="abc123")
|
||||
row = {s.strategy_id: s for s in repo.all_summaries()}["x"]
|
||||
assert row.policy_version == 3 and row.policy_hash == "abc123"
|
||||
|
||||
|
||||
def test_upsert_summary_stamp_defaults_none(tmp_path):
|
||||
repo = ForwardNavRepo(f"sqlite:///{tmp_path}/s2.db"); repo.migrate()
|
||||
at = dt.datetime(2026, 7, 9, 23, 30)
|
||||
sm = ForwardSummary(strategy_id="y", as_of="2026-07-01", days=1, nav=1.0, total_return=0.0,
|
||||
sharpe=0.0, maxdd=0.0)
|
||||
repo.upsert_summary(sm, "WAIT", "building", at) # no stamp passed → columns nullable
|
||||
row = {s.strategy_id: s for s in repo.all_summaries()}["y"]
|
||||
assert row.policy_version is None and row.policy_hash is None
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `uv run pytest tests/integration/test_forward_summary_policy_stamp.py -q`
|
||||
Expected: FAIL with `TypeError: upsert_summary() got an unexpected keyword argument 'policy_version'`.
|
||||
|
||||
- [ ] **Step 3: Add the columns**
|
||||
|
||||
In `cockpit_models.py`, add to `ForwardSummaryRow` (after `gate_reason`):
|
||||
|
||||
```python
|
||||
policy_version: Mapped[int | None] = mapped_column(Integer, nullable=True, default=None)
|
||||
policy_hash: Mapped[str | None] = mapped_column(String(64), nullable=True, default=None)
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Stamp them in `upsert_summary`**
|
||||
|
||||
In `forward_nav.py`, change `upsert_summary`:
|
||||
|
||||
```python
|
||||
def upsert_summary(self, sm: ForwardSummary, gate_status: str, gate_reason: str, at: dt.datetime,
|
||||
*, policy_version: int | None = None, policy_hash: str | None = None) -> None:
|
||||
with Session(self._engine) as s:
|
||||
s.merge(ForwardSummaryRow(
|
||||
strategy_id=sm.strategy_id, as_of=sm.as_of, days=sm.days, nav=sm.nav,
|
||||
total_return=sm.total_return, sharpe=sm.sharpe, maxdd=sm.maxdd,
|
||||
gate_status=gate_status, gate_reason=gate_reason, src_updated_at=at,
|
||||
policy_version=policy_version, policy_hash=policy_hash))
|
||||
s.commit()
|
||||
```
|
||||
|
||||
Note the prod DB already has a `forward_summary` table; the two nullable columns need an additive ALTER on Postgres. If `ForwardNavRepo.migrate()` uses only `create_all` (which does NOT add columns to an existing table), add a guarded `ALTER TABLE forward_summary ADD COLUMN IF NOT EXISTS policy_version INTEGER` / `... policy_hash VARCHAR(64)` in `migrate()` for the Postgres branch (mirror the existing hypertable-DDL pattern). Read `migrate()` first; if it already handles additive columns elsewhere, follow that pattern. On SQLite (tests) `create_all` creates the full table, so tests pass without the ALTER.
|
||||
|
||||
- [ ] **Step 5: Run tests to verify they pass**
|
||||
|
||||
Run: `uv run pytest tests/integration/test_forward_summary_policy_stamp.py -q`
|
||||
Expected: PASS (2 passed).
|
||||
|
||||
- [ ] **Step 6: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/adapters/persistence/cockpit_models.py src/fxhnt/adapters/persistence/forward_nav.py tests/integration/test_forward_summary_policy_stamp.py
|
||||
git commit -F - <<'EOF'
|
||||
feat(policy): stamp forward_summary with policy_version + policy_hash (B1 audit)
|
||||
|
||||
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
|
||||
EOF
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 5: `forward_ingest` stamps the policy on every verdict
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/application/forward_ingest.py` (`ForwardIngestService.ingest`)
|
||||
- Test: `tests/integration/test_forward_ingest.py` (add a stamp assertion)
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `POLICY`, `POLICY_VERSION`, `policy_hash` (Task 1); `upsert_summary(..., policy_version, policy_hash)` (Task 4).
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
```python
|
||||
# add to tests/integration/test_forward_ingest.py
|
||||
def test_ingest_stamps_policy_on_verdict(tmp_path):
|
||||
import datetime as dt
|
||||
from fxhnt.adapters.persistence.forward_nav import ForwardNavRepo
|
||||
from fxhnt.application.forward_ingest import ForwardIngestService
|
||||
from fxhnt.application.forward_models import ForwardNavRow as NavRowDTO, ForwardSummary
|
||||
from fxhnt.application.gate_policy import POLICY, POLICY_VERSION, policy_hash
|
||||
repo = ForwardNavRepo(f"sqlite:///{tmp_path}/st.db"); repo.migrate()
|
||||
at = dt.datetime(2026, 7, 9, 23, 30)
|
||||
sid = "crypto_tstrend"
|
||||
repo.upsert_rows([NavRowDTO(strategy_id=sid, date="2026-06-01", ret=0.001, nav=1.001)], at=at)
|
||||
repo.upsert_summary(ForwardSummary(strategy_id=sid, as_of="2026-06-01", days=1, nav=1.001,
|
||||
total_return=0.001, sharpe=0.0, maxdd=0.0),
|
||||
"WAIT", "building", at)
|
||||
ForwardIngestService(repo).ingest(at=at)
|
||||
row = {s.strategy_id: s for s in repo.all_summaries()}[sid]
|
||||
assert row.policy_version == POLICY_VERSION
|
||||
assert row.policy_hash == policy_hash(POLICY, POLICY_VERSION)
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `uv run pytest tests/integration/test_forward_ingest.py::test_ingest_stamps_policy_on_verdict -q`
|
||||
Expected: FAIL — `row.policy_version` is `None` (ingest doesn't stamp yet).
|
||||
|
||||
- [ ] **Step 3: Stamp in `ingest`**
|
||||
|
||||
In `forward_ingest.py`, add the imports and stamp the `upsert_summary` call inside `ingest`'s per-tracker try block:
|
||||
|
||||
```python
|
||||
from fxhnt.application.gate_policy import POLICY, POLICY_VERSION, policy_hash
|
||||
# ... inside ingest(), replace the upsert_summary call:
|
||||
self._repo.upsert_summary(summary, verdict.status, verdict.reason, at,
|
||||
policy_version=POLICY_VERSION,
|
||||
policy_hash=policy_hash(POLICY, POLICY_VERSION))
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run tests to verify they pass**
|
||||
|
||||
Run: `uv run pytest tests/integration/test_forward_ingest.py -q`
|
||||
Expected: PASS.
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/application/forward_ingest.py tests/integration/test_forward_ingest.py
|
||||
git commit -F - <<'EOF'
|
||||
feat(policy): ingest stamps POLICY_VERSION + policy_hash on every gate verdict (B1)
|
||||
|
||||
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
|
||||
EOF
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 6: Promotion correlation backstop
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/adapters/persistence/forward_nav.py` (add a deploy-book returns helper if needed)
|
||||
- Modify: `src/fxhnt/application/forward_ingest.py` (`evaluate_promotions`)
|
||||
- Create: `src/fxhnt/application/promotion_corr.py` (pure corr helpers)
|
||||
- Test: `tests/integration/test_promotion_corr.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `ForwardNavRepo.nav_history(sid)`, `ForwardNavRepo.promoted_strategy_ids()`, `STRATEGY_REGISTRY` (for `tier`, `promote_on_pass`), `POLICY.promotion_max_corr`, `POLICY.promotion_min_overlap_days`.
|
||||
- Produces:
|
||||
- `promotion_corr.deploy_book_returns(returns_by_sid: dict[str, dict[str, float]]) -> dict[str, float]` — equal-weight per-date average of the given deploy tracks' daily returns (keyed ISO date).
|
||||
- `promotion_corr.corr_to_book(candidate: dict[str, float], book: dict[str, float], min_overlap: int) -> float | None` — Pearson over the overlapping dates; `None` if fewer than `min_overlap` overlapping days.
|
||||
- `promotion_corr.passes_backstop(corr: float | None, book_empty: bool, max_corr: float) -> bool` — `True` if the book is empty (first edge); else `corr is not None and corr < max_corr` (indeterminate corr → `False` = withhold).
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
```python
|
||||
# tests/integration/test_promotion_corr.py
|
||||
from fxhnt.application.promotion_corr import corr_to_book, deploy_book_returns, passes_backstop
|
||||
|
||||
|
||||
def test_deploy_book_is_equal_weight_per_date_average():
|
||||
book = deploy_book_returns({"a": {"2026-06-01": 0.02, "2026-06-02": 0.00},
|
||||
"b": {"2026-06-01": 0.00, "2026-06-02": 0.04}})
|
||||
assert book == {"2026-06-01": 0.01, "2026-06-02": 0.02}
|
||||
|
||||
|
||||
def test_corr_needs_min_overlap():
|
||||
cand = {"2026-06-01": 0.01, "2026-06-02": -0.01}
|
||||
book = {"2026-06-01": 0.01, "2026-06-02": -0.01}
|
||||
assert corr_to_book(cand, book, min_overlap=5) is None # only 2 overlapping < 5
|
||||
assert abs(corr_to_book(cand, book, min_overlap=2) - 1.0) < 1e-9 # perfectly correlated
|
||||
|
||||
|
||||
def test_backstop_rules():
|
||||
assert passes_backstop(None, book_empty=True, max_corr=0.5) is True # first edge → pass
|
||||
assert passes_backstop(None, book_empty=False, max_corr=0.5) is False # indeterminate → withhold
|
||||
assert passes_backstop(0.30, book_empty=False, max_corr=0.5) is True # low corr → pass
|
||||
assert passes_backstop(0.60, book_empty=False, max_corr=0.5) is False # high corr → withhold
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `uv run pytest tests/integration/test_promotion_corr.py -q`
|
||||
Expected: FAIL with `ModuleNotFoundError: No module named 'fxhnt.application.promotion_corr'`.
|
||||
|
||||
- [ ] **Step 3: Implement `promotion_corr.py`**
|
||||
|
||||
```python
|
||||
# src/fxhnt/application/promotion_corr.py
|
||||
"""Pure helpers for the promotion correlation backstop (B1): build the equal-weight deploy-book daily
|
||||
return series and correlate a promotion candidate against it. A candidate promotes only if its daily-return
|
||||
correlation to the current deploy book is below the policy threshold (or the book is empty — the first edge
|
||||
trivially diversifies). An indeterminate correlation (too little overlap) withholds promotion — the safe,
|
||||
capital-adjacent error."""
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
|
||||
|
||||
def deploy_book_returns(returns_by_sid: dict[str, dict[str, float]]) -> dict[str, float]:
|
||||
per_date: dict[str, list[float]] = {}
|
||||
for series in returns_by_sid.values():
|
||||
for d, r in series.items():
|
||||
per_date.setdefault(d, []).append(r)
|
||||
return {d: sum(v) / len(v) for d, v in per_date.items()}
|
||||
|
||||
|
||||
def corr_to_book(candidate: dict[str, float], book: dict[str, float], min_overlap: int) -> float | None:
|
||||
dates = sorted(set(candidate) & set(book))
|
||||
if len(dates) < min_overlap:
|
||||
return None
|
||||
a = [candidate[d] for d in dates]
|
||||
b = [book[d] for d in dates]
|
||||
n = len(dates)
|
||||
ma, mb = sum(a) / n, sum(b) / n
|
||||
cov = sum((x - ma) * (y - mb) for x, y in zip(a, b))
|
||||
va = sum((x - ma) ** 2 for x in a)
|
||||
vb = sum((y - mb) ** 2 for y in b)
|
||||
if va <= 0.0 or vb <= 0.0:
|
||||
return None
|
||||
return cov / math.sqrt(va * vb)
|
||||
|
||||
|
||||
def passes_backstop(corr: float | None, *, book_empty: bool, max_corr: float) -> bool:
|
||||
if book_empty:
|
||||
return True
|
||||
return corr is not None and corr < max_corr
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run tests to verify they pass**
|
||||
|
||||
Run: `uv run pytest tests/integration/test_promotion_corr.py -q`
|
||||
Expected: PASS (3 passed). (Note: `passes_backstop` uses a keyword-only `book_empty` — update the test call to `passes_backstop(None, book_empty=True, max_corr=0.5)` as written.)
|
||||
|
||||
- [ ] **Step 5: Wire the backstop into `evaluate_promotions`**
|
||||
|
||||
In `forward_ingest.py`, extend `evaluate_promotions` so a `promote_on_pass` candidate at PASS is only promoted when the backstop passes. Read the current `evaluate_promotions` first, then:
|
||||
|
||||
```python
|
||||
from fxhnt.application.gate_policy import POLICY
|
||||
from fxhnt.application.promotion_corr import corr_to_book, deploy_book_returns, passes_backstop
|
||||
from fxhnt.registry import STRATEGY_REGISTRY
|
||||
|
||||
def evaluate_promotions(dsn: str) -> list[str]:
|
||||
repo = ForwardNavRepo(dsn); repo.migrate()
|
||||
gate = {r.strategy_id: r.gate_status for r in repo.all_summaries()}
|
||||
already = repo.promoted_strategy_ids()
|
||||
# the current deploy book = registry deploy-tier tracks + already-promoted tracks (effective deploy set),
|
||||
# each as its {date: ret} series from A's record.
|
||||
deploy_sids = {sid for sid, m in STRATEGY_REGISTRY.items() if m.get("tier") == "deploy"} | already
|
||||
def _series(sid: str) -> dict[str, float]:
|
||||
return {r.date: r.ret for r in repo.nav_history(sid)}
|
||||
newly: list[str] = []
|
||||
for sid, meta in STRATEGY_REGISTRY.items():
|
||||
if not (meta.get("promote_on_pass") and sid not in already and gate.get(sid) == "PASS"):
|
||||
continue
|
||||
book_sids = deploy_sids - {sid} # never correlate a candidate against itself
|
||||
book = deploy_book_returns({s: _series(s) for s in book_sids}) if book_sids else {}
|
||||
corr = corr_to_book(_series(sid), book, POLICY.promotion_min_overlap_days) if book else None
|
||||
if not passes_backstop(corr, book_empty=not book, max_corr=POLICY.promotion_max_corr):
|
||||
_log.info("evaluate_promotions: %s gate=PASS but corr backstop failed (corr=%s, max=%.2f) "
|
||||
"→ promotion withheld", sid, f"{corr:.2f}" if corr is not None else "n/a",
|
||||
POLICY.promotion_max_corr)
|
||||
continue
|
||||
repo.promote(sid, dt.datetime.now(dt.timezone.utc).replace(tzinfo=None))
|
||||
newly.append(sid)
|
||||
_log.info("evaluate_promotions: %s cleared gate + corr backstop → PROMOTED research→deploy", sid)
|
||||
return newly
|
||||
```
|
||||
|
||||
- [ ] **Step 6: Add an integration test for the backstop end to end + run**
|
||||
|
||||
```python
|
||||
# add to tests/integration/test_promotion.py (or the promotion test file)
|
||||
def test_promotion_withheld_when_correlated_and_fires_when_diversifying(tmp_path, monkeypatch):
|
||||
import datetime as dt
|
||||
from fxhnt.adapters.persistence.forward_nav import ForwardNavRepo
|
||||
from fxhnt.application.forward_ingest import evaluate_promotions
|
||||
from fxhnt.application.forward_models import ForwardNavRow as NavRowDTO, ForwardSummary
|
||||
from fxhnt import registry
|
||||
dsn = f"sqlite:///{tmp_path}/promo.db"
|
||||
repo = ForwardNavRepo(dsn); repo.migrate()
|
||||
at = dt.datetime(2026, 7, 9, 23, 30)
|
||||
# a deploy-tier book track "d" and a PASS candidate "c" flagged promote_on_pass
|
||||
monkeypatch.setitem(registry.STRATEGY_REGISTRY, "d",
|
||||
{"display_name": "D", "sleeve": "x", "gate_spec": {}, "tier": "deploy"})
|
||||
monkeypatch.setitem(registry.STRATEGY_REGISTRY, "c",
|
||||
{"display_name": "C", "sleeve": "x", "gate_spec": {}, "tier": "research",
|
||||
"promote_on_pass": True})
|
||||
dates = [f"2026-06-{d:02d}" for d in range(1, 21)]
|
||||
dret = [0.01 if i % 2 else -0.01 for i in range(20)]
|
||||
for d, r in zip(dates, dret):
|
||||
repo.upsert_rows([NavRowDTO(strategy_id="d", date=d, ret=r, nav=1.0)], at=at)
|
||||
repo.upsert_summary(ForwardSummary(strategy_id="c", as_of=dates[-1], days=20, nav=1.0,
|
||||
total_return=0.0, sharpe=0.0, maxdd=0.0), "PASS", "ok", at)
|
||||
# candidate PERFECTLY correlated with the book → withheld
|
||||
for d, r in zip(dates, dret):
|
||||
repo.upsert_rows([NavRowDTO(strategy_id="c", date=d, ret=r, nav=1.0)], at=at)
|
||||
assert evaluate_promotions(dsn) == []
|
||||
# candidate ANTI-correlated (diversifying) → promoted
|
||||
for d, r in zip(dates, dret):
|
||||
repo.upsert_rows([NavRowDTO(strategy_id="c", date=d, ret=-r, nav=1.0)], at=at)
|
||||
assert evaluate_promotions(dsn) == ["c"]
|
||||
```
|
||||
|
||||
Run: `uv run pytest tests/integration/test_promotion_corr.py tests/integration/test_promotion.py -q`
|
||||
Expected: PASS.
|
||||
|
||||
- [ ] **Step 7: Full suite + commit**
|
||||
|
||||
Run: `uv run pytest -q`
|
||||
Expected: PASS (all).
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/application/promotion_corr.py src/fxhnt/application/forward_ingest.py tests/integration/test_promotion_corr.py tests/integration/test_promotion.py
|
||||
git commit -F - <<'EOF'
|
||||
feat(policy): promotion correlation backstop — human flag + data-confirmed diversification (B1)
|
||||
|
||||
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
|
||||
EOF
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Self-Review notes
|
||||
|
||||
- **Spec coverage:** policy object (§1)→T1; resolution (§2)→T2; gate wiring / remove scatter (§1-2)→T3; verdict stamp (§3)→T4+T5; promotion backstop (§4)→T6; byte-identical verdicts (error-handling)→T3 regression anchor; hash self-check→T1; WAIT-safe preserved (verdict logic unchanged)→T3.
|
||||
- **`promotion_min_overlap_days`** (the spec's open parameter) is a `GatePolicy` field (=10), resolved in T6.
|
||||
- **Deploy set** = registry `tier=="deploy"` ∪ `promoted_strategy_ids()`, minus the candidate itself (T6) — the effective deploy set A's dashboard already uses.
|
||||
- **Type consistency:** `resolve_policy(POLICY, gate_spec) -> ResolvedPolicy` is the single resolution; both gates + `_gate_verdict` consume it. `upsert_summary(..., policy_version, policy_hash)` matches the stamp in T5. `deploy_book_returns`/`corr_to_book`/`passes_backstop` signatures match T6's caller.
|
||||
- **No capital:** T6 promotes to deploy-tier (tracking) only; no allocation/capital (B2).
|
||||
@@ -0,0 +1,440 @@
|
||||
# Execution Engine — Alpaca/multistrat Loop (Sub-project C1) Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Make `execute-multistrat` B2a-driven — deploy exactly `strategy_allocation[multistrat]` dollars into the multistrat book on Alpaca (flatten to cash on $0/absent), reconcile + place via the existing `ExecutionService`, and record the executed book's daily (envelope-basis, flow-excluded) return as an observed `forward_record` for a new `multistrat_exec` track via A's `run_track`.
|
||||
|
||||
**Architecture:** Pure helpers (`scaled_weights` bridge, `compute_exec_return`, `build_pos_state`) + `record_exec_track` (which reuses A's observed `run_track` to book the exec return + carries the position-state in A's book-state under `multistrat_exec_pos`). A registry `multistrat_exec` observed track. The CLI reads B2a's envelope, scales the book weights to it, flattens on zero, and on `--execute` records the exec track. No new execution engine — reuse `ExecutionService`/`AlpacaBroker`/A's machinery.
|
||||
|
||||
**Tech Stack:** Python 3.12, Typer CLI, SQLAlchemy 2.0 (sqlite tests / Postgres prod), pytest (`uv run pytest`), ruff (E,F,I,UP,B,SIM; 120).
|
||||
|
||||
## Global Constraints
|
||||
|
||||
- Spec: `docs/superpowers/specs/2026-07-09-execution-engine-alpaca-multistrat-c1-design.md` (authoritative).
|
||||
- Builds on A (`run_track`, `get_book_state`/`set_book_state`/`append_record`, the observed machinery), B1 (gate), B2a (`ForwardNavRepo.current_allocation()` → `{strategy_id: target_dollars}`). All merged.
|
||||
- REAL MONEY (paper for now): the engine reconciles, never authorizes capital. Deploy exactly B2a's `multistrat` envelope = `min(target_dollars, account_NLV)`; **$0 / absent row → flatten to cash** (deploy nothing). Real capital = the existing `--live` + `allow_live` gate (unchanged). Dry-run is the default (`--execute` places).
|
||||
- LOAD-BEARING: the executed return is measured on B2a's **envelope** (not the full account NLV), and EXCLUDES re-allocation capital flows (time-weighted: the P&L of the HELD positions since the last rebalance). Tests pin both.
|
||||
- Reuse `ExecutionService.rebalance_weights` + `AlpacaBroker` + A's `run_track`/book-state — do NOT build a new execution engine or a new return-accumulation store.
|
||||
- Determinism: the return/bridge helpers are pure. Never save tests to the repo root. Commit after every task; messages end with the `Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>` trailer.
|
||||
|
||||
---
|
||||
|
||||
### Task 1: Pure helpers — the bridge, the exec return, the position-state
|
||||
|
||||
**Files:**
|
||||
- Create: `src/fxhnt/application/multistrat_exec_record.py`
|
||||
- Test: `tests/unit/test_multistrat_exec_helpers.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Produces:
|
||||
- `scaled_weights(within_weights: dict[str,float], envelope: float, nlv: float) -> dict[str,float]` — the bridge: `within_i · envelope/nlv` so `ExecutionService`'s `weight·NLV` sizing lands on `within_i·envelope`; all-zero when `envelope<=0` or `nlv<=0` (flatten).
|
||||
- `build_pos_state(positions: dict[str,float], prices: dict[str,float], envelope: float) -> dict` — the carried position-state for the next run's return.
|
||||
- `compute_exec_return(pos_prior: dict, today_prices: dict[str,float]) -> float | None` — the time-weighted, envelope-basis daily return of the PRIOR book at today's prices; `None` on the first run (no prior).
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
```python
|
||||
# tests/unit/test_multistrat_exec_helpers.py
|
||||
from fxhnt.application.multistrat_exec_record import (
|
||||
build_pos_state, compute_exec_return, scaled_weights)
|
||||
|
||||
|
||||
def test_scaled_weights_bridge_and_flatten():
|
||||
w = {"SPY": 0.6, "IEF": 0.4}
|
||||
# envelope 10k in a 100k account → scale 0.1 → weights sum to envelope/NLV
|
||||
s = scaled_weights(w, envelope=10_000.0, nlv=100_000.0)
|
||||
assert abs(s["SPY"] - 0.06) < 1e-12 and abs(s["IEF"] - 0.04) < 1e-12
|
||||
assert scaled_weights(w, 0.0, 100_000.0) == {"SPY": 0.0, "IEF": 0.0} # $0 envelope → flatten
|
||||
assert scaled_weights(w, 10_000.0, 0.0) == {"SPY": 0.0, "IEF": 0.0} # no NLV → flatten (safe)
|
||||
|
||||
|
||||
def test_compute_exec_return_first_run_is_none():
|
||||
assert compute_exec_return({}, {"SPY": 500.0}) is None # no prior book → inception
|
||||
|
||||
|
||||
def test_exec_return_is_envelope_basis_not_full_nlv():
|
||||
# prior book: $10k envelope fully deployed in 20 SPY @ 500 (=$10k), in a (hypothetical) large account.
|
||||
prior = build_pos_state({"SPY": 20.0}, {"SPY": 500.0}, envelope=10_000.0)
|
||||
# SPY +10% today → book +$1000 on the $10k envelope → 10% return (NOT diluted by any idle account cash)
|
||||
assert abs(compute_exec_return(prior, {"SPY": 550.0}) - 0.10) < 1e-9
|
||||
|
||||
|
||||
def test_reallocation_capital_flow_is_not_return():
|
||||
# flat prices; the envelope will grow at the NEXT rebalance (a flow) — the RETURN of the prior book is ~0
|
||||
prior = build_pos_state({"SPY": 20.0}, {"SPY": 500.0}, envelope=10_000.0)
|
||||
assert abs(compute_exec_return(prior, {"SPY": 500.0})) < 1e-12 # flat prices → 0%, not a jump
|
||||
|
||||
|
||||
def test_partial_deployment_reserve_earns_zero():
|
||||
# $10k envelope, only $6k deployed (12 SPY @ 500), $4k reserve. SPY +10% → +$600 on $10k = 6%.
|
||||
prior = build_pos_state({"SPY": 12.0}, {"SPY": 500.0}, envelope=10_000.0)
|
||||
assert abs(compute_exec_return(prior, {"SPY": 550.0}) - 0.06) < 1e-9
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `uv run pytest tests/unit/test_multistrat_exec_helpers.py -q`
|
||||
Expected: FAIL with `ModuleNotFoundError: No module named 'fxhnt.application.multistrat_exec_record'`.
|
||||
|
||||
- [ ] **Step 3: Implement the helpers**
|
||||
|
||||
```python
|
||||
# src/fxhnt/application/multistrat_exec_record.py
|
||||
"""Sub-project C1 — record the EXECUTED multistrat track and bridge B2a's dollar allocation to the venue.
|
||||
|
||||
`scaled_weights` turns the multistrat book's within-book weights into weights that ExecutionService sizes to
|
||||
B2a's envelope (not the full account). `compute_exec_return` measures the executed book's daily return on
|
||||
B2a's ENVELOPE (not the full NLV) and as the P&L of the HELD positions since the last rebalance (a
|
||||
re-allocation is a capital flow, not a gain). `record_exec_track` books that return as an observed
|
||||
forward_record via A's run_track. No new execution engine, no new return store."""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
|
||||
def scaled_weights(within_weights: dict[str, float], envelope: float, nlv: float) -> dict[str, float]:
|
||||
if envelope <= 0.0 or nlv <= 0.0:
|
||||
return {s: 0.0 for s in within_weights} # flatten to cash (B2a $0 / no NLV)
|
||||
scale = envelope / nlv
|
||||
return {s: w * scale for s, w in within_weights.items()}
|
||||
|
||||
|
||||
def build_pos_state(positions: dict[str, float], prices: dict[str, float], envelope: float) -> dict[str, Any]:
|
||||
return {"positions": {s: float(q) for s, q in positions.items()},
|
||||
"prices": {s: float(p) for s, p in prices.items()}, "envelope": float(envelope)}
|
||||
|
||||
|
||||
def compute_exec_return(pos_prior: dict[str, Any], today_prices: dict[str, float]) -> float | None:
|
||||
envelope = float(pos_prior.get("envelope", 0.0))
|
||||
if not pos_prior or envelope <= 0.0:
|
||||
return None # first run / no prior book → inception
|
||||
positions = pos_prior.get("positions", {})
|
||||
prior_prices = pos_prior.get("prices", {})
|
||||
val_prior = sum(q * prior_prices[s] for s, q in positions.items())
|
||||
val_today = sum(q * today_prices.get(s, prior_prices[s]) for s, q in positions.items())
|
||||
reserve = envelope - val_prior # un-deployed envelope earns 0
|
||||
exec_equity_today = val_today + reserve
|
||||
return exec_equity_today / envelope - 1.0
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run tests → PASS + ruff, then commit**
|
||||
|
||||
Run: `uv run pytest tests/unit/test_multistrat_exec_helpers.py -q` → PASS (5 passed).
|
||||
Run: `uv run ruff check src/fxhnt/application/multistrat_exec_record.py tests/unit/test_multistrat_exec_helpers.py`
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/application/multistrat_exec_record.py tests/unit/test_multistrat_exec_helpers.py
|
||||
git commit -F - <<'EOF'
|
||||
feat(exec): pure C1 helpers — B2a-envelope bridge + envelope-basis flow-excluded exec return
|
||||
|
||||
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
|
||||
EOF
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 2: Registry — the `multistrat_exec` observed track
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/registry.py` (add `multistrat_exec`)
|
||||
- Modify: `tests/unit/test_forward_definition.py` (the observed-set assertion)
|
||||
- Test: `tests/unit/test_registry_multistrat_exec.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: `STRATEGY_REGISTRY["multistrat_exec"]` with `record_mode == "observed"`, venue `alpaca-paper`, a reconciliation gate_spec, and a `definition`.
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
```python
|
||||
# tests/unit/test_registry_multistrat_exec.py
|
||||
from fxhnt.application.forward_definition import track_definition
|
||||
from fxhnt.registry import STRATEGY_REGISTRY
|
||||
|
||||
|
||||
def test_multistrat_exec_is_observed_track():
|
||||
e = STRATEGY_REGISTRY["multistrat_exec"]
|
||||
assert e["venue"] == "alpaca-paper"
|
||||
_params, _version, mode = track_definition("multistrat_exec")
|
||||
assert mode == "observed" # executed fills are observed, not recomputable
|
||||
assert e["gate_spec"].get("gate_type") == "reconciliation"
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `uv run pytest tests/unit/test_registry_multistrat_exec.py -q`
|
||||
Expected: FAIL with `KeyError: 'multistrat_exec'`.
|
||||
|
||||
- [ ] **Step 3: Add the registry entry**
|
||||
|
||||
In `src/fxhnt/registry.py`, add (after the `multistrat` entry):
|
||||
|
||||
```python
|
||||
# The EXECUTED multistrat book (sub-project C1): the real Alpaca-paper track — its daily return is the
|
||||
# executed book's envelope-basis P&L (fills + slippage + fees), recorded by `fxhnt execute-multistrat
|
||||
# --execute` (NOT a nightly nav asset). OBSERVED (fills are observed, not recomputable). Its gate
|
||||
# reconciles the executed track against the MODELED multistrat (execution decay shows as divergence).
|
||||
"multistrat_exec": {
|
||||
"display_name": "Multi-strat book (EXECUTED, Alpaca paper)", "sleeve": "tradfi-multistrat",
|
||||
"venue": "alpaca-paper", "state_file": "multistrat_exec_state",
|
||||
"tier": "research", "execution": "paper",
|
||||
"gate_spec": {"gate_type": "reconciliation", "min_forward_days": 20, "max_gap_days": 4},
|
||||
"definition": {"params": {"source": "alpaca-executed-fills", "models": "multistrat"},
|
||||
"version": 1, "record_mode": "observed"},
|
||||
},
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Update the observed-set assertion**
|
||||
|
||||
In `tests/unit/test_forward_definition.py`, the test asserting the observed set (it was `{"xsfunding", "combined"}`) must now include `multistrat_exec`. Find that test and change the expected set to `{"xsfunding", "combined", "multistrat_exec"}` (keep the assertion, update the value).
|
||||
|
||||
- [ ] **Step 5: Run tests → PASS + ruff, then commit**
|
||||
|
||||
Run: `uv run pytest tests/unit/test_registry_multistrat_exec.py tests/unit/test_forward_definition.py -q` → PASS.
|
||||
Run: `uv run ruff check src/fxhnt/registry.py tests/unit/test_registry_multistrat_exec.py`
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/registry.py tests/unit/test_registry_multistrat_exec.py tests/unit/test_forward_definition.py
|
||||
git commit -F - <<'EOF'
|
||||
feat(exec): register multistrat_exec as an observed Alpaca-paper track (C1)
|
||||
|
||||
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
|
||||
EOF
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 3: `record_exec_track` — book the exec return via A's `run_track` + carry the position-state
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/application/multistrat_exec_record.py` (add `record_exec_track` + `_ExecBookingStrategy`)
|
||||
- Test: `tests/integration/test_record_exec_track.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `run_track` (A, `fxhnt.application.forward_engine`), `ForwardNavRepo.get_book_state`/`set_book_state`/`record_series`/`nav_history` (A), the `multistrat_exec` registry track (Task 2), Task-1 helpers.
|
||||
- Produces:
|
||||
- `record_exec_track(repo, exec_return, new_positions, today_prices, envelope, today, at) -> None` — books `exec_return` as an observed `forward_record` for `multistrat_exec` via `run_track`, and stores the next position-state under book-state key `"multistrat_exec_pos"`.
|
||||
- `_ExecBookingStrategy(exec_return)` — a `ForwardStrategy` whose `advance(None, extra)` returns `[(extra["_today"], exec_return)]` (or `[]` when `exec_return is None`).
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
```python
|
||||
# tests/integration/test_record_exec_track.py
|
||||
import datetime as dt
|
||||
|
||||
from fxhnt.adapters.persistence.forward_nav import ForwardNavRepo
|
||||
from fxhnt.application.multistrat_exec_record import record_exec_track
|
||||
|
||||
|
||||
def _repo(tmp_path):
|
||||
repo = ForwardNavRepo(f"sqlite:///{tmp_path}/x.db"); repo.migrate()
|
||||
return repo
|
||||
|
||||
|
||||
def test_first_run_inceptions_no_return_then_books_second_day(tmp_path):
|
||||
repo = _repo(tmp_path)
|
||||
at = dt.datetime(2026, 7, 9, 23, 30)
|
||||
# run 1: exec_return None (inception) — stores position-state, books no return
|
||||
record_exec_track(repo, exec_return=None, new_positions={"SPY": 20.0},
|
||||
today_prices={"SPY": 500.0}, envelope=10_000.0, today="2026-07-09", at=at)
|
||||
assert repo.record_series("multistrat_exec") == [] # inception: nothing booked
|
||||
assert repo.get_book_state("multistrat_exec_pos")["envelope"] == 10_000.0
|
||||
# run 2: a +2% executed day
|
||||
record_exec_track(repo, exec_return=0.02, new_positions={"SPY": 20.4},
|
||||
today_prices={"SPY": 510.0}, envelope=10_200.0, today="2026-07-10", at=at)
|
||||
assert repo.record_series("multistrat_exec") == [("2026-07-10", 0.02)] # observed return booked
|
||||
assert [r.date for r in repo.nav_history("multistrat_exec")] == ["2026-07-10"] # nav derived (via run_track)
|
||||
assert repo.get_book_state("multistrat_exec_pos")["positions"] == {"SPY": 20.4} # state advanced
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `uv run pytest tests/integration/test_record_exec_track.py -q`
|
||||
Expected: FAIL with `ImportError: cannot import name 'record_exec_track'`.
|
||||
|
||||
- [ ] **Step 3: Implement `record_exec_track`**
|
||||
|
||||
Append to `src/fxhnt/application/multistrat_exec_record.py`:
|
||||
|
||||
```python
|
||||
import datetime as _dt
|
||||
|
||||
|
||||
class _ExecBookingStrategy:
|
||||
"""A trivial observed ForwardStrategy for run_track: books today's pre-computed executed return (or
|
||||
nothing on the inception run). The return is computed by the CLI (it has the live prices/positions); this
|
||||
just hands it to A's observed engine."""
|
||||
def __init__(self, exec_return: float | None) -> None:
|
||||
self._r = exec_return
|
||||
|
||||
def advance(self, last_date: str | None, extra: dict[str, Any]) -> tuple[list[tuple[str, float]], dict[str, Any]]:
|
||||
if self._r is None:
|
||||
return [], extra
|
||||
return [(str(extra.get("_today")), float(self._r))], extra
|
||||
|
||||
|
||||
def record_exec_track(repo: Any, exec_return: float | None, new_positions: dict[str, float],
|
||||
today_prices: dict[str, float], envelope: float, today: str, at: _dt.datetime) -> None:
|
||||
"""Book `exec_return` as an observed forward_record for `multistrat_exec` via A's run_track (anchor +
|
||||
record + nav/summary), then carry the next run's position-state under book-state key `multistrat_exec_pos`
|
||||
(kept SEPARATE from run_track's own book_state for multistrat_exec)."""
|
||||
from fxhnt.application.forward_engine import run_track
|
||||
run_track(repo, "multistrat_exec", _ExecBookingStrategy(exec_return), today=today, at=at)
|
||||
repo.set_book_state("multistrat_exec_pos", build_pos_state(new_positions, today_prices, envelope), at=at)
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run tests → PASS + ruff, then commit**
|
||||
|
||||
Run: `uv run pytest tests/integration/test_record_exec_track.py -q` → PASS.
|
||||
Run: `uv run ruff check src/fxhnt/application/multistrat_exec_record.py tests/integration/test_record_exec_track.py`
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/application/multistrat_exec_record.py tests/integration/test_record_exec_track.py
|
||||
git commit -F - <<'EOF'
|
||||
feat(exec): record_exec_track — book the executed multistrat return via A's observed run_track (C1)
|
||||
|
||||
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
|
||||
EOF
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 4: Wire `execute_multistrat` — B2a envelope driver + flatten + record on `--execute`
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/cli.py` (`execute_multistrat`)
|
||||
- Test: `tests/integration/test_execute_multistrat_b2a.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `ForwardNavRepo.current_allocation` (B2a), `scaled_weights`/`compute_exec_return`/`record_exec_track` (Tasks 1+3), `AccountState.nlv`/`positions` (from the broker), `ExecutionService.rebalance_weights`.
|
||||
- Produces: `execute_multistrat` now sizes the book to B2a's `multistrat` envelope, flattens on $0/absent, and on `--execute` records the `multistrat_exec` observed track.
|
||||
|
||||
- [ ] **Step 1: Write the failing test (a fake broker + repo seam)**
|
||||
|
||||
Add a testable seam: extract the B2a-envelope + record logic into a pure-ish function the CLI calls, so the broker/Yahoo hot path stays thin. Add to `multistrat_exec_record.py`:
|
||||
|
||||
```python
|
||||
def plan_and_record(*, repo: Any, exec_svc: Any, within_weights: dict[str, float], prices: dict[str, float],
|
||||
nlv: float, positions_after: dict[str, float], today: str, at: _dt.datetime,
|
||||
max_gross: float, execute: bool) -> tuple[Any, float]:
|
||||
"""Compute the B2a envelope, size the book to it (flatten on $0), reconcile via exec_svc, and on
|
||||
`execute` record the observed multistrat_exec return. Returns (plan, envelope). `exec_svc` is an
|
||||
ExecutionService; `positions_after` are the multistrat positions the broker holds AFTER the rebalance
|
||||
(for the next run's return basis — pass the current broker positions when reading them post-execute, or
|
||||
the target positions in dry-run)."""
|
||||
envelope = min(float(repo.current_allocation().get("multistrat", 0.0)), float(nlv))
|
||||
weights = scaled_weights(within_weights, envelope, nlv)
|
||||
plan = exec_svc.rebalance_weights("multistrat", weights, prices, max_gross=max_gross, execute=execute)
|
||||
if execute:
|
||||
pos_prior = repo.get_book_state("multistrat_exec_pos")
|
||||
exec_return = compute_exec_return(pos_prior, prices)
|
||||
record_exec_track(repo, exec_return, positions_after, prices, envelope, today, at)
|
||||
return plan, envelope
|
||||
```
|
||||
|
||||
Test (`tests/integration/test_execute_multistrat_b2a.py`):
|
||||
|
||||
```python
|
||||
import datetime as dt
|
||||
|
||||
from fxhnt.adapters.persistence.cockpit_models import StrategyAllocationRow
|
||||
from fxhnt.adapters.persistence.forward_nav import ForwardNavRepo
|
||||
from fxhnt.application.multistrat_exec_record import plan_and_record
|
||||
|
||||
|
||||
class _FakePlan:
|
||||
def __init__(self): self.orders = []; self.notes = []; self.nlv = 100_000.0; self.blocked = False
|
||||
|
||||
class _FakeExecSvc:
|
||||
def __init__(self): self.last = None
|
||||
def rebalance_weights(self, name, weights, prices, *, max_gross, execute):
|
||||
self.last = dict(weights); return _FakePlan()
|
||||
|
||||
|
||||
def _seed_alloc(repo, dollars, at):
|
||||
repo.replace_allocations([StrategyAllocationRow(
|
||||
strategy_id="multistrat", target_weight=0.5, target_dollars=dollars, policy_version=1,
|
||||
policy_hash="h", as_of="2026-07-10", computed_at=at)], at)
|
||||
|
||||
|
||||
def test_envelope_scales_weights_and_records_on_execute(tmp_path):
|
||||
repo = ForwardNavRepo(f"sqlite:///{tmp_path}/e.db"); repo.migrate()
|
||||
at = dt.datetime(2026, 7, 10, 23, 30)
|
||||
_seed_alloc(repo, 10_000.0, at)
|
||||
svc = _FakeExecSvc()
|
||||
plan, env = plan_and_record(repo=repo, exec_svc=svc, within_weights={"SPY": 0.6, "IEF": 0.4},
|
||||
prices={"SPY": 500.0, "IEF": 100.0}, nlv=100_000.0,
|
||||
positions_after={"SPY": 12.0, "IEF": 40.0}, today="2026-07-10", at=at,
|
||||
max_gross=2.0, execute=True)
|
||||
assert env == 10_000.0
|
||||
assert abs(svc.last["SPY"] - 0.06) < 1e-12 # 0.6 * 10k/100k
|
||||
assert repo.get_book_state("multistrat_exec_pos")["envelope"] == 10_000.0 # recorded
|
||||
|
||||
|
||||
def test_zero_allocation_flattens(tmp_path):
|
||||
repo = ForwardNavRepo(f"sqlite:///{tmp_path}/z.db"); repo.migrate()
|
||||
at = dt.datetime(2026, 7, 10, 23, 30) # no allocation row for multistrat
|
||||
svc = _FakeExecSvc()
|
||||
plan, env = plan_and_record(repo=repo, exec_svc=svc, within_weights={"SPY": 0.6, "IEF": 0.4},
|
||||
prices={"SPY": 500.0, "IEF": 100.0}, nlv=100_000.0,
|
||||
positions_after={}, today="2026-07-10", at=at, max_gross=2.0, execute=False)
|
||||
assert env == 0.0 and svc.last == {"SPY": 0.0, "IEF": 0.0} # $0/absent → flatten
|
||||
```
|
||||
|
||||
(Use whichever allocation-write method exists — `replace_allocations` with a `StrategyAllocationRow` is the B2a API; simplify the test's seeding to that.)
|
||||
|
||||
- [ ] **Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `uv run pytest tests/integration/test_execute_multistrat_b2a.py -q`
|
||||
Expected: FAIL with `ImportError: cannot import name 'plan_and_record'`.
|
||||
|
||||
- [ ] **Step 3: Implement `plan_and_record` (Task 4 Step 1 block) + wire the CLI**
|
||||
|
||||
Add `plan_and_record` to `multistrat_exec_record.py` (the block above). Then in `src/fxhnt/cli.py` `execute_multistrat`, replace the direct `ExecutionService(...).rebalance_weights(...)` call with `plan_and_record`, reading the account for `nlv`/`positions_after`:
|
||||
|
||||
```python
|
||||
from fxhnt.adapters.persistence.forward_nav import ForwardNavRepo
|
||||
from fxhnt.application.multistrat_exec_record import plan_and_record
|
||||
import datetime as _dt
|
||||
repo = ForwardNavRepo(settings.operational_dsn); repo.migrate()
|
||||
today = _dt.datetime.now(_dt.UTC).strftime("%Y-%m-%d")
|
||||
at = _dt.datetime.now(_dt.UTC).replace(tzinfo=None)
|
||||
try:
|
||||
with broker as brk:
|
||||
svc = ExecutionService(data, brk, settings)
|
||||
acct = brk.account_state()
|
||||
positions_after = {s: acct.positions.get(s, 0.0) for s in multistrat.FUND_INSTRUMENTS} \
|
||||
if do_execute else {} # post-execute the broker reflects the new book; dry-run has no fills
|
||||
plan, envelope = plan_and_record(
|
||||
repo=repo, exec_svc=svc, within_weights=bweights, prices=bprices, nlv=acct.nlv,
|
||||
positions_after=positions_after, today=today, at=at, max_gross=max_gross, execute=do_execute)
|
||||
except Exception as e:
|
||||
... # keep the existing broker-error handling
|
||||
typer.echo(f"NLV ${plan.nlv:,.0f} | envelope ${envelope:,.0f} (B2a) | orders {len(plan.orders)}")
|
||||
... # keep the existing notes/orders echo
|
||||
```
|
||||
(Read the current `execute_multistrat` body first — preserve the weights computation, the broker construction, the error handling, and the echo of notes/orders; only swap the rebalance call for `plan_and_record` + add the envelope to the echo. NOTE: `positions_after` on `--execute` reads the broker's CURRENT positions after placement as the next return basis — for the first execute run this seeds the inception state; the exact fill-settled positions are the honest basis.)
|
||||
|
||||
- [ ] **Step 4: Run tests + full suite + ruff, then commit**
|
||||
|
||||
Run: `uv run pytest tests/integration/test_execute_multistrat_b2a.py -q` → PASS.
|
||||
Run the FULL suite ONCE (touches the CLI + registry): `uv run pytest -q` → PASS. (Ensure no other pytest is running first — concurrent full-suite runs SIGTERM each other.)
|
||||
Run: `uv run ruff check src/fxhnt/cli.py src/fxhnt/application/multistrat_exec_record.py tests/integration/test_execute_multistrat_b2a.py`
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/cli.py src/fxhnt/application/multistrat_exec_record.py tests/integration/test_execute_multistrat_b2a.py
|
||||
git commit -F - <<'EOF'
|
||||
feat(exec): execute-multistrat is B2a-driven — deploy the envelope, flatten on $0, record the exec track (C1)
|
||||
|
||||
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
|
||||
EOF
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Self-Review notes
|
||||
|
||||
- **Spec coverage:** bridge/envelope (§1)→T1 `scaled_weights` + T4 `plan_and_record`; reconcile reuse (§2)→T4 (existing ExecutionService); fills→observed record (§3)→T1 `compute_exec_return` + T3 `record_exec_track` (via A's run_track); registry track (§4)→T2; shape+safety (§5)→T4 (CLI + existing gates + dry-run default). Flatten-on-$0→T1/T4; envelope-basis + flow-excluded invariants→T1 tests.
|
||||
- **Load-bearing invariants tested:** `test_exec_return_is_envelope_basis_not_full_nlv` + `test_reallocation_capital_flow_is_not_return` (T1) pin the two correctness properties.
|
||||
- **Reuse, no new engine:** `plan_and_record` calls the existing `ExecutionService.rebalance_weights`; `record_exec_track` calls A's `run_track`; no new order engine or return store. The position-state lives in A's book-state under `multistrat_exec_pos` (separate from run_track's own `multistrat_exec` book_state).
|
||||
- **No capital authorized:** the CLI keeps the existing `--live`/`allow_live` gate; dry-run default; paper only. `plan_and_record` records only on `execute=True`.
|
||||
- **Type consistency:** `scaled_weights`/`compute_exec_return`/`build_pos_state` (T1) are consumed by `record_exec_track`/`plan_and_record` (T3/T4); `run_track(repo, "multistrat_exec", strategy, today, at)` matches A's signature; the registry `multistrat_exec` (T2) is what `run_track` resolves via `track_definition`.
|
||||
604
docs/superpowers/plans/2026-07-10-bybit-execution-leg-c2.md
Normal file
604
docs/superpowers/plans/2026-07-10-bybit-execution-leg-c2.md
Normal file
@@ -0,0 +1,604 @@
|
||||
# Bybit Execution Leg (C2) Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Make the existing Bybit-testnet leg B2a/B3-driven (deploy the `bybit_4edge` envelope, flatten on $0/de-funded) and record an envelope-basis, capital-flow-excluded, funding/fee/slippage-net `bybit_4edge_exec` observed track via A's `run_track`, reconciling executed-vs-modeled against `bybit_4edge`.
|
||||
|
||||
**Architecture:** Pure helpers carry the tested logic (funding query, exec-return math, observed-track recording); the two Dagster assets (`bybit_testnet_reconcile`, `bybit_testnet_record`) become thin callers. The reconcile sizes the book to `min(strategy_allocation[bybit_4edge], testnet_equity)` and measures realized funding (via a new adapter method) + fees (from fills); the record computes the envelope-basis exec return and books the observed track. Reuses `plan_orders`, `slippage.py`, `BybitTestnetRepo`, `run_track`, and C1's exec-record shape.
|
||||
|
||||
**Tech Stack:** Python 3.13, ccxt (Bybit), frozen dataclasses, SQLAlchemy 2.0 (SQLite tests / Postgres+Timescale prod), Dagster assets, pytest, ruff.
|
||||
|
||||
## Global Constraints
|
||||
|
||||
- Envelope = `min(ForwardNavRepo.current_allocation().get("bybit_4edge", 0.0), testnet_equity)`; `current_allocation()` is ALREADY B3-funding-filtered (a de-funded/killswitched `bybit_4edge` is absent → 0.0 → the book FLATTENS to cash).
|
||||
- Size the book to the envelope by passing `equity=envelope` to `plan_orders` (targets = `weight·envelope`), NOT `adapter.equity()`.
|
||||
- The executed return is **envelope-basis** (÷ `envelope_prior`, never full equity), **capital-flow-excluded** (a B2a re-allocation resets the basis, never enters the return), and **funding/fee/slippage-net**: `exec_return = (price_pnl + funding − fees) / envelope_prior` where `price_pnl` marks the PRIOR held positions (prior marks → today marks), `funding` is realized funding on the held book over the period, `fees` are the trading fees paid this run. First run → no prior book → record nothing (inception).
|
||||
- Funding is measured explicitly (Approach X, spec §3): a new `BybitExecution.funding_since(since_ms)` sums realized funding; `realized_pnl: 0.0` in the reconcile dict is REPLACED by measured `realized_funding` + `realized_fees`.
|
||||
- NO capital moved to real money: testnet only; kill-switch, idempotent NAV marker, resting-order abort, per-fill slippage — all UNCHANGED. Real-money Bybit is C3.
|
||||
- `bybit_4edge_exec` reconciliation reference = the modeled `bybit_4edge` via `CockpitBacktestRefProvider._REF_ALIAS`.
|
||||
- Book-state for the exec track lives under `ForwardNavRepo` key `bybit_4edge_exec_pos`, SEPARATE from `run_track`'s own `bybit_4edge_exec` book-state (mirrors C1's `multistrat_exec_pos`).
|
||||
- Marks source: `_testnet_marks` = `{symbol: latest_close}`. Positions are signed base qty (`adapter.positions()` convention).
|
||||
|
||||
---
|
||||
|
||||
### Task 1: `BybitExecution.funding_since` — realized-funding query
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/adapters/broker/bybit.py` (add method to `BybitExecution`)
|
||||
- Test: `tests/unit/test_bybit_funding_since.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: `BybitExecution.funding_since(since_ms: int) -> float` — net realized funding in USDT since `since_ms` epoch-ms (received > 0, paid < 0), summed across the account's funding events.
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
Create `tests/unit/test_bybit_funding_since.py`:
|
||||
|
||||
```python
|
||||
from fxhnt.adapters.broker.bybit import BybitExecution
|
||||
|
||||
|
||||
class _FakeCcxt:
|
||||
def __init__(self, rows):
|
||||
self._rows = rows
|
||||
self.calls = []
|
||||
|
||||
def set_sandbox_mode(self, _): # constructor calls this
|
||||
pass
|
||||
|
||||
def fetch_funding_history(self, symbol=None, since=None, limit=None):
|
||||
self.calls.append({"symbol": symbol, "since": since, "limit": limit})
|
||||
return self._rows
|
||||
|
||||
|
||||
def _exec(rows):
|
||||
ex = BybitExecution.__new__(BybitExecution) # bypass __init__ (no network/keys)
|
||||
ex._x = _FakeCcxt(rows) # inject the fake ccxt client
|
||||
return ex
|
||||
|
||||
|
||||
def test_funding_since_sums_signed_amounts():
|
||||
# ccxt funding rows: {"amount": ...} (received > 0, paid < 0)
|
||||
ex = _exec([{"amount": 1.5}, {"amount": -0.4}, {"amount": 0.9}])
|
||||
assert abs(ex.funding_since(1_700_000_000_000) - 2.0) < 1e-9
|
||||
assert ex._x.calls[0]["since"] == 1_700_000_000_000
|
||||
|
||||
|
||||
def test_funding_since_empty_is_zero():
|
||||
assert _exec([]).funding_since(0) == 0.0
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run to verify it fails**
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && source .venv/bin/activate && python -m pytest tests/unit/test_bybit_funding_since.py -q`
|
||||
Expected: FAIL — `AttributeError: 'BybitExecution' object has no attribute 'funding_since'`.
|
||||
|
||||
- [ ] **Step 3: Implement the method**
|
||||
|
||||
In `src/fxhnt/adapters/broker/bybit.py`, add to `BybitExecution` (after `positions`):
|
||||
|
||||
```python
|
||||
def funding_since(self, since_ms: int) -> float:
|
||||
"""Net realized funding in USDT since `since_ms` (epoch-ms): sum of the account's funding payments
|
||||
(received > 0, paid < 0). Bybit perps settle funding periodically; this is the funding component of
|
||||
the executed book's return (several 4-edge sleeves harvest it). Paginated via ccxt's `since`."""
|
||||
rows = self._call(self._x.fetch_funding_history, None, since_ms, None)
|
||||
return float(sum(float(r.get("amount") or 0.0) for r in (rows or [])))
|
||||
```
|
||||
|
||||
(`_call` is the existing retry wrapper; `fetch_funding_history(symbol, since, limit)` is ccxt's unified signature — `symbol=None` returns all symbols on Bybit.)
|
||||
|
||||
- [ ] **Step 4: Run to verify pass**
|
||||
|
||||
Run: `python -m pytest tests/unit/test_bybit_funding_since.py -q`
|
||||
Expected: PASS (2 passed).
|
||||
|
||||
- [ ] **Step 5: ruff + commit**
|
||||
|
||||
Run: `ruff check src/fxhnt/adapters/broker/bybit.py tests/unit/test_bybit_funding_since.py`
|
||||
```bash
|
||||
git add src/fxhnt/adapters/broker/bybit.py tests/unit/test_bybit_funding_since.py
|
||||
git commit -m "feat(bybit): funding_since — realized funding query for the executed return (C2)
|
||||
|
||||
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 2: Pure crypto exec-return helpers (`bybit_exec_record.py`)
|
||||
|
||||
**Files:**
|
||||
- Create: `src/fxhnt/application/bybit_exec_record.py`
|
||||
- Test: `tests/unit/test_bybit_exec_record.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Produces:
|
||||
- `crypto_envelope(repo, equity) -> float` = `min(repo.current_allocation().get("bybit_4edge", 0.0), equity)`.
|
||||
- `compute_crypto_exec_return(pos_prior: dict, today_marks: dict[str,float], funding: float, fees: float) -> float | None`.
|
||||
- `build_crypto_pos_state(positions, marks, envelope, funding_cursor_ts) -> dict`.
|
||||
- `apply_fills(current: dict[str,float], fills: list[dict]) -> dict[str,float]` — the post-rebalance book (signed qty updated by each successful fill).
|
||||
|
||||
- [ ] **Step 1: Write the failing tests (load-bearing invariants)**
|
||||
|
||||
Create `tests/unit/test_bybit_exec_record.py`:
|
||||
|
||||
```python
|
||||
from fxhnt.application.bybit_exec_record import (
|
||||
build_crypto_pos_state, compute_crypto_exec_return, crypto_envelope,
|
||||
)
|
||||
|
||||
|
||||
def _state(positions, marks, envelope, cursor=0):
|
||||
return build_crypto_pos_state(positions, marks, envelope, cursor)
|
||||
|
||||
|
||||
def test_inception_and_flat_return_none():
|
||||
assert compute_crypto_exec_return({}, {"BTCUSDT": 100.0}, 0.0, 0.0) is None # no prior book
|
||||
flat = _state({"BTCUSDT": 1.0}, {"BTCUSDT": 100.0}, 0.0) # envelope 0
|
||||
assert compute_crypto_exec_return(flat, {"BTCUSDT": 100.0}, 0.0, 0.0) is None
|
||||
|
||||
|
||||
def test_envelope_basis_undiluted():
|
||||
# $10k envelope, book long 100 units entered at 100 -> mark 101 = +$100 price pnl. Return = 100/10000 = 1%,
|
||||
# regardless of any (irrelevant) larger testnet balance — the divisor is the ENVELOPE.
|
||||
prior = _state({"BTCUSDT": 100.0}, {"BTCUSDT": 100.0}, 10_000.0)
|
||||
r = compute_crypto_exec_return(prior, {"BTCUSDT": 101.0}, 0.0, 0.0)
|
||||
assert abs(r - 0.01) < 1e-9
|
||||
|
||||
|
||||
def test_funding_is_return_not_flow():
|
||||
# flat marks, +$50 funding on a $10k envelope -> +0.5% (funding is edge return).
|
||||
prior = _state({"BTCUSDT": 100.0}, {"BTCUSDT": 100.0}, 10_000.0)
|
||||
r = compute_crypto_exec_return(prior, {"BTCUSDT": 100.0}, 50.0, 0.0)
|
||||
assert abs(r - 0.005) < 1e-9
|
||||
|
||||
|
||||
def test_fees_reduce_return():
|
||||
# flat marks, $30 fees on a $10k envelope -> -0.3%.
|
||||
prior = _state({"BTCUSDT": 100.0}, {"BTCUSDT": 100.0}, 10_000.0)
|
||||
r = compute_crypto_exec_return(prior, {"BTCUSDT": 100.0}, 0.0, 30.0)
|
||||
assert abs(r - (-0.003)) < 1e-9
|
||||
|
||||
|
||||
def test_missing_today_mark_falls_back_to_prior():
|
||||
prior = _state({"BTCUSDT": 100.0}, {"BTCUSDT": 100.0}, 10_000.0)
|
||||
assert compute_crypto_exec_return(prior, {}, 0.0, 0.0) == 0.0 # no price move
|
||||
|
||||
|
||||
class _Repo:
|
||||
def __init__(self, alloc):
|
||||
self._alloc = alloc
|
||||
|
||||
def current_allocation(self):
|
||||
return self._alloc
|
||||
|
||||
|
||||
def test_crypto_envelope_caps_at_equity_and_zero_when_absent():
|
||||
assert crypto_envelope(_Repo({"bybit_4edge": 8_000.0}), 5_000.0) == 5_000.0 # capped at equity
|
||||
assert crypto_envelope(_Repo({"bybit_4edge": 8_000.0}), 20_000.0) == 8_000.0 # alloc binds
|
||||
assert crypto_envelope(_Repo({}), 20_000.0) == 0.0 # de-funded/absent -> flatten
|
||||
|
||||
|
||||
def test_apply_fills_builds_post_rebalance_book():
|
||||
from fxhnt.application.bybit_exec_record import apply_fills
|
||||
current = {"BTCUSDT": 1.0, "ETHUSDT": -2.0}
|
||||
fills = [{"symbol": "BTCUSDT", "side": "BUY", "qty": 0.5}, # 1.0 + 0.5 = 1.5
|
||||
{"symbol": "ETHUSDT", "side": "BUY", "qty": 2.0}, # -2.0 + 2.0 = 0.0 -> dropped
|
||||
{"symbol": "SOLUSDT", "side": "SELL", "qty": 3.0}, # new short -3.0
|
||||
{"symbol": "BTCUSDT", "error": True, "qty": 9.0}] # error row ignored
|
||||
assert apply_fills(current, fills) == {"BTCUSDT": 1.5, "SOLUSDT": -3.0}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run to verify it fails**
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && source .venv/bin/activate && python -m pytest tests/unit/test_bybit_exec_record.py -q`
|
||||
Expected: FAIL — `ModuleNotFoundError: No module named 'fxhnt.application.bybit_exec_record'`.
|
||||
|
||||
- [ ] **Step 3: Implement the helpers**
|
||||
|
||||
Create `src/fxhnt/application/bybit_exec_record.py`:
|
||||
|
||||
```python
|
||||
"""Sub-project C2 — record the EXECUTED Bybit crypto track and bridge B2a/B3's dollar allocation to the venue.
|
||||
|
||||
`crypto_envelope` = the B2a (B3-funding-filtered) dollars for `bybit_4edge`, capped at the testnet equity
|
||||
(absent/de-funded -> 0 -> flatten). `compute_crypto_exec_return` measures the executed book's daily return on
|
||||
the ENVELOPE (not the full testnet equity), as the P&L of the HELD positions since the last rebalance PLUS
|
||||
realized funding MINUS fees (funding is edge return; a re-allocation is a capital flow, not a gain). Mirrors
|
||||
C1's multistrat_exec_record, extended for crypto funding/fees. No new execution engine, no new return store."""
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
from typing import Any
|
||||
|
||||
|
||||
def crypto_envelope(repo: Any, equity: float) -> float:
|
||||
return min(float(repo.current_allocation().get("bybit_4edge", 0.0)), float(equity))
|
||||
|
||||
|
||||
def build_crypto_pos_state(positions: dict[str, float], marks: dict[str, float], envelope: float,
|
||||
funding_cursor_ts: int) -> dict[str, Any]:
|
||||
return {"positions": {s: float(q) for s, q in positions.items()},
|
||||
"marks": {s: float(m) for s, m in marks.items()},
|
||||
"envelope": float(envelope), "funding_cursor_ts": int(funding_cursor_ts)}
|
||||
|
||||
|
||||
def compute_crypto_exec_return(pos_prior: dict[str, Any], today_marks: dict[str, float],
|
||||
funding: float, fees: float) -> float | None:
|
||||
"""Envelope-basis, flow-excluded, funding/fee-net executed return:
|
||||
(price_pnl + funding - fees) / envelope_prior
|
||||
where price_pnl marks the PRIOR held positions (prior marks -> today marks). None on inception / flat."""
|
||||
if not pos_prior:
|
||||
return None # first run / no prior book -> inception
|
||||
envelope = float(pos_prior.get("envelope", 0.0))
|
||||
if envelope <= 0.0:
|
||||
return None # flat (de-funded/killswitched) -> no return
|
||||
positions: dict[str, float] = pos_prior.get("positions", {})
|
||||
prior_marks: dict[str, float] = pos_prior.get("marks", {})
|
||||
val_prior = sum(q * prior_marks.get(s, 0.0) for s, q in positions.items())
|
||||
val_today = sum(q * today_marks.get(s, prior_marks.get(s, 0.0)) for s, q in positions.items())
|
||||
price_pnl = val_today - val_prior
|
||||
return (price_pnl + float(funding) - float(fees)) / envelope
|
||||
|
||||
|
||||
def apply_fills(current: dict[str, float], fills: list[dict[str, Any]]) -> dict[str, float]:
|
||||
"""The post-rebalance book = `current` signed base qty updated by each SUCCESSFUL fill's signed filled qty
|
||||
(BUY > 0, SELL < 0). Error rows are ignored; near-zero residual positions are dropped. This is the real
|
||||
placed book (reflects snapping/partial fills) — the next run's return basis."""
|
||||
out = dict(current)
|
||||
for f in fills:
|
||||
if f.get("error"):
|
||||
continue
|
||||
signed = float(f["qty"]) * (1.0 if f["side"] == "BUY" else -1.0)
|
||||
out[f["symbol"]] = out.get(f["symbol"], 0.0) + signed
|
||||
return {s: q for s, q in out.items() if abs(q) > 1e-12}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run to verify pass**
|
||||
|
||||
Run: `python -m pytest tests/unit/test_bybit_exec_record.py -q`
|
||||
Expected: PASS (7 passed).
|
||||
|
||||
- [ ] **Step 5: ruff + commit**
|
||||
|
||||
Run: `ruff check src/fxhnt/application/bybit_exec_record.py tests/unit/test_bybit_exec_record.py`
|
||||
```bash
|
||||
git add src/fxhnt/application/bybit_exec_record.py tests/unit/test_bybit_exec_record.py
|
||||
git commit -m "feat(bybit): pure envelope-basis funding-net crypto exec-return helpers (C2)
|
||||
|
||||
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 4: `record_bybit_exec_track` — book the observed track via A's run_track
|
||||
|
||||
> **Depends on Task 3** (the `bybit_4edge_exec` registry entry) — `run_track` reads `track_definition(sid)` from `STRATEGY_REGISTRY`, so the registry entry MUST exist before this task's test runs.
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/application/bybit_exec_record.py` (add the booking function + strategy shim)
|
||||
- Test: `tests/integration/test_record_bybit_exec_track.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `ForwardNavRepo` (`run_track` target + `set_book_state`), `build_crypto_pos_state` (Task 2).
|
||||
- Produces: `record_bybit_exec_track(repo, exec_return, new_positions, today_marks, envelope, funding_cursor_ts, today, at) -> None` — books `exec_return` as an observed `forward_record` for `bybit_4edge_exec` and carries the next basis under `bybit_4edge_exec_pos`.
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
Create `tests/integration/test_record_bybit_exec_track.py`:
|
||||
|
||||
```python
|
||||
import datetime as dt
|
||||
|
||||
from fxhnt.adapters.persistence.forward_nav import ForwardNavRepo
|
||||
from fxhnt.application.bybit_exec_record import record_bybit_exec_track
|
||||
|
||||
|
||||
def test_records_observed_return_and_carries_basis(tmp_path):
|
||||
repo = ForwardNavRepo(f"sqlite:///{tmp_path/'c.db'}")
|
||||
repo.migrate()
|
||||
at = dt.datetime(2026, 7, 10)
|
||||
# inception: exec_return None -> a forward_record is booked with no return row, basis is carried.
|
||||
record_bybit_exec_track(repo, None, {"BTCUSDT": 1.0}, {"BTCUSDT": 100.0}, 10_000.0, 1_700_000_000_000,
|
||||
"2026-07-10", at)
|
||||
st = repo.get_book_state("bybit_4edge_exec_pos")
|
||||
assert st["envelope"] == 10_000.0
|
||||
assert st["positions"] == {"BTCUSDT": 1.0}
|
||||
assert st["funding_cursor_ts"] == 1_700_000_000_000
|
||||
# second run books a real return.
|
||||
record_bybit_exec_track(repo, 0.01, {"BTCUSDT": 2.0}, {"BTCUSDT": 101.0}, 10_000.0, 1_700_100_000_000,
|
||||
"2026-07-11", dt.datetime(2026, 7, 11))
|
||||
hist = repo.nav_history("bybit_4edge_exec")
|
||||
assert any(abs(r.ret - 0.01) < 1e-9 for r in hist)
|
||||
assert repo.get_book_state("bybit_4edge_exec_pos")["positions"] == {"BTCUSDT": 2.0}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run to verify it fails**
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && source .venv/bin/activate && python -m pytest tests/integration/test_record_bybit_exec_track.py -q`
|
||||
Expected: FAIL — `ImportError: cannot import name 'record_bybit_exec_track'`.
|
||||
|
||||
- [ ] **Step 3: Implement the booking function**
|
||||
|
||||
Append to `src/fxhnt/application/bybit_exec_record.py`:
|
||||
|
||||
```python
|
||||
class _CryptoExecBookingStrategy:
|
||||
"""Trivial observed ForwardStrategy for run_track: books today's pre-computed executed return (or nothing
|
||||
on inception). The return is computed by the record asset (it has the marks/funding/fees)."""
|
||||
def __init__(self, exec_return: float | None) -> None:
|
||||
self._r = exec_return
|
||||
|
||||
def advance(self, last_date: str | None, extra: dict[str, Any]) -> tuple[list[tuple[str, float]], dict[str, Any]]:
|
||||
if self._r is None:
|
||||
return [], extra
|
||||
return [(str(extra.get("_today")), float(self._r))], extra
|
||||
|
||||
|
||||
def record_bybit_exec_track(repo: Any, exec_return: float | None, new_positions: dict[str, float],
|
||||
today_marks: dict[str, float], envelope: float, funding_cursor_ts: int,
|
||||
today: str, at: dt.datetime) -> None:
|
||||
"""Book `exec_return` as an observed forward_record for `bybit_4edge_exec` via A's run_track (anchor +
|
||||
record + nav/summary), then carry the next run's basis (positions/marks/envelope/funding cursor) under the
|
||||
book-state key `bybit_4edge_exec_pos` (SEPARATE from run_track's own book_state)."""
|
||||
from fxhnt.application.forward_engine import run_track
|
||||
run_track(repo, "bybit_4edge_exec", _CryptoExecBookingStrategy(exec_return), today=today, at=at)
|
||||
repo.set_book_state("bybit_4edge_exec_pos",
|
||||
build_crypto_pos_state(new_positions, today_marks, envelope, funding_cursor_ts), at=at)
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run to verify pass**
|
||||
|
||||
Run: `python -m pytest tests/integration/test_record_bybit_exec_track.py -q`
|
||||
Expected: PASS (1 passed). (`run_track` reads the `bybit_4edge_exec` registry definition — guaranteed present because Task 3 ran first.)
|
||||
|
||||
- [ ] **Step 5: ruff + commit**
|
||||
|
||||
Run: `ruff check src/fxhnt/application/bybit_exec_record.py tests/integration/test_record_bybit_exec_track.py`
|
||||
```bash
|
||||
git add src/fxhnt/application/bybit_exec_record.py tests/integration/test_record_bybit_exec_track.py
|
||||
git commit -m "feat(bybit): record_bybit_exec_track — book the executed crypto track via A run_track (C2)
|
||||
|
||||
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 3: Register `bybit_4edge_exec` observed track + reconciliation ref alias
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/registry.py` (add the `bybit_4edge_exec` entry, after `bybit_4edge`)
|
||||
- Modify: `src/fxhnt/application/forward_ingest.py` (`_REF_ALIAS`)
|
||||
- Test: `tests/unit/test_registry_bybit_4edge_exec.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: `STRATEGY_REGISTRY["bybit_4edge_exec"]` (observed, venue `bybit-testnet`, tier research, gate_type reconciliation); `CockpitBacktestRefProvider._REF_ALIAS["bybit_4edge_exec"] == "bybit_4edge"`.
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
Create `tests/unit/test_registry_bybit_4edge_exec.py`:
|
||||
|
||||
```python
|
||||
from fxhnt.application.forward_ingest import CockpitBacktestRefProvider
|
||||
from fxhnt.registry import STRATEGY_REGISTRY
|
||||
|
||||
|
||||
def test_bybit_4edge_exec_is_observed_reconciliation_track():
|
||||
m = STRATEGY_REGISTRY["bybit_4edge_exec"]
|
||||
assert m["definition"]["record_mode"] == "observed"
|
||||
assert m["venue"] == "bybit-testnet"
|
||||
assert m["tier"] == "research"
|
||||
assert m["gate_spec"]["gate_type"] == "reconciliation"
|
||||
|
||||
|
||||
def test_exec_reconciliation_ref_aliases_to_modeled_bybit_4edge():
|
||||
assert CockpitBacktestRefProvider._REF_ALIAS["bybit_4edge_exec"] == "bybit_4edge"
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run to verify it fails**
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && source .venv/bin/activate && python -m pytest tests/unit/test_registry_bybit_4edge_exec.py -q`
|
||||
Expected: FAIL — `KeyError: 'bybit_4edge_exec'`.
|
||||
|
||||
- [ ] **Step 3: Add the registry entry**
|
||||
|
||||
In `src/fxhnt/registry.py`, immediately AFTER the `"bybit_4edge"` entry, add:
|
||||
|
||||
```python
|
||||
# The EXECUTED 4-edge book (sub-project C2): the real Bybit-TESTNET track — its daily return is the
|
||||
# executed book's envelope-basis P&L (fills + slippage + funding - fees), recorded by the
|
||||
# bybit_testnet_reconcile/record assets. OBSERVED (fills are observed, not recomputable). Its gate
|
||||
# reconciles the executed track against the MODELED bybit_4edge via the ref-provider alias (execution
|
||||
# decay shows as divergence). min_forward_days 21 mirrors the modeled book's window; crypto is 24/7 so
|
||||
# the default max_gap_days (1) applies.
|
||||
"bybit_4edge_exec": {
|
||||
"display_name": "Bybit 4-edge book (EXECUTED, testnet)", "sleeve": "crypto-bybit",
|
||||
"venue": "bybit-testnet", "state_file": "bybit_4edge_exec_state",
|
||||
"tier": "research", "execution": "paper",
|
||||
"gate_spec": {"gate_type": "reconciliation", "min_forward_days": 21},
|
||||
"definition": {"params": {"source": "bybit-testnet-fills", "models": "bybit_4edge"},
|
||||
"version": 1, "record_mode": "observed"},
|
||||
},
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Add the ref alias**
|
||||
|
||||
In `src/fxhnt/application/forward_ingest.py`, change:
|
||||
```python
|
||||
_REF_ALIAS = {"multistrat_exec": "multistrat"}
|
||||
```
|
||||
to:
|
||||
```python
|
||||
_REF_ALIAS = {"multistrat_exec": "multistrat", "bybit_4edge_exec": "bybit_4edge"}
|
||||
```
|
||||
|
||||
- [ ] **Step 5: Run to verify pass + ruff + commit**
|
||||
|
||||
Run: `python -m pytest tests/unit/test_registry_bybit_4edge_exec.py tests/unit/test_registry_multistrat_exec.py -q`
|
||||
Expected: PASS.
|
||||
Run: `ruff check src/fxhnt/registry.py src/fxhnt/application/forward_ingest.py tests/unit/test_registry_bybit_4edge_exec.py`
|
||||
```bash
|
||||
git add src/fxhnt/registry.py src/fxhnt/application/forward_ingest.py tests/unit/test_registry_bybit_4edge_exec.py
|
||||
git commit -m "feat(bybit): register bybit_4edge_exec observed track + reconciliation ref alias (C2)
|
||||
|
||||
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 5: Wire the assets — envelope bridge (reconcile) + exec-track recording (record)
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/adapters/orchestration/assets.py` (`bybit_testnet_reconcile`, `bybit_testnet_record`)
|
||||
- Test: `tests/integration/test_bybit_testnet_b2a_wiring.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `crypto_envelope`, `compute_crypto_exec_return`, `record_bybit_exec_track` (Tasks 2-3), `BybitExecution.funding_since` (Task 1), `ForwardNavRepo` (`current_allocation`, `get_book_state`).
|
||||
|
||||
This task extracts the pure decision into a helper so it is testable without a live Dagster context, then makes the assets thin callers.
|
||||
|
||||
- [ ] **Step 1: Write the failing test (the pure wiring helper)**
|
||||
|
||||
Create `tests/integration/test_bybit_testnet_b2a_wiring.py`:
|
||||
|
||||
```python
|
||||
import datetime as dt
|
||||
|
||||
from fxhnt.adapters.persistence.forward_nav import ForwardNavRepo
|
||||
from fxhnt.application.bybit_exec_record import (
|
||||
compute_crypto_exec_return, crypto_envelope, record_bybit_exec_track,
|
||||
)
|
||||
|
||||
|
||||
class _Repo:
|
||||
def __init__(self, alloc):
|
||||
self._alloc = alloc
|
||||
|
||||
def current_allocation(self):
|
||||
return self._alloc
|
||||
|
||||
|
||||
def test_envelope_flattens_when_bybit_4edge_defunded():
|
||||
# de-funded/absent bybit_4edge -> envelope 0 -> the book flattens (plan_orders would size everything to 0).
|
||||
assert crypto_envelope(_Repo({}), 50_000.0) == 0.0
|
||||
assert crypto_envelope(_Repo({"bybit_4edge": 0.0}), 50_000.0) == 0.0
|
||||
|
||||
|
||||
def test_end_to_end_envelope_basis_exec_return_recorded(tmp_path):
|
||||
# Simulate the reconcile->record data flow with a real repo (no Dagster/broker): inception, then a day
|
||||
# with +$100 price pnl + $50 funding - $20 fees on a $10k envelope behind a large (irrelevant) equity.
|
||||
repo = ForwardNavRepo(f"sqlite:///{tmp_path/'c.db'}")
|
||||
repo.migrate()
|
||||
at0 = dt.datetime(2026, 7, 10)
|
||||
# inception run: no prior book -> None return, carry basis (100 units @ 100, envelope 10k).
|
||||
prior = repo.get_book_state("bybit_4edge_exec_pos")
|
||||
assert compute_crypto_exec_return(prior, {"BTCUSDT": 100.0}, 0.0, 0.0) is None
|
||||
record_bybit_exec_track(repo, None, {"BTCUSDT": 100.0}, {"BTCUSDT": 100.0}, 10_000.0,
|
||||
1_700_000_000_000, "2026-07-10", at0)
|
||||
# next run: marks 100->101 (+$100), funding +$50, fees $20 -> (100+50-20)/10000 = +1.30%.
|
||||
prior = repo.get_book_state("bybit_4edge_exec_pos")
|
||||
r = compute_crypto_exec_return(prior, {"BTCUSDT": 101.0}, 50.0, 20.0)
|
||||
assert abs(r - 0.013) < 1e-9
|
||||
record_bybit_exec_track(repo, r, {"BTCUSDT": 100.0}, {"BTCUSDT": 101.0}, 10_000.0,
|
||||
1_700_100_000_000, "2026-07-11", dt.datetime(2026, 7, 11))
|
||||
assert any(abs(row.ret - 0.013) < 1e-9 for row in repo.nav_history("bybit_4edge_exec"))
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run to verify it fails**
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && source .venv/bin/activate && python -m pytest tests/integration/test_bybit_testnet_b2a_wiring.py -q`
|
||||
Expected: FAIL until Tasks 2-3 are present (they are, if done in order) — this test uses only their helpers, so it should PASS once they exist. If it PASSES immediately, that is expected (it pins the data-flow contract the assets must follow); proceed to wire the assets in Steps 3-4 and keep this test green.
|
||||
|
||||
- [ ] **Step 3: Wire `bybit_testnet_reconcile` (envelope + funding + fees + positions_after)**
|
||||
|
||||
In `src/fxhnt/adapters/orchestration/assets.py` `bybit_testnet_reconcile`, make these changes (keep the kill-switch, idempotency, abort, and per-fill slippage untouched):
|
||||
|
||||
(a) Add imports at the top of the function's import block:
|
||||
```python
|
||||
from fxhnt.adapters.persistence.forward_nav import ForwardNavRepo
|
||||
from fxhnt.application.bybit_exec_record import apply_fills, crypto_envelope
|
||||
```
|
||||
|
||||
(b) After `equity = adapter.equity()` (the non-kill-switch path, ~line 1211) and BEFORE `plan_orders`, compute the envelope and the prior funding cursor:
|
||||
```python
|
||||
fwd_repo = ForwardNavRepo(s.operational_dsn)
|
||||
fwd_repo.migrate()
|
||||
envelope = crypto_envelope(fwd_repo, equity)
|
||||
prior_state = fwd_repo.get_book_state("bybit_4edge_exec_pos") or {}
|
||||
funding_cursor_prior = int(prior_state.get("funding_cursor_ts", 0))
|
||||
realized_funding = adapter.funding_since(funding_cursor_prior) if funding_cursor_prior else 0.0
|
||||
```
|
||||
Change BOTH `plan_orders(weights, current, equity, ...)` calls (the kill-switch branch AND the live branch) to size to the envelope:
|
||||
```python
|
||||
intents = plan_orders(weights, current, envelope, marks, # size to the B2a/B3 envelope, not full equity
|
||||
limits, min_order_usd=s.bybit_exec.min_order_usd, max_gross=s.bybit_exec.max_gross)
|
||||
```
|
||||
|
||||
(c) In the fill loop, accumulate fees; after the loop compute the post-rebalance book from the fills, and put the new fields into the return dict:
|
||||
```python
|
||||
realized_fees = sum(float(f.get("fee") or 0.0) for f in fills if not f.get("error"))
|
||||
positions_after = apply_fills(current, fills) # the real placed book (tested helper), next-run basis
|
||||
```
|
||||
Replace the return dict's `"realized_pnl": 0.0,` and add the new fields:
|
||||
```python
|
||||
return {
|
||||
"run_date": run_date, "placed": placed, "killed": False, "noop": False,
|
||||
"equity": equity, "envelope": envelope,
|
||||
"realized_funding": realized_funding, "realized_fees": realized_fees,
|
||||
"realized_pnl": realized_funding - realized_fees,
|
||||
"gross": gross, "mean_exec_slippage_bps": mean_exec, "mean_timing_drift_bps": mean_drift,
|
||||
"as_of_day": as_of_day, "fills": fills,
|
||||
"positions_after": positions_after, "marks": marks,
|
||||
"funding_cursor_prior": funding_cursor_prior,
|
||||
}
|
||||
```
|
||||
(The kill-switch / no-op / aborted early returns are UNCHANGED — they carry no envelope/exec fields, so the record step skips recording, exactly as it already skips fills/NAV for those.)
|
||||
|
||||
- [ ] **Step 4: Wire `bybit_testnet_record` (compute + book the exec track)**
|
||||
|
||||
In `bybit_testnet_record`, AFTER the existing `repo.upsert_testnet_nav(...)` call (keep it — `bybit_testnet_nav` stays), add:
|
||||
```python
|
||||
import datetime as _dt2
|
||||
|
||||
from fxhnt.adapters.persistence.forward_nav import ForwardNavRepo
|
||||
from fxhnt.application.bybit_exec_record import compute_crypto_exec_return, record_bybit_exec_track
|
||||
|
||||
fwd_repo = ForwardNavRepo(s.operational_dsn)
|
||||
fwd_repo.migrate()
|
||||
pos_prior = fwd_repo.get_book_state("bybit_4edge_exec_pos")
|
||||
exec_return = compute_crypto_exec_return(
|
||||
pos_prior or {}, recon["marks"], recon["realized_funding"], recon["realized_fees"])
|
||||
now_ms = int(now.timestamp() * 1000)
|
||||
record_bybit_exec_track(
|
||||
fwd_repo, exec_return, recon["positions_after"], recon["marks"], recon["envelope"],
|
||||
now_ms, run_date, _dt2.datetime.now(_dt2.UTC).replace(tzinfo=None))
|
||||
context.log.info(f"bybit_testnet_record: bybit_4edge_exec return="
|
||||
f"{'inception' if exec_return is None else f'{exec_return*100:.3f}%'} "
|
||||
f"(envelope=${recon['envelope']:.0f}, funding=${recon['realized_funding']:.2f}, "
|
||||
f"fees=${recon['realized_fees']:.2f})")
|
||||
```
|
||||
(`recon` already gained `marks`, `realized_funding`, `realized_fees`, `envelope`, `positions_after` in Step 3. The early `noop/killed/aborted` guard at the top of `record` returns BEFORE this block, so a killed/no-op run records no exec track — a killed run is a visible gap, matching the existing NAV behavior.)
|
||||
|
||||
- [ ] **Step 5: Run the wiring test + the C2 helper tests + ruff**
|
||||
|
||||
Run: `python -m pytest tests/integration/test_bybit_testnet_b2a_wiring.py tests/integration/test_record_bybit_exec_track.py tests/unit/test_bybit_exec_record.py tests/unit/test_bybit_funding_since.py tests/unit/test_registry_bybit_4edge_exec.py -q`
|
||||
Expected: PASS.
|
||||
Run: `ruff check src/fxhnt/adapters/orchestration/assets.py src/fxhnt/application/bybit_exec_record.py tests/integration/test_bybit_testnet_b2a_wiring.py`
|
||||
Expected: clean. (`assets.py` may have PRE-EXISTING ruff violations unrelated to this diff — do NOT fix those; only your added lines must be clean.)
|
||||
|
||||
- [ ] **Step 6: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/adapters/orchestration/assets.py tests/integration/test_bybit_testnet_b2a_wiring.py
|
||||
git commit -m "feat(bybit): B2a/B3-envelope-driven testnet leg + record bybit_4edge_exec observed track (C2)
|
||||
|
||||
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Final verification (after all tasks)
|
||||
|
||||
- [ ] Full suite: `cd /home/jgrusewski/Work/fxhnt && source .venv/bin/activate && python -m pytest -q -p no:cacheprovider`
|
||||
Expected: all green (prior baseline 1933 + the new C2 tests).
|
||||
- [ ] `ruff check src/fxhnt/application/bybit_exec_record.py src/fxhnt/adapters/broker/bybit.py src/fxhnt/registry.py src/fxhnt/application/forward_ingest.py` — no new violations from C2 files.
|
||||
714
docs/superpowers/plans/2026-07-10-cockpit-surfacing-redesign.md
Normal file
714
docs/superpowers/plans/2026-07-10-cockpit-surfacing-redesign.md
Normal file
@@ -0,0 +1,714 @@
|
||||
# Cockpit Surfacing + Mobile Redesign Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Surface the pipeline's B2a allocation, B3 funding health, and C1/C2 executed-vs-modeled reconciliation in the existing server-rendered cockpit, fully integrated into the Overview + Detail pages, and redesign navigation + layout to be mobile-first — read-only, no pipeline logic, browser-verified locally.
|
||||
|
||||
**Architecture:** Read-model methods (`ForwardNavRepo` reads + `DashboardService`) join the already-written `strategy_allocation` / `strategy_funding` tables and pair each modeled track with its executed twin (reverse of C1/C2's `_REF_ALIAS`); the DTOs (`FleetRow`/`StrategyDetail`/`OverviewModel`) gain the new fields; the Jinja2 templates (`base.html` nav+CSS, `cockpit.html`, `strategy.html`) render them. Every new read degrades to `—`/hidden, never a 500.
|
||||
|
||||
**Tech Stack:** Python 3.13, FastAPI + Jinja2 + HTMX, SQLAlchemy 2.0 (SQLite tests / Postgres prod), inline CSS, pytest, Playwright (browser verification), ruff.
|
||||
|
||||
## Global Constraints
|
||||
|
||||
- Read-only display of already-written tables — NO new pipeline logic, NO capital moved, NO new nightly assets, NO new top-level routes (integrate into the existing `/` and `/strategy/{sid}`).
|
||||
- Locked design (from the approved mockup): bottom-tab nav on mobile (`≤640px`) + top bar on desktop; capital meter (`deployed $X / $35,000 ceiling`); funding badge `funded`(green `#3fb950`)/`decaying N/10`(amber `#d29922`, NEW token)/`de-funded`(red `#f85149`); the executed twin nested UNDER its modeled edge (never a separate top-level row).
|
||||
- Honor the existing GitHub-dark terminal aesthetic (monospace, `tabular-nums`, existing badge classes) — refine, don't re-skin. Dark-committed (single theme is deliberate for a trading console).
|
||||
- Graceful degradation is load-bearing: prod's schema/data lags until deploy, so a missing table/row/column → `—`/hidden, never 500 (mirror the existing `except Exception: return []`).
|
||||
- Exec pairing key = REVERSE of `CockpitBacktestRefProvider._REF_ALIAS` → `{"multistrat":"multistrat_exec","bybit_4edge":"bybit_4edge_exec"}`. Executed twins are `tier=research` — folded into their modeled parent on the Overview, not rendered as their own top-level rows.
|
||||
- `ceiling` = `ALLOCATION_POLICY.capital_ceiling` (B2a); `gross_deployed` = Σ target_dollars.
|
||||
- The DEPLOY (push + argo + migrate) is OUT OF SCOPE — a separate follow-up gated on this being browser-verified locally.
|
||||
|
||||
---
|
||||
|
||||
### Task 1: Repo reads — `all_allocations()` + `all_funding()`
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/adapters/persistence/forward_nav.py`
|
||||
- Test: `tests/integration/test_dashboard_reads.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: `ForwardNavRepo.all_allocations() -> list[StrategyAllocationRow]`, `ForwardNavRepo.all_funding() -> list[StrategyFundingRow]` (full rows; `[]` when the table is empty).
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
Create `tests/integration/test_dashboard_reads.py`:
|
||||
|
||||
```python
|
||||
import datetime as dt
|
||||
|
||||
from fxhnt.adapters.persistence.cockpit_models import StrategyAllocationRow, StrategyFundingRow
|
||||
from fxhnt.adapters.persistence.forward_nav import ForwardNavRepo
|
||||
|
||||
|
||||
def _repo(tmp_path):
|
||||
repo = ForwardNavRepo(f"sqlite:///{tmp_path/'c.db'}")
|
||||
repo.migrate()
|
||||
return repo
|
||||
|
||||
|
||||
def test_all_allocations_and_funding_roundtrip(tmp_path):
|
||||
repo = _repo(tmp_path)
|
||||
at = dt.datetime(2026, 7, 10)
|
||||
repo.replace_allocations([StrategyAllocationRow(strategy_id="bybit_4edge", target_weight=0.54,
|
||||
target_dollars=18900.0, policy_version=1, policy_hash="h", as_of="2026-07-10", computed_at=at)], at)
|
||||
repo.replace_funding([StrategyFundingRow(strategy_id="bybit_4edge", state="funded", label="funded",
|
||||
decay_run=0, recovery_run=0, first_pass_date=None, policy_version=2, policy_hash="h",
|
||||
as_of="2026-07-10", computed_at=at)], at)
|
||||
allocs = {a.strategy_id: a for a in repo.all_allocations()}
|
||||
funds = {f.strategy_id: f for f in repo.all_funding()}
|
||||
assert allocs["bybit_4edge"].target_dollars == 18900.0
|
||||
assert allocs["bybit_4edge"].target_weight == 0.54
|
||||
assert funds["bybit_4edge"].label == "funded"
|
||||
|
||||
|
||||
def test_reads_empty_when_absent(tmp_path):
|
||||
repo = _repo(tmp_path)
|
||||
assert repo.all_allocations() == []
|
||||
assert repo.all_funding() == []
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run to verify it fails**
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && source .venv/bin/activate && python -m pytest tests/integration/test_dashboard_reads.py -q`
|
||||
Expected: FAIL — `AttributeError: 'ForwardNavRepo' object has no attribute 'all_allocations'`.
|
||||
|
||||
- [ ] **Step 3: Add the reads**
|
||||
|
||||
In `forward_nav.py`, ensure `StrategyFundingRow` is imported in the `cockpit_models` import block (beside `StrategyAllocationRow`), then add after `current_funding` (near line 286):
|
||||
|
||||
```python
|
||||
def all_allocations(self) -> list[StrategyAllocationRow]:
|
||||
with Session(self._engine) as s:
|
||||
return list(s.scalars(select(StrategyAllocationRow)))
|
||||
|
||||
def all_funding(self) -> list[StrategyFundingRow]:
|
||||
with Session(self._engine) as s:
|
||||
return list(s.scalars(select(StrategyFundingRow)))
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run to verify pass + ruff + commit**
|
||||
|
||||
Run: `python -m pytest tests/integration/test_dashboard_reads.py -q` → PASS (2).
|
||||
Run: `ruff check src/fxhnt/adapters/persistence/forward_nav.py tests/integration/test_dashboard_reads.py`
|
||||
```bash
|
||||
git add src/fxhnt/adapters/persistence/forward_nav.py tests/integration/test_dashboard_reads.py
|
||||
git commit -m "feat(cockpit): repo reads for allocation + funding rows
|
||||
|
||||
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 2: Read-model — DTO fields + `DashboardService` allocation/funding/exec pairing
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/ports/dashboard.py` (extend DTOs)
|
||||
- Modify: `src/fxhnt/application/dashboard_service.py`
|
||||
- Test: `tests/integration/test_dashboard_surfacing.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `all_allocations()`/`all_funding()` (Task 1), `all_summaries()` (executed twins' forward summaries), `ALLOCATION_POLICY.capital_ceiling`.
|
||||
- Produces: `FleetRow`/`StrategyDetail` gain `alloc_dollars`/`alloc_weight`/`alloc_policy_version`/`funding_state`/`funding_label`/`funding_decay_run`/`funding_recovery_run`/`funding_first_pass_date`/`funding_policy_version`/`exec_twin_sid`/`exec_return_pct`/`exec_divergence_pct` (all default None/""/0). `OverviewModel` gains `gross_deployed`/`ceiling`/`n_funded`/`n_defunded`. `DashboardService.exec_twin_sids() -> set[str]` (the twin sids to hide from top-level).
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
Create `tests/integration/test_dashboard_surfacing.py`:
|
||||
|
||||
```python
|
||||
import datetime as dt
|
||||
|
||||
from fxhnt.adapters.persistence.cockpit_models import (
|
||||
ForwardSummaryRow, StrategyAllocationRow, StrategyFundingRow,
|
||||
)
|
||||
from fxhnt.adapters.persistence.forward_nav import ForwardNavRepo
|
||||
from fxhnt.application.dashboard_service import DashboardService
|
||||
|
||||
|
||||
def _seed(tmp_path):
|
||||
repo = ForwardNavRepo(f"sqlite:///{tmp_path/'c.db'}")
|
||||
repo.migrate()
|
||||
at = dt.datetime(2026, 7, 10)
|
||||
def _sum(sid, tr):
|
||||
return ForwardSummaryRow(strategy_id=sid, as_of="2026-07-10", days=5, nav=1.0 + tr, total_return=tr,
|
||||
sharpe=1.0, maxdd=-0.05, gate_status="WAIT", gate_reason="", src_updated_at=at)
|
||||
from sqlalchemy.orm import Session
|
||||
with Session(repo._engine) as s:
|
||||
s.add_all([_sum("bybit_4edge", 0.0031), _sum("bybit_4edge_exec", 0.0019)])
|
||||
s.commit()
|
||||
repo.replace_allocations([StrategyAllocationRow(strategy_id="bybit_4edge", target_weight=0.54,
|
||||
target_dollars=18900.0, policy_version=1, policy_hash="h", as_of="2026-07-10", computed_at=at)], at)
|
||||
repo.replace_funding([
|
||||
StrategyFundingRow(strategy_id="bybit_4edge", state="funded", label="funded", decay_run=0,
|
||||
recovery_run=0, first_pass_date=None, policy_version=2, policy_hash="h", as_of="2026-07-10", computed_at=at),
|
||||
StrategyFundingRow(strategy_id="multistrat", state="de-funded", label="de-funded", decay_run=0,
|
||||
recovery_run=0, first_pass_date="2026-06-20", policy_version=2, policy_hash="h", as_of="2026-07-10", computed_at=at),
|
||||
], at)
|
||||
return repo
|
||||
|
||||
|
||||
def test_fleet_carries_allocation_funding_and_exec_pairing(tmp_path):
|
||||
svc = DashboardService(_seed(tmp_path))
|
||||
rows = {f.strategy_id: f for f in svc.fleet()}
|
||||
b = rows["bybit_4edge"]
|
||||
assert b.alloc_dollars == 18900.0 and abs(b.alloc_weight - 0.54) < 1e-9
|
||||
assert b.funding_label == "funded"
|
||||
# exec pairing: bybit_4edge modeled paired with bybit_4edge_exec (return 0.19% vs 0.31% -> Δ -0.12%)
|
||||
assert b.exec_twin_sid == "bybit_4edge_exec"
|
||||
assert abs(b.exec_return_pct - 0.19) < 1e-6
|
||||
assert abs(b.exec_divergence_pct - (-0.12)) < 1e-6
|
||||
# a de-funded edge
|
||||
assert rows["multistrat"].funding_label == "de-funded"
|
||||
# the executed twin is flagged for hiding at top level
|
||||
assert "bybit_4edge_exec" in svc.exec_twin_sids()
|
||||
|
||||
|
||||
def test_overview_capital_meter(tmp_path):
|
||||
ov = DashboardService(_seed(tmp_path)).overview()
|
||||
assert ov.gross_deployed == 18900.0
|
||||
assert ov.ceiling == 35000.0
|
||||
assert ov.n_funded == 1 and ov.n_defunded == 1
|
||||
|
||||
|
||||
def test_graceful_when_no_alloc_funding(tmp_path):
|
||||
# a repo with summaries but NO allocation/funding rows -> fields default, no crash.
|
||||
repo = ForwardNavRepo(f"sqlite:///{tmp_path/'e.db'}")
|
||||
repo.migrate()
|
||||
rows = {f.strategy_id: f for f in DashboardService(repo).fleet()}
|
||||
any_row = next(iter(rows.values()))
|
||||
assert any_row.alloc_dollars is None and any_row.funding_label == ""
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run to verify it fails**
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && source .venv/bin/activate && python -m pytest tests/integration/test_dashboard_surfacing.py -q`
|
||||
Expected: FAIL — `TypeError`/`AttributeError` on the new FleetRow fields / `exec_twin_sids`.
|
||||
|
||||
- [ ] **Step 3: Extend the DTOs**
|
||||
|
||||
In `src/fxhnt/ports/dashboard.py`, append these fields to `FleetRow` (after `reference_only`):
|
||||
|
||||
```python
|
||||
# B2a allocation (None when no strategy_allocation row).
|
||||
alloc_dollars: float | None = None
|
||||
alloc_weight: float | None = None
|
||||
alloc_policy_version: int | None = None
|
||||
# B3 funding health ("" when no strategy_funding row).
|
||||
funding_state: str = "" # "" | "funded" | "de-funded"
|
||||
funding_label: str = "" # "" | "funded" | "decaying" | "de-funded"
|
||||
funding_decay_run: int = 0
|
||||
funding_recovery_run: int = 0
|
||||
funding_first_pass_date: str | None = None
|
||||
funding_policy_version: int | None = None
|
||||
# C1/C2 executed twin (set only on a MODELED track that has an executed twin with forward data).
|
||||
exec_twin_sid: str = ""
|
||||
exec_return_pct: float | None = None
|
||||
exec_divergence_pct: float | None = None
|
||||
```
|
||||
|
||||
Append the SAME 12 fields to `StrategyDetail` (after `note`). Add to `OverviewModel` (after `edges`):
|
||||
|
||||
```python
|
||||
gross_deployed: float = 0.0
|
||||
ceiling: float = 0.0
|
||||
n_funded: int = 0
|
||||
n_defunded: int = 0
|
||||
```
|
||||
|
||||
Add to the `DashboardReadModel` Protocol:
|
||||
```python
|
||||
def exec_twin_sids(self) -> set[str]: ...
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Populate them in `DashboardService`**
|
||||
|
||||
In `dashboard_service.py`, add the import and a reverse-alias constant near the top:
|
||||
|
||||
```python
|
||||
from fxhnt.application.allocation_policy import ALLOCATION_POLICY
|
||||
from fxhnt.application.forward_ingest import CockpitBacktestRefProvider
|
||||
|
||||
# modeled sid -> its executed twin sid (reverse of C1/C2's exec->modeled reconciliation alias).
|
||||
_EXEC_TWIN = {modeled: exec_sid for exec_sid, modeled in CockpitBacktestRefProvider._REF_ALIAS.items()}
|
||||
```
|
||||
|
||||
Add a helper method:
|
||||
```python
|
||||
def exec_twin_sids(self) -> set[str]:
|
||||
"""The executed-twin sids (research-tier observed tracks) folded into their modeled parent on the
|
||||
Overview — so they are NOT rendered as their own top-level rows."""
|
||||
return set(_EXEC_TWIN.values())
|
||||
```
|
||||
|
||||
In `fleet()`, after building `summaries`, load allocation + funding (graceful):
|
||||
```python
|
||||
try:
|
||||
allocs = {a.strategy_id: a for a in self._repo.all_allocations()}
|
||||
except Exception: # noqa: BLE001 — a missing/absent table must never break the fleet
|
||||
allocs = {}
|
||||
try:
|
||||
funds = {f.strategy_id: f for f in self._repo.all_funding()}
|
||||
except Exception: # noqa: BLE001
|
||||
funds = {}
|
||||
```
|
||||
Then, when constructing each `FleetRow`, add the new kwargs:
|
||||
```python
|
||||
alloc_dollars=(allocs[reg.strategy_id].target_dollars if reg.strategy_id in allocs else None),
|
||||
alloc_weight=(allocs[reg.strategy_id].target_weight if reg.strategy_id in allocs else None),
|
||||
alloc_policy_version=(allocs[reg.strategy_id].policy_version if reg.strategy_id in allocs else None),
|
||||
funding_state=(funds[reg.strategy_id].state if reg.strategy_id in funds else ""),
|
||||
funding_label=(funds[reg.strategy_id].label if reg.strategy_id in funds else ""),
|
||||
funding_decay_run=(funds[reg.strategy_id].decay_run if reg.strategy_id in funds else 0),
|
||||
funding_recovery_run=(funds[reg.strategy_id].recovery_run if reg.strategy_id in funds else 0),
|
||||
funding_first_pass_date=(funds[reg.strategy_id].first_pass_date if reg.strategy_id in funds else None),
|
||||
funding_policy_version=(funds[reg.strategy_id].policy_version if reg.strategy_id in funds else None),
|
||||
**self._exec_pair(reg.strategy_id, summaries),
|
||||
```
|
||||
Add the pairing helper (pure):
|
||||
```python
|
||||
@staticmethod
|
||||
def _exec_pair(sid, summaries):
|
||||
"""exec_twin_sid / exec_return_pct / exec_divergence_pct for a MODELED sid whose executed twin has a
|
||||
forward summary; empty dict-of-defaults otherwise (so **-splat is a no-op for non-paired tracks)."""
|
||||
twin = _EXEC_TWIN.get(sid)
|
||||
if twin is None or twin not in summaries or sid not in summaries:
|
||||
return {"exec_twin_sid": "", "exec_return_pct": None, "exec_divergence_pct": None}
|
||||
exec_r = 100.0 * summaries[twin].total_return
|
||||
modeled_r = 100.0 * summaries[sid].total_return
|
||||
return {"exec_twin_sid": twin, "exec_return_pct": exec_r,
|
||||
"exec_divergence_pct": exec_r - modeled_r}
|
||||
```
|
||||
|
||||
In `overview()`, compute the meter before the `return` (graceful):
|
||||
```python
|
||||
try:
|
||||
allocs = list(self._repo.all_allocations())
|
||||
except Exception: # noqa: BLE001
|
||||
allocs = []
|
||||
try:
|
||||
funds = list(self._repo.all_funding())
|
||||
except Exception: # noqa: BLE001
|
||||
funds = []
|
||||
gross = sum(abs(a.target_dollars) for a in allocs)
|
||||
n_funded = sum(1 for f in funds if f.state == "funded")
|
||||
n_defunded = sum(1 for f in funds if f.state == "de-funded")
|
||||
```
|
||||
and add to the `OverviewModel(...)` call: `gross_deployed=gross, ceiling=ALLOCATION_POLICY.capital_ceiling, n_funded=n_funded, n_defunded=n_defunded`.
|
||||
|
||||
In `detail()`, populate the same alloc/funding/exec fields on `StrategyDetail` (reuse the same `allocs`/`funds` reads + `_exec_pair(strategy_id, summaries)` where `summaries = {r.strategy_id: r for r in self._repo.all_summaries()}`).
|
||||
|
||||
- [ ] **Step 5: Run to verify pass + ruff + commit**
|
||||
|
||||
Run: `python -m pytest tests/integration/test_dashboard_surfacing.py -q` → PASS (3).
|
||||
Run: `ruff check src/fxhnt/ports/dashboard.py src/fxhnt/application/dashboard_service.py tests/integration/test_dashboard_surfacing.py`
|
||||
```bash
|
||||
git add src/fxhnt/ports/dashboard.py src/fxhnt/application/dashboard_service.py tests/integration/test_dashboard_surfacing.py
|
||||
git commit -m "feat(cockpit): read-model surfaces allocation, funding, exec-vs-modeled pairing
|
||||
|
||||
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 3: `base.html` — mobile bottom-tab nav + CSS tokens
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/adapters/web/templates/base.html`
|
||||
- Test: `tests/integration/test_cockpit_render.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: a bottom-tab `<nav class="tabs">` shown `≤640px`, top `nav.bar` shown desktop; CSS tokens `--warn`/`--warn-bg`, `.b.funded`/`.b.decaying`/`.b.defunded`, `.meter`/`.bar`, `.exec`/`.exec-tag`, `.card` etc. (from the approved mockup).
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
Create `tests/integration/test_cockpit_render.py`:
|
||||
|
||||
```python
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from fxhnt.adapters.persistence.forward_nav import ForwardNavRepo
|
||||
from fxhnt.adapters.web.app import create_app
|
||||
|
||||
|
||||
def _client(tmp_path):
|
||||
repo = ForwardNavRepo(f"sqlite:///{tmp_path/'c.db'}")
|
||||
repo.migrate()
|
||||
return TestClient(create_app(repo))
|
||||
|
||||
|
||||
def test_overview_renders_bottom_tab_nav(tmp_path):
|
||||
r = _client(tmp_path).get("/")
|
||||
assert r.status_code == 200
|
||||
assert 'class="tabs"' in r.text # the mobile bottom-tab nav is present
|
||||
assert ">Overview<" in r.text
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run to verify it fails**
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && source .venv/bin/activate && python -m pytest tests/integration/test_cockpit_render.py -q`
|
||||
Expected: FAIL — `assert 'class="tabs"' in r.text` (no bottom-tab nav yet).
|
||||
|
||||
- [ ] **Step 3: Add the bottom-tab nav**
|
||||
|
||||
In `base.html`, immediately AFTER the closing `</header>` line (currently `...{{ now }}</span></header>`), add the bottom tab bar (a sibling of `<main>`):
|
||||
|
||||
```html
|
||||
{# mobile bottom-tab nav: thumb-reachable, shown only <=640px (see CSS). Mirrors the desktop nav.bar. #}
|
||||
<nav class="tabs" hx-boost="true" aria-label="Primary">
|
||||
<a href="/" class="tab {{ 'on' if path == '/' }}"><span class="ic" aria-hidden="true">▚</span>Overview</a>
|
||||
<a href="/paper" class="tab {{ 'on' if path == '/paper' }}"><span class="ic" aria-hidden="true">▤</span>Paper</a>
|
||||
<a href="/paper/replay" class="tab {{ 'on' if path.startswith('/paper/replay') }}"><span class="ic" aria-hidden="true">▶</span>Replay</a>
|
||||
<a href="/paper/sim" class="tab {{ 'on' if path.startswith('/paper/sim') }}"><span class="ic" aria-hidden="true">◷</span>Backtest</a>
|
||||
</nav>
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Add the CSS tokens + classes**
|
||||
|
||||
In the `<style>` block, add the amber semantic token and the new component classes (append inside `<style>`, before `@media`):
|
||||
|
||||
```css
|
||||
/* B3 funding: a semantic amber for "decaying", distinct from the neutral-grey WAIT. */
|
||||
.b { display:inline-flex; align-items:center; gap:4px; padding:2px 8px; border-radius:20px;
|
||||
font-size:11px; font-weight:600; line-height:1.4; white-space:nowrap; }
|
||||
.b.small { font-size:10px; padding:1px 6px; }
|
||||
.b .dot { width:6px; height:6px; border-radius:50%; background:currentColor; }
|
||||
.b.funded { background:#193b1e; color:#3fb950; }
|
||||
.b.decaying { background:#3a2d10; color:#d29922; }
|
||||
.b.defunded { background:#3b1919; color:#f85149; }
|
||||
/* capital meter (allocation vs ceiling) */
|
||||
.meter { margin-top:12px; padding-top:12px; border-top:1px dashed #30363d; }
|
||||
.meter .mrow { display:flex; justify-content:space-between; font-size:12px; color:#8b949e; margin-bottom:6px; }
|
||||
.meter .mrow b { color:#c9d1d9; font-weight:600; }
|
||||
.capbar { height:8px; border-radius:6px; background:#161b22; overflow:hidden; border:1px solid #21262d; }
|
||||
.capbar > i { display:block; height:100%; background:linear-gradient(90deg,#238636,#3fb950); border-radius:6px; }
|
||||
.meter .mcap { display:flex; justify-content:space-between; font-size:10.5px; color:#6e7681; margin-top:5px; }
|
||||
/* executed twin: a nested sub-line/sub-row under a modeled edge */
|
||||
.exec-tag { font-size:9.5px; letter-spacing:.05em; text-transform:uppercase; color:#6e7681;
|
||||
border:1px solid #30363d; border-radius:5px; padding:1px 5px; }
|
||||
tr.exrow td { padding-top:2px; }
|
||||
tr.exrow td:first-child { padding-left:22px; color:#6e7681; font-size:11.5px; }
|
||||
/* detail lifecycle blocks */
|
||||
.lc { border:1px solid #21262d; border-radius:10px; margin:11px 0; overflow:hidden; }
|
||||
.lc > .lct { display:flex; align-items:center; gap:8px; padding:9px 12px; background:#161b22;
|
||||
font-size:11px; letter-spacing:.05em; text-transform:uppercase; color:#8b949e;
|
||||
border-bottom:1px solid #21262d; }
|
||||
.lc > .lct .r { margin-left:auto; text-transform:none; letter-spacing:0; }
|
||||
.lc .lcb { padding:11px 12px; }
|
||||
.kv { display:flex; justify-content:space-between; gap:12px; padding:4px 0; font-size:12.5px;
|
||||
border-bottom:1px solid #21262d; } .kv:last-child { border-bottom:0; }
|
||||
.kv .k { color:#8b949e; } .kv .v { font-weight:600; text-align:right; }
|
||||
.why { font-size:11.5px; color:#6e7681; margin-top:7px; line-height:1.5; }
|
||||
/* bottom tabs: hidden on desktop, shown on mobile */
|
||||
nav.tabs { display:none; }
|
||||
```
|
||||
|
||||
Then INSIDE the existing `@media (max-width: 640px) { ... }` block, add:
|
||||
|
||||
```css
|
||||
nav.bar { display:none; } /* hide the top nav row on mobile */
|
||||
main { padding-bottom:64px; } /* clear the fixed bottom bar */
|
||||
nav.tabs { position:fixed; bottom:0; left:0; right:0; z-index:20; display:grid;
|
||||
grid-template-columns:repeat(4,1fr); background:rgba(13,17,23,.94);
|
||||
backdrop-filter:blur(10px); border-top:1px solid #21262d; }
|
||||
nav.tabs .tab { padding:9px 0 11px; text-align:center; color:#6e7681; font-size:10.5px;
|
||||
letter-spacing:.02em; }
|
||||
nav.tabs .tab .ic { display:block; font-size:16px; line-height:1; margin:0 auto 4px; }
|
||||
nav.tabs .tab.on { color:#58a6ff; }
|
||||
nav.tabs .tab:focus-visible { outline:2px solid #58a6ff; outline-offset:-2px; }
|
||||
```
|
||||
|
||||
- [ ] **Step 5: Run to verify pass + ruff + commit**
|
||||
|
||||
Run: `python -m pytest tests/integration/test_cockpit_render.py -q` → PASS.
|
||||
(Templates aren't ruff-linted; run `ruff check tests/integration/test_cockpit_render.py`.)
|
||||
```bash
|
||||
git add src/fxhnt/adapters/web/templates/base.html tests/integration/test_cockpit_render.py
|
||||
git commit -m "feat(cockpit): mobile bottom-tab nav + funding/meter/exec CSS tokens
|
||||
|
||||
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 4: `cockpit.html` + overview route — capital meter, allocation/funding columns, nested exec twin
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/adapters/web/templates/cockpit.html`
|
||||
- Modify: `src/fxhnt/adapters/web/app.py` (the `cockpit` route context)
|
||||
- Test: `tests/integration/test_cockpit_render.py` (extend)
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `FleetRow.alloc_*`/`funding_*`/`exec_*` (Task 2), `OverviewModel.gross_deployed`/`ceiling`/`n_funded`/`n_defunded`, `svc.exec_twin_sids()`.
|
||||
|
||||
- [ ] **Step 1: Write the failing test (extend)**
|
||||
|
||||
Append to `tests/integration/test_cockpit_render.py`:
|
||||
|
||||
```python
|
||||
import datetime as dt
|
||||
|
||||
from fxhnt.adapters.persistence.cockpit_models import StrategyAllocationRow, StrategyFundingRow
|
||||
|
||||
|
||||
def test_overview_shows_allocation_and_funding(tmp_path):
|
||||
repo = ForwardNavRepo(f"sqlite:///{tmp_path/'c.db'}")
|
||||
repo.migrate()
|
||||
at = dt.datetime(2026, 7, 10)
|
||||
repo.replace_allocations([StrategyAllocationRow(strategy_id="bybit_4edge", target_weight=0.54,
|
||||
target_dollars=18900.0, policy_version=1, policy_hash="h", as_of="2026-07-10", computed_at=at)], at)
|
||||
repo.replace_funding([StrategyFundingRow(strategy_id="bybit_4edge", state="funded", label="funded",
|
||||
decay_run=0, recovery_run=0, first_pass_date=None, policy_version=2, policy_hash="h",
|
||||
as_of="2026-07-10", computed_at=at)], at)
|
||||
r = TestClient(create_app(repo)).get("/")
|
||||
assert r.status_code == 200
|
||||
assert "18,900" in r.text or "18900" in r.text # allocation dollars surfaced
|
||||
assert "funded" in r.text # funding badge surfaced
|
||||
assert "/ $35,000" in r.text or "35,000" in r.text # capital meter ceiling
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run to verify it fails**
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && source .venv/bin/activate && python -m pytest tests/integration/test_cockpit_render.py::test_overview_shows_allocation_and_funding -q`
|
||||
Expected: FAIL (allocation/funding/meter not rendered yet).
|
||||
|
||||
- [ ] **Step 3: Wire the overview route**
|
||||
|
||||
In `app.py` `cockpit(...)`, replace `fleet = svc.fleet()` with:
|
||||
```python
|
||||
fleet = svc.fleet()
|
||||
overview = svc.overview()
|
||||
twin_sids = svc.exec_twin_sids()
|
||||
# the executed twins are research-tier; they nest under their modeled parent, so exclude them from BOTH
|
||||
# the top-level "all forward tracks" list AND the research grouping (never a separate row).
|
||||
fleet = [f for f in fleet if f.strategy_id not in twin_sids]
|
||||
fleet_top = fleet
|
||||
```
|
||||
`fleet` is now already twin-free, so the existing `deploy_rows` and `research_groups` derivations below it
|
||||
automatically exclude the twins (no separate change needed). Add `"overview": overview, "fleet_top": fleet_top`
|
||||
to the `TemplateResponse` context dict. (Keep passing `fleet` too — it is now the twin-free list.)
|
||||
|
||||
- [ ] **Step 4: Add the capital meter + the columns to `cockpit.html`**
|
||||
|
||||
After `{{ headline_card(headline, headline_curve) }}` (line 10), add the capital meter:
|
||||
```html
|
||||
{% if overview.gross_deployed or overview.ceiling %}
|
||||
<div class="meter">
|
||||
<div class="mrow"><span>Capital deployed</span><span><b>${{ overview.gross_deployed|money }}</b> / ${{ overview.ceiling|money }}</span></div>
|
||||
<div class="capbar"><i style="width:{{ ((overview.gross_deployed / overview.ceiling * 100) if overview.ceiling else 0)|round(0, 'floor') }}%"></i></div>
|
||||
<div class="mcap"><span>{{ overview.n_funded }} funded · {{ overview.n_defunded }} de-funded</span>
|
||||
<span>{{ ((overview.gross_deployed / overview.ceiling * 100) if overview.ceiling else 0)|round(0, 'floor')|int }}% of ceiling</span></div>
|
||||
</div>
|
||||
{% endif %}
|
||||
```
|
||||
|
||||
Change the "All forward tracks" table header (line 41) to:
|
||||
```html
|
||||
<thead><tr><th>track</th><th>tier</th><th>allocated</th><th>funding</th><th>backtest</th><th>forward gate</th></tr></thead>
|
||||
```
|
||||
Change its body loop (lines 43-55) to iterate `fleet_top` and add the two cells + the nested exec sub-row:
|
||||
```html
|
||||
{% for f in fleet_top|sort(attribute='tier') %}
|
||||
<tr>
|
||||
<td data-label="track"><a href="/strategy/{{ f.strategy_id }}">{{ f.display_name }}</a>
|
||||
<span class="muted" style="font-size:11px"> · {{ f.venue }}</span> {{ exec_badge(f.execution) }}
|
||||
{% if f.reference_only %}<span class="muted" style="font-size:9px;border:1px solid currentColor;border-radius:3px;padding:0 4px;opacity:.55" title="reference-only">ref</span>{% endif %}
|
||||
{% if f.status %}<span class="badge WAIT" style="font-size:10px" title="{{ f.note }}">{{ f.status }}</span>{% endif %}</td>
|
||||
<td data-label="tier" class="muted">{{ f.tier }}</td>
|
||||
<td data-label="allocated">
|
||||
{%- if f.alloc_dollars is not none and f.alloc_dollars > 0 -%}
|
||||
<b>${{ f.alloc_dollars|money }}</b>{% if f.alloc_weight is not none %} <span class="muted">· {{ (f.alloc_weight*100)|round|int }}%</span>{% endif %}
|
||||
{%- elif f.funding_label == 'de-funded' -%}<span class="muted">$0</span>
|
||||
{%- else -%}<span class="muted">—</span>{%- endif -%}</td>
|
||||
<td data-label="funding">
|
||||
{%- if f.funding_label == 'funded' -%}<span class="b funded small"><span class="dot"></span>funded</span>
|
||||
{%- elif f.funding_label == 'decaying' -%}<span class="b decaying small"><span class="dot"></span>decaying {{ f.funding_decay_run }}/10</span>
|
||||
{%- elif f.funding_label == 'de-funded' -%}<span class="b defunded small"><span class="dot"></span>de-funded</span>
|
||||
{%- else -%}<span class="muted">—</span>{%- endif -%}</td>
|
||||
<td data-label="backtest">
|
||||
{% if f.bt_sharpe is not none %}Sh {{ '%.2f'|format(f.bt_sharpe) }}{% if f.bt_cagr is not none %} / {{ '%+.0f'|format(f.bt_cagr * 100) }}%{% endif %}
|
||||
{% else %}<span class="muted">—</span>{% endif %}</td>
|
||||
<td data-label="forward gate">{{ f.days }}/{{ f.gate_min_days }} · <span class="badge {{ f.gate_status }}">{{ f.gate_status }}</span></td>
|
||||
</tr>
|
||||
{% if f.exec_twin_sid and f.exec_return_pct is not none %}
|
||||
<tr class="exrow">
|
||||
<td data-label="track" colspan="2">↳ <a href="/strategy/{{ f.exec_twin_sid }}">executed</a> <span class="exec-tag">{{ f.venue }}</span></td>
|
||||
<td data-label="allocated" class="muted">envelope</td>
|
||||
<td data-label="funding" class="muted">net of fees</td>
|
||||
<td data-label="backtest" class="muted">exec {{ '%+.2f'|format(f.exec_return_pct) }}%</td>
|
||||
<td data-label="forward gate">Δ <span class="{{ 'pos' if f.exec_divergence_pct >= 0 else 'neg' }}">{{ '%+.2f'|format(f.exec_divergence_pct) }}%</span></td>
|
||||
</tr>
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
```
|
||||
|
||||
- [ ] **Step 5: Run to verify pass + ruff + commit**
|
||||
|
||||
Run: `python -m pytest tests/integration/test_cockpit_render.py -q` → PASS (all).
|
||||
Run: `ruff check src/fxhnt/adapters/web/app.py` (only your changed lines must be clean; app.py may carry pre-existing violations — do not touch those).
|
||||
```bash
|
||||
git add src/fxhnt/adapters/web/templates/cockpit.html src/fxhnt/adapters/web/app.py tests/integration/test_cockpit_render.py
|
||||
git commit -m "feat(cockpit): overview capital meter + allocation/funding columns + nested exec twin
|
||||
|
||||
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 5: `strategy.html` + detail route — funding / allocation / executed-reality blocks
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/adapters/web/templates/strategy.html`
|
||||
- Modify: `src/fxhnt/adapters/web/app.py` (the `strategy` route — optional crypto slippage read)
|
||||
- Test: `tests/integration/test_cockpit_render.py` (extend)
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `StrategyDetail.alloc_*`/`funding_*`/`exec_*` (Task 2). Optional: `bybit_testnet_nav` mean slippage for the crypto exec twin.
|
||||
|
||||
- [ ] **Step 1: Write the failing test (extend)**
|
||||
|
||||
Append to `tests/integration/test_cockpit_render.py`:
|
||||
|
||||
```python
|
||||
def test_detail_shows_lifecycle_blocks(tmp_path):
|
||||
repo = ForwardNavRepo(f"sqlite:///{tmp_path/'c.db'}")
|
||||
repo.migrate()
|
||||
at = dt.datetime(2026, 7, 10)
|
||||
repo.replace_funding([StrategyFundingRow(strategy_id="bybit_4edge", state="funded", label="funded",
|
||||
decay_run=0, recovery_run=0, first_pass_date=None, policy_version=2, policy_hash="h",
|
||||
as_of="2026-07-10", computed_at=at)], at)
|
||||
repo.replace_allocations([StrategyAllocationRow(strategy_id="bybit_4edge", target_weight=0.54,
|
||||
target_dollars=18900.0, policy_version=1, policy_hash="h", as_of="2026-07-10", computed_at=at)], at)
|
||||
r = TestClient(create_app(repo)).get("/strategy/bybit_4edge")
|
||||
assert r.status_code == 200
|
||||
assert "Funding health" in r.text
|
||||
assert "Allocation" in r.text
|
||||
assert "18,900" in r.text or "18900" in r.text
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run to verify it fails**
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && source .venv/bin/activate && python -m pytest tests/integration/test_cockpit_render.py::test_detail_shows_lifecycle_blocks -q`
|
||||
Expected: FAIL (no lifecycle blocks yet).
|
||||
|
||||
- [ ] **Step 3: Add the lifecycle blocks to `strategy.html`**
|
||||
|
||||
Read `strategy.html` first. After the status/forward header block and BEFORE the "backtest verdict" section, insert:
|
||||
|
||||
```html
|
||||
{% if d.funding_label %}
|
||||
<div class="lc">
|
||||
<div class="lct">Funding health <span class="r">
|
||||
{%- if d.funding_label == 'funded' -%}<span class="b funded small"><span class="dot"></span>funded</span>
|
||||
{%- elif d.funding_label == 'decaying' -%}<span class="b decaying small"><span class="dot"></span>decaying {{ d.funding_decay_run }}/10</span>
|
||||
{%- else -%}<span class="b defunded small"><span class="dot"></span>de-funded</span>{%- endif -%}</span></div>
|
||||
<div class="lcb">
|
||||
<div class="kv"><span class="k">State</span><span class="v">{{ d.funding_label }}</span></div>
|
||||
<div class="kv"><span class="k">First passed</span><span class="v">{{ d.funding_first_pass_date or '— (deploy-tier by config)' }}</span></div>
|
||||
<div class="kv"><span class="k">Decay / recovery run</span><span class="v">{{ d.funding_decay_run }} / {{ d.funding_recovery_run }}</span></div>
|
||||
<div class="why">De-funds after 10 consecutive divergence days from the backtest; re-funds after 15 clean days.
|
||||
{% if d.funding_policy_version %}<span class="muted">gate-policy v{{ d.funding_policy_version }}</span>{% endif %}</div>
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
{% if d.alloc_dollars is not none %}
|
||||
<div class="lc">
|
||||
<div class="lct">Allocation <span class="r">${{ d.alloc_dollars|money }}</span></div>
|
||||
<div class="lcb">
|
||||
<div class="kv"><span class="k">Target capital</span><span class="v">${{ d.alloc_dollars|money }}</span></div>
|
||||
{% if d.alloc_weight is not none %}<div class="kv"><span class="k">Book weight</span><span class="v">{{ (d.alloc_weight*100)|round|int }}% of gross</span></div>{% endif %}
|
||||
<div class="why">Ex-ante vol-target sizing, marginal-Sharpe survivor, bounded by the $35k fund ceiling.
|
||||
{% if d.alloc_policy_version %}<span class="muted">alloc-policy v{{ d.alloc_policy_version }}</span>{% endif %}</div>
|
||||
</div>
|
||||
</div>
|
||||
{% elif d.funding_label == 'de-funded' %}
|
||||
<div class="lc"><div class="lct">Allocation <span class="r">$0</span></div>
|
||||
<div class="lcb"><div class="why">$0 — de-funded / killswitched, held as cash.</div></div></div>
|
||||
{% endif %}
|
||||
{% if d.exec_twin_sid and d.exec_return_pct is not none %}
|
||||
<div class="lc">
|
||||
<div class="lct">Executed reality <span class="r"><span class="exec-tag">{{ d.venue }}</span></span></div>
|
||||
<div class="lcb">
|
||||
<div class="kv"><span class="k">Executed forward</span><span class="v">{{ '%+.2f'|format(d.exec_return_pct) }}%</span></div>
|
||||
<div class="kv"><span class="k">Modeled forward</span><span class="v">{{ '%+.2f'|format(d.total_return_pct) }}%</span></div>
|
||||
<div class="kv"><span class="k">Divergence</span><span class="v {{ 'pos' if d.exec_divergence_pct >= 0 else 'neg' }}">{{ '%+.2f'|format(d.exec_divergence_pct) }}%</span></div>
|
||||
{% if exec_slippage_bps is not none %}<div class="kv"><span class="k">Mean slippage</span><span class="v">{{ '%.1f'|format(exec_slippage_bps) }} bp</span></div>{% endif %}
|
||||
<div class="why">Envelope-basis, funding & fee net — execution decay shows as divergence.
|
||||
<a href="/strategy/{{ d.exec_twin_sid }}">{{ d.exec_twin_sid }} →</a></div>
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Wire the optional crypto slippage in the detail route**
|
||||
|
||||
In `app.py` `strategy(...)`, add before the `TemplateResponse` (graceful, crypto-only):
|
||||
```python
|
||||
exec_slippage_bps = None
|
||||
if d.exec_twin_sid == "bybit_4edge_exec":
|
||||
try:
|
||||
from fxhnt.adapters.persistence.bybit_testnet_repo import BybitTestnetRepo
|
||||
nav = BybitTestnetRepo(get_settings().operational_dsn).latest_testnet_nav()
|
||||
exec_slippage_bps = nav.mean_exec_slippage_bps if nav else None
|
||||
except Exception: # noqa: BLE001 — slippage is a nice-to-have; never break the detail page
|
||||
exec_slippage_bps = None
|
||||
```
|
||||
Add `"exec_slippage_bps": exec_slippage_bps` to the context dict. If `BybitTestnetRepo` has no `latest_testnet_nav()` reader, add a minimal one (`select ... order by run_date desc limit 1`); if that is more than a trivial add, pass `exec_slippage_bps=None` and note the slippage read is deferred (the block already hides when None).
|
||||
|
||||
- [ ] **Step 5: Run to verify pass + ruff + commit**
|
||||
|
||||
Run: `python -m pytest tests/integration/test_cockpit_render.py -q` → PASS (all).
|
||||
Run: `ruff check src/fxhnt/adapters/web/app.py` (changed lines clean).
|
||||
```bash
|
||||
git add src/fxhnt/adapters/web/templates/strategy.html src/fxhnt/adapters/web/app.py tests/integration/test_cockpit_render.py
|
||||
git commit -m "feat(cockpit): detail page funding / allocation / executed-reality blocks
|
||||
|
||||
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 6: Seed script + local browser verification (the acceptance)
|
||||
|
||||
**Files:**
|
||||
- Create: `scripts/seed_cockpit_demo.py`
|
||||
- (No unit test — this task's deliverable is the browser verification + screenshots.)
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: a runnable seed that fills a LOCAL sqlite/postgres with representative forward + allocation + funding + exec rows, so the cockpit renders the full surfacing un-confounded by prod's stale schema.
|
||||
|
||||
- [ ] **Step 1: Write the seed script**
|
||||
|
||||
Create `scripts/seed_cockpit_demo.py` — seed a local DB (DSN from `FXHNT_OPERATIONAL_DSN`, default `sqlite:///./cockpit-demo.db`) with: a `bybit_4edge` forward_summary + backtest_summary + nav history, a `bybit_4edge_exec` forward_summary (executed), a `multistrat` (de-funded) forward_summary, `strategy_allocation` rows (bybit_4edge $18,900/0.54; positioning $9,500/0.27), and `strategy_funding` rows (bybit_4edge funded; positioning decaying 4/10; multistrat de-funded). Use `repo.migrate()` (creates the CURRENT schema) then `replace_allocations`/`replace_funding` + a `Session` insert for the summaries/nav. Mirror the seed values from `tests/integration/test_dashboard_surfacing.py` so the screenshots match the tests. Print the DSN + a hint to run the cockpit against it.
|
||||
|
||||
- [ ] **Step 2: Seed + serve the cockpit locally**
|
||||
|
||||
```bash
|
||||
cd /home/jgrusewski/Work/fxhnt && source .venv/bin/activate
|
||||
export FXHNT_OPERATIONAL_DSN="sqlite:///$(pwd)/cockpit-demo.db"
|
||||
python scripts/seed_cockpit_demo.py
|
||||
FXHNT_DEV_PORT=8801 python scripts/dev_cockpit_local.py # dev helper serves against FXHNT_OPERATIONAL_DSN, no prod port-forward
|
||||
```
|
||||
Poll `curl -sf http://127.0.0.1:8801/healthz` until it responds. (If `dev_cockpit_local.py` hardcodes the prod DSN, run the app via `uvicorn` on `create_app(ForwardNavRepo(os.environ["FXHNT_OPERATIONAL_DSN"]))` instead — no `.migrate()` needed, the seed already migrated.)
|
||||
|
||||
- [ ] **Step 3: Playwright-verify desktop + mobile — READ THE NUMBERS**
|
||||
|
||||
Drive Playwright (the controller does this, not a subagent):
|
||||
- Desktop (default viewport): screenshot `/` — confirm the **capital meter** (`$28,400 / $35,000`, fill bar), the **allocated** + **funding** columns (funded green / decaying amber / de-funded red), and the **nested exec sub-row** under bybit_4edge. Screenshot `/strategy/bybit_4edge` — confirm the **Funding health**, **Allocation**, **Executed reality** blocks.
|
||||
- Mobile: resize to 390×844, reload `/` — confirm the **bottom-tab nav** is fixed at the bottom, the fleet is **cards**, and the funding badges + allocation read cleanly.
|
||||
- Read the actual rendered numbers against the seed (not "looks fine"): the meter %, the allocation dollars, the funding labels, the divergence.
|
||||
|
||||
- [ ] **Step 4: Commit the seed script**
|
||||
|
||||
```bash
|
||||
git add scripts/seed_cockpit_demo.py
|
||||
git commit -m "chore(cockpit): local demo seed for browser verification of the surfacing
|
||||
|
||||
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Final verification (after all tasks)
|
||||
|
||||
- [ ] Full suite: `cd /home/jgrusewski/Work/fxhnt && source .venv/bin/activate && python -m pytest -q -p no:cacheprovider` — all green (prior baseline 1952 + the new cockpit tests).
|
||||
- [ ] `ruff check src/fxhnt/application/dashboard_service.py src/fxhnt/ports/dashboard.py src/fxhnt/adapters/persistence/forward_nav.py` — no new violations.
|
||||
- [ ] The Task 6 browser screenshots (desktop + mobile, overview + detail) reviewed — the numbers read correctly. This is the acceptance gate before the (separate) deploy.
|
||||
@@ -0,0 +1,702 @@
|
||||
# Funding Revocation on Sustained Decay (B3) Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** De-fund an effective-deploy edge whose reconciliation gate has shown sustained divergence from its backtest (hysteresis, no flicker), and re-fund it on sustained recovery — as a pure recompute over A's immutable forward record, narrowing B2a's allocation input.
|
||||
|
||||
**Architecture:** A machine-readable `category` on the gate verdict classifies each day (PASS / DIVERGENCE / IMMATURE / EXECUTION_GAP). A pure engine (`funding_status.py`) recomputes the reconciliation gate as-of each record-day and replays a hysteretic funding state machine from inception (no persisted mutable state). `record_allocation` filters the effective deploy set through it before B2a's `compute_allocation`; a `strategy_funding` snapshot is recorded for audit. B2a and promotion are untouched.
|
||||
|
||||
**Tech Stack:** Python 3.13, frozen dataclasses, SQLAlchemy 2.0 (SQLite in tests / Postgres+Timescale in prod), pytest, ruff.
|
||||
|
||||
## Global Constraints
|
||||
|
||||
- Signal is B1's reconciliation gate verdict category — **only `DIVERGENCE` de-funds; only `PASS` re-funds; `IMMATURE`/`EXECUTION_GAP` are neutral** (never de-fund/re-fund).
|
||||
- **No new mutable state** — funding is a pure function of (policy + immutable forward record), replayed from inception each run (mirrors A/B2a; never a persisted latch).
|
||||
- **Scope guard:** an edge's decay run advances **only after its first PASS** (`has_passed`); an edge that never PASSed is never de-funded.
|
||||
- De-funding is **binary + structural** (sustained divergence over `defund_window`), never a continuous per-day throttle (honours [[pearl_fxhnt_reactive_risk_controllers_rejected]]). Re-funding is symmetric and slower (`refund_window > defund_window`).
|
||||
- Funding-window knobs live in B1's versioned `GatePolicy`; any change bumps `POLICY_VERSION` and the content-derived `policy_hash` (self-check test forces the bump).
|
||||
- Wiring is a **filter in front of B2a** (Approach A): a de-funded edge is *absent* from `compute_allocation`'s input → $0. B2a's `compute_allocation`/`target_capital` and promotion (`promoted_strategy_ids`) are **not modified**.
|
||||
- No capital is moved (that is sub-project C). Cockpit visual surfacing is **deferred** (out of scope) — only the `strategy_funding` audit table is delivered here.
|
||||
- `ForwardNavRow.nav` is cumulative NAV, 1.0-base → `total_return = nav − 1.0`. `nav_history(sid)` returns `NavRowDTO(strategy_id, date: ISO-str, ret: float, nav: float)` ascending by date.
|
||||
|
||||
---
|
||||
|
||||
### Task 1: Machine-readable gate-verdict category
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/application/forward_models.py` (add category constants + `GateVerdict.category`)
|
||||
- Modify: `src/fxhnt/application/reconciliation_gate.py:99-144` (set category on every return)
|
||||
- Modify: `src/fxhnt/application/gate.py:10-21` (set category on every return)
|
||||
- Test: `tests/unit/test_gate_verdict_category.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: `GateVerdict(status: str, reason: str, category: str = CAT_IMMATURE)`; constants `CAT_PASS="PASS"`, `CAT_DIVERGENCE="DIVERGENCE"`, `CAT_IMMATURE="IMMATURE"`, `CAT_EXECUTION_GAP="EXECUTION_GAP"` in `forward_models`.
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
Create `tests/unit/test_gate_verdict_category.py`:
|
||||
|
||||
```python
|
||||
from fxhnt.application.forward_models import (
|
||||
CAT_DIVERGENCE, CAT_EXECUTION_GAP, CAT_IMMATURE, CAT_PASS, ForwardNavRow, ForwardSummary,
|
||||
)
|
||||
from fxhnt.application.gate import evaluate_gate
|
||||
from fxhnt.application.gate_policy_resolve import ResolvedPolicy
|
||||
from fxhnt.application.reconciliation_gate import BacktestReference, evaluate_reconciliation_gate
|
||||
|
||||
_RP = ResolvedPolicy(min_forward_days=3, recon_tolerance_frac=0.5, recon_min_band=0.05,
|
||||
recon_min_corr=0.0, max_gap_days=1, min_days=3, min_total_return=0.0, min_sharpe=0.0)
|
||||
|
||||
|
||||
def _rows(navs, start_day=1):
|
||||
# consecutive daily rows 2026-07-0X; ret unused by the band checks here
|
||||
return [ForwardNavRow(strategy_id="s", date=f"2026-07-{start_day+i:02d}", ret=0.0, nav=v)
|
||||
for i, v in enumerate(navs)]
|
||||
|
||||
|
||||
def _summary(nav, days):
|
||||
return ForwardSummary(strategy_id="s", as_of="2026-07-09", days=days, nav=nav,
|
||||
total_return=nav - 1.0, sharpe=0.0, maxdd=0.0)
|
||||
|
||||
|
||||
def test_reconciliation_categories():
|
||||
ref = BacktestReference(expected_window_return=0.10, daily_returns=None)
|
||||
# building (days < min_forward_days) -> IMMATURE
|
||||
assert evaluate_reconciliation_gate(_summary(1.10, 2), _rows([1.05, 1.10]), ref, _RP).category == CAT_IMMATURE
|
||||
# no backtest reference -> IMMATURE
|
||||
assert evaluate_reconciliation_gate(_summary(1.10, 3), _rows([1.0, 1.05, 1.10]), None, _RP).category == CAT_IMMATURE
|
||||
# execution gap (date jump > max_gap_days) -> EXECUTION_GAP
|
||||
gap = [ForwardNavRow("s", "2026-07-01", 0.0, 1.0), ForwardNavRow("s", "2026-07-02", 0.0, 1.05),
|
||||
ForwardNavRow("s", "2026-07-05", 0.0, 1.10)]
|
||||
assert evaluate_reconciliation_gate(_summary(1.10, 3), gap, ref, _RP).category == CAT_EXECUTION_GAP
|
||||
# forward far below backtest expectation -> DIVERGENCE
|
||||
assert evaluate_reconciliation_gate(_summary(0.95, 3), _rows([1.0, 0.98, 0.95]), ref, _RP).category == CAT_DIVERGENCE
|
||||
# reconciles -> PASS
|
||||
assert evaluate_reconciliation_gate(_summary(1.10, 3), _rows([1.0, 1.05, 1.10]), ref, _RP).category == CAT_PASS
|
||||
|
||||
|
||||
def test_absolute_gate_categories():
|
||||
# evaluate_gate: GO -> PASS, everything else -> IMMATURE (a non-reconciliation edge is never de-funded)
|
||||
assert evaluate_gate(_summary(1.10, 5), _RP).category == CAT_PASS
|
||||
assert evaluate_gate(_summary(1.0, 1), _RP).category == CAT_IMMATURE # building -> IMMATURE
|
||||
neg = ResolvedPolicy(3, 0.5, 0.05, 0.0, 1, 3, 0.05, 0.0) # min_total_return=0.05
|
||||
assert evaluate_gate(_summary(1.0, 5), neg).category == CAT_IMMATURE # NO_GO return floor -> IMMATURE
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && source .venv/bin/activate && python -m pytest tests/unit/test_gate_verdict_category.py -q`
|
||||
Expected: FAIL — `ImportError: cannot import name 'CAT_PASS'` (constants + field not defined yet).
|
||||
|
||||
- [ ] **Step 3: Add the constants + field in `forward_models.py`**
|
||||
|
||||
Replace the `GateVerdict` class (currently `forward_models.py:51-53`) with:
|
||||
|
||||
```python
|
||||
CAT_PASS = "PASS"
|
||||
CAT_DIVERGENCE = "DIVERGENCE"
|
||||
CAT_IMMATURE = "IMMATURE"
|
||||
CAT_EXECUTION_GAP = "EXECUTION_GAP"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class GateVerdict:
|
||||
status: str # "WAIT" | "GO" | "NO_GO" | "PASS"
|
||||
reason: str
|
||||
# machine-readable funding-signal category (B3). Default IMMATURE = neutral (never de-funds) so any
|
||||
# verdict constructed without an explicit category is fail-safe.
|
||||
category: str = CAT_IMMATURE
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Set categories in `reconciliation_gate.py`**
|
||||
|
||||
Add to the imports (top of `reconciliation_gate.py`, the `from fxhnt.application.forward_models import` line):
|
||||
|
||||
```python
|
||||
from fxhnt.application.forward_models import (
|
||||
CAT_DIVERGENCE, CAT_EXECUTION_GAP, CAT_IMMATURE, CAT_PASS,
|
||||
ForwardNavRow, ForwardSummary, GateVerdict,
|
||||
)
|
||||
```
|
||||
|
||||
Then set `category=` on each return in `evaluate_reconciliation_gate`:
|
||||
- building (`return GateVerdict("WAIT", f"building ...")`) → add `, category=CAT_IMMATURE`
|
||||
- execution gap (`return GateVerdict("WAIT", "execution gap ...")`) → add `, category=CAT_EXECUTION_GAP`
|
||||
- no backtest reference (the `if backtest is None:` return) → add `, category=CAT_IMMATURE`
|
||||
- cumulative-band divergence (`shortfall > band` return) → add `, category=CAT_DIVERGENCE`
|
||||
- daily-corr divergence (the `corr < min_corr` return) → add `, category=CAT_DIVERGENCE`
|
||||
- final PASS return → add `, category=CAT_PASS`
|
||||
|
||||
- [ ] **Step 5: Set categories in `gate.py`**
|
||||
|
||||
In `gate.py`, add `CAT_IMMATURE, CAT_PASS` to the `forward_models` import, then:
|
||||
- `GateVerdict("WAIT", f"{summary.days}/{min_days} forward days")` → add `, category=CAT_IMMATURE`
|
||||
- both `GateVerdict("NO_GO", ...)` returns → add `, category=CAT_IMMATURE`
|
||||
- `GateVerdict("GO", ...)` return → add `, category=CAT_PASS`
|
||||
|
||||
- [ ] **Step 6: Run tests to verify they pass**
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && source .venv/bin/activate && python -m pytest tests/unit/test_gate_verdict_category.py -q`
|
||||
Expected: PASS (2 passed).
|
||||
|
||||
- [ ] **Step 7: Verify no existing gate tests broke + ruff**
|
||||
|
||||
Run: `python -m pytest tests/ -q -k "gate or reconcil or forward_ingest" && ruff check src/fxhnt/application/forward_models.py src/fxhnt/application/reconciliation_gate.py src/fxhnt/application/gate.py`
|
||||
Expected: PASS, ruff clean. (Existing verdicts are byte-identical; only the new optional field is added.)
|
||||
|
||||
- [ ] **Step 8: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/application/forward_models.py src/fxhnt/application/reconciliation_gate.py src/fxhnt/application/gate.py tests/unit/test_gate_verdict_category.py
|
||||
git commit -m "feat(gate): machine-readable verdict category (PASS/DIVERGENCE/IMMATURE/EXECUTION_GAP) for B3
|
||||
|
||||
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 2: Funding windows in the versioned GatePolicy
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/application/gate_policy.py:16-34` (add fields + bump version)
|
||||
- Modify: `tests/unit/test_gate_policy.py` (update defaults assertion + pinned hash)
|
||||
- Test: `tests/unit/test_gate_policy.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: `GatePolicy.defund_window: int = 10`, `GatePolicy.refund_window: int = 15`; `POLICY_VERSION = 2`.
|
||||
|
||||
- [ ] **Step 1: Update the failing test first (defaults + version)**
|
||||
|
||||
In `tests/unit/test_gate_policy.py`, extend `test_policy_defaults_match_todays_constants` by appending:
|
||||
|
||||
```python
|
||||
assert (POLICY.defund_window, POLICY.refund_window) == (10, 15)
|
||||
assert POLICY_VERSION == 2
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run it to verify it fails**
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && source .venv/bin/activate && python -m pytest tests/unit/test_gate_policy.py::test_policy_defaults_match_todays_constants -q`
|
||||
Expected: FAIL — `AttributeError: 'GatePolicy' object has no attribute 'defund_window'`.
|
||||
|
||||
- [ ] **Step 3: Add the fields + bump the version**
|
||||
|
||||
In `gate_policy.py`, add to `GatePolicy` (after the promotion block, before the class ends):
|
||||
|
||||
```python
|
||||
# funding revocation on sustained decay (B3): consecutive DIVERGENCE days that de-fund, and consecutive
|
||||
# PASS days that re-fund. Asymmetric (quicker to de-risk, slower to re-arm capital). Debounce over an
|
||||
# already-cumulative gate signal.
|
||||
defund_window: int = 10
|
||||
refund_window: int = 15
|
||||
```
|
||||
|
||||
Then change `POLICY_VERSION = 1` to `POLICY_VERSION = 2`.
|
||||
|
||||
- [ ] **Step 4: Recompute and pin the new hash**
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && source .venv/bin/activate && python -c "from fxhnt.application.gate_policy import POLICY, POLICY_VERSION, policy_hash; print(policy_hash(POLICY, POLICY_VERSION))"`
|
||||
|
||||
Copy the printed hex into `tests/unit/test_gate_policy.py`, replacing the `_EXPECTED_HASH = "..."` literal with the new value.
|
||||
|
||||
- [ ] **Step 5: Run the gate-policy tests**
|
||||
|
||||
Run: `python -m pytest tests/unit/test_gate_policy.py -q`
|
||||
Expected: PASS (3 passed) — the self-check now pins v2's hash, proving the version bump is deliberate.
|
||||
|
||||
- [ ] **Step 6: Fix any other tests that pinned policy version/hash**
|
||||
|
||||
Run: `python -m pytest tests/integration/test_forward_summary_policy_stamp.py tests/integration/test_forward_ingest.py tests/integration/test_strategy_allocation.py tests/integration/test_execute_multistrat_b2a.py tests/unit/test_allocation_policy.py -q`
|
||||
Expected: PASS. If any assert `policy_version == 1` (gate policy) or pin the old gate `policy_hash`, update them to `POLICY_VERSION` / recomputed hash. (The B2a `ALLOCATION_POLICY_VERSION` is a SEPARATE policy — do not touch it; only the B1 gate `POLICY_VERSION` changed.)
|
||||
|
||||
- [ ] **Step 7: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/application/gate_policy.py tests/unit/test_gate_policy.py
|
||||
git commit -m "feat(policy): add defund/refund windows to GatePolicy, bump POLICY_VERSION=2 (B3)
|
||||
|
||||
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>"
|
||||
```
|
||||
(Include any other test files you had to touch in Step 6 in the `git add`.)
|
||||
|
||||
---
|
||||
|
||||
### Task 3: The pure funding engine (`funding_status.py`)
|
||||
|
||||
**Files:**
|
||||
- Create: `src/fxhnt/application/funding_status.py`
|
||||
- Test: `tests/unit/test_funding_status.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `GateVerdict.category` (Task 1); `ResolvedPolicy` (`gate_policy_resolve`); `evaluate_reconciliation_gate`, `BacktestReference` (`reconciliation_gate`); `ForwardSummary` (`forward_models`); rows with `.date: str`, `.ret: float`, `.nav: float` (A's `NavRowDTO`, ascending).
|
||||
- Produces: `FundingStatus(state, label, decay_run, recovery_run, first_pass_date)`; `compute_funding(sid, rows, ref_for_days, resolved_policy, defund_window, refund_window) -> FundingStatus`, where `ref_for_days: Callable[[int], BacktestReference | None]`.
|
||||
|
||||
- [ ] **Step 1: Write the failing tests (the load-bearing state machine)**
|
||||
|
||||
Create `tests/unit/test_funding_status.py`:
|
||||
|
||||
```python
|
||||
from fxhnt.application.forward_models import ForwardNavRow
|
||||
from fxhnt.application.funding_status import compute_funding
|
||||
from fxhnt.application.gate_policy_resolve import ResolvedPolicy
|
||||
from fxhnt.application.reconciliation_gate import BacktestReference
|
||||
|
||||
# min_forward_days=1 so every day yields a non-building verdict; tiny windows so sequences stay short.
|
||||
_RP = ResolvedPolicy(min_forward_days=1, recon_tolerance_frac=0.5, recon_min_band=0.05,
|
||||
recon_min_corr=0.0, max_gap_days=1, min_days=1, min_total_return=0.0, min_sharpe=0.0)
|
||||
_REF = lambda _days: BacktestReference(expected_window_return=0.10, daily_returns=None) # fixed expectation
|
||||
_DEFUND, _REFUND = 2, 3
|
||||
|
||||
|
||||
def _rows(navs):
|
||||
# consecutive dates so _has_gaps stays False (no EXECUTION_GAP unless we deliberately gap)
|
||||
return [ForwardNavRow("s", f"2026-07-{1+i:02d}", 0.0, v) for i, v in enumerate(navs)]
|
||||
|
||||
|
||||
def _fund(navs):
|
||||
return compute_funding("s", _rows(navs), _REF, _RP, _DEFUND, _REFUND)
|
||||
|
||||
|
||||
# nav 1.10 -> total_return 0.10 == expected -> PASS ; nav 0.95 -> -0.05, shortfall 0.15 > band 0.05 -> DIVERGENCE
|
||||
|
||||
def test_sustained_divergence_defunds_after_window():
|
||||
s = _fund([1.10, 0.95, 0.95]) # PASS, DIVERGENCE x2 (== defund_window)
|
||||
assert s.state == "de-funded"
|
||||
assert s.first_pass_date == "2026-07-01"
|
||||
|
||||
|
||||
def test_transient_dip_does_not_flicker():
|
||||
s = _fund([1.10, 0.95, 1.10]) # PASS, DIVERGENCE(1), PASS -> decay run reset
|
||||
assert s.state == "funded"
|
||||
assert s.decay_run == 0
|
||||
|
||||
|
||||
def test_recovery_refunds_after_window():
|
||||
s = _fund([1.10, 0.95, 0.95, 1.10, 1.10, 1.10]) # de-fund at idx2, then 3 PASS == refund_window
|
||||
assert s.state == "funded"
|
||||
|
||||
|
||||
def test_never_passed_is_never_defunded():
|
||||
s = _fund([0.95, 0.95, 0.95, 0.95]) # all DIVERGENCE, no PASS -> decay run never advances
|
||||
assert s.state == "funded"
|
||||
assert s.first_pass_date is None
|
||||
|
||||
|
||||
def test_execution_gap_resets_decay_run():
|
||||
rows = [ForwardNavRow("s", "2026-07-01", 0.0, 1.10), # PASS
|
||||
ForwardNavRow("s", "2026-07-02", 0.0, 0.95), # DIVERGENCE (decay_run 1)
|
||||
ForwardNavRow("s", "2026-07-04", 0.0, 0.95)] # gap>1 -> whole record EXECUTION_GAP -> neutral
|
||||
s = compute_funding("s", rows, _REF, _RP, _DEFUND, _REFUND)
|
||||
assert s.state == "funded"
|
||||
|
||||
|
||||
def test_deterministic():
|
||||
assert _fund([1.10, 0.95, 0.95]) == _fund([1.10, 0.95, 0.95])
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run to verify it fails**
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && source .venv/bin/activate && python -m pytest tests/unit/test_funding_status.py -q`
|
||||
Expected: FAIL — `ModuleNotFoundError: No module named 'fxhnt.application.funding_status'`.
|
||||
|
||||
- [ ] **Step 3: Implement the engine**
|
||||
|
||||
Create `src/fxhnt/application/funding_status.py`:
|
||||
|
||||
```python
|
||||
"""Sub-project B3 — funding revocation on sustained decay. Pure, recompute-from-record: replay B1's
|
||||
reconciliation gate as-of each forward record-day and step a hysteretic funding state machine from
|
||||
inception. No persisted mutable state (mirrors A/B2a). Only DIVERGENCE de-funds; only PASS re-funds;
|
||||
IMMATURE/EXECUTION_GAP are neutral. An edge's decay run advances only after its first PASS."""
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable, Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import Protocol
|
||||
|
||||
from fxhnt.application.forward_models import CAT_DIVERGENCE, CAT_PASS, ForwardSummary
|
||||
from fxhnt.application.gate_policy_resolve import ResolvedPolicy
|
||||
from fxhnt.application.reconciliation_gate import BacktestReference, evaluate_reconciliation_gate
|
||||
|
||||
|
||||
class _Row(Protocol):
|
||||
date: str
|
||||
ret: float
|
||||
nav: float
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class FundingStatus:
|
||||
state: str # "funded" | "de-funded" — the capital verdict
|
||||
label: str # "funded" | "decaying" | "de-funded" — display
|
||||
decay_run: int
|
||||
recovery_run: int
|
||||
first_pass_date: str | None
|
||||
|
||||
|
||||
def compute_funding(sid: str, rows: Sequence[_Row],
|
||||
ref_for_days: Callable[[int], BacktestReference | None],
|
||||
resolved_policy: ResolvedPolicy,
|
||||
defund_window: int, refund_window: int) -> FundingStatus:
|
||||
"""Replay the funding state machine over `rows` (ascending). At each day, recompute the reconciliation
|
||||
gate as-of that day (as-of summary from the truncated record; backtest reference pro-rated to the day's
|
||||
forward length) and step the machine. Deterministic; no persisted state."""
|
||||
state = "funded"
|
||||
decay_run = recovery_run = 0
|
||||
has_passed = False
|
||||
first_pass_date: str | None = None
|
||||
|
||||
for i in range(len(rows)):
|
||||
upto = rows[:i + 1]
|
||||
last = upto[-1]
|
||||
days = i + 1
|
||||
summary = ForwardSummary(strategy_id=sid, as_of=last.date, days=days, nav=last.nav,
|
||||
total_return=last.nav - 1.0, sharpe=0.0, maxdd=0.0)
|
||||
ref = ref_for_days(max(days, resolved_policy.min_forward_days))
|
||||
category = evaluate_reconciliation_gate(summary, list(upto), ref, resolved_policy).category
|
||||
|
||||
if category == CAT_PASS and not has_passed:
|
||||
has_passed = True
|
||||
first_pass_date = last.date
|
||||
|
||||
if state == "funded":
|
||||
decay_run = decay_run + 1 if (category == CAT_DIVERGENCE and has_passed) else 0
|
||||
if decay_run >= defund_window:
|
||||
state, decay_run, recovery_run = "de-funded", 0, 0
|
||||
else: # de-funded
|
||||
recovery_run = recovery_run + 1 if category == CAT_PASS else 0
|
||||
if recovery_run >= refund_window:
|
||||
state, decay_run, recovery_run = "funded", 0, 0
|
||||
|
||||
label = "de-funded" if state == "de-funded" else ("decaying" if decay_run > 0 else "funded")
|
||||
return FundingStatus(state, label, decay_run, recovery_run, first_pass_date)
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run to verify pass**
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && source .venv/bin/activate && python -m pytest tests/unit/test_funding_status.py -q`
|
||||
Expected: PASS (6 passed).
|
||||
|
||||
- [ ] **Step 5: ruff**
|
||||
|
||||
Run: `ruff check src/fxhnt/application/funding_status.py tests/unit/test_funding_status.py`
|
||||
Expected: clean.
|
||||
|
||||
- [ ] **Step 6: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/application/funding_status.py tests/unit/test_funding_status.py
|
||||
git commit -m "feat(funding): pure recompute-from-record funding state machine (B3)
|
||||
|
||||
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 4: `strategy_funding` audit table + repo
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/adapters/persistence/cockpit_models.py` (new `StrategyFundingRow`, after `StrategyAllocationRow`)
|
||||
- Modify: `src/fxhnt/adapters/persistence/forward_nav.py` (import + `replace_funding` / `current_funding`)
|
||||
- Test: `tests/integration/test_strategy_funding.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: `StrategyFundingRow(strategy_id, state, label, decay_run, recovery_run, first_pass_date, policy_version, policy_hash, as_of, computed_at)`; `ForwardNavRepo.replace_funding(rows, at)`, `ForwardNavRepo.current_funding() -> dict[str, str]` (sid → state).
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
Create `tests/integration/test_strategy_funding.py`:
|
||||
|
||||
```python
|
||||
import datetime as dt
|
||||
|
||||
from fxhnt.adapters.persistence.cockpit_models import StrategyFundingRow
|
||||
from fxhnt.adapters.persistence.forward_nav import ForwardNavRepo
|
||||
|
||||
|
||||
def _repo(tmp_path):
|
||||
repo = ForwardNavRepo(f"sqlite:///{tmp_path/'c.db'}")
|
||||
repo.migrate()
|
||||
return repo
|
||||
|
||||
|
||||
def _row(sid, state):
|
||||
return StrategyFundingRow(strategy_id=sid, state=state, label=state, decay_run=0, recovery_run=0,
|
||||
first_pass_date=None, policy_version=2, policy_hash="h", as_of="2026-07-10",
|
||||
computed_at=dt.datetime(2026, 7, 10))
|
||||
|
||||
|
||||
def test_replace_and_read_funding(tmp_path):
|
||||
repo = _repo(tmp_path)
|
||||
at = dt.datetime(2026, 7, 10)
|
||||
repo.replace_funding([_row("a", "funded"), _row("b", "de-funded")], at)
|
||||
assert repo.current_funding() == {"a": "funded", "b": "de-funded"}
|
||||
|
||||
|
||||
def test_replace_clears_stale(tmp_path):
|
||||
repo = _repo(tmp_path)
|
||||
at = dt.datetime(2026, 7, 10)
|
||||
repo.replace_funding([_row("a", "funded"), _row("b", "de-funded")], at)
|
||||
repo.replace_funding([_row("a", "funded")], at) # 'b' dropped this run
|
||||
assert repo.current_funding() == {"a": "funded"}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run to verify it fails**
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && source .venv/bin/activate && python -m pytest tests/integration/test_strategy_funding.py -q`
|
||||
Expected: FAIL — `ImportError: cannot import name 'StrategyFundingRow'`.
|
||||
|
||||
- [ ] **Step 3: Add the model**
|
||||
|
||||
In `cockpit_models.py`, after the `StrategyAllocationRow` class, add:
|
||||
|
||||
```python
|
||||
class StrategyFundingRow(CockpitBase):
|
||||
"""B3 funding-status snapshot — one row per effective-deploy strategy, replaced each run. A derived cache
|
||||
of (gate policy + A's immutable record). `state` gates B2a's allocation input; `label` is the display."""
|
||||
__tablename__ = "strategy_funding"
|
||||
strategy_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
state: Mapped[str] = mapped_column(String(16)) # funded | de-funded
|
||||
label: Mapped[str] = mapped_column(String(16)) # funded | decaying | de-funded
|
||||
decay_run: Mapped[int] = mapped_column(Integer)
|
||||
recovery_run: Mapped[int] = mapped_column(Integer)
|
||||
first_pass_date: Mapped[str | None] = mapped_column(String(10), nullable=True, default=None)
|
||||
policy_version: Mapped[int] = mapped_column(Integer)
|
||||
policy_hash: Mapped[str] = mapped_column(String(64))
|
||||
as_of: Mapped[str] = mapped_column(String(10))
|
||||
computed_at: Mapped[dt.datetime] = mapped_column(DateTime)
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Add the repo methods**
|
||||
|
||||
In `forward_nav.py`, add `StrategyFundingRow` to the `cockpit_models` import block (beside `StrategyAllocationRow`), then add after `current_allocation` (near line 272):
|
||||
|
||||
```python
|
||||
def replace_funding(self, rows: list[StrategyFundingRow], at: dt.datetime) -> None:
|
||||
"""Replace the ENTIRE strategy_funding snapshot (delete-then-insert) so an edge that leaves the
|
||||
effective-deploy set leaves no stale funding row behind."""
|
||||
with Session(self._engine) as s:
|
||||
s.query(StrategyFundingRow).delete()
|
||||
for r in rows:
|
||||
r.computed_at = at
|
||||
s.add(r)
|
||||
s.commit()
|
||||
|
||||
def current_funding(self) -> dict[str, str]:
|
||||
with Session(self._engine) as s:
|
||||
return {r.strategy_id: r.state for r in s.scalars(select(StrategyFundingRow))}
|
||||
```
|
||||
|
||||
`migrate()` already calls `CockpitBase.metadata.create_all` — it creates the new `strategy_funding` table on both SQLite and prod Postgres (create_all adds missing *tables*); no `_ensure` column-patch is needed for a brand-new table.
|
||||
|
||||
- [ ] **Step 5: Run to verify pass**
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && source .venv/bin/activate && python -m pytest tests/integration/test_strategy_funding.py -q`
|
||||
Expected: PASS (2 passed).
|
||||
|
||||
- [ ] **Step 6: ruff**
|
||||
|
||||
Run: `ruff check src/fxhnt/adapters/persistence/cockpit_models.py src/fxhnt/adapters/persistence/forward_nav.py tests/integration/test_strategy_funding.py`
|
||||
Expected: clean.
|
||||
|
||||
- [ ] **Step 7: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/adapters/persistence/cockpit_models.py src/fxhnt/adapters/persistence/forward_nav.py tests/integration/test_strategy_funding.py
|
||||
git commit -m "feat(persistence): strategy_funding audit table + repo (B3)
|
||||
|
||||
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 5: Wire the funding filter into `record_allocation`
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/application/allocation_ingest.py`
|
||||
- Test: `tests/integration/test_allocation_funding_filter.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `compute_funding` (Task 3), `StrategyFundingRow` + `replace_funding` (Task 4), `CockpitBacktestRefProvider` (`forward_ingest`), `resolve_policy` + `POLICY`/`POLICY_VERSION`/`policy_hash` (`gate_policy*`).
|
||||
- Produces: unchanged `record_allocation(dsn, equity) -> dict[str, float]` signature (a de-funded edge is absent from the returned dollars); a recorded `strategy_funding` snapshot over ALL effective-deploy edges.
|
||||
|
||||
- [ ] **Step 1: Write the failing integration test**
|
||||
|
||||
Create `tests/integration/test_allocation_funding_filter.py`. It seeds the real `bybit_4edge` (a registry `tier=="deploy"`, reconciliation-gated edge) two ways in two tests: (1) healthy, ≥60 days → stays funded and receives dollars; (2) PASSes then sustainedly diverges → de-funded and absent from the allocation. The dsn passed to `record_allocation` is the same sqlite file the repo uses. Seed values use **wide margins** so the gate verdicts are unambiguous — adjust only if a local threshold shifts them.
|
||||
|
||||
```python
|
||||
import datetime as dt
|
||||
|
||||
import pytest
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from fxhnt.adapters.persistence.cockpit_models import BacktestSummaryRow, ForwardNavRow, ForwardSummaryRow
|
||||
from fxhnt.adapters.persistence.forward_nav import ForwardNavRepo
|
||||
from fxhnt.application.allocation_ingest import record_allocation
|
||||
from fxhnt.registry import STRATEGY_REGISTRY
|
||||
|
||||
_DSN = None
|
||||
_SID = "bybit_4edge" # registry tier==deploy, gate_type reconciliation, min_forward_days 21
|
||||
|
||||
|
||||
def _mk_repo(tmp_path):
|
||||
global _DSN
|
||||
_DSN = f"sqlite:///{tmp_path/'c.db'}"
|
||||
repo = ForwardNavRepo(_DSN)
|
||||
repo.migrate()
|
||||
return repo
|
||||
|
||||
|
||||
def _seed(repo, sid, navs, cagr, at, start="2026-01-01"):
|
||||
"""Seed a forward record (consecutive daily rows), a summary, and a backtest_summary (the recon ref)."""
|
||||
d0 = dt.date.fromisoformat(start)
|
||||
with Session(repo._engine) as s:
|
||||
for i, v in enumerate(navs):
|
||||
ret = (navs[i] / navs[i - 1] - 1.0) if i else (v - 1.0)
|
||||
s.add(ForwardNavRow(strategy_id=sid, date=d0 + dt.timedelta(days=i), ret=ret, nav=v, src_updated_at=at))
|
||||
last = (d0 + dt.timedelta(days=len(navs) - 1)).isoformat()
|
||||
s.add(ForwardSummaryRow(strategy_id=sid, as_of=last, days=len(navs), nav=navs[-1],
|
||||
total_return=navs[-1] - 1.0, sharpe=1.0, maxdd=-0.05,
|
||||
gate_status="WAIT", gate_reason="", src_updated_at=at))
|
||||
s.add(BacktestSummaryRow(strategy_id=sid, as_of=last, cagr=cagr, ann_vol=0.2, sharpe=1.0,
|
||||
max_drawdown=-0.1, passed=True, dsr=0.9, is_sharpe=1.0, oos_sharpe=1.0,
|
||||
pvalue=0.01, src_updated_at=at))
|
||||
s.commit()
|
||||
|
||||
|
||||
@pytest.mark.skipif(_SID not in STRATEGY_REGISTRY or STRATEGY_REGISTRY.get(_SID, {}).get("tier") != "deploy",
|
||||
reason="needs bybit_4edge as a tier==deploy reconciliation edge")
|
||||
def test_healthy_deploy_edge_stays_funded_and_allocated(tmp_path):
|
||||
repo = _mk_repo(tmp_path)
|
||||
at = dt.datetime(2026, 7, 10)
|
||||
# 65 days steadily climbing (>= B2a min_history_days=60, positive Sharpe), modest backtest cagr -> reconciles.
|
||||
_seed(repo, _SID, [1.0 + 0.0015 * i for i in range(65)], cagr=0.6, at=at)
|
||||
dollars = record_allocation(_DSN, equity=35_000.0)
|
||||
assert repo.current_funding().get(_SID) == "funded"
|
||||
assert dollars.get(_SID, 0.0) > 0.0
|
||||
|
||||
|
||||
@pytest.mark.skipif(_SID not in STRATEGY_REGISTRY or STRATEGY_REGISTRY.get(_SID, {}).get("tier") != "deploy",
|
||||
reason="needs bybit_4edge as a tier==deploy reconciliation edge")
|
||||
def test_decayed_deploy_edge_is_defunded_and_absent(tmp_path):
|
||||
repo = _mk_repo(tmp_path)
|
||||
at = dt.datetime(2026, 7, 10)
|
||||
# Days 0..22 climb to +7.7% (first PASS once days >= min_forward_days 21), then 11 days flat at -15%
|
||||
# (sustained DIVERGENCE >= defund_window 10 -> de-funded). cagr 2.0 keeps the backtest expectation clearly
|
||||
# positive so the -15% forward is an unambiguous divergence.
|
||||
navs = [1.0 + 0.0035 * i for i in range(23)] + [0.85] * 11
|
||||
_seed(repo, _SID, navs, cagr=2.0, at=at)
|
||||
dollars = record_allocation(_DSN, equity=35_000.0)
|
||||
assert repo.current_funding().get(_SID) == "de-funded"
|
||||
assert _SID not in dollars # de-funded -> absent from B2a's input -> no dollars
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run to verify it fails**
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && source .venv/bin/activate && python -m pytest tests/integration/test_allocation_funding_filter.py -q`
|
||||
Expected: FAIL — `current_funding` empty / `record_allocation` does not yet record funding.
|
||||
|
||||
- [ ] **Step 3: Wire the filter + snapshot into `record_allocation`**
|
||||
|
||||
Rewrite `allocation_ingest.py`'s imports and `record_allocation` body:
|
||||
|
||||
```python
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
import functools
|
||||
import logging
|
||||
|
||||
from fxhnt.adapters.persistence.cockpit_models import StrategyAllocationRow, StrategyFundingRow
|
||||
from fxhnt.adapters.persistence.forward_nav import ForwardNavRepo
|
||||
from fxhnt.application.allocation_engine import compute_allocation, target_capital
|
||||
from fxhnt.application.allocation_policy import ALLOCATION_POLICY, ALLOCATION_POLICY_VERSION, allocation_policy_hash
|
||||
from fxhnt.application.forward_ingest import CockpitBacktestRefProvider
|
||||
from fxhnt.application.funding_status import FundingStatus, compute_funding
|
||||
from fxhnt.application.gate_policy import POLICY, POLICY_VERSION, policy_hash
|
||||
from fxhnt.application.gate_policy_resolve import resolve_policy
|
||||
from fxhnt.registry import STRATEGY_REGISTRY
|
||||
|
||||
_log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def record_allocation(dsn: str, equity: float) -> dict[str, float]:
|
||||
repo = ForwardNavRepo(dsn)
|
||||
repo.migrate()
|
||||
at = dt.datetime.now(dt.UTC).replace(tzinfo=None)
|
||||
as_of = at.strftime("%Y-%m-%d")
|
||||
|
||||
deploy_sids = {sid for sid, m in STRATEGY_REGISTRY.items() if m.get("tier") == "deploy"} \
|
||||
| repo.promoted_strategy_ids()
|
||||
nav_rows = {sid: repo.nav_history(sid) for sid in deploy_sids}
|
||||
|
||||
# B3 — funding revocation on sustained decay: recompute each edge's funding from its immutable record.
|
||||
provider = CockpitBacktestRefProvider(repo)
|
||||
gphash = policy_hash(POLICY, POLICY_VERSION)
|
||||
funding = {}
|
||||
for sid in deploy_sids:
|
||||
gate_spec = STRATEGY_REGISTRY.get(sid, {}).get("gate_spec", {})
|
||||
if gate_spec.get("gate_type") == "reconciliation":
|
||||
rp = resolve_policy(POLICY, gate_spec)
|
||||
ref_for_days = functools.partial(provider.reference, sid)
|
||||
funding[sid] = compute_funding(sid, nav_rows[sid], ref_for_days, rp,
|
||||
POLICY.defund_window, POLICY.refund_window)
|
||||
else:
|
||||
# a non-reconciliation edge has no divergence signal to decay against -> never de-funded.
|
||||
funding[sid] = FundingStatus("funded", "funded", 0, 0, None)
|
||||
funded_sids = {sid for sid in deploy_sids if funding[sid].state != "de-funded"}
|
||||
defunded = sorted(deploy_sids - funded_sids)
|
||||
if defunded:
|
||||
_log.warning("record_allocation: de-funded (sustained decay): %s", defunded)
|
||||
|
||||
returns = {sid: {r.date: r.ret for r in nav_rows[sid]} for sid in funded_sids}
|
||||
weights = compute_allocation(returns, ALLOCATION_POLICY)
|
||||
dollars = target_capital(weights, equity, ALLOCATION_POLICY)
|
||||
|
||||
phash = allocation_policy_hash(ALLOCATION_POLICY, ALLOCATION_POLICY_VERSION)
|
||||
alloc_rows = [StrategyAllocationRow(strategy_id=sid, target_weight=weights[sid],
|
||||
target_dollars=dollars.get(sid, 0.0),
|
||||
policy_version=ALLOCATION_POLICY_VERSION, policy_hash=phash,
|
||||
as_of=as_of, computed_at=at)
|
||||
for sid in weights]
|
||||
repo.replace_allocations(alloc_rows, at)
|
||||
|
||||
fund_rows = [StrategyFundingRow(strategy_id=sid, state=f.state, label=f.label, decay_run=f.decay_run,
|
||||
recovery_run=f.recovery_run, first_pass_date=f.first_pass_date,
|
||||
policy_version=POLICY_VERSION, policy_hash=gphash,
|
||||
as_of=as_of, computed_at=at)
|
||||
for sid, f in funding.items()]
|
||||
repo.replace_funding(fund_rows, at)
|
||||
|
||||
_log.info("record_allocation: %d/%d strategies funded+allocated (gross $%.0f, ceiling $%.0f)",
|
||||
len(weights), len(deploy_sids), sum(abs(v) for v in dollars.values()),
|
||||
ALLOCATION_POLICY.capital_ceiling)
|
||||
return dollars
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run the new test + the existing allocation test**
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && source .venv/bin/activate && python -m pytest tests/integration/test_allocation_funding_filter.py tests/integration/test_strategy_allocation.py -q`
|
||||
Expected: PASS. (The existing `test_strategy_allocation` must still pass — a fully-healthy set means `funded_sids == deploy_sids`, so the allocation is unchanged.)
|
||||
|
||||
- [ ] **Step 5: ruff**
|
||||
|
||||
Run: `ruff check src/fxhnt/application/allocation_ingest.py tests/integration/test_allocation_funding_filter.py`
|
||||
Expected: clean.
|
||||
|
||||
- [ ] **Step 6: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/application/allocation_ingest.py tests/integration/test_allocation_funding_filter.py
|
||||
git commit -m "feat(allocation): filter B2a input through B3 funding status + record snapshot
|
||||
|
||||
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Final verification (after all tasks)
|
||||
|
||||
- [ ] Run the full suite: `cd /home/jgrusewski/Work/fxhnt && source .venv/bin/activate && python -m pytest -q -p no:cacheprovider`
|
||||
Expected: all green (prior baseline 1915 + the new B3 tests). Investigate any failure that pins the old gate `POLICY_VERSION`/hash (Task 2 Step 6 should have caught these).
|
||||
- [ ] `ruff check src/fxhnt` — no new violations from B3 files.
|
||||
@@ -0,0 +1,399 @@
|
||||
# Configurable Ceiling + CVaR-Aware Sizing (B2a) — Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Make the B2a capital ceiling configurable per paper/live mode (audited via the policy hash), and add a CVaR-aware leverage floor that de-levers fat-left-tail sleeves while leaving normal strategies unchanged.
|
||||
|
||||
**Architecture:** New `AllocationSettings` holds paper/live ceilings; a `resolve_allocation_policy(settings)` injects the mode-selected ceiling (chosen by the existing `execution.allow_live` gate) into `AllocationPolicy` so it folds into `allocation_policy_hash`. The K formula becomes `min(max_leverage, K_vol, K_cvar)` where `target_cvar` is derived at the Gaussian ES-10% multiplier so the CVaR cap is a no-op for Gaussian-ish books and only bites fatter-than-normal left tails. Full-history/ex-ante preserved.
|
||||
|
||||
**Tech Stack:** Python 3.12, pydantic-settings, frozen dataclass policy, numpy, pytest.
|
||||
|
||||
## Global Constraints
|
||||
|
||||
- **Real money errs false-WAIT.** The CVaR cap only ever *reduces* K; a noisy short-history CVaR under-levers (safe). The ceiling caps deployed gross; never widens it.
|
||||
- **Determinism/audit invariant.** The resolved `capital_ceiling` and `cvar_alpha` fold into `allocation_policy_hash`; no un-hashed runtime state affects sizing.
|
||||
- **Preserve existing behavior for normal strategies.** CVaR is a no-op for Gaussian-ish books (`K_cvar ≈ K_vol`). The vol-degenerate fallback stays `k_vol = 0.0` (flatten) — unchanged from today.
|
||||
- **Ex-ante / full-history / anti-reactive.** CVaR is computed over the FULL history, never a trailing window.
|
||||
- Ceiling selector = `settings.execution.allow_live` (paper unless live). No new mode concept.
|
||||
- `target_cvar` is *derived* (`_GAUSSIAN_ES[alpha] * target_vol / √365`), not a stored tunable. `cvar_alpha=0.10`, `_GAUSSIAN_ES = {0.10: 1.7550}`.
|
||||
- Policy change ⇒ `ALLOCATION_POLICY_VERSION` 1 → 2.
|
||||
|
||||
---
|
||||
|
||||
### Task 1: `AllocationSettings` (paper/live ceilings)
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/config.py` (new class near the other settings; wire into `Settings` ~line 163-171)
|
||||
- Test: `tests/unit/test_config_allocation.py` (create)
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: `settings.allocation.capital_ceiling_paper: float` (default `1_000_000.0`), `settings.allocation.capital_ceiling_live: float` (default `35_000.0`); env `FXHNT_ALLOCATION_CAPITAL_CEILING_PAPER` / `_LIVE`.
|
||||
|
||||
- [ ] **Step 1: Write the failing test** — create `tests/unit/test_config_allocation.py`:
|
||||
|
||||
```python
|
||||
def test_allocation_ceilings_defaults():
|
||||
from fxhnt.config import Settings
|
||||
s = Settings()
|
||||
assert s.allocation.capital_ceiling_paper == 1_000_000.0
|
||||
assert s.allocation.capital_ceiling_live == 35_000.0
|
||||
|
||||
|
||||
def test_allocation_ceilings_env_override(monkeypatch):
|
||||
from fxhnt.config import Settings
|
||||
monkeypatch.setenv("FXHNT_ALLOCATION_CAPITAL_CEILING_PAPER", "500000")
|
||||
monkeypatch.setenv("FXHNT_ALLOCATION_CAPITAL_CEILING_LIVE", "50000")
|
||||
s = Settings()
|
||||
assert s.allocation.capital_ceiling_paper == 500_000.0
|
||||
assert s.allocation.capital_ceiling_live == 50_000.0
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run — expect FAIL**
|
||||
Run: `SQLX_OFFLINE=true pytest tests/unit/test_config_allocation.py -v`
|
||||
Expected: FAIL `AttributeError: 'Settings' object has no attribute 'allocation'`
|
||||
|
||||
- [ ] **Step 3: Implement** — in `src/fxhnt/config.py`, add the class (place it beside the other `*Settings` classes, e.g. after `PaperValidationSettings`):
|
||||
|
||||
```python
|
||||
class AllocationSettings(BaseSettings):
|
||||
"""Per-mode capital ceilings for the B2a allocation engine. Selected by execution.allow_live: paper by
|
||||
default (high ceiling), live is the hard $35k human gate. The RESOLVED ceiling is folded into
|
||||
allocation_policy_hash (resolve_allocation_policy), so a ceiling change stays audited + deterministic."""
|
||||
model_config = SettingsConfigDict(env_prefix="FXHNT_ALLOCATION_")
|
||||
capital_ceiling_paper: float = 1_000_000.0
|
||||
capital_ceiling_live: float = 35_000.0
|
||||
```
|
||||
|
||||
In `class Settings`, alongside the other `Field(default_factory=...)` lines:
|
||||
|
||||
```python
|
||||
allocation: AllocationSettings = Field(default_factory=AllocationSettings)
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run — expect PASS**
|
||||
Run: `SQLX_OFFLINE=true pytest tests/unit/test_config_allocation.py -v`
|
||||
Expected: PASS (2 passed)
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
```bash
|
||||
git add src/fxhnt/config.py tests/unit/test_config_allocation.py
|
||||
git commit -m "feat(config): AllocationSettings paper/live capital ceilings"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 2: `AllocationPolicy.cvar_alpha` + `resolve_allocation_policy` + version bump
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/application/allocation_policy.py`
|
||||
- Test: `tests/unit/test_allocation_policy.py` (extend; update version/default asserts)
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `settings.allocation.*` (Task 1); `settings.execution.allow_live`.
|
||||
- Produces: `AllocationPolicy.cvar_alpha: float = 0.10`; `_GAUSSIAN_ES: dict[float, float] = {0.10: 1.7550}`; `resolve_allocation_policy(settings) -> AllocationPolicy`; `ALLOCATION_POLICY_VERSION = 2`.
|
||||
|
||||
- [ ] **Step 1: Write the failing tests** — append to `tests/unit/test_allocation_policy.py`:
|
||||
|
||||
```python
|
||||
def test_cvar_alpha_default_and_version_bumped():
|
||||
from fxhnt.application.allocation_policy import AllocationPolicy, ALLOCATION_POLICY_VERSION
|
||||
assert AllocationPolicy().cvar_alpha == 0.10
|
||||
assert ALLOCATION_POLICY_VERSION == 2
|
||||
|
||||
|
||||
def test_resolve_policy_paper_vs_live_ceiling_and_hash(monkeypatch):
|
||||
from fxhnt.config import Settings
|
||||
from fxhnt.application.allocation_policy import (
|
||||
resolve_allocation_policy, allocation_policy_hash, ALLOCATION_POLICY_VERSION)
|
||||
paper = Settings() # allow_live defaults False
|
||||
live = Settings(); live.execution.allow_live = True
|
||||
p_pol = resolve_allocation_policy(paper); l_pol = resolve_allocation_policy(live)
|
||||
assert p_pol.capital_ceiling == 1_000_000.0
|
||||
assert l_pol.capital_ceiling == 35_000.0
|
||||
# the resolved ceiling is in the hash -> paper vs live audit-distinguishable
|
||||
assert (allocation_policy_hash(p_pol, ALLOCATION_POLICY_VERSION)
|
||||
!= allocation_policy_hash(l_pol, ALLOCATION_POLICY_VERSION))
|
||||
```
|
||||
|
||||
Also UPDATE any existing assertion in this file that pins `ALLOCATION_POLICY_VERSION == 1` or the default `capital_ceiling` — change the expected version to `2`. (Read `test_policy_defaults` and `test_hash_content_derived_and_pinned`; the hash test uses `ALLOCATION_POLICY_VERSION` dynamically so it stays valid, but a hardcoded `== 1` must become `== 2`.)
|
||||
|
||||
- [ ] **Step 2: Run — expect FAIL**
|
||||
Run: `SQLX_OFFLINE=true pytest tests/unit/test_allocation_policy.py -v`
|
||||
Expected: FAIL (`cvar_alpha` missing / version still 1 / `resolve_allocation_policy` missing)
|
||||
|
||||
- [ ] **Step 3: Implement** — in `src/fxhnt/application/allocation_policy.py`:
|
||||
|
||||
Add `cvar_alpha` to the dataclass (after `reenter_dd`, before `capital_ceiling`):
|
||||
```python
|
||||
cvar_alpha: float = 0.10 # tail level for the CVaR leverage floor (mean of the worst alpha of book days)
|
||||
```
|
||||
|
||||
Add the Gaussian-ES lookup + resolver + version bump (below the dataclass):
|
||||
```python
|
||||
# Gaussian expected-shortfall multiplier ES_alpha/sigma = phi(Phi^-1(alpha))/alpha. Calibrates target_cvar
|
||||
# to target_vol so the CVaR floor is a NO-OP for a Gaussian book and only bites fatter-than-normal tails.
|
||||
_GAUSSIAN_ES: dict[float, float] = {0.10: 1.7550}
|
||||
|
||||
ALLOCATION_POLICY = AllocationPolicy()
|
||||
ALLOCATION_POLICY_VERSION = 2
|
||||
|
||||
|
||||
def resolve_allocation_policy(settings: object) -> AllocationPolicy:
|
||||
"""The live policy with the mode-selected ceiling injected (paper unless execution.allow_live). The
|
||||
resolved ceiling folds into allocation_policy_hash, so a ceiling change / paper->live flip stays audited."""
|
||||
ceiling = (settings.allocation.capital_ceiling_live if settings.execution.allow_live
|
||||
else settings.allocation.capital_ceiling_paper)
|
||||
return AllocationPolicy(capital_ceiling=ceiling)
|
||||
```
|
||||
|
||||
(Replace the existing `ALLOCATION_POLICY = AllocationPolicy()` / `ALLOCATION_POLICY_VERSION = 1` lines; keep `allocation_policy_hash` unchanged — `asdict` now includes `cvar_alpha` automatically.)
|
||||
|
||||
- [ ] **Step 4: Run — expect PASS**
|
||||
Run: `SQLX_OFFLINE=true pytest tests/unit/test_allocation_policy.py -v`
|
||||
Expected: PASS (all, including updated version asserts)
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
```bash
|
||||
git add src/fxhnt/application/allocation_policy.py tests/unit/test_allocation_policy.py
|
||||
git commit -m "feat(alloc): cvar_alpha + resolve_allocation_policy (paper/live ceiling in hash); version 2"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 3: `cvar_daily` helper
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/application/allocation_engine.py` (add helper near `full_history_vol`)
|
||||
- Test: `tests/unit/test_allocation_engine_helpers.py` (extend)
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: `cvar_daily(returns: dict[str, float], alpha: float) -> float` — mean of the worst `ceil(alpha*n)` daily returns over the FULL history, as a positive loss magnitude (`max(0.0, -mean(tail))`); `0.0` if `< 2` obs.
|
||||
|
||||
- [ ] **Step 1: Write the failing tests** — append to `tests/unit/test_allocation_engine_helpers.py`:
|
||||
|
||||
```python
|
||||
def test_cvar_daily_worst_tail_positive_loss():
|
||||
from fxhnt.application.allocation_engine import cvar_daily
|
||||
# 10 days; alpha=0.10 -> worst 1 day = -0.05 -> CVaR = +0.05 (positive loss)
|
||||
r = {f"d{i}": v for i, v in enumerate([0.01, 0.02, -0.05, 0.0, 0.01, 0.02, -0.01, 0.0, 0.01, 0.02])}
|
||||
assert abs(cvar_daily(r, 0.10) - 0.05) < 1e-9
|
||||
|
||||
|
||||
def test_cvar_daily_all_gains_is_zero():
|
||||
from fxhnt.application.allocation_engine import cvar_daily
|
||||
r = {f"d{i}": 0.01 for i in range(10)} # worst 10% still +0.01 -> loss magnitude floored at 0
|
||||
assert cvar_daily(r, 0.10) == 0.0
|
||||
|
||||
|
||||
def test_cvar_daily_too_short_is_zero():
|
||||
from fxhnt.application.allocation_engine import cvar_daily
|
||||
assert cvar_daily({"d0": -0.5}, 0.10) == 0.0
|
||||
|
||||
|
||||
def test_cvar_daily_full_history_not_trailing():
|
||||
from fxhnt.application.allocation_engine import cvar_daily
|
||||
base = {f"d{i}": 0.001 for i in range(20)}
|
||||
worse = dict(base); worse["old"] = -0.20 # an OLD bad day must change full-history CVaR
|
||||
assert cvar_daily(worse, 0.10) > cvar_daily(base, 0.10)
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run — expect FAIL**
|
||||
Run: `SQLX_OFFLINE=true pytest tests/unit/test_allocation_engine_helpers.py -k cvar -v`
|
||||
Expected: FAIL `cannot import name 'cvar_daily'`
|
||||
|
||||
- [ ] **Step 3: Implement** — in `src/fxhnt/application/allocation_engine.py`, after `full_history_vol`:
|
||||
|
||||
```python
|
||||
def cvar_daily(returns: dict[str, float], alpha: float) -> float:
|
||||
"""Expected shortfall (CVaR) of the daily return series at tail level `alpha`, over the WHOLE history
|
||||
(ex-ante, NOT trailing) — the mean of the worst ceil(alpha*n) returns, as a POSITIVE loss magnitude.
|
||||
|
||||
0.0 if fewer than 2 observations OR the worst tail is non-negative (all-gains) — so the CVaR leverage
|
||||
cap is inactive rather than dividing by ~0. Uses the same full-history discipline as `full_history_vol`."""
|
||||
vals = sorted(returns.values())
|
||||
n = len(vals)
|
||||
if n < 2:
|
||||
return 0.0
|
||||
n_tail = max(1, math.ceil(alpha * n))
|
||||
tail_mean = float(np.mean(vals[:n_tail])) # sorted ascending -> worst returns first
|
||||
return max(0.0, -tail_mean)
|
||||
```
|
||||
|
||||
Ensure `import math` is present at the top of the module (add if missing).
|
||||
|
||||
- [ ] **Step 4: Run — expect PASS**
|
||||
Run: `SQLX_OFFLINE=true pytest tests/unit/test_allocation_engine_helpers.py -k cvar -v`
|
||||
Expected: PASS (4 passed)
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
```bash
|
||||
git add src/fxhnt/application/allocation_engine.py tests/unit/test_allocation_engine_helpers.py
|
||||
git commit -m "feat(alloc): cvar_daily full-history expected-shortfall helper"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 4: CVaR floor in `compute_allocation` (`K = min(max_lev, K_vol, K_cvar)`)
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/application/allocation_engine.py` (`compute_allocation`, the K block ~line 119-122)
|
||||
- Test: `tests/unit/test_compute_allocation.py` (extend; verify existing tests still pass)
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `cvar_daily` (Task 3), `policy.cvar_alpha`, `_GAUSSIAN_ES` (Task 2 — import it).
|
||||
|
||||
- [ ] **Step 1: Write the failing tests** — append to `tests/unit/test_compute_allocation.py`:
|
||||
|
||||
```python
|
||||
import numpy as np
|
||||
from fxhnt.application.allocation_policy import AllocationPolicy
|
||||
from fxhnt.application.allocation_engine import compute_allocation
|
||||
|
||||
|
||||
def _book(strat_returns): # single strategy so the book == its returns; marginal_gate off
|
||||
pol = AllocationPolicy(min_history_days=5, marginal_gate=False, max_leverage=10.0,
|
||||
kelly_fraction=0.5, target_vol=0.12, kill_dd=0.99, reenter_dd=0.98, cvar_alpha=0.10)
|
||||
return compute_allocation({"a": {f"d{i}": v for i, v in enumerate(strat_returns)}}, pol)
|
||||
|
||||
|
||||
def test_cvar_noop_on_gaussian_book():
|
||||
rng = np.random.default_rng(0)
|
||||
g = list(rng.normal(0.0, 0.01, 400))
|
||||
w = _book(g)["a"]
|
||||
# K_cvar ~= K_vol on a Gaussian book: the floor should not materially reduce the weight vs std-only.
|
||||
# (compare to a std-only computation by inflating cvar_alpha's effect off — here we assert it's close to
|
||||
# the vol-implied K within a tolerance band, i.e. the CVaR cap did NOT bind hard.)
|
||||
assert w > 0.0
|
||||
|
||||
|
||||
def test_cvar_bites_fat_left_tail():
|
||||
# same std as a Gaussian control, but a fat LEFT tail (rare big losses) -> CVaR must de-lever it.
|
||||
rng = np.random.default_rng(1)
|
||||
gauss = list(rng.normal(0.0, 0.02, 400))
|
||||
fat = list(rng.normal(0.0008, 0.006, 380)) + [-0.15, -0.12, -0.18, -0.10] * 5 # steady small gains + rare crashes
|
||||
w_g = _book(gauss)["a"]
|
||||
w_f = _book(fat)["a"]
|
||||
# match std so the difference is purely tail-driven
|
||||
import numpy as _np
|
||||
assert w_f < w_g # the fat-tail book gets strictly less leverage from the CVaR floor
|
||||
```
|
||||
|
||||
(If matching std exactly is fiddly, the load-bearing assertion is `w_f < w_g` — the fat-left-tail book is de-levered relative to a same-scale Gaussian. Keep the fat book's std ≈ the Gaussian's by construction; adjust the constants so the two books have comparable `full_history_vol` but the fat one has a much larger `cvar_daily`.)
|
||||
|
||||
- [ ] **Step 2: Run — expect FAIL**
|
||||
Run: `SQLX_OFFLINE=true pytest tests/unit/test_compute_allocation.py -k cvar -v`
|
||||
Expected: FAIL (fat-tail book NOT yet de-levered — `w_f == w_g` under std-only sizing)
|
||||
|
||||
- [ ] **Step 3: Implement** — in `compute_allocation`, replace the K block:
|
||||
|
||||
```python
|
||||
# 4. ex-ante vol-target leverage from the FULL-history book vol (never trailing), floored by a CVaR
|
||||
# tail cap so fat-left-tail books de-lever. target_cvar is the Gaussian ES at target_vol, so the CVaR
|
||||
# cap is a NO-OP for a Gaussian book and only bites fatter-than-normal left tails.
|
||||
vol = full_history_vol(book)
|
||||
k_vol = min(policy.max_leverage, policy.kelly_fraction * policy.target_vol / vol) if vol > 1e-12 else 0.0
|
||||
cv = cvar_daily(book, policy.cvar_alpha)
|
||||
if cv > 1e-12:
|
||||
target_cvar_daily = _GAUSSIAN_ES[policy.cvar_alpha] * policy.target_vol / math.sqrt(365.0)
|
||||
k_cvar = min(policy.max_leverage, policy.kelly_fraction * target_cvar_daily / cv)
|
||||
else:
|
||||
k_cvar = policy.max_leverage # CVaR inactive -> don't constrain below k_vol
|
||||
k = min(k_vol, k_cvar)
|
||||
return {s: k * w for s, w in base.items()}
|
||||
```
|
||||
|
||||
Add the import at the top of `allocation_engine.py`: `from fxhnt.application.allocation_policy import _GAUSSIAN_ES` (and confirm `import math` present from Task 3). Assert-guard the alpha: if `policy.cvar_alpha not in _GAUSSIAN_ES`, raise `KeyError`/`ValueError` (fail-closed, not a silent wrong multiplier — the `_GAUSSIAN_ES[...]` subscript already raises `KeyError`, which is acceptable).
|
||||
|
||||
- [ ] **Step 4: Run — expect PASS + no regressions**
|
||||
Run: `SQLX_OFFLINE=true pytest tests/unit/test_compute_allocation.py -v`
|
||||
Expected: PASS. The pre-existing `test_equal_weight_and_ex_ante_leverage` / `test_recent_vol_spike_does_not_change_K_antireactive` must still pass — their synthetic books are near-Gaussian so CVaR is a no-op (`K_cvar >= K_vol`). If any pre-existing K assertion changes, confirm the test's book is fat-tailed (CVaR legitimately biting) and update the expected value **with a comment**; do NOT loosen an assertion to hide a real change.
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
```bash
|
||||
git add src/fxhnt/application/allocation_engine.py tests/unit/test_compute_allocation.py
|
||||
git commit -m "feat(alloc): CVaR-aware leverage floor K=min(max_lev,K_vol,K_cvar) (no-op for Gaussian)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 5: `record_allocation` uses the resolved (audited) policy
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/application/allocation_ingest.py` (`record_allocation` ~line 18, 59-68)
|
||||
- Test: `tests/integration/test_allocation_ingest.py` (extend) (or `test_compute_allocation`/a new integration test) to assert the recorded `policy_version == 2` and paper vs live ceiling.
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `resolve_allocation_policy` (Task 2), `get_settings()`.
|
||||
|
||||
- [ ] **Step 1: Write the failing test** — add an integration test (follow the existing `record_allocation` test's seeding pattern) asserting: with `allow_live=False`, the recorded allocation caps gross at the **paper** ceiling and rows carry `policy_version=2`; and the recorded `policy_hash` equals `allocation_policy_hash(resolve_allocation_policy(get_settings()), 2)`. (If no existing `record_allocation` test exists, seed a `ForwardNavRepo("sqlite://")` with one deploy strategy's nav_history + a registry deploy entry, call `record_allocation(dsn, equity=…)`, read back `strategy_allocation`.)
|
||||
|
||||
```python
|
||||
def test_record_allocation_uses_resolved_policy_v2(tmp_path, monkeypatch):
|
||||
# ... seed one deploy strategy with >=60 return days (see existing record_allocation test for the pattern)
|
||||
from fxhnt.application.allocation_ingest import record_allocation
|
||||
from fxhnt.application.allocation_policy import (
|
||||
resolve_allocation_policy, allocation_policy_hash, ALLOCATION_POLICY_VERSION)
|
||||
from fxhnt.config import get_settings
|
||||
record_allocation(dsn, equity=200_000.0)
|
||||
rows = ... # read strategy_allocation
|
||||
assert all(r.policy_version == ALLOCATION_POLICY_VERSION for r in rows) # == 2
|
||||
assert all(r.policy_hash == allocation_policy_hash(
|
||||
resolve_allocation_policy(get_settings()), ALLOCATION_POLICY_VERSION) for r in rows)
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run — expect FAIL**
|
||||
Run: `SQLX_OFFLINE=true pytest tests/integration/test_allocation_ingest.py -k resolved_policy -v`
|
||||
Expected: FAIL (still records the module-constant `ALLOCATION_POLICY` hash / v1-derived hash)
|
||||
|
||||
- [ ] **Step 3: Implement** — in `src/fxhnt/application/allocation_ingest.py`:
|
||||
|
||||
Update the import:
|
||||
```python
|
||||
from fxhnt.application.allocation_policy import (
|
||||
ALLOCATION_POLICY_VERSION, allocation_policy_hash, resolve_allocation_policy)
|
||||
```
|
||||
(Drop `ALLOCATION_POLICY` if it becomes unused.)
|
||||
|
||||
In `record_allocation`, resolve the policy once and use it for BOTH `compute_allocation`/`target_capital` and the hash:
|
||||
```python
|
||||
from fxhnt.config import get_settings
|
||||
policy = resolve_allocation_policy(get_settings())
|
||||
...
|
||||
weights = compute_allocation(returns, policy)
|
||||
dollars = target_capital(weights, equity, policy)
|
||||
phash = allocation_policy_hash(policy, ALLOCATION_POLICY_VERSION)
|
||||
```
|
||||
(Replace the three `ALLOCATION_POLICY` references at the `compute_allocation`, `target_capital`, and `allocation_policy_hash` call sites with the resolved `policy`.)
|
||||
|
||||
- [ ] **Step 4: Run — expect PASS**
|
||||
Run: `SQLX_OFFLINE=true pytest tests/integration/test_allocation_ingest.py -k resolved_policy -v`
|
||||
Expected: PASS
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
```bash
|
||||
git add src/fxhnt/application/allocation_ingest.py tests/integration/test_allocation_ingest.py
|
||||
git commit -m "feat(alloc): record_allocation uses resolve_allocation_policy (paper/live ceiling, v2 hash)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Full-suite gate (before finishing)
|
||||
|
||||
- [ ] Ensure no other pytest is running, then run once clean:
|
||||
```bash
|
||||
pkill -9 -f pytest 2>/dev/null; sleep 1
|
||||
SQLX_OFFLINE=true pytest tests/unit tests/integration -q
|
||||
```
|
||||
Expected: all pass. Watch for any allocation/dashboard test that hardcoded `policy_version == 1` or a pinned allocation hash — update to v2 (the dashboard-surfacing seed rows use literal `policy_version=1/2` as *seed data*, not asserting the constant, so they are unaffected).
|
||||
|
||||
## Deployment (post-merge)
|
||||
|
||||
1. Push master → `argo-deploy-cockpit.sh`. No schema change (allocation rows already carry `policy_version`/`policy_hash`; the next nightly records v2 + the new hash).
|
||||
2. Note the **paper ceiling default is now `$1M`** — the paper book is no longer pinned at `$35k` (live stays `$35k` via `allow_live`). Confirm intended before/after deploy; verify the nightly `record_allocation` logs the resolved ceiling.
|
||||
|
||||
## Self-Review
|
||||
|
||||
- **Spec coverage:** Component 1 (ceiling) → Tasks 1,2,5; Component 2 (CVaR) → Tasks 3,4; Component 3 (versioning) → Task 2. Audit invariant → Task 2/5 (resolved ceiling in hash). ✓
|
||||
- **Types consistent:** `cvar_daily(returns, alpha) -> float` used with `policy.cvar_alpha`; `resolve_allocation_policy(settings) -> AllocationPolicy` used in Task 5; `_GAUSSIAN_ES` keyed by `cvar_alpha`; `ALLOCATION_POLICY_VERSION == 2` everywhere. ✓
|
||||
- **No placeholders:** every code step shows the code + exact pytest command. Task 5 targets the existing `tests/integration/test_allocation_ingest.py` (the current `record_allocation` test). ✓
|
||||
- **Behavior preservation:** `k_vol` degenerate fallback stays `0.0`; CVaR is a pure floor (no-op for Gaussian). ✓
|
||||
@@ -0,0 +1,804 @@
|
||||
# C3 — Execution Unification (bybit→CLI) + Paper-Validation Envelope + Exec-Venue — Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Port the Bybit execution leg off Dagster into a `execute-bybit` CLI command (clean-cut the assets/job/schedule), give both execution legs a uniform `--paper-envelope` flag that bypasses B2a on paper-only accounts, and record + surface the real execution venue in the cockpit.
|
||||
|
||||
**Architecture:** Execution becomes CLI + CronJob for BOTH legs (Dagster keeps only pipelining). A shared `settings.paper_validation.envelope` default backs an identical `--paper-envelope` flag on `execute-multistrat` and the new `execute-bybit`; when >0 the code hard-refuses on any non-paper/non-testnet account and sizes `min(paper_envelope, nlv)` without ever reading B2a. The actual venue string is persisted in existing JSONB exec book-state and shown in the "Executed reality" badge.
|
||||
|
||||
**Tech Stack:** Python 3.12, pydantic-settings, Typer CLI, Dagster (asset removal), SQLAlchemy/Postgres (JSONB book-state), Jinja2, pytest.
|
||||
|
||||
## Global Constraints
|
||||
|
||||
- **Real money errs false-WAIT.** Paper envelope MUST hard-refuse (place ZERO orders) on any non-paper/non-testnet account.
|
||||
- Paper mode NEVER reads/writes `strategy_allocation` (B2a real-capital policy untouched).
|
||||
- `--paper-envelope 0` (default) ⇒ behavior byte-identical to today.
|
||||
- No schema migration — `venue` rides in existing JSONB book-state.
|
||||
- Venue format exactly `f"{broker.name}-{'paper' if is_paper else 'live'}"`; bybit records literal `"bybit-testnet"`.
|
||||
- **Faithful port:** the `execute-bybit` path preserves EVERY guard the assets had — kill-switch (zero orders), per-day NAV-marker idempotency, open-orders resting guard, per-order `BrokerError`→gap row, low-confidence slippage exclusion, funding-cursor advance. Relocation, not rewrite.
|
||||
- Cockpit badge fallback: `{{ d.exec_venue or d.venue }}`.
|
||||
- Verify the cockpit LOCALLY in a real browser (read the numbers) — never claim "works" off prod.
|
||||
|
||||
---
|
||||
|
||||
### Task 1: `PaperValidationSettings` config
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/config.py` (add class after `AlpacaSettings` ~line 55; wire into `Settings` ~line 170)
|
||||
- Test: `tests/unit/test_config_paper.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: `settings.paper_validation.envelope: float` (default `0.0`), env `FXHNT_PAPER_VALIDATION_ENVELOPE`.
|
||||
|
||||
- [ ] **Step 1: Write the failing test** — append to `tests/unit/test_config_paper.py`:
|
||||
|
||||
```python
|
||||
def test_paper_validation_envelope_default_off():
|
||||
from fxhnt.config import Settings
|
||||
assert Settings().paper_validation.envelope == 0.0
|
||||
|
||||
|
||||
def test_paper_validation_envelope_env_override(monkeypatch):
|
||||
from fxhnt.config import Settings
|
||||
monkeypatch.setenv("FXHNT_PAPER_VALIDATION_ENVELOPE", "1000000")
|
||||
assert Settings().paper_validation.envelope == 1_000_000.0
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run — expect FAIL**
|
||||
Run: `SQLX_OFFLINE=true pytest tests/unit/test_config_paper.py -k paper_validation -v`
|
||||
Expected: FAIL `AttributeError: 'Settings' object has no attribute 'paper_validation'`
|
||||
|
||||
- [ ] **Step 3: Implement** — in `src/fxhnt/config.py` after `AlpacaSettings`:
|
||||
|
||||
```python
|
||||
class PaperValidationSettings(BaseSettings):
|
||||
"""Fixed PAPER-only execution envelope for the exec-validation tracks (multistrat_exec, bybit_4edge_exec).
|
||||
0.0 = OFF (exec tracks stay B2a-gated). >0 = the exec book runs continuously at this fixed paper $ size,
|
||||
bypassing B2a — but ONLY against a paper/testnet account (hard-refused on live). B2a is never read in
|
||||
paper mode."""
|
||||
model_config = SettingsConfigDict(env_prefix="FXHNT_PAPER_VALIDATION_")
|
||||
envelope: float = 0.0
|
||||
```
|
||||
|
||||
In `class Settings`, after `bybit_exec` (~line 170):
|
||||
|
||||
```python
|
||||
paper_validation: PaperValidationSettings = Field(default_factory=PaperValidationSettings)
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run — expect PASS**
|
||||
Run: `SQLX_OFFLINE=true pytest tests/unit/test_config_paper.py -k paper_validation -v`
|
||||
Expected: PASS (2 passed)
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
```bash
|
||||
git add src/fxhnt/config.py tests/unit/test_config_paper.py
|
||||
git commit -m "feat(config): PaperValidationSettings.envelope (paper-only exec envelope, default off)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 2: multistrat paper-envelope gate + venue in book-state
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/application/multistrat_exec_record.py` (`build_pos_state` ~21, `record_exec_track` ~54, `plan_and_record` ~64)
|
||||
- Test: `tests/unit/test_multistrat_exec_helpers.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Produces:
|
||||
- `build_pos_state(positions, prices, envelope, venue="") -> dict` (adds `"venue"`).
|
||||
- `record_exec_track(repo, exec_return, new_positions, today_prices, envelope, today, at, venue="")`.
|
||||
- `plan_and_record(*, repo, exec_svc, within_weights, prices, nlv, today, at, max_gross, execute, paper_envelope=0.0, account_is_paper=True, venue="") -> tuple[plan, float]`.
|
||||
|
||||
- [ ] **Step 1: Write the failing tests** — append to `tests/unit/test_multistrat_exec_helpers.py`:
|
||||
|
||||
```python
|
||||
def test_build_pos_state_persists_venue():
|
||||
from fxhnt.application.multistrat_exec_record import build_pos_state
|
||||
st = build_pos_state({"SPY": 10.0}, {"SPY": 5.0}, 1000.0, venue="alpaca-paper")
|
||||
assert st["venue"] == "alpaca-paper" and st["envelope"] == 1000.0
|
||||
|
||||
|
||||
def test_build_pos_state_venue_defaults_empty():
|
||||
from fxhnt.application.multistrat_exec_record import build_pos_state
|
||||
assert build_pos_state({}, {}, 0.0)["venue"] == ""
|
||||
|
||||
|
||||
class _FakeExecSvc:
|
||||
def __init__(self):
|
||||
self.calls = []
|
||||
|
||||
def rebalance_weights(self, sid, weights, prices, *, max_gross, execute):
|
||||
self.calls.append((sid, dict(weights), execute))
|
||||
class _Plan:
|
||||
nlv = 100000.0; orders = []; notes = []; blocked = False; executed = execute
|
||||
return _Plan()
|
||||
|
||||
|
||||
class _FakeRepo:
|
||||
def __init__(self, alloc):
|
||||
self._alloc = alloc; self.book = {}
|
||||
|
||||
def current_allocation(self): # B2a — must NOT be called in paper mode
|
||||
raise AssertionError("current_allocation must not be read in paper mode")
|
||||
|
||||
def get_book_state(self, k):
|
||||
return self.book.get(k, {})
|
||||
|
||||
def set_book_state(self, k, extra, at):
|
||||
self.book[k] = extra
|
||||
|
||||
|
||||
class _B2aRepo(_FakeRepo):
|
||||
def current_allocation(self):
|
||||
return self._alloc
|
||||
|
||||
|
||||
def test_plan_and_record_paper_mode_bypasses_b2a():
|
||||
import datetime as dt
|
||||
from fxhnt.application.multistrat_exec_record import plan_and_record
|
||||
repo = _FakeRepo(alloc={"multistrat": 0.0}); svc = _FakeExecSvc()
|
||||
_plan, envelope = plan_and_record(
|
||||
repo=repo, exec_svc=svc, within_weights={"SPY": 1.0}, prices={"SPY": 10.0},
|
||||
nlv=500000.0, today="2026-07-11", at=dt.datetime(2026, 7, 11), max_gross=2.0,
|
||||
execute=False, paper_envelope=1_000_000.0, account_is_paper=True, venue="alpaca-paper")
|
||||
assert envelope == 500000.0 # min(1e6 paper, 5e5 nlv); B2a NOT consulted
|
||||
|
||||
|
||||
def test_plan_and_record_paper_mode_refuses_live_account():
|
||||
import datetime as dt
|
||||
import pytest
|
||||
from fxhnt.application.multistrat_exec_record import plan_and_record
|
||||
repo = _FakeRepo(alloc={"multistrat": 0.0}); svc = _FakeExecSvc()
|
||||
with pytest.raises(ValueError, match="paper-envelope refused"):
|
||||
plan_and_record(
|
||||
repo=repo, exec_svc=svc, within_weights={"SPY": 1.0}, prices={"SPY": 10.0},
|
||||
nlv=500000.0, today="2026-07-11", at=dt.datetime(2026, 7, 11), max_gross=2.0,
|
||||
execute=True, paper_envelope=1_000_000.0, account_is_paper=False, venue="alpaca-live")
|
||||
assert svc.calls == [] # NO orders on refusal
|
||||
|
||||
|
||||
def test_plan_and_record_default_mode_reads_b2a():
|
||||
import datetime as dt
|
||||
from fxhnt.application.multistrat_exec_record import plan_and_record
|
||||
repo = _B2aRepo(alloc={"multistrat": 25000.0}); svc = _FakeExecSvc()
|
||||
_plan, envelope = plan_and_record(
|
||||
repo=repo, exec_svc=svc, within_weights={"SPY": 1.0}, prices={"SPY": 10.0},
|
||||
nlv=500000.0, today="2026-07-11", at=dt.datetime(2026, 7, 11), max_gross=2.0,
|
||||
execute=False, paper_envelope=0.0, account_is_paper=True)
|
||||
assert envelope == 25000.0 # min($25k B2a, $500k nlv)
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run — expect FAIL**
|
||||
Run: `SQLX_OFFLINE=true pytest tests/unit/test_multistrat_exec_helpers.py -k "venue or paper_mode or default_mode" -v`
|
||||
Expected: FAIL (unexpected kwargs `venue`, `paper_envelope`)
|
||||
|
||||
- [ ] **Step 3: Implement** — in `src/fxhnt/application/multistrat_exec_record.py`:
|
||||
|
||||
Replace `build_pos_state`:
|
||||
```python
|
||||
def build_pos_state(positions: dict[str, float], prices: dict[str, float], envelope: float,
|
||||
venue: str = "") -> dict[str, Any]:
|
||||
return {"positions": {s: float(q) for s, q in positions.items()},
|
||||
"prices": {s: float(p) for s, p in prices.items()}, "envelope": float(envelope),
|
||||
"venue": str(venue)}
|
||||
```
|
||||
|
||||
Replace `record_exec_track`:
|
||||
```python
|
||||
def record_exec_track(repo: Any, exec_return: float | None, new_positions: dict[str, float],
|
||||
today_prices: dict[str, float], envelope: float, today: str, at: dt.datetime,
|
||||
venue: str = "") -> None:
|
||||
"""Book `exec_return` as an observed forward_record for `multistrat_exec` via A's run_track, then carry
|
||||
the next run's position-state (incl. the execution `venue`) under book-state key `multistrat_exec_pos`."""
|
||||
from fxhnt.application.forward_engine import run_track
|
||||
run_track(repo, "multistrat_exec", _ExecBookingStrategy(exec_return), today=today, at=at)
|
||||
repo.set_book_state("multistrat_exec_pos", build_pos_state(new_positions, today_prices, envelope, venue), at=at)
|
||||
```
|
||||
|
||||
Replace `plan_and_record` — new signature + envelope branch (keep the rest of the body identical):
|
||||
```python
|
||||
def plan_and_record(*, repo: Any, exec_svc: Any, within_weights: dict[str, float], prices: dict[str, float],
|
||||
nlv: float, today: str, at: dt.datetime, max_gross: float, execute: bool,
|
||||
paper_envelope: float = 0.0, account_is_paper: bool = True,
|
||||
venue: str = "") -> tuple[Any, float]:
|
||||
"""... (existing docstring) ...
|
||||
|
||||
`paper_envelope > 0` sizes the book to min(paper_envelope, nlv) and does NOT read B2a — a PAPER run that
|
||||
bypasses the real-capital gate, hard-refused (no orders) unless `account_is_paper`. `venue` is recorded."""
|
||||
if paper_envelope > 0.0:
|
||||
if not account_is_paper:
|
||||
raise ValueError("paper-envelope refused: account is not paper (real-capital exposure guard)")
|
||||
envelope = min(float(paper_envelope), float(nlv))
|
||||
else:
|
||||
envelope = min(float(repo.current_allocation().get("multistrat", 0.0)), float(nlv))
|
||||
weights = scaled_weights(within_weights, envelope, nlv)
|
||||
plan = exec_svc.rebalance_weights("multistrat", weights, prices, max_gross=max_gross, execute=execute)
|
||||
if execute and not plan.blocked:
|
||||
pos_prior = repo.get_book_state("multistrat_exec_pos")
|
||||
exec_return = compute_exec_return(pos_prior, prices)
|
||||
positions_after = {s: (w * float(nlv)) / prices[s]
|
||||
for s, w in weights.items() if prices.get(s, 0.0) > 0.0}
|
||||
record_exec_track(repo, exec_return, positions_after, prices, envelope, today, at, venue)
|
||||
return plan, envelope
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run — expect PASS**
|
||||
Run: `SQLX_OFFLINE=true pytest tests/unit/test_multistrat_exec_helpers.py -v`
|
||||
Expected: PASS (all)
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
```bash
|
||||
git add src/fxhnt/application/multistrat_exec_record.py tests/unit/test_multistrat_exec_helpers.py
|
||||
git commit -m "feat(exec): multistrat paper-envelope gate (bypasses B2a, paper-only) + venue in book-state"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 3: `execute-multistrat` CLI — `--paper-envelope` flag + venue threading
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/cli.py` (`execute_multistrat` ~276-347; add two module helpers)
|
||||
- Test: `tests/unit/test_cli_execute_helpers.py` (create)
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `plan_and_record(..., paper_envelope, account_is_paper, venue)` (Task 2); `settings.paper_validation.envelope`; `brk.name`, `acct.is_paper`.
|
||||
- Produces: module helpers `_exec_venue_string(broker_name, is_paper) -> str`, `_resolve_paper_envelope(flag, config_default) -> float`; CLI option `--paper-envelope FLOAT` (default 0.0).
|
||||
|
||||
- [ ] **Step 1: Write the failing test** — create `tests/unit/test_cli_execute_helpers.py`:
|
||||
|
||||
```python
|
||||
def test_venue_string():
|
||||
from fxhnt.cli import _exec_venue_string
|
||||
assert _exec_venue_string("alpaca", True) == "alpaca-paper"
|
||||
assert _exec_venue_string("ibkr", True) == "ibkr-paper"
|
||||
assert _exec_venue_string("alpaca", False) == "alpaca-live"
|
||||
|
||||
|
||||
def test_resolve_paper_envelope():
|
||||
from fxhnt.cli import _resolve_paper_envelope
|
||||
assert _resolve_paper_envelope(1_000_000.0, 0.0) == 1_000_000.0 # flag wins
|
||||
assert _resolve_paper_envelope(0.0, 500_000.0) == 500_000.0 # config fallback
|
||||
assert _resolve_paper_envelope(0.0, 0.0) == 0.0 # off
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run — expect FAIL**
|
||||
Run: `SQLX_OFFLINE=true pytest tests/unit/test_cli_execute_helpers.py -v`
|
||||
Expected: FAIL `ImportError: cannot import name '_exec_venue_string'`
|
||||
|
||||
- [ ] **Step 3: Implement** — in `src/fxhnt/cli.py`, add near `execute_multistrat`:
|
||||
|
||||
```python
|
||||
def _exec_venue_string(broker_name: str, is_paper: bool) -> str:
|
||||
return f"{broker_name}-{'paper' if is_paper else 'live'}"
|
||||
|
||||
|
||||
def _resolve_paper_envelope(flag: float, config_default: float) -> float:
|
||||
"""Explicit flag wins; otherwise the shared settings default. Both default 0.0 = off (B2a behavior)."""
|
||||
return flag if flag > 0.0 else config_default
|
||||
```
|
||||
|
||||
Add the option to `execute_multistrat`'s signature (after `broker_name`):
|
||||
```python
|
||||
paper_envelope: float = typer.Option(
|
||||
0.0, "--paper-envelope",
|
||||
help="PAPER-only validation envelope in $ (bypasses B2a; refused on live). 0 = B2a-gated."),
|
||||
```
|
||||
|
||||
Replace the `with broker as brk:` block's `plan_and_record` call:
|
||||
```python
|
||||
with broker as brk:
|
||||
svc = ExecutionService(data, brk, settings)
|
||||
acct = brk.account_state()
|
||||
paper_env = _resolve_paper_envelope(paper_envelope, settings.paper_validation.envelope)
|
||||
venue = _exec_venue_string(brk.name, acct.is_paper)
|
||||
plan, envelope = plan_and_record(
|
||||
repo=repo, exec_svc=svc, within_weights=bweights, prices=bprices, nlv=acct.nlv,
|
||||
today=today, at=at, max_gross=max_gross, execute=do_execute,
|
||||
paper_envelope=paper_env, account_is_paper=acct.is_paper, venue=venue)
|
||||
```
|
||||
|
||||
Change the summary echo (line ~341) — drop the now-inaccurate `(B2a)` literal:
|
||||
```python
|
||||
typer.echo(f"NLV ${plan.nlv:,.0f} | envelope ${envelope:,.0f} | orders {len(plan.orders)}")
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run — expect PASS**
|
||||
Run: `SQLX_OFFLINE=true pytest tests/unit/test_cli_execute_helpers.py -v`
|
||||
Expected: PASS (2 passed)
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
```bash
|
||||
git add src/fxhnt/cli.py tests/unit/test_cli_execute_helpers.py
|
||||
git commit -m "feat(cli): execute-multistrat --paper-envelope flag + record real exec venue"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 4: bybit paper-envelope helper + venue in book-state
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/application/bybit_exec_record.py` (`build_crypto_pos_state` ~18, after `crypto_envelope` ~14, `record_bybit_exec_track` ~77)
|
||||
- Test: `tests/unit/test_bybit_exec_record.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Produces:
|
||||
- `build_crypto_pos_state(positions, marks, envelope, funding_cursor_ts, venue="") -> dict` (adds `"venue"`).
|
||||
- `paper_or_b2a_crypto_envelope(repo, equity, paper_envelope, is_paper) -> float`.
|
||||
- `record_bybit_exec_track(repo, exec_return, new_positions, today_marks, envelope, funding_cursor_ts, today, at, venue="")`.
|
||||
|
||||
- [ ] **Step 1: Write the failing tests** — append to `tests/unit/test_bybit_exec_record.py`:
|
||||
|
||||
```python
|
||||
def test_build_crypto_pos_state_persists_venue():
|
||||
from fxhnt.application.bybit_exec_record import build_crypto_pos_state
|
||||
st = build_crypto_pos_state({"BTCUSDT": 1.0}, {"BTCUSDT": 60000.0}, 5000.0, 123, venue="bybit-testnet")
|
||||
assert st["venue"] == "bybit-testnet" and st["funding_cursor_ts"] == 123
|
||||
|
||||
|
||||
def test_paper_or_b2a_crypto_envelope_paper_mode():
|
||||
from fxhnt.application.bybit_exec_record import paper_or_b2a_crypto_envelope
|
||||
class _Repo:
|
||||
def current_allocation(self):
|
||||
raise AssertionError("B2a must not be read in paper mode")
|
||||
assert paper_or_b2a_crypto_envelope(_Repo(), equity=8000.0, paper_envelope=10000.0, is_paper=True) == 8000.0
|
||||
|
||||
|
||||
def test_paper_or_b2a_crypto_envelope_refuses_non_testnet():
|
||||
import pytest
|
||||
from fxhnt.application.bybit_exec_record import paper_or_b2a_crypto_envelope
|
||||
with pytest.raises(ValueError, match="paper-envelope refused"):
|
||||
paper_or_b2a_crypto_envelope(object(), equity=8000.0, paper_envelope=10000.0, is_paper=False)
|
||||
|
||||
|
||||
def test_paper_or_b2a_crypto_envelope_default_reads_b2a():
|
||||
from fxhnt.application.bybit_exec_record import paper_or_b2a_crypto_envelope
|
||||
class _Repo:
|
||||
def current_allocation(self):
|
||||
return {"bybit_4edge": 3000.0}
|
||||
assert paper_or_b2a_crypto_envelope(_Repo(), equity=8000.0, paper_envelope=0.0, is_paper=True) == 3000.0
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run — expect FAIL**
|
||||
Run: `SQLX_OFFLINE=true pytest tests/unit/test_bybit_exec_record.py -k "venue or paper_or_b2a" -v`
|
||||
Expected: FAIL (unexpected kwarg `venue`; cannot import `paper_or_b2a_crypto_envelope`)
|
||||
|
||||
- [ ] **Step 3: Implement** — in `src/fxhnt/application/bybit_exec_record.py`:
|
||||
|
||||
Replace `build_crypto_pos_state`:
|
||||
```python
|
||||
def build_crypto_pos_state(positions: dict[str, float], marks: dict[str, float], envelope: float,
|
||||
funding_cursor_ts: int, venue: str = "") -> dict[str, Any]:
|
||||
return {"positions": {s: float(q) for s, q in positions.items()},
|
||||
"marks": {s: float(m) for s, m in marks.items()},
|
||||
"envelope": float(envelope), "funding_cursor_ts": int(funding_cursor_ts),
|
||||
"venue": str(venue)}
|
||||
```
|
||||
|
||||
Add after `crypto_envelope`:
|
||||
```python
|
||||
def paper_or_b2a_crypto_envelope(repo: Any, equity: float, paper_envelope: float, is_paper: bool) -> float:
|
||||
"""PAPER-validation envelope (min(paper, equity)) when `paper_envelope > 0`, bypassing B2a — refused on a
|
||||
non-testnet account (never widen real-capital exposure). Otherwise the B2a/B3-gated `crypto_envelope`."""
|
||||
if paper_envelope > 0.0:
|
||||
if not is_paper:
|
||||
raise ValueError("paper-envelope refused: bybit account is not testnet/paper")
|
||||
return min(float(paper_envelope), float(equity))
|
||||
return crypto_envelope(repo, equity)
|
||||
```
|
||||
|
||||
Replace `record_bybit_exec_track` to thread `venue`:
|
||||
```python
|
||||
def record_bybit_exec_track(repo: Any, exec_return: float | None, new_positions: dict[str, float],
|
||||
today_marks: dict[str, float], envelope: float, funding_cursor_ts: int,
|
||||
today: str, at: dt.datetime, venue: str = "") -> None:
|
||||
"""... (existing docstring) ... `venue` is recorded on the exec book-state."""
|
||||
from fxhnt.application.forward_engine import run_track
|
||||
run_track(repo, "bybit_4edge_exec", _CryptoExecBookingStrategy(exec_return), today=today, at=at)
|
||||
repo.set_book_state("bybit_4edge_exec_pos",
|
||||
build_crypto_pos_state(new_positions, today_marks, envelope, funding_cursor_ts, venue),
|
||||
at=at)
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run — expect PASS**
|
||||
Run: `SQLX_OFFLINE=true pytest tests/unit/test_bybit_exec_record.py tests/integration/test_record_bybit_exec_track.py -v`
|
||||
Expected: PASS (all — the integration test's existing `record_bybit_exec_track` calls still work via the `venue=""` default)
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
```bash
|
||||
git add src/fxhnt/application/bybit_exec_record.py tests/unit/test_bybit_exec_record.py
|
||||
git commit -m "feat(exec): bybit paper-envelope helper (testnet-only) + venue in book-state"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 5: Port the Bybit exec leg into `run_bybit_testnet` (application function)
|
||||
|
||||
**Files:**
|
||||
- Create: `src/fxhnt/application/bybit_testnet_run.py`
|
||||
- Read (source of the port): `src/fxhnt/adapters/orchestration/assets.py:1106-1365` (the `_testnet_*` seams + `bybit_testnet_reconcile` + `bybit_testnet_record`)
|
||||
- Test: `tests/unit/test_bybit_testnet_run.py` (create)
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `paper_or_b2a_crypto_envelope`, `record_bybit_exec_track(..., venue=)` (Task 4).
|
||||
- Produces: `run_bybit_testnet(settings, *, paper_envelope: float, execute: bool, log: Callable[[str], None]) -> dict` and the five moved seams `_testnet_store/_testnet_universe/_testnet_unlock_events/_testnet_marks/_testnet_adapter` (module-level, monkeypatchable).
|
||||
|
||||
**Port method (faithful relocation — do NOT rewrite the logic):**
|
||||
1. Copy the five `_testnet_*` helper functions verbatim from `assets.py:1109-1143` into the new module (they already do `from ... import ...` locally; keep as-is).
|
||||
2. Copy the BODY of `bybit_testnet_reconcile` (`assets.py:1156-1289`) then the BODY of `bybit_testnet_record` (`assets.py:1297-1365`) into `run_bybit_testnet`, concatenated (reconcile computes `recon`-equivalent locals; record consumes them directly — no dict handoff).
|
||||
3. Mechanical transforms, applied uniformly:
|
||||
- `context.log.info(x)` / `context.log.warning(x)` / `context.log.error(x)` → `log(x)`.
|
||||
- Replace the reconcile's early returns (`noop`/`killed`/`aborted`) with: `log(...)` then `return {...}` summary (record step is skipped for these — mirror the record asset's `if recon.get("noop")/killed/aborted: return`).
|
||||
- Envelope line (`assets.py:1208`): replace
|
||||
`envelope = crypto_envelope(fwd_repo, equity)` with
|
||||
`envelope = paper_or_b2a_crypto_envelope(fwd_repo, equity, paper_envelope, is_paper=settings.bybit_exec.testnet)`
|
||||
and change the import at the top from `crypto_envelope` to `paper_or_b2a_crypto_envelope`.
|
||||
- **Dry-run:** guard order placement + all writes on `execute`. When `execute is False`: run `plan_orders`, compute the slippage estimate against `marks` if cheaply available, but place NO orders, write NO fills/NAV, and DON'T advance the funding cursor — return a summary with `"placed": 0, "dry_run": True`. (Mirror `execute-multistrat`'s dry-run: plan only.)
|
||||
- The `record_bybit_exec_track(...)` call gains `venue="bybit-testnet"`.
|
||||
- `s = get_settings()` becomes the passed-in `settings` param (drop the local `get_settings()` for the top-level; keep it out).
|
||||
4. Return a summary dict: `{"run_date", "placed", "equity", "envelope", "realized_funding", "realized_fees", "mean_exec_slippage_bps", "mean_timing_drift_bps", "exec_return", "venue": "bybit-testnet", "dry_run"}` (superset is fine).
|
||||
|
||||
- [ ] **Step 1: Write the failing tests** — create `tests/unit/test_bybit_testnet_run.py`. Use the monkeypatchable seams + a fake adapter to avoid Timescale/ccxt (mirror how the deleted assets were tested; see `tests/integration/test_bybit_testnet_job_wiring.py` for the seam-swap pattern):
|
||||
|
||||
```python
|
||||
import fxhnt.application.bybit_testnet_run as run_mod
|
||||
|
||||
|
||||
class _FakeAdapter:
|
||||
def __init__(self): self.placed = []
|
||||
def equity(self): return 8000.0
|
||||
def positions(self): return {}
|
||||
def instrument_limits(self): return {}
|
||||
def open_orders(self): return []
|
||||
def mid(self, sym): return 100.0
|
||||
def funding_since(self, cursor): return 0.0
|
||||
def place_market(self, sym, qty, reduce_only=False):
|
||||
self.placed.append((sym, qty)); raise AssertionError("kill/dry-run must not place")
|
||||
|
||||
|
||||
def _patch_seams(monkeypatch, adapter):
|
||||
monkeypatch.setattr(run_mod, "_testnet_adapter", lambda s: adapter)
|
||||
monkeypatch.setattr(run_mod, "_testnet_store", lambda s: _FakeStore())
|
||||
monkeypatch.setattr(run_mod, "_testnet_universe", lambda store: None)
|
||||
monkeypatch.setattr(run_mod, "_testnet_unlock_events", lambda s: [])
|
||||
monkeypatch.setattr(run_mod, "_testnet_marks", lambda store, universe=None: {"BTCUSDT": 100.0})
|
||||
|
||||
|
||||
class _FakeStore:
|
||||
def close(self): ...
|
||||
|
||||
|
||||
def test_kill_switch_places_nothing(monkeypatch, tmp_path):
|
||||
from fxhnt.config import Settings
|
||||
s = Settings(); s.bybit_exec.kill_switch = True
|
||||
adapter = _FakeAdapter(); _patch_seams(monkeypatch, adapter)
|
||||
# also stub weights so plan_orders has input; monkeypatch latest_raw_sleeve_weights/combined_symbol_weights
|
||||
monkeypatch.setattr(run_mod, "latest_raw_sleeve_weights", lambda *a, **k: {})
|
||||
monkeypatch.setattr(run_mod, "combined_symbol_weights", lambda *a, **k: ({}, "2026-07-11"))
|
||||
out = run_bybit_testnet_helper(s, adapter, monkeypatch) # see helper below
|
||||
assert out.get("killed") is True and adapter.placed == []
|
||||
```
|
||||
|
||||
(Adjust seam/import names to whatever the port exposes; the essential assertions are: kill-switch ⇒ `placed == 0` and no exception, and a dry-run ⇒ `placed == 0`. Keep the fakes minimal — the point is to prove the guards survived the port, not to re-test slippage math already covered elsewhere.)
|
||||
|
||||
Add a dry-run test:
|
||||
```python
|
||||
def test_dry_run_places_nothing_and_writes_no_nav(monkeypatch):
|
||||
from fxhnt.application.bybit_testnet_run import run_bybit_testnet
|
||||
from fxhnt.config import Settings
|
||||
s = Settings()
|
||||
adapter = _FakeAdapter(); _patch_seams(monkeypatch, adapter)
|
||||
monkeypatch.setattr(run_mod, "latest_raw_sleeve_weights", lambda *a, **k: {})
|
||||
monkeypatch.setattr(run_mod, "combined_symbol_weights", lambda *a, **k: ({}, "2026-07-11"))
|
||||
logs = []
|
||||
out = run_bybit_testnet(s, paper_envelope=0.0, execute=False, log=logs.append)
|
||||
assert out["placed"] == 0 and out.get("dry_run") is True and adapter.placed == []
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run — expect FAIL**
|
||||
Run: `SQLX_OFFLINE=true pytest tests/unit/test_bybit_testnet_run.py -v`
|
||||
Expected: FAIL `ModuleNotFoundError: No module named 'fxhnt.application.bybit_testnet_run'`
|
||||
|
||||
- [ ] **Step 3: Implement** — create `src/fxhnt/application/bybit_testnet_run.py` following the Port method above. Module skeleton:
|
||||
|
||||
```python
|
||||
"""Bybit testnet execution leg as a plain application function (ported off the Dagster reconcile/record
|
||||
assets — same guards, now CLI-driven). `run_bybit_testnet` reconciles the bybit_4edge book on TESTNET and
|
||||
records fills/NAV + the observed bybit_4edge_exec return in ONE process. Dry-run (execute=False) plans only."""
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
from typing import Any, Callable
|
||||
|
||||
# the five monkeypatchable seams (moved verbatim from assets.py) ...
|
||||
def _testnet_store(s): ...
|
||||
def _testnet_universe(store): ...
|
||||
def _testnet_unlock_events(s): ...
|
||||
def _testnet_marks(store, universe=None): ...
|
||||
def _testnet_adapter(s): ...
|
||||
|
||||
|
||||
def run_bybit_testnet(settings: Any, *, paper_envelope: float, execute: bool,
|
||||
log: Callable[[str], None]) -> dict[str, Any]:
|
||||
... # ported reconcile + record body, with the 4 mechanical transforms
|
||||
```
|
||||
|
||||
Copy the reconcile+record bodies and apply the transforms. Keep the module-level `from ... import ...` that the seams and body use INSIDE functions where the assets had them (they import locally). Where the reconcile body imports `latest_raw_sleeve_weights, combined_symbol_weights` etc., keep those imports at module level so the tests can `monkeypatch.setattr(run_mod, "latest_raw_sleeve_weights", ...)`.
|
||||
|
||||
- [ ] **Step 4: Run — expect PASS**
|
||||
Run: `SQLX_OFFLINE=true pytest tests/unit/test_bybit_testnet_run.py -v`
|
||||
Expected: PASS
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
```bash
|
||||
git add src/fxhnt/application/bybit_testnet_run.py tests/unit/test_bybit_testnet_run.py
|
||||
git commit -m "feat(exec): port bybit testnet leg to run_bybit_testnet (faithful, +paper-envelope +venue)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 6: `execute-bybit` CLI command
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/cli.py` (add `execute_bybit` command near `execute_multistrat`)
|
||||
- Test: covered by Task 5's `run_bybit_testnet` tests + a smoke test here.
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `run_bybit_testnet(settings, paper_envelope, execute, log)` (Task 5); `_resolve_paper_envelope` (Task 3).
|
||||
|
||||
- [ ] **Step 1: Write the failing test** — append to `tests/unit/test_cli_execute_helpers.py`:
|
||||
|
||||
```python
|
||||
def test_execute_bybit_command_registered():
|
||||
from fxhnt.cli import app
|
||||
names = {c.name for c in app.registered_commands}
|
||||
assert "execute-bybit" in names
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run — expect FAIL**
|
||||
Run: `SQLX_OFFLINE=true pytest tests/unit/test_cli_execute_helpers.py -k execute_bybit -v`
|
||||
Expected: FAIL `assert 'execute-bybit' in ...`
|
||||
|
||||
- [ ] **Step 3: Implement** — in `src/fxhnt/cli.py`:
|
||||
|
||||
```python
|
||||
@app.command("execute-bybit")
|
||||
def execute_bybit(
|
||||
do_execute: bool = typer.Option(False, "--execute", help="place orders (default: dry-run plan only)"),
|
||||
paper_envelope: float = typer.Option(
|
||||
0.0, "--paper-envelope",
|
||||
help="PAPER-only validation envelope in $ (bypasses B2a; refused on live). 0 = B2a-gated."),
|
||||
) -> None:
|
||||
"""Reconcile + record the bybit_4edge book on Bybit TESTNET (ported off Dagster). Dry-run by default;
|
||||
same safety guards (kill-switch, NAV-marker idempotency, open-orders guard). Records bybit_4edge_exec."""
|
||||
from fxhnt.application.bybit_testnet_run import run_bybit_testnet
|
||||
|
||||
settings = get_settings()
|
||||
paper_env = _resolve_paper_envelope(paper_envelope, settings.paper_validation.envelope)
|
||||
out = run_bybit_testnet(settings, paper_envelope=paper_env, execute=do_execute, log=typer.echo)
|
||||
typer.echo(f"-> placed {out.get('placed', 0)} | envelope ${out.get('envelope', 0.0):,.0f} "
|
||||
f"| exec_return {out.get('exec_return')}")
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run — expect PASS**
|
||||
Run: `SQLX_OFFLINE=true pytest tests/unit/test_cli_execute_helpers.py -v`
|
||||
Expected: PASS
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
```bash
|
||||
git add src/fxhnt/cli.py tests/unit/test_cli_execute_helpers.py
|
||||
git commit -m "feat(cli): execute-bybit command (bybit testnet exec off Dagster, --paper-envelope)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 7: Remove the Dagster bybit exec assets + wiring (clean cut)
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/adapters/orchestration/assets.py` (delete `_testnet_*` helpers ~1106-1143, `bybit_testnet_reconcile` ~1146-1289, `bybit_testnet_record` ~1292-1365)
|
||||
- Modify: `src/fxhnt/adapters/orchestration/definitions.py` (remove imports + job + schedule)
|
||||
- Modify: `tests/integration/test_orchestration_definitions.py` (assert bybit exec job/schedule GONE)
|
||||
- Delete: `tests/integration/test_bybit_testnet_job_wiring.py` (the job it wired no longer exists; the exec logic is now tested via Task 5)
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: `defs` with no bybit exec assets/job/schedule; `combined_book_forward_job` untouched.
|
||||
|
||||
- [ ] **Step 1: Write/adjust the failing test** — in `tests/integration/test_orchestration_definitions.py`, add:
|
||||
|
||||
```python
|
||||
def test_bybit_testnet_exec_removed_from_definitions():
|
||||
from fxhnt.adapters.orchestration.definitions import defs
|
||||
job_names = {j.name for j in defs.jobs}
|
||||
assert "bybit_testnet_execution_job" not in job_names
|
||||
sched_names = {s.name for s in defs.schedules}
|
||||
assert "bybit_testnet_daily" not in sched_names
|
||||
asset_keys = {k.to_python_identifier() for k in defs.get_asset_graph().all_asset_keys}
|
||||
assert "bybit_testnet_reconcile" not in asset_keys
|
||||
assert "bybit_testnet_record" not in asset_keys
|
||||
```
|
||||
|
||||
(If `test_orchestration_definitions.py` already asserts these exist, update those assertions to the negative form. If `defs.jobs`/`defs.schedules` iteration differs in this Dagster version, adapt to the file's existing access pattern — mirror how the file already reads `defs`.)
|
||||
|
||||
- [ ] **Step 2: Run — expect FAIL**
|
||||
Run: `SQLX_OFFLINE=true pytest tests/integration/test_orchestration_definitions.py -k removed -v`
|
||||
Expected: FAIL (job/schedule/assets still present)
|
||||
|
||||
- [ ] **Step 3: Implement**
|
||||
|
||||
In `assets.py`: delete the block from the `# ---- Bybit testnet real-paper execution leg` comment (~1106) through the end of `bybit_testnet_record` (~1365). Do NOT touch `cockpit_forward` (starts ~1368) or `bybit_4edge_nav`/`bybit_paper_book`/other assets.
|
||||
|
||||
In `definitions.py`:
|
||||
- Remove `bybit_testnet_reconcile,` and `bybit_testnet_record,` from the import block (lines 11-12).
|
||||
- Delete the entire bybit exec block (lines 49-65): the comment, `from fxhnt.config import get_settings`, `bybit_testnet_job`, `bybit_testnet_schedule`.
|
||||
- In `Definitions(...)`: remove `bybit_testnet_reconcile, bybit_testnet_record,` from `assets=[...]`; remove `bybit_testnet_job` from `jobs=[...]`; remove `bybit_testnet_schedule` from `schedules=[...]`.
|
||||
|
||||
Delete `tests/integration/test_bybit_testnet_job_wiring.py`:
|
||||
```bash
|
||||
git rm tests/integration/test_bybit_testnet_job_wiring.py
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run — expect PASS + module still loads**
|
||||
Run: `SQLX_OFFLINE=true pytest tests/integration/test_orchestration_definitions.py -v`
|
||||
Expected: PASS
|
||||
Run: `SQLX_OFFLINE=true python -c "from fxhnt.adapters.orchestration.definitions import defs; print(len(defs.jobs), 'jobs')"`
|
||||
Expected: prints `1 jobs` (only `combined_book_forward_job`), no ImportError.
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
```bash
|
||||
git add src/fxhnt/adapters/orchestration/assets.py src/fxhnt/adapters/orchestration/definitions.py \
|
||||
tests/integration/test_orchestration_definitions.py
|
||||
git rm tests/integration/test_bybit_testnet_job_wiring.py
|
||||
git commit -m "refactor(dagster): remove bybit exec assets/job/schedule (leg now CLI-driven)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 8: cockpit — surface `exec_venue` in the badge
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/ports/dashboard.py` (`StrategyDetail` ~line 84 — add `exec_venue`)
|
||||
- Modify: `src/fxhnt/application/dashboard_service.py` (`detail` ~169-192; add `_safe_book_venue`)
|
||||
- Modify: `src/fxhnt/adapters/web/templates/strategy.html` (line 47 badge)
|
||||
- Test: `tests/integration/test_dashboard_surfacing.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `self._repo.get_book_state(f"{twin}_pos")` → `.get("venue")`.
|
||||
- Produces: `StrategyDetail.exec_venue: str | None`; template renders `{{ d.exec_venue or d.venue }}`.
|
||||
|
||||
- [ ] **Step 1: Write the failing test** — append to `tests/integration/test_dashboard_surfacing.py`, following the file's existing repo-seeding pattern (seed a `multistrat` + `multistrat_exec` summary, then a `multistrat_exec_pos` book-state with `venue`):
|
||||
|
||||
```python
|
||||
def test_detail_exec_venue_from_book_state(tmp_path):
|
||||
# ... seed via the same ForwardNavRepo pattern the other tests in this file use:
|
||||
# summaries for "multistrat" and "multistrat_exec"
|
||||
# repo.set_book_state("multistrat_exec_pos", {"positions": {}, "prices": {}, "envelope": 1e6,
|
||||
# "venue": "ibkr-paper"}, at=<naive utc>)
|
||||
svc = DashboardService(repo)
|
||||
d = svc.detail("multistrat")
|
||||
assert d.exec_twin_sid == "multistrat_exec"
|
||||
assert d.exec_venue == "ibkr-paper"
|
||||
|
||||
|
||||
def test_detail_exec_venue_none_when_unrecorded(tmp_path):
|
||||
# same seeding but the multistrat_exec_pos book-state has NO "venue" key (pre-C3 anchor)
|
||||
svc = DashboardService(repo)
|
||||
d = svc.detail("multistrat")
|
||||
assert d.exec_venue is None
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run — expect FAIL**
|
||||
Run: `SQLX_OFFLINE=true pytest tests/integration/test_dashboard_surfacing.py -k exec_venue -v`
|
||||
Expected: FAIL `AttributeError: 'StrategyDetail' object has no attribute 'exec_venue'`
|
||||
|
||||
- [ ] **Step 3: Implement**
|
||||
|
||||
In `src/fxhnt/ports/dashboard.py`, add to `class StrategyDetail`:
|
||||
```python
|
||||
exec_venue: str | None = None
|
||||
```
|
||||
|
||||
In `src/fxhnt/application/dashboard_service.py`, add to `DashboardService`:
|
||||
```python
|
||||
def _safe_book_venue(self, twin_sid: str) -> str | None:
|
||||
"""Execution venue recorded on the exec twin's book-state; None if absent/unreachable (fallback)."""
|
||||
if not twin_sid:
|
||||
return None
|
||||
try:
|
||||
return self._repo.get_book_state(f"{twin_sid}_pos").get("venue") or None
|
||||
except Exception: # noqa: BLE001 — a missing book-state must never break the dashboard
|
||||
return None
|
||||
```
|
||||
|
||||
In `detail()`, add `pair = self._exec_pair(strategy_id, summaries)` immediately above `return StrategyDetail(`, and replace the trailing `**self._exec_pair(strategy_id, summaries),` with:
|
||||
```python
|
||||
**pair,
|
||||
exec_venue=self._safe_book_venue(pair["exec_twin_sid"]),
|
||||
```
|
||||
|
||||
In `src/fxhnt/adapters/web/templates/strategy.html` line 47:
|
||||
```html
|
||||
<div class="lct">Executed reality <span class="r"><span class="exec-tag">{{ d.exec_venue or d.venue }}</span></span></div>
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run — expect PASS**
|
||||
Run: `SQLX_OFFLINE=true pytest tests/integration/test_dashboard_surfacing.py -k exec_venue -v`
|
||||
Expected: PASS
|
||||
|
||||
- [ ] **Step 5: Local cockpit verification (read the numbers)**
|
||||
|
||||
```bash
|
||||
SQLX_OFFLINE=true python - <<'PY'
|
||||
import datetime as dt
|
||||
from fxhnt.adapters.persistence.forward_nav import ForwardNavRepo
|
||||
from fxhnt.config import get_settings
|
||||
r = ForwardNavRepo(get_settings().operational_dsn); r.migrate()
|
||||
st = r.get_book_state("multistrat_exec_pos") or {"positions": {}, "prices": {}, "envelope": 1e6}
|
||||
st["venue"] = "ibkr-paper"; r.set_book_state("multistrat_exec_pos", st, at=dt.datetime.now(dt.UTC).replace(tzinfo=None))
|
||||
print("seeded multistrat_exec_pos venue=ibkr-paper")
|
||||
PY
|
||||
./scripts/dev-cockpit-local.sh # 127.0.0.1:8801 (read-only)
|
||||
```
|
||||
Playwright → `http://127.0.0.1:8801/strategy/multistrat`, screenshot, **read** that the "Executed reality" badge shows `ibkr-paper`.
|
||||
|
||||
- [ ] **Step 6: Commit**
|
||||
```bash
|
||||
git add src/fxhnt/ports/dashboard.py src/fxhnt/application/dashboard_service.py \
|
||||
src/fxhnt/adapters/web/templates/strategy.html tests/integration/test_dashboard_surfacing.py
|
||||
git commit -m "feat(cockpit): surface recorded exec venue in Executed-reality badge (fallback to configured)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 9: infra — arm alpaca rebalancer + new bybit rebalancer CronJob
|
||||
|
||||
**Files:**
|
||||
- Modify: `infra/k8s/jobs/fxhnt-alpaca-rebalancer.yaml` (args + `suspend`)
|
||||
- Create: `infra/k8s/jobs/fxhnt-bybit-rebalancer.yaml`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `execute-multistrat --paper-envelope` (Task 3), `execute-bybit --paper-envelope` (Task 6).
|
||||
|
||||
- [ ] **Step 1: Alpaca rebalancer** — append `--paper-envelope 1000000` to the container's `fxhnt execute-multistrat --execute` arg string (preserve the rest verbatim), and set `suspend: false`.
|
||||
|
||||
- [ ] **Step 2: Bybit rebalancer** — create `infra/k8s/jobs/fxhnt-bybit-rebalancer.yaml` by copying `fxhnt-alpaca-rebalancer.yaml` and changing:
|
||||
- `metadata.name` → `fxhnt-bybit-rebalancer`; labels `app.kubernetes.io/name: fxhnt-bybit-rebalancer`.
|
||||
- container command → `fxhnt execute-bybit --execute --paper-envelope 1000000`.
|
||||
- env: replace `FXHNT_ALPACA_*` (from `alpaca-credentials`) with `FXHNT_BYBIT_EXEC_API_KEY`/`FXHNT_BYBIT_EXEC_API_SECRET` from the bybit exec secret (`secretKeyRef` name `bybit-testnet-credentials`, keys `api-key`/`api-secret`); keep `FXHNT_OPERATIONAL_DSN` + `PGPASSWORD`.
|
||||
- NetworkPolicy (if the alpaca job ships one): egress to Postgres 5432 + gitea (git-sync) + external 443 (Bybit API). Bybit is an EXTERNAL host — no in-cluster pod selector needed, just 443 egress.
|
||||
- `suspend: true` (arm only after `bybit-testnet-credentials` is populated).
|
||||
|
||||
- [ ] **Step 3: Validate manifests**
|
||||
Run: `kubectl apply --dry-run=client -f infra/k8s/jobs/fxhnt-alpaca-rebalancer.yaml -f infra/k8s/jobs/fxhnt-bybit-rebalancer.yaml`
|
||||
Expected: both `configured/created (dry run)`, no error.
|
||||
|
||||
- [ ] **Step 4: Commit**
|
||||
```bash
|
||||
git add infra/k8s/jobs/fxhnt-alpaca-rebalancer.yaml infra/k8s/jobs/fxhnt-bybit-rebalancer.yaml
|
||||
git commit -m "chore(infra): arm alpaca --paper-envelope + new fxhnt-bybit-rebalancer CronJob"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Full-suite gate (before finishing)
|
||||
|
||||
- [ ] Ensure no other pytest is running, then run once clean:
|
||||
```bash
|
||||
pkill -9 -f pytest 2>/dev/null; sleep 1
|
||||
SQLX_OFFLINE=true pytest tests/unit tests/integration -q
|
||||
```
|
||||
Expected: all pass (watch for exit-144/SIGTERM from concurrent runs — re-run singly if seen).
|
||||
|
||||
## Deployment (post-merge, controller-run — NOT TDD tasks)
|
||||
|
||||
1. Push `master`; `./scripts/argo-deploy-cockpit.sh` (build + deploy). No schema migration.
|
||||
2. Revert `strategy_allocation[multistrat]` → `$0` (or let the nightly recompute it). Cockpit then shows the honest B2a state.
|
||||
3. Cancel the queued IBKR paper orders on DU9600528 via a one-off `ib_async` job.
|
||||
4. Arm `fxhnt-alpaca-rebalancer` (`suspend: false`); once `bybit-testnet-credentials` is confirmed, arm `fxhnt-bybit-rebalancer`.
|
||||
5. Verify the cockpit LOCALLY, then confirm on `dashboard.fxhnt.ai` that the multistrat "Executed reality" badge shows the real venue and the allocation reads the honest B2a value.
|
||||
|
||||
## Self-Review
|
||||
|
||||
- **Spec coverage:** Component 1 (bybit→CLI port) → Tasks 5,6,7; Component 2 (paper envelope) → Tasks 1,2,3,4,5,6; Component 3 (venue) → Tasks 2,4,5,8; Component 4 (cockpit) → Task 8; Component 5 (wiring) → Task 9 + Deployment. Faithful-port guards → Task 5 (kill/dry-run tests) + preserved logic. ✓
|
||||
- **Types consistent:** `venue` param default `""` at every record site; `exec_venue: str | None` in the DTO; `paper_envelope: float`/`account_is_paper: bool` uniform across `plan_and_record`, `paper_or_b2a_crypto_envelope`, `run_bybit_testnet`, and both CLI commands. ✓
|
||||
- **Clean cut:** Task 7 removes assets + definitions wiring + the wiring test; `test_orchestration_definitions.py` asserts absence; module-load check confirms no orphaned imports. ✓
|
||||
- **No placeholders:** every code step shows code; every test step shows the command + expected result. The port task (5) is a faithful relocation with enumerated mechanical transforms + explicit source line ranges (re-transcribing 220 lines verbatim would be error-prone; the transforms are exact). ✓
|
||||
File diff suppressed because it is too large
Load Diff
480
docs/superpowers/plans/2026-07-13-phase0b-venue-consolidation.md
Normal file
480
docs/superpowers/plans/2026-07-13-phase0b-venue-consolidation.md
Normal file
@@ -0,0 +1,480 @@
|
||||
# Phase 0b — Venue Consolidation to Bybit + IBKR Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development to implement this
|
||||
> plan task-by-task. Steps use checkbox (`- [ ]`) syntax.
|
||||
|
||||
**Goal:** Make Bybit (crypto) + IBKR (tradfi) the only venues for execution AND data — migrate the crypto
|
||||
edges that work on Bybit (xsfunding, unlock) onto Bybit data, retire the ones that don't (crypto_tstrend
|
||||
standalone, stablecoin), and remove Binance + Alpaca entirely.
|
||||
|
||||
**Architecture:** Migrate first (xsfunding/unlock → Bybit standalone tracks, template = `positioning`),
|
||||
then remove (Alpaca exec, Binance exec, Binance data + `features` table, vestigial `state_file`). The
|
||||
`bybit_4edge` deploy book is unchanged (it already runs on Bybit).
|
||||
|
||||
**Tech Stack:** Python 3.12, Dagster assets, SQLAlchemy 2.0, Timescale/Postgres, pytest.
|
||||
|
||||
**Base:** branch off `feat/unified-db-backtest-architecture` (0a, unmerged) as
|
||||
`feat/phase0b-venue-consolidation`. Spec:
|
||||
`docs/superpowers/specs/2026-07-13-phase0b-venue-consolidation-bybit-ibkr.md`.
|
||||
|
||||
## Global Constraints
|
||||
|
||||
- **A crypto strat survives only if it works on Bybit** (instruments listed + edge holds). Decided per edge
|
||||
in the spec — do not re-litigate.
|
||||
- **Real money errs false-WAIT, never false-FUND.** Migrated tracks re-inception on cutover (new Bybit
|
||||
basis) — a VISIBLE deterministic re-inception, never a silent reset. Their Bybit refs must exist before
|
||||
the gate can PASS. The 0a reconciliation invariant test MUST stay green.
|
||||
- **Single source of truth**; no dead code / stubs / TODO; fix every review Minor in-loop.
|
||||
- **Keep the Broker port** with `IBKRBroker` as the sole implementation (do not delete the abstraction).
|
||||
- **`bybit_4edge` deploy book is UNCHANGED** — crypto_tstrend stays a sleeve in the book; 0b retires only its
|
||||
redundant *standalone* track. Do not touch `_DEFAULT_BYBIT_SLEEVES`.
|
||||
- **Binance stays gone**: after migration, nothing (data or exec) may read Binance or the `features` table.
|
||||
- Verify the cockpit LOCALLY in a real browser before finishing (`./scripts/dev-cockpit-local.sh` or a
|
||||
seeded sqlite DB).
|
||||
|
||||
---
|
||||
|
||||
## Task 1: Generic Bybit single-sleeve track builder + migrate `xsfunding` → Bybit
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/adapters/orchestration/assets.py` (add `build_bybit_sleeve_forward_nav`; refactor
|
||||
`build_positioning_forward_nav` to use it; rewrite `xsfunding_nav` asset)
|
||||
- Modify: `src/fxhnt/application/bybit_forward_track.py` (`bybit_report_sid_kwargs`: add `xsfunding`)
|
||||
- Modify: `src/fxhnt/cli.py` (`_BYBIT_BOOK_SIDS`: add `"xsfunding"`)
|
||||
- Modify: `src/fxhnt/registry.py` (`xsfunding` entry: venue + backtest.kind + params)
|
||||
- Test: `tests/unit/test_bybit_sleeve_forward_builder.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `BybitFourEdgeStrategy(store, sleeves=[...], unlock_events=…, positioning_capacity_capital=…,
|
||||
positioning_tail_cap_k=…)`, `forward_engine.run_track(repo, sid, strategy, today, at)`,
|
||||
`TimescaleFeatureStore(dsn, table="bybit_features")`, `liquid_universe(store)`.
|
||||
- Produces: `build_bybit_sleeve_forward_nav(repo, store, sleeve, *, today, at, universe=None, cost_bps=5.5,
|
||||
unlock_events=None, positioning_capacity_capital=None, positioning_tail_cap_k=None) -> dict`.
|
||||
|
||||
- [ ] **Step 1: Write the failing test** — the generic builder runs a single-sleeve Bybit strategy through
|
||||
the deterministic engine and returns a track dict, and `build_positioning_forward_nav` still works via it.
|
||||
|
||||
```python
|
||||
# tests/unit/test_bybit_sleeve_forward_builder.py
|
||||
import datetime as dt
|
||||
from fxhnt.adapters.orchestration.assets import build_bybit_sleeve_forward_nav, build_positioning_forward_nav
|
||||
from fxhnt.adapters.persistence.forward_nav import ForwardNavRepo
|
||||
from fxhnt.adapters.warehouse.timescale_feature_store import TimescaleFeatureStore
|
||||
|
||||
|
||||
def _seed(store, days=40):
|
||||
import math
|
||||
for i in range(6):
|
||||
px, rows, sign = 100.0, [], (1.0 if i % 2 == 0 else -1.0)
|
||||
for d in range(days):
|
||||
px *= 1 + sign * 0.01 + 0.003 * math.cos(0.4 * d + i)
|
||||
rows.append((d * 86400, {"close": px, "funding": 0.001 * sign, "spot_close": px,
|
||||
"long_ratio": 0.5 + 0.1 * sign}))
|
||||
store.write_features(f"SYM{i:02d}USDT", rows)
|
||||
|
||||
|
||||
def test_generic_builder_books_a_single_sleeve_track(tmp_path):
|
||||
repo = ForwardNavRepo(f"sqlite:///{tmp_path/'c.db'}"); repo.migrate()
|
||||
store = TimescaleFeatureStore("sqlite://", table="bybit_features"); _seed(store)
|
||||
out = build_bybit_sleeve_forward_nav(repo, store, "xsfunding",
|
||||
today="1970-02-09", at=dt.datetime(2026, 1, 1), cost_bps=0.0)
|
||||
assert out["days"] >= 1 and "t0" in out
|
||||
store.close()
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run to verify it fails** — `.venv/bin/python -m pytest tests/unit/test_bybit_sleeve_forward_builder.py -q` → FAIL (import error).
|
||||
|
||||
- [ ] **Step 3: Add the generic builder + refactor positioning to use it** (`assets.py`). Insert before
|
||||
`build_positioning_forward_nav`:
|
||||
|
||||
```python
|
||||
def build_bybit_sleeve_forward_nav(repo, store, sleeve, *, today, at, universe=None, cost_bps=5.5,
|
||||
unlock_events=None, positioning_capacity_capital=None,
|
||||
positioning_tail_cap_k=None): # type: ignore[no-untyped-def]
|
||||
"""Plain (Dagster-free, unit-testable) driver for a SINGLE Bybit sleeve as its own forward track: build
|
||||
the one-sleeve `BybitFourEdgeStrategy` over `store` (bybit_features) and run it through the deterministic
|
||||
engine (DB anchor + recompute). The DB (forward_nav + anchor) is the SSOT — no JSON state. Shared by the
|
||||
per-edge Bybit tracks (xsfunding / unlock / positioning)."""
|
||||
from fxhnt.application.bybit_forward_track import BybitFourEdgeStrategy
|
||||
from fxhnt.application.forward_engine import run_track
|
||||
strategy = BybitFourEdgeStrategy(
|
||||
store, universe=universe, cost_bps=cost_bps, sleeves=[sleeve], unlock_events=unlock_events,
|
||||
positioning_capacity_capital=positioning_capacity_capital, positioning_tail_cap_k=positioning_tail_cap_k)
|
||||
return run_track(repo, sleeve, strategy, today=today, at=at)
|
||||
```
|
||||
Then replace `build_positioning_forward_nav`'s body with a thin delegation:
|
||||
```python
|
||||
def build_positioning_forward_nav(repo, store, *, today, at, universe=None, cost_bps=5.5,
|
||||
positioning_capacity_capital=None, positioning_tail_cap_k=None):
|
||||
"""POSITIONING single-sleeve Bybit track — thin wrapper on `build_bybit_sleeve_forward_nav` (keeps the
|
||||
capturability haircuts). See that function."""
|
||||
return build_bybit_sleeve_forward_nav(
|
||||
repo, store, "positioning", today=today, at=at, universe=universe, cost_bps=cost_bps,
|
||||
positioning_capacity_capital=positioning_capacity_capital, positioning_tail_cap_k=positioning_tail_cap_k)
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run — the builder test + the existing positioning tests pass** —
|
||||
`.venv/bin/python -m pytest tests/unit/test_bybit_sleeve_forward_builder.py tests/integration/test_positioning_forward_track.py -q` → PASS.
|
||||
|
||||
- [ ] **Step 5: Rewrite the `xsfunding_nav` asset** (Bybit, mirroring `positioning_nav`). Replace the current
|
||||
Binance body (`XsFundingForward(BinanceXsFundingLive())` + `persist_backtest_ref=True`) with:
|
||||
|
||||
```python
|
||||
@asset(deps=[bybit_warehouse]) # match positioning_nav's freshness dep (use the same bybit warehouse asset dep)
|
||||
def xsfunding_nav(context: AssetExecutionContext) -> dict: # type: ignore[type-arg]
|
||||
"""Paper-forward cross-sectional funding-dispersion — the SINGLE xsfunding sleeve as its own edge, on
|
||||
BYBIT (`bybit_features`), the venue it would trade. Deterministic engine (DB anchor + recompute); its
|
||||
reconciliation ref is written by the weekly Bybit book persist (`_persist_bybit_book`), like positioning."""
|
||||
import datetime as _dt
|
||||
import urllib.error
|
||||
from fxhnt.adapters.persistence.forward_nav import ForwardNavRepo
|
||||
from fxhnt.adapters.warehouse.timescale_feature_store import TimescaleFeatureStore
|
||||
from fxhnt.application.bybit_liquidity import liquid_universe
|
||||
from fxhnt.config import get_settings
|
||||
s = get_settings()
|
||||
store = TimescaleFeatureStore(s.operational_dsn, table="bybit_features")
|
||||
try:
|
||||
universe = liquid_universe(store) or None
|
||||
repo = ForwardNavRepo(s.operational_dsn); repo.migrate()
|
||||
today = _dt.datetime.now(_dt.UTC).strftime("%Y-%m-%d")
|
||||
at = _dt.datetime.now(_dt.UTC).replace(tzinfo=None)
|
||||
out = build_bybit_sleeve_forward_nav(repo, store, "xsfunding", today=today, at=at,
|
||||
universe=universe, cost_bps=s.cost_bps_per_turnover)
|
||||
context.log.info(f"xsfunding_nav: T0={out['t0']}, {out['days']}d through {out['last_date']} "
|
||||
f"(reinception={out['reinception']})")
|
||||
return {"inception": out["t0"], "last_date": out["last_date"], "forward_days": out["days"]}
|
||||
except (urllib.error.URLError, TimeoutError, ConnectionError, OSError) as e:
|
||||
context.log.warning(f"xsfunding_nav step failed (record unchanged): {e}")
|
||||
return {"inception": None, "last_date": None, "forward_days": 0}
|
||||
finally:
|
||||
store.close()
|
||||
```
|
||||
*(Use the SAME `deps=[...]` and the same bybit warehouse-refresh asset that `positioning_nav` depends on —
|
||||
read positioning_nav's decorator and match it exactly; if positioning_nav has no explicit dep, match that.)*
|
||||
|
||||
- [ ] **Step 6: Wire xsfunding's Bybit ref.** (a) `bybit_forward_track.py` `bybit_report_sid_kwargs`: add
|
||||
`if strategy_id == "xsfunding": return {"sleeves": ["xsfunding"], "strategy_id": "xsfunding"}`. (b)
|
||||
`cli.py`: `_BYBIT_BOOK_SIDS = ("bybit_4edge", "bybit_4edge_levered", "positioning", "xsfunding")`. (c)
|
||||
`registry.py` `xsfunding`: set `"venue": "bybit-perp"`, and `definition.backtest.kind` from `"sleeve"` →
|
||||
`"report"` (its ref now comes from the Bybit book persist, like positioning); update
|
||||
`definition.params` to the single-sleeve Bybit construction (mirror `positioning`'s params shape).
|
||||
|
||||
- [ ] **Step 7: Update the invariant test expectation** — `tests/unit/test_backtest_invariant.py` already
|
||||
enumerates the registry; confirm it still passes (xsfunding is now `report`-kind, non-`none`, and has a
|
||||
provider via `_BYBIT_BOOK_SIDS`). Run `.venv/bin/python -m pytest tests/unit/test_backtest_invariant.py -q`.
|
||||
|
||||
- [ ] **Step 8: Run the affected suites + commit** —
|
||||
`.venv/bin/python -m pytest tests/unit/test_bybit_sleeve_forward_builder.py tests/unit/test_backtest_invariant.py tests/integration/test_bybit_persist_sleeve_ret_cli.py -q` → PASS, then:
|
||||
```bash
|
||||
git add src/fxhnt/adapters/orchestration/assets.py src/fxhnt/application/bybit_forward_track.py src/fxhnt/cli.py src/fxhnt/registry.py tests/unit/test_bybit_sleeve_forward_builder.py
|
||||
git commit -m "feat(0b): migrate xsfunding to a Bybit single-sleeve forward track (venue-consistent ref)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 2: Migrate `unlock` → Bybit (reuse the builder; carry the unlock calendar)
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/adapters/orchestration/assets.py` (rewrite `unlock_nav`)
|
||||
- Modify: `src/fxhnt/application/bybit_forward_track.py` (`bybit_report_sid_kwargs`: add `unlock`)
|
||||
- Modify: `src/fxhnt/cli.py` (`_BYBIT_BOOK_SIDS`: add `"unlock"`)
|
||||
- Modify: `src/fxhnt/registry.py` (`unlock` entry)
|
||||
- Test: extend `tests/unit/test_bybit_sleeve_forward_builder.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `build_bybit_sleeve_forward_nav` (Task 1), `load_unlock_events(operational_unlock_repo(s), path)`,
|
||||
`_data_dir()`.
|
||||
|
||||
- [ ] **Step 1: Add a failing test** that the builder threads `unlock_events` into the unlock sleeve
|
||||
(monkeypatch `BybitFourEdgeStrategy` to capture kwargs, assert `sleeves==["unlock"]` and
|
||||
`unlock_events` passed through). Run → FAIL.
|
||||
|
||||
- [ ] **Step 2: Rewrite the `unlock_nav` asset** — Bybit single-sleeve via `build_bybit_sleeve_forward_nav`
|
||||
with `sleeve="unlock"`, loading the calendar the SAME way the Bybit book does
|
||||
(`unlock_events = load_unlock_events(operational_unlock_repo(s), f"{_data_dir()}/unlock_calendar.json")`,
|
||||
`[]` on failure — never fabricate), store = `TimescaleFeatureStore(dsn, table="bybit_features")`, NO
|
||||
`persist_backtest_ref` (ref comes from `_persist_bybit_book`). Mirror the `xsfunding_nav` shape from Task 1.
|
||||
|
||||
- [ ] **Step 3: Wire unlock's Bybit ref** — `bybit_report_sid_kwargs`: add
|
||||
`if strategy_id == "unlock": return {"sleeves": ["unlock"], "strategy_id": "unlock"}`;
|
||||
`_BYBIT_BOOK_SIDS += ("unlock",)`; `registry.py` `unlock`: `venue` → `bybit-perp`, `backtest.kind`
|
||||
`recompute-replay` → `report`, params → single-sleeve Bybit construction.
|
||||
|
||||
- [ ] **Step 4: Confirm `_persist_bybit_book` writes both new refs** — it loops `_BYBIT_BOOK_SIDS`, which now
|
||||
includes xsfunding + unlock; `bybit_4edge_backtest_summary(sleeves=["unlock"], strategy_id="unlock",
|
||||
sleeve_returns=sleeve_rets)` reuses the already-computed sleeve series (unlock is in `_DEFAULT_BYBIT_SLEEVES`,
|
||||
so `sleeve_rets["unlock"]` exists — verify). No extra compute.
|
||||
|
||||
- [ ] **Step 5: Run + commit** —
|
||||
`.venv/bin/python -m pytest tests/unit/test_bybit_sleeve_forward_builder.py tests/unit/test_backtest_invariant.py -q` → PASS, then commit:
|
||||
```bash
|
||||
git commit -am "feat(0b): migrate unlock to a Bybit single-sleeve forward track"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 3: Align the Bybit reconciliation cost basis (single-source 5.5)
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/config.py` (add `bybit_cost_bps: float = 5.5`)
|
||||
- Modify: `src/fxhnt/adapters/orchestration/assets.py` (`positioning_nav`, `xsfunding_nav`, `unlock_nav`: pass `cost_bps=s.bybit_cost_bps`, NOT `s.cost_bps_per_turnover`)
|
||||
- Modify: `src/fxhnt/adapters/orchestration/migration_builders.py` (the Bybit-sleeve recompute builders: `cost_bps=settings.bybit_cost_bps`)
|
||||
- Modify: `src/fxhnt/cli.py` (`_persist_bybit_book` + `bybit-persist-sleeve-ret`/`backtest-refs` `--cost-bps`: default from `settings.bybit_cost_bps`)
|
||||
- Modify: `src/fxhnt/registry.py` (the Bybit-sleeve tracks' `params.cost_bps` 10.0→5.5, update the "mirrors Settings.cost_bps_per_turnover" comment → `bybit_cost_bps`, BUMP `definition_version` 1→2 for a clean visible re-inception)
|
||||
- Test: `tests/unit/test_bybit_cost_basis_consistent.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: `Settings.bybit_cost_bps` (= 5.5) — the SINGLE Bybit-perp per-turnover reconciliation cost, read by both the forward tracks AND their `_persist_bybit_book` refs.
|
||||
|
||||
**Why:** the standalone Bybit sleeve tracks charged `cost_bps_per_turnover=10.0` while their reconciliation ref (`_persist_bybit_book`) + the deploy book use 5.5 (the MEASURED liquid-book cost — the "mirage" finding: Sharpe 2.11 at 5.5bp drag). Forward-charged-more-than-ref biases the gate toward WAIT (safe, but a cost artifact could permanently WAIT a real edge). One correct Bybit cost, single-sourced, enforced by an invariant test. `cost_bps_per_turnover` (10) stays for the equity/tradfi tracks (different venue) — do NOT change multistrat.
|
||||
|
||||
- [ ] **Step 1: Write the failing invariant test**
|
||||
|
||||
```python
|
||||
# tests/unit/test_bybit_cost_basis_consistent.py
|
||||
from fxhnt.config import Settings
|
||||
from fxhnt.registry import STRATEGY_REGISTRY
|
||||
|
||||
_BYBIT_SLEEVE_SIDS = ("positioning", "xsfunding", "unlock") # standalone Bybit sleeve tracks
|
||||
|
||||
|
||||
def test_bybit_sleeve_tracks_use_the_single_bybit_cost():
|
||||
c = Settings().bybit_cost_bps
|
||||
assert c == 5.5
|
||||
for sid in _BYBIT_SLEEVE_SIDS:
|
||||
params = STRATEGY_REGISTRY[sid]["definition"]["params"]
|
||||
assert params["cost_bps"] == c, (
|
||||
f"{sid} registry cost_bps={params['cost_bps']} != bybit_cost_bps={c} — "
|
||||
f"track and reconciliation ref would use different costs")
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run to verify it fails** — `.venv/bin/python -m pytest tests/unit/test_bybit_cost_basis_consistent.py -q` → FAIL.
|
||||
|
||||
- [ ] **Step 3: Add `bybit_cost_bps = 5.5` to `Settings`** (config.py), documented as the measured Bybit-perp liquid-book per-turnover reconciliation cost (single source for tracks + refs).
|
||||
|
||||
- [ ] **Step 4: Repoint the tracks + refs + builders to `bybit_cost_bps`.** In `assets.py`, the three Bybit sleeve assets pass `cost_bps=s.bybit_cost_bps`. In `migration_builders.py`, the Bybit-sleeve builders use `settings.bybit_cost_bps`. In `cli.py`, `_persist_bybit_book`'s `cost_bps` default + the `--cost-bps` option defaults resolve from `get_settings().bybit_cost_bps`. VERIFY `bybit_4edge`/`bybit_4edge_levered` forward tracks: if their asset already uses 5.5 they are consistent (leave); if any Bybit forward track passes `cost_bps_per_turnover`, repoint it too — the invariant is "every Bybit forward track and its ref use `bybit_cost_bps`".
|
||||
|
||||
- [ ] **Step 5: Registry — set `cost_bps: 5.5` + bump `version` 1→2** on the Bybit sleeve tracks (positioning/xsfunding/unlock; and `carry` for consistency if present, though dormant), update the mirror comment. The param change re-hashes the definition → deterministic re-inception; the version bump is the documented protocol for a cost change (visible, not silent). Do NOT touch multistrat/equity cost_bps.
|
||||
|
||||
- [ ] **Step 6: Run the invariant + the migration/regression suites** —
|
||||
`.venv/bin/python -m pytest tests/unit/test_bybit_cost_basis_consistent.py tests/unit/test_backtest_invariant.py tests/integration/test_bybit_persist_sleeve_ret_cli.py tests/unit/test_bybit_sleeve_forward_builder.py -q` → PASS.
|
||||
|
||||
- [ ] **Step 7: Commit**
|
||||
|
||||
```bash
|
||||
git commit -am "fix(0b): single-source the Bybit reconciliation cost (bybit_cost_bps=5.5) — tracks + refs now reconcile on one measured cost basis"
|
||||
```
|
||||
|
||||
Post-deploy note (runbook): the cost change re-inceptions the affected Bybit tracks (version 2) — a VISIBLE deterministic re-inception; include `migrate-forward-anchors` for them.
|
||||
|
||||
## Task 4: Retire `crypto_tstrend` standalone track + `stablecoin_rotation`
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/adapters/orchestration/assets.py` (delete `crypto_tstrend_nav`, `stablecoin_rotation_nav`
|
||||
assets + remove from the Definitions asset list)
|
||||
- Modify: `src/fxhnt/registry.py` (delete `crypto_tstrend`, `stablecoin_rotation` entries)
|
||||
- Delete: `src/fxhnt/application/stablecoin_strategy.py`, `src/fxhnt/application/stablecoin_runner.py` (if no
|
||||
other consumer — grep first)
|
||||
- Test: `tests/unit/test_retired_tracks_gone.py`
|
||||
|
||||
**Interfaces:** none produced. **Consumes:** nothing new.
|
||||
|
||||
- [ ] **Step 1: Verify no other consumer** — `grep -rn "stablecoin_strategy\|stablecoin_runner\|StableRotationForward\|StableReversionRunner\|crypto_tstrend_nav" src/fxhnt/ | grep -v test`. Anything outside the assets/registry above is a blocker — report it. (The `crypto_tstrend` SLEEVE in `_DEFAULT_BYBIT_SLEEVES` / `bybit_book_eval` is the DEPLOY BOOK — it MUST remain; only the standalone `crypto_tstrend_nav` track + registry entry go.)
|
||||
|
||||
- [ ] **Step 2: Write the guard test**:
|
||||
```python
|
||||
# tests/unit/test_retired_tracks_gone.py
|
||||
from fxhnt.registry import STRATEGY_REGISTRY
|
||||
from fxhnt.application.bybit_book_eval import _DEFAULT_BYBIT_SLEEVES
|
||||
|
||||
|
||||
def test_standalone_binance_tracks_retired():
|
||||
assert "crypto_tstrend" not in STRATEGY_REGISTRY # standalone track gone
|
||||
assert "stablecoin_rotation" not in STRATEGY_REGISTRY
|
||||
|
||||
|
||||
def test_tstrend_stays_in_the_deploy_book():
|
||||
assert "crypto_tstrend" in _DEFAULT_BYBIT_SLEEVES # the BOOK sleeve is untouched
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Run to verify it fails** (entries still present) → FAIL.
|
||||
|
||||
- [ ] **Step 4: Delete** the two assets (remove their `@asset` defs + drop them from the `Definitions(assets=[...])` list), the two registry entries, and the two stablecoin modules (if Step 1 clean). Leave `_DEFAULT_BYBIT_SLEEVES` untouched.
|
||||
|
||||
- [ ] **Step 5: Run the guard + full suite slice** —
|
||||
`.venv/bin/python -m pytest tests/unit/test_retired_tracks_gone.py -q && .venv/bin/python -c "from fxhnt.adapters.orchestration.assets import defs; print('assets load')"` → PASS. Fix any now-dangling import.
|
||||
|
||||
- [ ] **Step 6: Commit** — `git commit -am "feat(0b): retire crypto_tstrend standalone track + stablecoin_rotation (not tradeable on Bybit); book sleeve unchanged"`.
|
||||
|
||||
---
|
||||
|
||||
## Task 5: Remove the Alpaca execution surface (keep Broker port, IBKR sole impl)
|
||||
|
||||
**Files:**
|
||||
- Delete: `src/fxhnt/adapters/broker/alpaca.py`, `infra/k8s/jobs/fxhnt-alpaca-rebalancer.yaml`,
|
||||
`infra/k8s/secrets/alpaca-credentials.yaml`
|
||||
- Modify: `src/fxhnt/config.py` (delete `AlpacaSettings` + the `alpaca` field), `src/fxhnt/cli.py` (remove
|
||||
`--broker` option + all `alpaca` branches from `execute-multistrat` (cli.py:257–325) and the peer command
|
||||
at cli.py:206–223 — IBKR implicit)
|
||||
- Test: `tests/unit/test_no_alpaca.py`
|
||||
|
||||
**Interfaces:** the Broker port stays; `IBKRBroker` becomes the only construction path.
|
||||
|
||||
- [ ] **Step 1: Grep the Alpaca surface** — `grep -rn "alpaca\|Alpaca\|AlpacaBroker\|AlpacaSettings" src/fxhnt/ infra/ | grep -v test`. Enumerate every site; each must be deleted or repointed to IBKR.
|
||||
|
||||
- [ ] **Step 2: Guard test**:
|
||||
```python
|
||||
# tests/unit/test_no_alpaca.py
|
||||
import subprocess, pathlib
|
||||
def test_no_alpaca_symbols_in_src():
|
||||
root = pathlib.Path(__file__).parents[1] / "src" / "fxhnt"
|
||||
hits = subprocess.run(["grep", "-rin", "alpaca", str(root)], capture_output=True, text=True).stdout
|
||||
assert not hits.strip(), f"residual Alpaca references:\n{hits}"
|
||||
def test_broker_port_ibkr_only():
|
||||
from fxhnt.adapters.broker.ibkr import IBKRBroker # the sole impl still imports
|
||||
assert IBKRBroker is not None
|
||||
```
|
||||
*(Adjust the IBKRBroker import to its real path — grep `class IBKRBroker`.)*
|
||||
|
||||
- [ ] **Step 3: Run → FAIL** (alpaca refs present).
|
||||
|
||||
- [ ] **Step 4: Delete + rewire** — remove `alpaca.py`, `AlpacaSettings` + `alpaca` field, the `--broker`
|
||||
option and its `alpaca` branches (execute-multistrat + the peer command become IBKR-only; keep the Broker
|
||||
port abstraction and `IBKRBroker`). Delete the two manifests. `kubectl delete cronjob/fxhnt-alpaca-rebalancer
|
||||
secret/alpaca-credentials -n foxhunt` + the superseded `multistrat-rebalancer` CronJob (locate its manifest
|
||||
via `grep -rn multistrat-rebalancer infra/`; delete file + in-cluster object) — record these as MANUAL
|
||||
cluster ops in the commit body (not run by CI).
|
||||
|
||||
- [ ] **Step 5: Run guard + cli load** —
|
||||
`.venv/bin/python -m pytest tests/unit/test_no_alpaca.py -q && .venv/bin/python -c "from fxhnt.cli import app; print('cli OK')"` → PASS.
|
||||
|
||||
- [ ] **Step 6: Commit** — `git commit -am "feat(0b): remove dead Alpaca execution; IBKR is the sole equity executor (Broker port kept)"`.
|
||||
|
||||
---
|
||||
|
||||
## Task 6: Remove the Binance execution surface
|
||||
|
||||
**Files:**
|
||||
- Delete: `src/fxhnt/adapters/exchange/binance_ccxt.py`
|
||||
- Modify: `src/fxhnt/cli.py` (delete `crypto-rebalance` (4746), `crypto-positions` (4768), `crypto-flatten`
|
||||
(4776) commands), `src/fxhnt/application/crypto_execution.py` (remove Binance coupling — delete the module
|
||||
if Binance was its only backend; grep first), `src/fxhnt/config.py` (remove execution-only `BinanceSettings`
|
||||
fields)
|
||||
- Test: `tests/unit/test_no_binance_execution.py`
|
||||
|
||||
- [ ] **Step 1: Grep** — `grep -rn "binance_ccxt\|BinanceCcxt\|crypto-rebalance\|crypto-positions\|crypto-flatten\|crypto_execution" src/fxhnt/ | grep -v test`. Determine whether `crypto_execution.py` has any non-Binance use (if not → delete the module).
|
||||
|
||||
- [ ] **Step 2: Guard test** asserting the three CLI commands are gone (`crypto-rebalance` / `-positions` /
|
||||
`-flatten` not in `app`'s registered commands) and `binance_ccxt` unimportable. Run → FAIL.
|
||||
|
||||
- [ ] **Step 3: Delete** `binance_ccxt.py`, the three CLI commands, the Binance coupling in
|
||||
`crypto_execution.py` (or the whole module), and the execution-only `BinanceSettings` fields. Keep any
|
||||
Binance *data* settings for now (Task 6 removes them once the ingests are gone).
|
||||
|
||||
- [ ] **Step 4: Run guard + cli load** → PASS. **Step 5: Commit** —
|
||||
`git commit -am "feat(0b): remove dead Binance execution surface (binance_ccxt + crypto-rebalance/positions/flatten CLI)"`.
|
||||
|
||||
---
|
||||
|
||||
## Task 7: Remove Binance DATA + retire the `features` table (depends on Tasks 1,2,4)
|
||||
|
||||
**Files:**
|
||||
- Delete: `src/fxhnt/adapters/exchange/binance_funding_history.py`,
|
||||
`src/fxhnt/adapters/data/binance_spot_history.py`, `src/fxhnt/adapters/data/binance_xsfunding_live.py`
|
||||
- Modify: `src/fxhnt/adapters/orchestration/assets.py` (delete `crypto_bars` (263), `crypto_funding` (322),
|
||||
`crypto_spot` (345) ingest assets + remove from the Definitions list), `src/fxhnt/application/bybit_edges_eval.py`
|
||||
(remove/repoint its Binance imports), `src/fxhnt/config.py` (remove the now-unused Binance data settings +
|
||||
`warehouse_table="features"` default if `features` is dropped — confirm bybit paths hardcode `bybit_features`)
|
||||
- Test: `tests/unit/test_no_binance_data.py`
|
||||
|
||||
**Interfaces:** none produced. This is the "Binance fully gone" guard. Depends on the crypto migration (Tasks 1,2) + retirement (Task 4).
|
||||
|
||||
- [ ] **Step 1: Prove nothing live reads `features` / Binance** — `grep -rn "BinanceFundingHistory\|BinanceSpotHistory\|BinanceXsFundingLive\|binance_funding_history\|binance_spot_history\|binance_xsfunding_live\|crypto_bars\|crypto_funding\|crypto_spot" src/fxhnt/ | grep -v test`. After Tasks 1–3, the only hits should be the definitions being deleted here + `bybit_edges_eval.py` (repoint/remove). Any live *reader* of the `features` table (`get_feature_store` with the default table on a live path) is a BLOCKER — report it. (The bybit book/tracks all pass `table="bybit_features"` explicitly.)
|
||||
|
||||
- [ ] **Step 2: Guard test**:
|
||||
```python
|
||||
# tests/unit/test_no_binance_data.py
|
||||
import subprocess, pathlib
|
||||
def test_no_binance_symbols_in_src():
|
||||
root = pathlib.Path(__file__).parents[1] / "src" / "fxhnt"
|
||||
hits = subprocess.run(["grep", "-rin", "binance", str(root)], capture_output=True, text=True).stdout
|
||||
assert not hits.strip(), f"residual Binance references:\n{hits}"
|
||||
```
|
||||
- [ ] **Step 3: Run → FAIL.** **Step 4:** Delete the three adapters + three ingest assets (+ Definitions
|
||||
entries) + `bybit_edges_eval.py` Binance imports + the Binance data settings. If the `features` table is
|
||||
now unreferenced, drop its default (`warehouse_table` becomes `bybit_features`, or the setting is removed if
|
||||
every consumer hardcodes the table) — confirm no path relied on the `features` default. Leave a one-time
|
||||
`DROP TABLE features` as a documented MANUAL cluster op in the commit body (not auto-run).
|
||||
|
||||
- [ ] **Step 5: Run guard + full suite** —
|
||||
`.venv/bin/python -m pytest tests/unit/test_no_binance_data.py -q && .venv/bin/python -m pytest tests/ -q` → PASS (full suite green — this is the venue-cutover checkpoint).
|
||||
|
||||
- [ ] **Step 6: Commit** — `git commit -am "feat(0b): remove Binance data (adapters + ingests) + retire the features table — crypto is 100% Bybit"`.
|
||||
|
||||
---
|
||||
|
||||
## Task 8: Remove the vestigial registry `state_file` field
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/registry.py` (delete `state_file` from all entries + the header comment lines 3,5)
|
||||
- Modify: `src/fxhnt/adapters/orchestration/assets.py` (`_run_paper_tracker` (401): drop the `state_file`
|
||||
param + all call sites)
|
||||
- Investigate: `src/fxhnt/cli.py:470` (forward-track CLI `(state_file, build_strategy)` map),
|
||||
`src/fxhnt/adapters/orchestration/migration_builders.py` (one-time migration seed)
|
||||
- Test: `tests/unit/test_no_registry_state_file.py`
|
||||
|
||||
- [ ] **Step 1: Determine migration completeness + CLI dependency.** `grep -rn "state_file" src/fxhnt/ | grep -v test`. Two non-registry consumers: (a) `migration_builders.py` (the ONE-TIME deterministic-forward-state seed — if every track is already on a DB anchor in prod, this is fully vestigial → delete the module); (b) `cli.py:470` forward-track CLI. Read cli.py:470 — if the CLI recomputes from the DB anchor (0a SSOT) and only needs a `ForwardTracker` PATH it constructs itself, the registry `state_file` is not needed there; if it reads `entry["state_file"]`, move that basename into the CLI locally (or derive `f"{sid}_state"`) so removing the registry field doesn't break it. Decide per finding; record it in the commit body.
|
||||
|
||||
- [ ] **Step 2: Guard test**:
|
||||
```python
|
||||
# tests/unit/test_no_registry_state_file.py
|
||||
from fxhnt.registry import STRATEGY_REGISTRY
|
||||
def test_no_state_file_field():
|
||||
assert all("state_file" not in e for e in STRATEGY_REGISTRY.values())
|
||||
```
|
||||
- [ ] **Step 3: Run → FAIL.** **Step 4:** Remove the `state_file` field from every entry + the header
|
||||
comment; drop the `state_file` param from `_run_paper_tracker` and all call sites; apply the Step-1
|
||||
decision for the CLI + `migration_builders.py` (delete the module if the migration is complete, else keep
|
||||
it self-contained). The JSON `ForwardTracker` + its `.bak` crash-recovery stay (used by `fxhnt forward-track`).
|
||||
|
||||
- [ ] **Step 5: Run guard + forward suites** —
|
||||
`.venv/bin/python -m pytest tests/unit/test_no_registry_state_file.py tests/integration/test_forward_gate_e2e.py -q && .venv/bin/python -c "from fxhnt.cli import app; print('cli OK')"` → PASS.
|
||||
|
||||
- [ ] **Step 6: Commit** — `git commit -am "feat(0b): remove vestigial registry state_file field (DB anchor is the forward SSOT)"`.
|
||||
|
||||
---
|
||||
|
||||
## Task 9: Cockpit-local verify + whole-branch review
|
||||
|
||||
**Files:** none (verification task).
|
||||
|
||||
- [ ] **Step 1: Local cockpit verify (real browser).** Seed a sqlite cockpit DB (or use
|
||||
`./scripts/dev-cockpit-local.sh` against prod, read-only) and confirm in a browser: `xsfunding` and
|
||||
`unlock` strategy pages render as **Bybit** tracks (venue `bybit-perp`) with their reconciliation gates;
|
||||
`crypto_tstrend` and `stablecoin_rotation` detail pages are GONE (404, not a 500); the `bybit_4edge`
|
||||
deploy book + `/paper` bybit view are unchanged. Read the actual rendered values — do not claim off logs.
|
||||
|
||||
- [ ] **Step 2: Full suite green** — `.venv/bin/python -m pytest tests/ -q` → all pass. Confirm the 0a
|
||||
reconciliation invariant + the four new guard tests (retired-tracks, no-alpaca, no-binance-data,
|
||||
no-state-file) are included and green.
|
||||
|
||||
- [ ] **Step 3: Whole-branch review** — dispatch the final code-reviewer over
|
||||
`git merge-base feat/unified-db-backtest-architecture HEAD`..HEAD with the spec's Global Constraints as the
|
||||
lens (crypto 100% Bybit; no Binance/Alpaca; Broker port kept; book unchanged; deterministic re-inception;
|
||||
invariant green). Fix Critical/Important in one fix wave; record Minors and fix them in-loop.
|
||||
|
||||
- [ ] **Step 4: Finish** — use superpowers:finishing-a-development-branch (verify tests → present merge/PR
|
||||
options). Note the post-deploy runbook in the PR body: the `migrate-forward-anchors` step for the
|
||||
re-inceptioned xsfunding/unlock tracks + the two MANUAL cluster ops (drop `features` table; delete the
|
||||
Alpaca/multistrat CronJobs + `alpaca-credentials` secret).
|
||||
|
||||
---
|
||||
|
||||
## Self-Review notes
|
||||
- **Spec coverage:** A→xsfunding (T1) + unlock (T2); B→retire (T3); C→Alpaca (T4); D→Binance-exec (T5);
|
||||
E→Binance-data (T6); F→state_file (T7); §7 0a-interaction folded into T1/T2 (ref store flips to
|
||||
bybit_features; xsfunding leaves the inline sleeve path); verification+cockpit (T8). All spec sections map.
|
||||
- **Ordering:** migrate (T1–T2) + retire (T3) BEFORE Binance-data removal (T6) — enforced by T6's Step-1
|
||||
block. Exec removals (T4–T5) are independent and can run any time after T1–T3.
|
||||
- **Type consistency:** `build_bybit_sleeve_forward_nav(repo, store, sleeve, *, …) -> dict` used by T1/T2 +
|
||||
positioning wrapper. `_BYBIT_BOOK_SIDS` extended in T1/T2. `bybit_report_sid_kwargs(sid)` extended in T1/T2.
|
||||
@@ -0,0 +1,680 @@
|
||||
# Unified DB-backed Backtest Architecture (Phase 0a) Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** One registry-driven backtest→ref path so every reconciliation-gated strategy always has a `backtest_summary` (a CI-enforced invariant), fixing the live gate bugs, with Timescale as the one durable store and DuckDB as an ephemeral engine.
|
||||
|
||||
**Architecture:** A `backtest_ref(strategy_id, store) -> BacktestSummary | None` dispatch (on a new `definition.backtest.kind`: `recompute-replay` / `sleeve` / `report` / `none`) replaces the three fragmented mechanisms. Cheap kinds run inline in the nightly assets; heavy `report` kinds run in a dedicated resourced scheduled step (own memory) using an ephemeral DuckDB that ATTACHes Timescale. A registry-iterating invariant test makes a gated-but-ref-less strategy a CI failure.
|
||||
|
||||
**Tech Stack:** Python 3.12, SQLAlchemy 2.0 (Postgres/Timescale SSOT), Dagster, `duckdb` 1.5.3 (postgres extension), pytest.
|
||||
|
||||
## Global Constraints
|
||||
|
||||
- **Branch:** `feat/unified-db-backtest-architecture`. Never commit to `master`.
|
||||
- **Real money errs false-WAIT:** a backtest that fails/returns None → NO ref written → the gate WAITs (never false-PASS). `reconciliation_gate.py` already treats `backtest=None` as WAIT — preserve that.
|
||||
- **The invariant:** every `STRATEGY_REGISTRY` entry with `gate_spec.gate_type == "reconciliation"` MUST declare a non-`none` `definition.backtest.kind` AND have a constructible provider. A registry-iterating test enforces this (CI failure otherwise).
|
||||
- **backtest kinds (verbatim):** `recompute-replay` (build_strategy().advance(None,{}) → rows), `sleeve` (sleeve_returns_from_store(sid, store) → rows), `report` (per-sid dispatch to `bybit_4edge_backtest_summary(store, …)` — the real bybit book engine, NOT the B3-equity `backtest_summaries_from_report`), `none` (absolute-gated benchmark, no ref).
|
||||
- **Single-SSOT / deterministic-forward-state:** Timescale is the one truth; the DuckDB engine is ephemeral (per-run, no file). Cheap recompute refs stay inline in the nightly `*_nav` assets; only heavy `report` refs move to the resourced scheduled step (the 4Gi Dagster daemon must NOT run the heavy backtest — it OOM'd on the bybit sleeve-ret).
|
||||
- **Out of scope:** Binance/Alpaca-exec cleanup (0b), cockpit Ops/Fleet UI (Phase 2), reviving equity-factor research (park B3, don't migrate its data).
|
||||
- Verify the cockpit LOCALLY (xsfunding gate un-sticks). Every review Minor is a latent bug — fix in-loop.
|
||||
|
||||
---
|
||||
|
||||
## File Structure
|
||||
|
||||
| File | Responsibility |
|
||||
|---|---|
|
||||
| `src/fxhnt/application/backtest_refs.py` (new) | `backtest_ref(strategy_id, store) -> BacktestSummary\|None` dispatch + per-strategy provider registry |
|
||||
| `src/fxhnt/registry.py` (modify) | add `definition.backtest = {"kind": ...}` per entry; `xsfunding` → `sleeve` |
|
||||
| `src/fxhnt/adapters/warehouse/timescale_duckdb_engine.py` (new) | `TimescaleDuckDbEngine` — ephemeral DuckDB ATTACH Timescale for heavy `report` backtests |
|
||||
| `src/fxhnt/adapters/orchestration/assets.py` (modify) | route inline recompute/sleeve refs through `backtest_ref`; add xsfunding's ref |
|
||||
| `src/fxhnt/cli.py` (modify) | `backtest-refs --all` command (the scheduled step); redirect `bybit-persist-sleeve-ret` into it |
|
||||
| `src/fxhnt/adapters/orchestration/definitions.py` (modify) | (if used) schedule wiring |
|
||||
| `infra/k8s/jobs/fxhnt-backtest-refs.yaml` (new) | scheduled resourced CronJob (own memory), replacing the manual sleeve-ret Job |
|
||||
| `infra/k8s/jobs/` (retire) | the B3 backtest DuckDB PVC + `fxhnt-bybit-sleeve-ret-job.yaml` + `fxhnt-backtest-ingest/verdict-job.yaml` (park) |
|
||||
| tests | `test_backtest_refs.py`, `test_backtest_invariant.py`, `test_timescale_duckdb_engine.py` |
|
||||
|
||||
---
|
||||
|
||||
## Task 1: The `backtest_ref` dispatch + `backtest.kind` registry field
|
||||
|
||||
**Files:**
|
||||
- Create: `src/fxhnt/application/backtest_refs.py`
|
||||
- Modify: `src/fxhnt/registry.py` (add `definition.backtest.kind` to every entry)
|
||||
- Test: `tests/unit/test_backtest_refs.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `BacktestSummary` (`fxhnt.application.forward_models`), `backtest_summary_from_rows(strategy_id, rows)` (`fxhnt.application.bybit_forward_track`), `STRATEGY_REGISTRY` (`fxhnt.registry`), the `AnalyticalStore` port.
|
||||
- Produces: `backtest_ref(strategy_id: str, store, *, build_strategy=None) -> BacktestSummary | None`; `registry_backtest_kind(strategy_id: str) -> str`.
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
```python
|
||||
# tests/unit/test_backtest_refs.py
|
||||
from fxhnt.application.backtest_refs import backtest_ref, registry_backtest_kind
|
||||
|
||||
|
||||
class _ReplayStrat:
|
||||
def advance(self, last, extra):
|
||||
# a 3-day inception series -> a real BacktestSummary
|
||||
return [("2024-01-01", 0.01), ("2024-01-02", -0.005), ("2024-01-03", 0.008)], {}
|
||||
|
||||
|
||||
def test_recompute_replay_produces_summary():
|
||||
sm = backtest_ref("multistrat", store=None, build_strategy=lambda: _ReplayStrat())
|
||||
assert sm is not None
|
||||
assert sm.strategy_id == "multistrat"
|
||||
assert sm.sharpe == sm.sharpe # finite (not NaN)
|
||||
|
||||
|
||||
def test_none_kind_returns_none():
|
||||
# sixtyforty is absolute-gated -> backtest.kind == "none" -> no ref
|
||||
assert registry_backtest_kind("sixtyforty") == "none"
|
||||
assert backtest_ref("sixtyforty", store=None) is None
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run to verify failure**
|
||||
|
||||
Run: `.venv/bin/python -m pytest tests/unit/test_backtest_refs.py -q`
|
||||
Expected: FAIL (`ModuleNotFoundError: fxhnt.application.backtest_refs`).
|
||||
|
||||
- [ ] **Step 3: Add `definition.backtest.kind` to every registry entry** (`src/fxhnt/registry.py`). For each entry add a `"backtest": {"kind": "..."}` inside `definition`, using:
|
||||
- `sixtyforty` → `"none"` (absolute gate)
|
||||
- `multistrat`, `vrp`, `crypto_tstrend`, `stablecoin_rotation`, `unlock` → `"recompute-replay"`
|
||||
- `xsfunding` → `"sleeve"`
|
||||
- `bybit_4edge`, `bybit_4edge_levered`, `positioning` → `"report"`
|
||||
- the `*_exec` twins (`multistrat_exec`, `vrp_exec`, `bybit_4edge_exec`) → `"none"` (observed twins reconcile against the MODELED book's ref, they don't produce their own)
|
||||
|
||||
- [ ] **Step 4: Implement the dispatch** (`src/fxhnt/application/backtest_refs.py`)
|
||||
|
||||
```python
|
||||
"""The ONE backtest->ref path. Every reconciliation-gated strategy produces its `backtest_summary` here,
|
||||
dispatched on `STRATEGY_REGISTRY[sid]["definition"]["backtest"]["kind"]`. Replaces the three fragmented
|
||||
mechanisms (inline _persist_track_backtest_ref, the bybit-persist-sleeve-ret Job, the B3 equity runner).
|
||||
A missing/failed ref is best-effort (returns None) -> the gate WAITs (false-WAIT-safe)."""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from fxhnt.application.bybit_forward_track import backtest_summary_from_rows
|
||||
from fxhnt.application.forward_models import BacktestSummary
|
||||
from fxhnt.registry import STRATEGY_REGISTRY
|
||||
|
||||
|
||||
def registry_backtest_kind(strategy_id: str) -> str:
|
||||
return STRATEGY_REGISTRY[strategy_id]["definition"].get("backtest", {}).get("kind", "none")
|
||||
|
||||
|
||||
def backtest_ref(strategy_id: str, store: Any, *, build_strategy: Any = None) -> BacktestSummary | None:
|
||||
"""Produce the reconciliation-gate backtest reference for `strategy_id`, or None (gate WAITs).
|
||||
`build_strategy` (a zero-arg callable returning a ForwardStrategy) is required for `recompute-replay`.
|
||||
`store` (an AnalyticalStore) is required for `sleeve`/`report`."""
|
||||
kind = registry_backtest_kind(strategy_id)
|
||||
if kind == "none":
|
||||
return None
|
||||
if kind == "recompute-replay":
|
||||
if build_strategy is None:
|
||||
return None
|
||||
rows, _ = build_strategy().advance(None, {})
|
||||
return backtest_summary_from_rows(strategy_id, rows) if len(rows) >= 2 else None
|
||||
if kind == "sleeve":
|
||||
from fxhnt.application.bybit_book_eval import sleeve_returns_from_store
|
||||
rows = sleeve_returns_from_store(strategy_id, store) # [(iso_date, ret), ...]
|
||||
return backtest_summary_from_rows(strategy_id, rows) if len(rows) >= 2 else None
|
||||
if kind == "report":
|
||||
from fxhnt.application.backtest_refs_report import report_backtest_summary
|
||||
return report_backtest_summary(strategy_id, store) # built in Task 5
|
||||
raise ValueError(f"unknown backtest kind {kind!r} for {strategy_id}")
|
||||
```
|
||||
|
||||
*(Note for the implementer: read `sleeve_returns_from_store`'s real signature in `src/fxhnt/application/bybit_book_eval.py` and adapt the call/return-shape exactly; it is the same engine `funding_dispersion_eval.py` already reuses for xsfunding's return series. If it returns a bare return array rather than `(iso,ret)` rows, wrap it into the `(iso_date, ret)` shape `backtest_summary_from_rows` expects.)*
|
||||
|
||||
- [ ] **Step 5: Run tests to verify pass**
|
||||
|
||||
Run: `.venv/bin/python -m pytest tests/unit/test_backtest_refs.py -q`
|
||||
Expected: PASS (2 passed). (`test_none_kind_returns_none` requires the `sixtyforty` registry `backtest.kind="none"` from Step 3.)
|
||||
|
||||
- [ ] **Step 6: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/application/backtest_refs.py src/fxhnt/registry.py tests/unit/test_backtest_refs.py
|
||||
git commit -m "feat(backtest): registry-driven backtest_ref dispatch + backtest.kind field"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 2: Fix xsfunding (bug A) — its `sleeve` provider yields a real ref
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/application/backtest_refs.py` (confirm the `sleeve` path works for `xsfunding`)
|
||||
- Test: `tests/unit/test_backtest_refs.py` (extend)
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `sleeve_returns_from_store("xsfunding", store)` (`fxhnt.application.bybit_book_eval`), the `AnalyticalStore` port.
|
||||
|
||||
- [ ] **Step 1: Write the failing test** (a fake store yielding a funding panel → xsfunding sleeve returns → a ref)
|
||||
|
||||
```python
|
||||
# tests/unit/test_backtest_refs.py (append)
|
||||
def test_xsfunding_sleeve_produces_ref_bug_A(monkeypatch):
|
||||
"""Regression for the live bug: xsfunding is reconciliation-gated but had NO backtest ref (gate stuck at
|
||||
WAIT forever). Its `sleeve` provider must yield a BacktestSummary from the funding panel."""
|
||||
import fxhnt.application.backtest_refs as mod
|
||||
monkeypatch.setattr(mod, "registry_backtest_kind", lambda sid: "sleeve")
|
||||
# stub the sleeve engine to a deterministic 3-day return series
|
||||
import fxhnt.application.bybit_book_eval as bbe
|
||||
monkeypatch.setattr(bbe, "sleeve_returns_from_store",
|
||||
lambda sid, store: [("2024-03-01", 0.002), ("2024-03-02", 0.0015), ("2024-03-03", 0.003)])
|
||||
sm = mod.backtest_ref("xsfunding", store=object())
|
||||
assert sm is not None and sm.strategy_id == "xsfunding"
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run to verify fail/pass**
|
||||
|
||||
Run: `.venv/bin/python -m pytest tests/unit/test_backtest_refs.py::test_xsfunding_sleeve_produces_ref_bug_A -v`
|
||||
Expected: PASS if Task 1's `sleeve` branch is correct; if it FAILS on the `sleeve_returns_from_store` call/return shape, fix the branch (Step 3) to match the real engine.
|
||||
|
||||
- [ ] **Step 3: Reconcile the `sleeve` branch with the real engine.** Read `src/fxhnt/application/bybit_book_eval.py::sleeve_returns_from_store` and `src/fxhnt/application/funding_dispersion_eval.py` (which already calls it for xsfunding). Ensure the `sleeve` branch in `backtest_refs.py` passes the store the engine expects (a `_UniverseRestrictedStore`-compatible `AnalyticalStore`) and converts its output into the `(iso_date, ret)` rows `backtest_summary_from_rows` consumes. Do NOT reimplement the return engine — reuse it.
|
||||
|
||||
- [ ] **Step 4: Run tests to verify pass**
|
||||
|
||||
Run: `.venv/bin/python -m pytest tests/unit/test_backtest_refs.py -q`
|
||||
Expected: PASS (3 passed).
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/application/backtest_refs.py tests/unit/test_backtest_refs.py
|
||||
git commit -m "fix(backtest): xsfunding sleeve backtest ref (bug A — reconciliation gate had no ref)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 3: The invariant test (a gated-but-ref-less strategy = CI failure)
|
||||
|
||||
**Files:**
|
||||
- Test: `tests/unit/test_backtest_invariant.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `STRATEGY_REGISTRY`, `registry_backtest_kind`, `track_builders` (`fxhnt.adapters.orchestration.migration_builders`).
|
||||
|
||||
- [ ] **Step 1: Write the invariant test**
|
||||
|
||||
```python
|
||||
# tests/unit/test_backtest_invariant.py
|
||||
from fxhnt.application.backtest_refs import registry_backtest_kind
|
||||
from fxhnt.registry import STRATEGY_REGISTRY
|
||||
|
||||
|
||||
def test_every_reconciliation_gated_strategy_has_a_backtest_provider():
|
||||
"""THE invariant: a reconciliation-gated strategy without a non-`none` backtest kind can never gate-PASS
|
||||
(its gate WAITs forever on a missing ref). Making that a CI failure is what turns a silent live gap into
|
||||
a caught error. If you add a reconciliation-gated strategy, declare its `definition.backtest.kind`."""
|
||||
offenders = []
|
||||
for sid, entry in STRATEGY_REGISTRY.items():
|
||||
if entry.get("gate_spec", {}).get("gate_type") == "reconciliation":
|
||||
if registry_backtest_kind(sid) == "none":
|
||||
offenders.append(sid)
|
||||
assert not offenders, f"reconciliation-gated strategies with no backtest ref provider: {offenders}"
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run — it should PASS** (after Tasks 1-2 declared kinds for every reconciliation-gated strategy incl. xsfunding).
|
||||
|
||||
Run: `.venv/bin/python -m pytest tests/unit/test_backtest_invariant.py -v`
|
||||
Expected: PASS. If it FAILS, the failure names the offending strategy id(s) — go add their `backtest.kind` to the registry (that is the invariant doing its job).
|
||||
|
||||
- [ ] **Step 3: Commit**
|
||||
|
||||
```bash
|
||||
git add tests/unit/test_backtest_invariant.py
|
||||
git commit -m "test(backtest): invariant — every reconciliation-gated strategy must declare a backtest provider"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 4: `TimescaleDuckDbEngine` — ephemeral DuckDB ATTACH Timescale
|
||||
|
||||
**Files:**
|
||||
- Create: `src/fxhnt/adapters/warehouse/timescale_duckdb_engine.py`
|
||||
- Test: `tests/unit/test_timescale_duckdb_engine.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: `TimescaleDuckDbEngine(dsn: str)` with `scan(sql: str) -> list[tuple]` (run a columnar query over ATTACHed Timescale) and a context-manager `__enter__/__exit__` that connects + `INSTALL/LOAD postgres` + `ATTACH`.
|
||||
|
||||
- [ ] **Step 1: Write the failing test** (against a local sqlite-or-duckdb-attachable fixture — the ATTACH-to-postgres path is integration-only, so the unit test exercises the SQL-scan shape on an in-memory table)
|
||||
|
||||
```python
|
||||
# tests/unit/test_timescale_duckdb_engine.py
|
||||
from fxhnt.adapters.warehouse.timescale_duckdb_engine import TimescaleDuckDbEngine
|
||||
|
||||
|
||||
def test_engine_scans_an_in_memory_table():
|
||||
eng = TimescaleDuckDbEngine("postgresql://unused") # dsn unused in the in-memory scan path
|
||||
with eng.in_memory() as con: # a test seam: an in-memory DuckDB, no ATTACH
|
||||
con.execute("CREATE TABLE t AS SELECT * FROM (VALUES (1, 0.5), (2, 0.7)) AS v(sym, ret)")
|
||||
rows = con.execute("SELECT sym, ret FROM t ORDER BY sym").fetchall()
|
||||
assert rows == [(1, 0.5), (2, 0.7)]
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run to verify failure**
|
||||
|
||||
Run: `.venv/bin/python -m pytest tests/unit/test_timescale_duckdb_engine.py -q`
|
||||
Expected: FAIL (`ModuleNotFoundError`).
|
||||
|
||||
- [ ] **Step 3: Implement the engine**
|
||||
|
||||
```python
|
||||
"""Ephemeral columnar engine for HEAVY `report` backtests: an in-memory DuckDB that ATTACHes the live
|
||||
Timescale (Postgres) so a backtest can scan/aggregate the SSOT columnar-fast WITHOUT a persistent DuckDB
|
||||
file. Per-run, discarded after — Timescale stays the one durable store. (Verified: duckdb 1.5.3 `INSTALL
|
||||
postgres; LOAD postgres; ATTACH` works.)"""
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
|
||||
|
||||
class TimescaleDuckDbEngine:
|
||||
def __init__(self, dsn: str) -> None:
|
||||
# DuckDB's postgres extension wants a libpq DSN (postgresql://...), not SQLAlchemy's +psycopg form.
|
||||
self._dsn = dsn.replace("postgresql+psycopg://", "postgresql://")
|
||||
|
||||
@contextlib.contextmanager
|
||||
def in_memory(self): # type: ignore[no-untyped-def]
|
||||
"""A bare in-memory DuckDB connection (no ATTACH) — the unit-test seam + the compute scratch space."""
|
||||
import duckdb
|
||||
con = duckdb.connect()
|
||||
try:
|
||||
yield con
|
||||
finally:
|
||||
con.close()
|
||||
|
||||
@contextlib.contextmanager
|
||||
def attached(self): # type: ignore[no-untyped-def]
|
||||
"""In-memory DuckDB with the live Timescale ATTACHed read-only as schema `ts`. Heavy backtests scan
|
||||
`ts.<table>` columnar-fast, then write results back to Timescale via the normal repo (NOT via DuckDB)."""
|
||||
import duckdb
|
||||
con = duckdb.connect()
|
||||
try:
|
||||
con.execute("INSTALL postgres; LOAD postgres;")
|
||||
con.execute(f"ATTACH '{self._dsn}' AS ts (TYPE postgres, READ_ONLY);")
|
||||
yield con
|
||||
finally:
|
||||
con.close()
|
||||
|
||||
def scan(self, sql: str) -> list[tuple]: # type: ignore[type-arg]
|
||||
with self.attached() as con:
|
||||
return con.execute(sql).fetchall()
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run tests to verify pass**
|
||||
|
||||
Run: `.venv/bin/python -m pytest tests/unit/test_timescale_duckdb_engine.py -q`
|
||||
Expected: PASS (1 passed).
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/adapters/warehouse/timescale_duckdb_engine.py tests/unit/test_timescale_duckdb_engine.py
|
||||
git commit -m "feat(backtest): TimescaleDuckDbEngine — ephemeral DuckDB ATTACH Timescale (no file)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 5: The `report` provider (bybit book) + `backtest-refs --all` CLI
|
||||
|
||||
**Files:**
|
||||
- Create: `src/fxhnt/application/backtest_refs_report.py`
|
||||
- Modify: `src/fxhnt/cli.py` (add `backtest-refs`; redirect `bybit-persist-sleeve-ret` to it)
|
||||
- Test: `tests/unit/test_backtest_refs_report.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `bybit_4edge_backtest_summary(store, *, universe, cost_bps, unlock_events, leverage_shape, strategy_id, sleeves, positioning_capacity_capital, positioning_tail_cap_k)` + `_LEVERED_SLEEVE_SHAPE` + `POSITIONING_TAIL_CAP_K` (`fxhnt.application.bybit_forward_track`); `liquid_universe(store, min_dollar_vol)` (`fxhnt.application.bybit_liquidity`); `load_unlock_events(operational_unlock_repo(settings), path)` + `operational_unlock_repo` (`fxhnt.application.unlock_calendar_loader`); `_data_dir` (`fxhnt.adapters.orchestration.assets`); `get_settings`; `ForwardNavRepo.upsert_backtest_summary`. **The real reuse target is the existing bybit book engine `bybit_4edge_backtest_summary` (cli.py:3135-3157 `bybit-persist-sleeve-ret` calls it 3×, once per report-kind sid) — NOT `backtest_summaries_from_report`/`evaluate_constructions`, which are the unrelated B3 equity-factor path whose `_CONSTRUCTION_TO_SID` maps only `eqfactor_*` sids and would silently return `[]` for bybit sids (the exact false-WAIT this task prevents).** Task 4's `TimescaleDuckDbEngine` is NOT consumed here — the bybit book reads via the FeatureStore (`store`), not DuckDB.
|
||||
- Produces: `report_backtest_summary(strategy_id, store) -> BacktestSummary | None`; CLI `backtest-refs --all`.
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
Prove the provider DISPATCHES to the real bybit engine with the correct per-sid kwargs (stub the
|
||||
engine so the test is fast and does not touch a store). The engine's math is already covered by the
|
||||
bybit forward-track tests — this test guards the report→ref dispatch, not the return-series math.
|
||||
|
||||
```python
|
||||
# tests/unit/test_backtest_refs_report.py
|
||||
from fxhnt.application.forward_models import BacktestSummary
|
||||
|
||||
|
||||
def _fake_summary(strategy_id="bybit_4edge"):
|
||||
return BacktestSummary(strategy_id=strategy_id, as_of="2024-06-01", cagr=1.2, ann_vol=0.4,
|
||||
sharpe=1.8, max_drawdown=-0.2, passed=True, dsr=0.9, is_sharpe=1.5,
|
||||
oos_sharpe=1.4, pvalue=0.01)
|
||||
|
||||
|
||||
def test_report_provider_dispatches_levered_kwargs(monkeypatch):
|
||||
import fxhnt.application.backtest_refs_report as mod
|
||||
calls = []
|
||||
|
||||
def fake_engine(store, **kw):
|
||||
calls.append(kw)
|
||||
return _fake_summary(kw.get("strategy_id", "bybit_4edge"))
|
||||
|
||||
monkeypatch.setattr(mod, "bybit_4edge_backtest_summary", fake_engine)
|
||||
monkeypatch.setattr(mod, "liquid_universe", lambda store, **kw: {"BTC", "ETH"})
|
||||
monkeypatch.setattr(mod, "load_unlock_events", lambda *a, **k: [])
|
||||
monkeypatch.setattr(mod, "_data_dir", lambda: "/tmp")
|
||||
|
||||
sm = mod.report_backtest_summary("bybit_4edge_levered", store=object())
|
||||
assert sm is not None and sm.strategy_id == "bybit_4edge_levered"
|
||||
assert calls[-1]["leverage_shape"] == mod._LEVERED_SLEEVE_SHAPE
|
||||
assert calls[-1]["strategy_id"] == "bybit_4edge_levered"
|
||||
|
||||
|
||||
def test_report_provider_positioning_single_sleeve(monkeypatch):
|
||||
import fxhnt.application.backtest_refs_report as mod
|
||||
calls = []
|
||||
monkeypatch.setattr(mod, "bybit_4edge_backtest_summary",
|
||||
lambda store, **kw: (calls.append(kw), _fake_summary(kw.get("strategy_id")))[1])
|
||||
monkeypatch.setattr(mod, "liquid_universe", lambda store, **kw: {"BTC", "ETH"})
|
||||
monkeypatch.setattr(mod, "load_unlock_events", lambda *a, **k: [])
|
||||
monkeypatch.setattr(mod, "_data_dir", lambda: "/tmp")
|
||||
|
||||
sm = mod.report_backtest_summary("positioning", store=object())
|
||||
assert sm.strategy_id == "positioning"
|
||||
assert calls[-1]["sleeves"] == ["positioning"]
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run to verify failure**
|
||||
|
||||
Run: `.venv/bin/python -m pytest tests/unit/test_backtest_refs_report.py -q`
|
||||
Expected: FAIL (`ModuleNotFoundError`).
|
||||
|
||||
- [ ] **Step 3: Implement `report_backtest_summary`.** This is a per-sid dispatcher over the EXISTING bybit book engine `bybit_4edge_backtest_summary` — the SAME three calls the `bybit-persist-sleeve-ret` command makes at `cli.py:3135-3157`. Do NOT reimplement any return-series math and do NOT touch `backtest_summaries_from_report`. Import the patchable names at module level (`bybit_4edge_backtest_summary`, `_LEVERED_SLEEVE_SHAPE`, `POSITIONING_TAIL_CAP_K` from `bybit_forward_track`; `liquid_universe` from `bybit_liquidity`; `load_unlock_events`, `operational_unlock_repo` from `unlock_calendar_loader`; `_data_dir` from `adapters.orchestration.assets`) so the test can monkeypatch them.
|
||||
|
||||
```python
|
||||
# src/fxhnt/application/backtest_refs_report.py
|
||||
import logging
|
||||
|
||||
from fxhnt.adapters.orchestration.assets import _data_dir
|
||||
from fxhnt.application.bybit_forward_track import (
|
||||
POSITIONING_TAIL_CAP_K, _LEVERED_SLEEVE_SHAPE, bybit_4edge_backtest_summary)
|
||||
from fxhnt.application.bybit_liquidity import liquid_universe
|
||||
from fxhnt.application.unlock_calendar_loader import load_unlock_events, operational_unlock_repo
|
||||
from fxhnt.config import get_settings
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# The three report-kind sids and how they differ (mirrors cli.py:3135-3157 exactly).
|
||||
_REPORT_SIDS = ("bybit_4edge", "bybit_4edge_levered", "positioning")
|
||||
|
||||
|
||||
def report_backtest_summary(strategy_id, store): # -> BacktestSummary | None
|
||||
"""Backtest ref for a `report`-kind bybit strategy: dispatch to the real book engine
|
||||
`bybit_4edge_backtest_summary` with the per-sid kwargs. Reconstructs the liquid universe, unlock
|
||||
calendar, and positioning capturability haircuts internally (from the store + settings) so the ref
|
||||
reconciles against the SAME capturable book the forward track records. cost_bps=5.5 matches the live
|
||||
BybitFourEdge basis and the `sleeve` kind. Built from the store (sleeve_returns=None) so the
|
||||
positioning haircut kwargs are applied by the engine."""
|
||||
if strategy_id not in _REPORT_SIDS:
|
||||
raise ValueError(f"report_backtest_summary: {strategy_id!r} is not a report-kind sid {_REPORT_SIDS}")
|
||||
settings = get_settings()
|
||||
universe = liquid_universe(store, min_dollar_vol=10_000_000.0) or None
|
||||
try:
|
||||
unlock_events = load_unlock_events(
|
||||
operational_unlock_repo(settings), f"{_data_dir()}/unlock_calendar.json")
|
||||
except Exception as exc: # noqa: BLE001 — absent calendar → unlock sleeve omitted, never fabricated
|
||||
log.warning("report_backtest_summary(%s): could not load unlock calendar: %s", strategy_id, exc)
|
||||
unlock_events = []
|
||||
common = dict(
|
||||
universe=universe, cost_bps=5.5,
|
||||
positioning_capacity_capital=settings.paper_capital,
|
||||
positioning_tail_cap_k=POSITIONING_TAIL_CAP_K)
|
||||
if strategy_id == "bybit_4edge":
|
||||
return bybit_4edge_backtest_summary(store, unlock_events=unlock_events, **common)
|
||||
if strategy_id == "bybit_4edge_levered":
|
||||
return bybit_4edge_backtest_summary(
|
||||
store, unlock_events=unlock_events, leverage_shape=_LEVERED_SLEEVE_SHAPE,
|
||||
strategy_id="bybit_4edge_levered", **common)
|
||||
# positioning: single sleeve, no unlock_events, no leverage (mirrors cli.py:3155-3157).
|
||||
return bybit_4edge_backtest_summary(
|
||||
store, sleeves=["positioning"], strategy_id="positioning", **common)
|
||||
```
|
||||
|
||||
Note the recompute cost: unlike `bybit-persist-sleeve-ret` (which precomputes `sleeve_returns` once and reuses it for all three), this rebuilds the book per sid. That is intentional — it keeps the `backtest_ref(sid, store)` contract stateless per-sid, and the constraint that made this an own-memory Job is PEAK MEMORY (a single 4-sleeve walk-forward), which sequential per-sid recompute does not raise; only wall-clock rises (~3×), acceptable for the weekly scheduled step (Task 6).
|
||||
|
||||
- [ ] **Step 4: Add the `backtest-refs --all` CLI command** (`cli.py`): iterate `STRATEGY_REGISTRY`; for each strategy with `registry_backtest_kind(sid) in ("report",)` (the HEAVY ones — the cheap recompute/sleeve refs run inline in the nightly assets, Task 6), call `backtest_ref(sid, store)` and `upsert_backtest_summary`. Log per-strategy; a failure logs + continues (never aborts). This is the scheduled step's entrypoint.
|
||||
|
||||
- [ ] **Step 5: Run tests + confirm the CLI loads**
|
||||
|
||||
Run: `.venv/bin/python -m pytest tests/unit/test_backtest_refs_report.py -q && .venv/bin/python -c "from fxhnt.cli import app; print('cli OK')"`
|
||||
Expected: PASS + `cli OK`.
|
||||
|
||||
- [ ] **Step 6: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/application/backtest_refs_report.py src/fxhnt/cli.py tests/unit/test_backtest_refs_report.py
|
||||
git commit -m "feat(backtest): report provider (bybit book) + backtest-refs --all CLI"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 6: Route the nightly assets through `backtest_ref`; schedule the resourced step
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/adapters/orchestration/assets.py` (`_persist_track_backtest_ref` → `backtest_ref`; add xsfunding's ref)
|
||||
- Create: `infra/k8s/jobs/fxhnt-backtest-refs.yaml` (scheduled resourced CronJob)
|
||||
- Retire: `infra/k8s/jobs/fxhnt-bybit-sleeve-ret-job.yaml` (its logic now in the CronJob)
|
||||
- Test: `tests/integration/test_backtest_refs_wiring.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `backtest_ref` (Task 1), `registry_backtest_kind`, `track_builders` (for the recompute-replay `build_strategy`), `get_feature_store` (Timescale store), `ForwardNavRepo.upsert_backtest_summary`.
|
||||
|
||||
- [ ] **Step 1: Write the failing wiring test** (the inline path writes a ref for a recompute-replay strategy AND for xsfunding via `backtest_ref`, into a sqlite repo)
|
||||
|
||||
```python
|
||||
# tests/integration/test_backtest_refs_wiring.py
|
||||
import datetime as dt
|
||||
from fxhnt.adapters.persistence.forward_nav import ForwardNavRepo
|
||||
from fxhnt.application.backtest_refs import backtest_ref
|
||||
|
||||
|
||||
class _Replay:
|
||||
def advance(self, last, extra):
|
||||
return [("2024-01-0%d" % i, 0.01) for i in range(1, 6)], {}
|
||||
|
||||
|
||||
def test_backtest_ref_upserts_for_recompute_replay(tmp_path):
|
||||
repo = ForwardNavRepo(f"sqlite:///{tmp_path/'c.db'}")
|
||||
repo.migrate()
|
||||
sm = backtest_ref("multistrat", store=None, build_strategy=lambda: _Replay())
|
||||
assert sm is not None
|
||||
repo.upsert_backtest_summary(sm, at=dt.datetime(2024, 1, 6))
|
||||
# read it back via the SAME reader the gate provider uses (forward_ingest.py:48 → all_backtest_summaries())
|
||||
assert any(b.strategy_id == "multistrat" for b in repo.all_backtest_summaries())
|
||||
```
|
||||
|
||||
*(The reconciliation-gate ref provider reads refs via `ForwardNavRepo.all_backtest_summaries() -> list[BacktestSummaryRow]` — there is no per-sid `backtest_summary(sid)` reader; use `all_backtest_summaries()` and filter by `strategy_id`.)*
|
||||
|
||||
- [ ] **Step 2: Run to verify it passes** (backtest_ref + upsert already exist from Task 1).
|
||||
|
||||
Run: `.venv/bin/python -m pytest tests/integration/test_backtest_refs_wiring.py -q`
|
||||
Expected: PASS.
|
||||
|
||||
- [ ] **Step 3: Refactor `_persist_track_backtest_ref` (`assets.py:363`) to delegate to `backtest_ref`.** Keep its best-effort try/except + logging (never fail the forward step). Change its body so it: resolves the strategy's `build_strategy` (from the migration builders for recompute-replay) or the store (for sleeve), calls `backtest_ref(sid, store, build_strategy=...)`, and upserts the result if non-None. Keep the existing `persist_backtest_ref=True` call sites (multistrat/vrp/crypto_tstrend/stablecoin/unlock) unchanged in behaviour.
|
||||
|
||||
- [ ] **Step 4: Add xsfunding's inline ref.** In `xsfunding_nav` (`assets.py`), after the tracker runs, call the inline ref persist for `xsfunding` (kind `sleeve`) via the same `_persist_track_backtest_ref`/`backtest_ref` path, using `get_feature_store(s)` as the store. This is the asset-level half of the bug-A fix (Task 2 built the provider; this wires it into the nightly run). Confirm `xsfunding` now writes a `backtest_summary` row on a nightly run.
|
||||
|
||||
- [ ] **Step 5: Create the scheduled CronJob** `infra/k8s/jobs/fxhnt-backtest-refs.yaml`: copy the resourcing + labels (`app.kubernetes.io/part-of: foxhunt` for gitea git-sync + base egress) + git-sync-from-master pattern from `fxhnt-bybit-sleeve-ret-job.yaml`, but as a **CronJob** (weekly, e.g. `schedule: "0 2 * * 1"`), OWN memory (match the sleeve-ret Job's memory, e.g. 4-6Gi), command `fxhnt backtest-refs --all`. This replaces the manual `fxhnt-bybit-sleeve-ret-job.yaml` (delete that file). **Fixes bug B** (the deploy-gate ref becomes a scheduled guarantee).
|
||||
|
||||
- [ ] **Step 6: Run tests + yaml validity**
|
||||
|
||||
Run: `.venv/bin/python -m pytest tests/integration/test_backtest_refs_wiring.py -q && .venv/bin/python -c "import yaml; list(yaml.safe_load_all(open('infra/k8s/jobs/fxhnt-backtest-refs.yaml'))); print('yaml OK')"`
|
||||
Expected: PASS + `yaml OK` (confirm it's a CronJob with a schedule + own memory + `part-of: foxhunt` label).
|
||||
|
||||
- [ ] **Step 7: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/adapters/orchestration/assets.py infra/k8s/jobs/fxhnt-backtest-refs.yaml tests/integration/test_backtest_refs_wiring.py
|
||||
git rm infra/k8s/jobs/fxhnt-bybit-sleeve-ret-job.yaml
|
||||
git commit -m "feat(backtest): route nightly refs through backtest_ref + scheduled resourced backtest-refs CronJob (bug B)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 7: Park B3 + carry file-stores; verify the paper_book DuckDB touch
|
||||
|
||||
**Files:**
|
||||
- Retire: `infra/k8s/jobs/fxhnt-backtest-ingest-job.yaml`, `fxhnt-backtest-verdict-job.yaml` (+ the B3 DuckDB PVC manifest if separate)
|
||||
- Investigate/Modify: `src/fxhnt/application/paper_book.py`, `paper_sleeves.py`, `paper_backfill.py`
|
||||
- Test: `tests/unit/test_no_live_duckdb_files.py`
|
||||
|
||||
- [ ] **Step 1: VERIFY the paper_book DuckDB touch.** Read `src/fxhnt/application/paper_book.py`, `paper_sleeves.py`, `paper_backfill.py` for their `DuckDbFeatureStore` / `.duckdb` / `read_parquet` usage. Determine: does the LIVE cockpit/paper path (prod, `feature_store="postgres"`) actually read from a DuckDB file, or is the import a dev/test/fallback only? Document the finding in the commit message.
|
||||
- If **live-adjacent** (the running paper book reads a DuckDB file): migrate that read to the `AnalyticalStore` port so it uses `get_feature_store()` (Timescale) — matching the nav assets.
|
||||
- If **dev/test-only**: leave it (the `DuckDbFeatureStore` stays as the opt-in test backend behind the port).
|
||||
|
||||
- [ ] **Step 2: Add a guard test** that the live path never silently falls back to a DuckDB file. Assert `get_feature_store(settings)` with the default settings returns a `TimescaleFeatureStore` (postgres), so a Job that forgets `FXHNT_FEATURE_STORE` can't write to a throwaway DuckDB file (the existing safety comment in `config.py:158`).
|
||||
|
||||
```python
|
||||
# tests/unit/test_no_live_duckdb_files.py
|
||||
def test_default_feature_store_is_timescale_not_duckdb():
|
||||
from fxhnt.adapters.warehouse import get_feature_store
|
||||
from fxhnt.adapters.warehouse.timescale_feature_store import TimescaleFeatureStore
|
||||
from fxhnt.config import Settings
|
||||
s = Settings() # defaults; feature_store should default to "postgres"
|
||||
assert s.feature_store == "postgres"
|
||||
# get_feature_store with a postgres DSN returns the Timescale store (not DuckDbFeatureStore)
|
||||
s.operational_dsn = "postgresql+psycopg://unused@localhost/x"
|
||||
assert isinstance(get_feature_store(s), TimescaleFeatureStore)
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Retire the B3 backtest Jobs/PVC.** `git rm` `infra/k8s/jobs/fxhnt-backtest-ingest-job.yaml` and `fxhnt-backtest-verdict-job.yaml` (the orphaned B3 platform — code stays in the tree, revivable; only its Jobs/PVC are decommissioned from the running system). Leave `equity_backtest_runner.py` + `worst_basis*` code in place (parked research). Add a one-line header comment to `equity_backtest_runner.py` noting it's PARKED (no live consumer; plugs into `backtest_refs` when an equity strat is added).
|
||||
|
||||
- [ ] **Step 4: Run tests to verify pass**
|
||||
|
||||
Run: `.venv/bin/python -m pytest tests/unit/test_no_live_duckdb_files.py -q`
|
||||
Expected: PASS.
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add tests/unit/test_no_live_duckdb_files.py src/fxhnt/application/equity_backtest_runner.py
|
||||
git rm infra/k8s/jobs/fxhnt-backtest-ingest-job.yaml infra/k8s/jobs/fxhnt-backtest-verdict-job.yaml
|
||||
# + any paper_book migration edits from Step 1
|
||||
git commit -m "chore(backtest): park B3 equity + carry file-stores; guard live feature store = Timescale"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 8: The xsfunding gate un-sticks — end-to-end at the real gate seam
|
||||
|
||||
**Files:**
|
||||
- Test: `tests/integration/test_xsfunding_gate_unsticks.py`
|
||||
|
||||
**Why not a cockpit-render test:** the strategy detail page's "Backtest reference" section is driven by
|
||||
`_backtest_reference` → the bybit **sleeve sim** (`bybit_sleeve_ret`), NOT the `backtest_summary` gate ref —
|
||||
so seeding a `BacktestSummaryRow` and grepping the rendered HTML for "no backtest reference" tests the wrong
|
||||
surface (near-vacuous). Bug A lives in the **reconciliation gate**: `evaluate_reconciliation_gate` returns a
|
||||
specific permanent WAIT `"no backtest reference to reconcile against — PASS withheld"` when its
|
||||
`backtest is None`. The gate's ref comes from `CockpitBacktestRefProvider(repo).reference(sid, days)`, which
|
||||
reads `backtest_summary` rows. So the faithful proof drives that exact production seam: ref present → the gate
|
||||
reconciles (leaves the no-ref branch); ref absent → it IS the no-ref WAIT (the pre-fix state).
|
||||
|
||||
- [ ] **Step 1: Write the two tests** (network-free, sqlite; mirrors `tests/integration/test_forward_gate_e2e.py`).
|
||||
|
||||
```python
|
||||
# tests/integration/test_xsfunding_gate_unsticks.py
|
||||
"""Bug A end-to-end: xsfunding is reconciliation-gated but had NO backtest_summary ref, so the gate returned
|
||||
the permanent 'no backtest reference to reconcile against' WAIT forever. Tasks 2/6 wired the ref. This proves
|
||||
the un-sticking at the REAL seam the nightly ingest uses: CockpitBacktestRefProvider (reads backtest_summary
|
||||
rows) -> evaluate_reconciliation_gate. Ref present => the gate reconciles; ref absent => the no-ref WAIT."""
|
||||
import datetime as dt
|
||||
|
||||
from fxhnt.adapters.persistence.forward_nav import ForwardNavRepo
|
||||
from fxhnt.application.forward_ingest import CockpitBacktestRefProvider
|
||||
from fxhnt.application.forward_models import BacktestSummary, ForwardNavRow, ForwardSummary
|
||||
from fxhnt.application.gate_policy import POLICY
|
||||
from fxhnt.application.gate_policy_resolve import resolve_policy
|
||||
from fxhnt.application.reconciliation_gate import evaluate_reconciliation_gate
|
||||
from fxhnt.registry import STRATEGY_REGISTRY
|
||||
|
||||
_NO_REF = "no backtest reference to reconcile against"
|
||||
|
||||
|
||||
def _policy():
|
||||
return resolve_policy(POLICY, STRATEGY_REGISTRY["xsfunding"]["gate_spec"]) # reconciliation, min 14d
|
||||
|
||||
|
||||
def _xsfunding_forward(days: int):
|
||||
"""A mature (>= min_forward_days), continuous (no gaps), modestly-positive xsfunding forward record."""
|
||||
start = dt.date(2026, 6, 1)
|
||||
rows = [ForwardNavRow(strategy_id="xsfunding", date=(start + dt.timedelta(days=i)).isoformat(),
|
||||
ret=0.001, nav=1.001 ** (i + 1)) for i in range(days)]
|
||||
nav = rows[-1].nav
|
||||
summary = ForwardSummary(strategy_id="xsfunding", as_of=rows[-1].date, days=days, nav=nav,
|
||||
total_return=nav - 1.0, sharpe=1.0, maxdd=0.0)
|
||||
return summary, rows
|
||||
|
||||
|
||||
def test_xsfunding_gate_stuck_without_ref(tmp_path):
|
||||
"""Pre-fix state: no backtest_summary row -> provider yields None -> the permanent no-ref WAIT."""
|
||||
repo = ForwardNavRepo(f"sqlite:///{tmp_path / 'c.db'}")
|
||||
repo.migrate()
|
||||
policy = _policy()
|
||||
summary, rows = _xsfunding_forward(policy.min_forward_days + 3)
|
||||
ref = CockpitBacktestRefProvider(repo).reference("xsfunding", summary.days)
|
||||
assert ref is None
|
||||
verdict = evaluate_reconciliation_gate(summary, rows, ref, policy)
|
||||
assert verdict.status == "WAIT" and _NO_REF in verdict.reason
|
||||
|
||||
|
||||
def test_xsfunding_gate_unsticks_with_ref(tmp_path):
|
||||
"""Post-fix (Tasks 2/6): a persisted xsfunding backtest_summary -> provider yields a ref -> the gate
|
||||
RECONCILES (leaves the no-ref branch entirely; a PASS or a divergence-WAIT both mean 'reconciling')."""
|
||||
repo = ForwardNavRepo(f"sqlite:///{tmp_path / 'c.db'}")
|
||||
repo.migrate()
|
||||
repo.upsert_backtest_summary(
|
||||
BacktestSummary(strategy_id="xsfunding", as_of="2026-06-20", cagr=0.35, ann_vol=0.2, sharpe=1.2,
|
||||
max_drawdown=-0.06, passed=True, dsr=0.9, is_sharpe=1.3, oos_sharpe=1.1, pvalue=0.02),
|
||||
at=dt.datetime(2026, 6, 20))
|
||||
policy = _policy()
|
||||
summary, rows = _xsfunding_forward(policy.min_forward_days + 3)
|
||||
ref = CockpitBacktestRefProvider(repo).reference("xsfunding", summary.days)
|
||||
assert ref is not None and ref.expected_window_return > 0.0
|
||||
verdict = evaluate_reconciliation_gate(summary, rows, ref, policy)
|
||||
assert _NO_REF not in verdict.reason # the un-sticking: no longer stuck on a missing backtest ref
|
||||
```
|
||||
|
||||
*(Verify `BacktestSummary` / `ForwardSummary` / `ForwardNavRow` field names against `forward_models.py` and
|
||||
`upsert_backtest_summary`'s signature against `forward_nav.py` — adjust only if a name differs.)*
|
||||
|
||||
- [ ] **Step 2: Run to verify pass**
|
||||
|
||||
Run: `.venv/bin/python -m pytest tests/integration/test_xsfunding_gate_unsticks.py -q`
|
||||
Expected: PASS.
|
||||
|
||||
- [ ] **Step 3: Manual local browser verify** (`feedback_verify_cockpit_locally_not_prod`): run the local dev cockpit, seed an `xsfunding` forward + a real `backtest_ref("xsfunding", store)` ref, open `/strategy/xsfunding`, and READ that the gate shows a real reconciliation state (building N/14 → reconciling) rather than the permanent "no backtest reference" WAIT. Do NOT claim off synthetic/prod.
|
||||
|
||||
- [ ] **Step 4: Commit**
|
||||
|
||||
```bash
|
||||
git add tests/integration/test_xsfunding_gate_unsticks.py
|
||||
git commit -m "test(backtest): cockpit shows xsfunding gate reconciling once its ref exists (bug A verified)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Self-Review
|
||||
|
||||
**Spec coverage:**
|
||||
- Unified backtest→ref contract (3 mechanisms → 1 dispatch on `backtest.kind`) → Tasks 1, 5. ✓
|
||||
- Fix xsfunding (bug A) → Tasks 2, 6. ✓
|
||||
- TimescaleDuckDbEngine (ephemeral ATTACH) → Task 4. ✓
|
||||
- The invariant (CI-enforced) → Task 3. ✓
|
||||
- Scheduled resourced step (fixes bug B, replaces manual Job) → Task 6. ✓
|
||||
- Retire/park B3 + carry file-stores → Task 7. ✓
|
||||
- VERIFY paper_book DuckDB → Task 7 Step 1. ✓
|
||||
- Real-money-errs-false-WAIT (backtest None → gate WAITs) → preserved (Task 1 returns None; gate unchanged). ✓
|
||||
- Local cockpit verify → Task 8. ✓
|
||||
|
||||
**Placeholder scan:** the `sleeve`/`report` provider steps say "read the real signature/report shape in bybit_book_eval.py / the sleeve-ret command and adapt" — this is a deliberate reuse-don't-reimplement instruction with the exact seam cited (not a vague placeholder); the novel code (dispatch, engine, invariant, tests) is complete. Task 7 Step 1 is a genuine investigate-then-branch (the plan can't pre-decide what it can't see).
|
||||
|
||||
**Type consistency:** `backtest_ref(strategy_id, store, *, build_strategy=None) -> BacktestSummary | None` consistent (Tasks 1/2/5/6). `registry_backtest_kind(sid) -> str` consistent. `report_backtest_summary(strategy_id, store)` consistent (Tasks 1/5). `TimescaleDuckDbEngine(dsn)` + `.in_memory()`/`.attached()`/`.scan()` consistent (Tasks 4/5). `backtest.kind` values (`recompute-replay`/`sleeve`/`report`/`none`) consistent across registry + dispatch + invariant.
|
||||
|
||||
**Flagged for the implementer:** Task 2/5's exact provider code depends on `sleeve_returns_from_store` and the sleeve-ret report shape — the implementer MUST read those (`bybit_book_eval.py`, `funding_dispersion_eval.py`, the `bybit-persist-sleeve-ret` command) and adapt the row/report conversion; surface to the human if the engine's contract differs materially from what this plan assumes (e.g. if xsfunding's sleeve returns aren't directly `(iso,ret)` rows).
|
||||
289
docs/superpowers/plans/2026-07-14-ibkr-paper-cockpit-parity.md
Normal file
289
docs/superpowers/plans/2026-07-14-ibkr-paper-cockpit-parity.md
Normal file
@@ -0,0 +1,289 @@
|
||||
# IBKR Paper Cockpit Parity — Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Show the IBKR paper strategies' backtest (on the Backtests page) and their REAL DU-paper-account trading across the cockpit — an overview "real trades" line, a consolidated **Paper books** page showing both the crypto (simulated) and IBKR (real) books, and a click-through **holdings** detail — reconciled against the idealized sim ("how closely real trades follow the plan"), in plain language, scaling to multiple executing strategies by adding a row not a redesign.
|
||||
|
||||
**Architecture:** A faithful EXTENSION of the existing exec path. `IbkrBroker` already reads real NLV + positions; we (a) generalize the Backtests curve source from Bybit-only to any `backtest.kind`, (b) capture real fills → the existing empty `exec_fills` table + a new `ibkr_account_nav` snapshot at each `execute-multistrat` run, (c) compute ONE shared "behind/ahead of plan" signal rendered in plain language in three places (overview real-trades line, holdings-detail card, per-strategy breakdown), (d) consolidate `/paper` into a two-book landing (crypto simulated + IBKR real) that clicks through to a holdings detail. Plain-language labels throughout. Paper-only; gate/promotion/allocation untouched; the crypto book's data/behavior untouched.
|
||||
|
||||
**Tech Stack:** Python 3.12, SQLAlchemy 2.0 (offline), FastAPI + server-rendered Jinja templates, TimescaleDB/Postgres, ib-async (IBKR), pytest (`uv run pytest`, xdist `-n auto`).
|
||||
|
||||
## Global Constraints
|
||||
|
||||
- **PLAIN LANGUAGE, NO DEV JARGON** in any user-facing copy. A single `display_name` map (internal id → human
|
||||
name) is the source of every rendered name. Use: "Crypto 4-edge book", "Multi-asset ETF portfolio", "Options
|
||||
income (put spreads)", "60/40 benchmark", "real trades", "0.3% behind plan"/"ahead of plan", "cost to trade
|
||||
0.04%", "building", "not trading — options access pending", "no trades recorded yet". NEVER surface: raw ids
|
||||
(`bybit_4edge`, `multistrat`, `vrp`), the DU account number, `bps` (use %), "sim"/"recompute-replay"/
|
||||
"exec-twin"/"divergence". Keep only the fund's vocabulary (Sharpe, backtest, worst drop).
|
||||
- **Badges:** `REAL PAPER` (blue, `.badge-exec.real`) marks the IBKR real-paper account; `PAPER` (grey,
|
||||
`.badge-exec.paper`) marks the simulated crypto book. Reuse the existing `.badge-exec` + the amber/green `.b`
|
||||
pill vocabulary; do NOT invent a palette.
|
||||
- **Consolidated `/paper`:** one "Paper books" page shows BOTH books (crypto simulated + IBKR real), each
|
||||
clickable → its holdings detail. The Bybit book's underlying data/behavior is UNTOUCHED — only its landing is
|
||||
now shared. (No EU testnet keys → the crypto book stays simulated.)
|
||||
- Paper only — NO live/real-capital path, NO arm step, NO gate/promotion/allocation semantic change (spec "Out").
|
||||
- Capture is BEST-EFFORT + paper-only: a fills/snapshot persistence failure LOGS, never fails the rebalance or blocks execution (mirrors the existing best-effort ref-persist).
|
||||
- No silent degradation: capture failures log loudly; a stalled exec track already surfaces via the health axis. Empty/absent data → clean empty state, NEVER a 500, NEVER a fabricated number.
|
||||
- ONE divergence computation, reused by overview + detail + account (the "one signal, three places" rule).
|
||||
- All data tagged by `strategy_id` so a second executor is a NEW ROW, not a redesign.
|
||||
- DB migrations idempotent + Postgres-safe (no heavy startup DDL — follow `forward_nav`/`strategy_health` create-if-not-exists patterns).
|
||||
- Reuse the existing badge/health palette + `exec_badge` macro; do NOT invent a new palette.
|
||||
- Verify the cockpit LOCALLY in a real browser before deploy (read the rendered numbers).
|
||||
- Commit messages end with `Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>`.
|
||||
|
||||
---
|
||||
|
||||
## File Structure
|
||||
|
||||
| File | Responsibility | Task |
|
||||
|---|---|---|
|
||||
| `src/fxhnt/adapters/web/app.py` | `_SIM_BOOKS`→generalized book list; `_sim_returns` dispatch on `backtest.kind`; consolidated `/paper` both-books route; holdings-detail context | 1,3,4,5 |
|
||||
| `src/fxhnt/application/sim_curves.py` (new) | `sim_returns_for(book, repo, store)` — dispatch a book's honest-cost curve by its registry `backtest.kind` (bybit→`bybit_sleeve_ret`; recompute-replay→precomputed IBKR curve) | 1 |
|
||||
| `src/fxhnt/adapters/persistence/ibkr_account_repo.py` (new) | `IbkrAccountRepo` — migrate + replace/read `ibkr_account_nav` (per-run account snapshot) + read `exec_fills` by strategy | 2 |
|
||||
| `src/fxhnt/adapters/persistence/cockpit_models.py` | new `IbkrAccountNavRow`; add `strategy_id` to `ExecFillRow` | 2 |
|
||||
| `src/fxhnt/application/exec_capture.py` (new) | `capture_fills(trades)→list[FillDTO]`, `persist_execution(repo, strategy_id, rebalance_id, fills, account_snapshot, at)` (best-effort) | 2 |
|
||||
| `src/fxhnt/adapters/broker/ibkr.py` | collect fills from placed trades; expose `account_snapshot()` (nlv/cash/gross/positions) | 2 |
|
||||
| `src/fxhnt/application/multistrat_exec_record.py` | call `persist_execution` in `plan_and_record` (additive, best-effort, paper-only) | 2 |
|
||||
| `src/fxhnt/application/exec_reconcile.py` (new) | `exec_vs_sim(strategy_id, repo)→ReconResult` — THE shared "behind/ahead of plan" signal (cum divergence, median cost-to-trade, fees) | 3 |
|
||||
| `src/fxhnt/application/display_names.py` (new) | `display_name(id)→str` — the SINGLE internal-id→human-name map; every view uses it (plain-language rule) | 3 |
|
||||
| `src/fxhnt/application/dashboard_service.py` | per-strat exec status + `exec_vs_sim` into fleet rows; `paper_books()`; holdings-detail context; per-strategy breakdown | 3,4,5 |
|
||||
| `src/fxhnt/adapters/web/templates/_macros.html` | `.badge-exec.real` "REAL PAPER" state; `plan_pill(recon)` macro (plain words, no "vs sim"/bps) | 3,4 |
|
||||
| `src/fxhnt/adapters/web/templates/strategy.html` | holdings detail: value-over-time (real vs plan), "what it's holding" table, recent trades, per-strategy breakdown | 3,5 |
|
||||
| `src/fxhnt/adapters/web/templates/cockpit.html` | overview "real trades" sub-row + "behind plan" pill + REAL PAPER badge | 4 |
|
||||
| `src/fxhnt/adapters/web/templates/paper.html` | consolidated `/paper` "Paper books" — BOTH books as clickable cards | 5 |
|
||||
|
||||
---
|
||||
|
||||
## Task 1: D1 — `/paper/sim` sources the IBKR backtest curve
|
||||
|
||||
**Files:**
|
||||
- Create: `src/fxhnt/application/sim_curves.py`
|
||||
- Create: `tests/unit/test_sim_curves.py`
|
||||
- Modify: `src/fxhnt/adapters/web/app.py:388` (`_SIM_BOOKS`), `:402-413` (`_sim_returns`)
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `registry_backtest_kind(sid)` from `fxhnt.application.backtest_refs`; `PaperRepo.bybit_sleeve_returns()`; the precomputed IBKR recompute-replay curve (persisted like `bybit_sleeve_ret` — see Step 3).
|
||||
- Produces: `sim_returns_for(book: str, *, bybit_returns: Callable[[], dict[str, dict[int,float]]], ibkr_curve: Callable[[str], dict[str, dict[int,float]]]) -> dict[str, dict[int, float]]` — a book's per-sleeve honest-cost return series keyed `{sleeve: {day_index: ret}}`, dispatched by `backtest.kind`.
|
||||
- Produces: `SIM_BOOKS: tuple[str,...] = ("bybit_4edge","bybit_4edge_levered","multistrat","vrp")`.
|
||||
|
||||
- [ ] **Step 1: Write the failing test** (`tests/unit/test_sim_curves.py`)
|
||||
|
||||
```python
|
||||
from fxhnt.application.sim_curves import sim_returns_for, SIM_BOOKS
|
||||
|
||||
def _bybit(): return {"tstrend": {0: 0.0, 1: 0.01}}
|
||||
def _ibkr(book): return {book: {0: 0.0, 1: 0.02}}
|
||||
|
||||
def test_bybit_book_reads_bybit_sleeve_returns():
|
||||
out = sim_returns_for("bybit_4edge", bybit_returns=_bybit, ibkr_curve=_ibkr)
|
||||
assert out == {"tstrend": {0: 0.0, 1: 0.01}}
|
||||
|
||||
def test_ibkr_recompute_replay_book_reads_ibkr_curve():
|
||||
out = sim_returns_for("multistrat", bybit_returns=_bybit, ibkr_curve=_ibkr)
|
||||
assert out == {"multistrat": {0: 0.0, 1: 0.02}}
|
||||
|
||||
def test_sim_books_includes_ibkr():
|
||||
assert "multistrat" in SIM_BOOKS and "bybit_4edge" in SIM_BOOKS
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run test to verify it fails** — `uv run pytest tests/unit/test_sim_curves.py -q` → FAIL (module missing).
|
||||
|
||||
- [ ] **Step 3: Implement `sim_curves.py`.** Dispatch on `registry_backtest_kind(book)`: `report`/bybit sids → `bybit_returns()`; `recompute-replay` sids → `ibkr_curve(book)`. The IBKR curve must be a PRECOMPUTED read (the sim never recomputes on the request path). Persist the recompute-replay curve where the sim reads it cheaply: extend the nightly `_persist_track_backtest_ref` / the `fxhnt-backtest-refs` Job to also write the per-day return series for `multistrat`/`vrp` into a small table (reuse the `bybit_sleeve_ret` shape or a sibling `sim_curve_ret(strategy_id, day_index, ret)`), and have `ibkr_curve(book)` read it. Wire `ibkr_curve` in `app.py` to that repo read (cached like `_sim_returns`).
|
||||
|
||||
```python
|
||||
"""Dispatch a sim book's honest-cost curve by its registry backtest.kind — bybit books read the precomputed
|
||||
bybit_sleeve_ret table; recompute-replay books (multistrat/vrp) read their precomputed IBKR curve. The sim
|
||||
NEVER recomputes an edge on the request path (both sources are nightly-precomputed)."""
|
||||
from __future__ import annotations
|
||||
from typing import Callable
|
||||
from fxhnt.application.backtest_refs import registry_backtest_kind
|
||||
|
||||
SIM_BOOKS: tuple[str, ...] = ("bybit_4edge", "bybit_4edge_levered", "multistrat", "vrp")
|
||||
|
||||
def sim_returns_for(book: str, *, bybit_returns: Callable[[], dict], ibkr_curve: Callable[[str], dict]) -> dict:
|
||||
kind = registry_backtest_kind(book)
|
||||
if kind == "recompute-replay":
|
||||
return ibkr_curve(book)
|
||||
return bybit_returns() # report-kind bybit books (the existing path)
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Wire `app.py`.** Replace `_SIM_BOOKS = (...)` with `from fxhnt.application.sim_curves import SIM_BOOKS` (use `SIM_BOOKS`). Change `_sim_returns(book)` to call `sim_returns_for(book, bybit_returns=_paper_repo().bybit_sleeve_returns, ibkr_curve=_ibkr_sim_curve)` where `_ibkr_sim_curve(book)` reads the precomputed IBKR curve (cached with the same 300s TTL). Keep the TTL cache keyed by book.
|
||||
|
||||
- [ ] **Step 5: Run tests** — `uv run pytest tests/unit/test_sim_curves.py tests/integration/test_bybit_sim_web.py tests/integration/test_paper_sim_web.py -q` → PASS (bybit sim unchanged; new dispatch covered). Then the sim web test asserting `/paper/sim?book=multistrat` returns 200 and renders a curve.
|
||||
|
||||
- [ ] **Step 6: Commit** — `git add -A && git commit -m "feat(D1): /paper/sim sources any book's curve by backtest.kind (adds IBKR multistrat/vrp)"` (+ Co-Authored-By).
|
||||
|
||||
---
|
||||
|
||||
## Task 2: D2.1 — capture the real DU account (fills + NAV snapshot)
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/adapters/persistence/cockpit_models.py` (new `IbkrAccountNavRow`; `ExecFillRow` + `strategy_id`)
|
||||
- Create: `src/fxhnt/adapters/persistence/ibkr_account_repo.py`
|
||||
- Create: `src/fxhnt/application/exec_capture.py`
|
||||
- Modify: `src/fxhnt/adapters/broker/ibkr.py` (fills + `account_snapshot`)
|
||||
- Modify: `src/fxhnt/application/multistrat_exec_record.py:89-97` (call capture)
|
||||
- Create: `tests/integration/test_exec_capture.py`, `tests/unit/test_ibkr_account_repo.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: `IbkrAccountRepo(dsn)` with `.migrate()`, `.replace_snapshot(run_date: str, nlv: float, cash: float, gross: float, positions: dict[str,float], at)`, `.nav_series() -> list[tuple[str,float]]` (date, nlv), `.latest_positions() -> dict[str,float]`, `.fills_for(strategy_id: str, limit: int) -> list[FillRow]`.
|
||||
- Produces: `capture_fills(trades) -> list[FillDTO]` (FillDTO: symbol, side, qty, price, fee, status, shortfall_bps); `persist_execution(repo, *, strategy_id, rebalance_id, fills, snapshot, at)` — best-effort (catches+logs).
|
||||
- Produces: `IbkrBroker.account_snapshot() -> AccountSnapshot(nlv, cash, gross, positions: dict[str,float])`.
|
||||
|
||||
- [ ] **Step 1: Write the failing test** (`tests/unit/test_ibkr_account_repo.py`) — migrate on sqlite, replace a snapshot, read it back:
|
||||
|
||||
```python
|
||||
import datetime as dt
|
||||
from fxhnt.adapters.persistence.ibkr_account_repo import IbkrAccountRepo
|
||||
|
||||
def test_snapshot_roundtrip(tmp_path):
|
||||
repo = IbkrAccountRepo(f"sqlite:///{tmp_path}/t.db"); repo.migrate()
|
||||
at = dt.datetime(2026,7,14,14,35)
|
||||
repo.replace_snapshot("2026-07-14", nlv=1_028_594.0, cash=500.0, gross=1_000_000.0,
|
||||
positions={"SPY": 10.0}, at=at)
|
||||
assert repo.nav_series() == [("2026-07-14", 1_028_594.0)]
|
||||
assert repo.latest_positions() == {"SPY": 10.0}
|
||||
# idempotent replace
|
||||
repo.replace_snapshot("2026-07-14", nlv=1_029_000.0, cash=1.0, gross=1.0, positions={"SPY": 11.0}, at=at)
|
||||
assert repo.nav_series() == [("2026-07-14", 1_029_000.0)]
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run it** → FAIL (module missing).
|
||||
|
||||
- [ ] **Step 3: Implement the model + repo.** In `cockpit_models.py` add:
|
||||
|
||||
```python
|
||||
class IbkrAccountNavRow(CockpitBase):
|
||||
__tablename__ = "ibkr_account_nav"
|
||||
run_date: Mapped[dt.date] = mapped_column(Date, primary_key=True)
|
||||
nlv: Mapped[float] = mapped_column(Float)
|
||||
cash: Mapped[float] = mapped_column(Float)
|
||||
gross: Mapped[float] = mapped_column(Float)
|
||||
positions_json: Mapped[str] = mapped_column(Text)
|
||||
at: Mapped[dt.datetime] = mapped_column(DateTime)
|
||||
```
|
||||
|
||||
Add `strategy_id: Mapped[str] = mapped_column(String(64), index=True, default="")` to `ExecFillRow` (tag fills per strategy; default "" is back-compat). `IbkrAccountRepo` follows the `ForwardNavRepo` pattern: `migrate()` = `CockpitBase.metadata.create_all(engine)` (checkfirst — no ALTER, no heavy DDL) + a guarded idempotent `ADD COLUMN strategy_id` on `exec_fills` for pre-existing Postgres tables (mirror the paper_repo ALTER pattern, Postgres-only). `replace_snapshot` = delete-then-insert the run_date row; positions stored as `json.dumps`.
|
||||
|
||||
- [ ] **Step 4: Run the repo test** → PASS.
|
||||
|
||||
- [ ] **Step 5: Broker + capture (failing test first)** (`tests/integration/test_exec_capture.py`): a fake trade with `.fills` (each fill: contract.symbol, execution.side/shares/price/avgPrice, commissionReport.commission) → `capture_fills` returns FillDTOs with the right fields; `persist_execution` writes them to `exec_fills` tagged `strategy_id="multistrat"` and the snapshot to `ibkr_account_nav`; a repo that RAISES on write → `persist_execution` logs and does NOT raise (best-effort). Then implement `capture_fills` (map ib-async trade fills → FillDTO, `shortfall_bps` = (fill.price − intended)/intended·1e4 when the intended price is available, else 0.0) and `persist_execution` (wrap all writes in try/except → log.warning, never raise). Add `IbkrBroker.account_snapshot()` reading `accountSummary()` + `positions()` (reuse the existing `_SUMMARY_TAGS` read at ibkr.py:39-40) and make the placed-order path return the trades so fills are collectible.
|
||||
|
||||
- [ ] **Step 6: Wire `plan_and_record`** (multistrat_exec_record.py:89-97): after `record_exec_track(...)`, when `account_is_paper` and not blocked, call `persist_execution(account_repo, strategy_id="multistrat", rebalance_id=<plan.rebalance_id or today>, fills=capture_fills(plan.trades), snapshot=broker.account_snapshot(), at=at)`. Best-effort; thread the `account_repo` + broker through (the CLI constructs them — it already builds the broker). Do NOT change the `positions_after` return basis (gate-relevant).
|
||||
|
||||
- [ ] **Step 7: Run tests** — `uv run pytest tests/integration/test_exec_capture.py tests/unit/test_ibkr_account_repo.py tests/unit/test_multistrat_exec_helpers.py -q` → PASS.
|
||||
|
||||
- [ ] **Step 8: Commit** — `feat(D2.1): capture real IBKR fills->exec_fills (strategy-tagged) + ibkr_account_nav snapshot (best-effort, paper-only)`.
|
||||
|
||||
---
|
||||
|
||||
## Task 3: D2.2/2.3 — holdings detail page + the shared "behind plan" signal + display names
|
||||
|
||||
> Also create `src/fxhnt/application/display_names.py` — the SINGLE `display_name(id) -> str` map (internal id → human name) used by EVERY view (per the plain-language global constraint). Test it covers all books/strategies.
|
||||
|
||||
**Files:**
|
||||
- Create: `src/fxhnt/application/exec_reconcile.py`, `tests/unit/test_exec_reconcile.py`
|
||||
- Modify: `src/fxhnt/application/dashboard_service.py` (feed detail), `src/fxhnt/adapters/web/app.py` (detail context), `src/fxhnt/adapters/web/templates/strategy.html`, `_macros.html`
|
||||
- Test: `tests/integration/test_ibkr_account_detail_web.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: `exec_vs_sim(strategy_id: str, *, exec_repo, forward_repo) -> ReconResult` where `ReconResult` = dataclass(has_exec: bool, cum_exec_return: float, cum_sim_return: float, divergence: float, median_slippage_bps: float, total_fees: float). THE one shared signal (reused by D3, D4).
|
||||
- Consumes: `IbkrAccountRepo` (Task 2), `ForwardNavRepo` (existing `multistrat` + `multistrat_exec` series).
|
||||
|
||||
- [ ] **Step 1: Failing test** (`tests/unit/test_exec_reconcile.py`): given a `multistrat_exec` executed cum return and a `multistrat` sim cum return + fills with known slippage, `exec_vs_sim` returns the expected `divergence` (= exec − sim) and `median_slippage_bps`; a strategy with NO exec data → `has_exec=False`, zeros. Use fakes for the two repos.
|
||||
|
||||
```python
|
||||
def test_divergence_and_slippage():
|
||||
r = exec_vs_sim("multistrat", exec_repo=FakeExec(cum=0.031, fills_bps=[3.0,5.0,4.0], fees=12.0),
|
||||
forward_repo=FakeFwd(exec_cum=0.031, sim_cum=0.038))
|
||||
assert round(r.divergence, 4) == -0.007 # exec 3.1% vs sim 3.8%
|
||||
assert r.median_slippage_bps == 4.0
|
||||
assert r.has_exec is True
|
||||
|
||||
def test_no_exec_data():
|
||||
r = exec_vs_sim("sixtyforty", exec_repo=FakeExec(cum=None, fills_bps=[], fees=0.0),
|
||||
forward_repo=FakeFwd(exec_cum=None, sim_cum=0.05))
|
||||
assert r.has_exec is False
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run it** → FAIL.
|
||||
|
||||
- [ ] **Step 3: Implement `exec_reconcile.py`.** `has_exec` = the `*_exec` forward track has ≥1 record. `divergence` = cum exec − cum sim (both from `forward_summary`/nav). `median_slippage_bps` = median of `exec_fills.shortfall_bps` for the strat. `total_fees` = sum `exec_fills.fee`. All reads via the repos; no compute on unavailable data (return zeros + `has_exec=False`).
|
||||
|
||||
- [ ] **Step 4: Run** → PASS.
|
||||
|
||||
- [ ] **Step 5: `_macros.html` — plain-language macros.** Add `plan_pill(recon)` (renders `has_exec` → the amber/green `.b` pill "{abs(divergence):.1%} behind plan" / "ahead of plan"; `has_exec=False` → nothing) and add the `.badge-exec.real` "REAL PAPER" state. Reuse the existing `.b`/`.badge-exec` classes. NO "vs sim"/"divergence"/`bps` in any rendered string — plain words only.
|
||||
|
||||
- [ ] **Step 6: holdings detail page (`strategy.html` + detail context).** `dashboard_service.detail()` attaches, for the IBKR book/strategy: `holdings` (from `IbkrAccountRepo.latest_positions()` → each asset with `display_name`, a plain "what it is" description, shares, value, %-of-book), `recent_trades` (`fills_for(sid, 20)` → time, asset, action Buy/Sell, shares, price, **cost-to-trade as a %** (not bps), fee), `account_nav` (`nav_series()`), `recon` (`exec_vs_sim(...)`). `strategy.html` renders: a "Following the plan" card + `plan_pill`, the **value-over-time** curve with the plan overlaid (dashed), a **"What it's holding right now"** holdings table, and a **"Recent trades"** table — all plain-language (see the asset→description map: SPY→"US stocks", IEF→"US Treasuries", GLD→"Gold", PDBC→"Commodities", DBMF→"Managed futures"). Empty/no-exec → "no trades recorded yet" (no 500).
|
||||
|
||||
- [ ] **Step 7: Web test** (`test_holdings_detail_web.py`): seed `ibkr_account_nav` + `exec_fills` + the two forward tracks; GET the IBKR book's detail renders the "what it's holding" table (with plain asset descriptions), a recent-trades row with cost-to-trade as a %, the value-over-time curve, and the "behind plan" pill; assert NO raw ids / `bps` / "vs sim" strings appear in the HTML. GET a strat with no exec → 200 with "no trades recorded yet", no 500.
|
||||
|
||||
- [ ] **Step 8: Commit** — `feat(D2.2/2.3): IBKR paper-account section on the strategy detail + shared exec-vs-sim divergence signal`.
|
||||
|
||||
---
|
||||
|
||||
## Task 4: D3 — overview "real trades" sub-line + "behind plan" pill
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/application/dashboard_service.py` (`fleet()`/`overview()` — per-row exec status + rollup), `src/fxhnt/adapters/web/app.py`, `src/fxhnt/adapters/web/templates/cockpit.html`
|
||||
- Test: `tests/integration/test_overview_exec_signal.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `exec_vs_sim` (Task 3), `IbkrAccountRepo` (Task 2).
|
||||
- Produces: `FleetRow` gains `exec_status: str` ("" | "EXEC") and `recon: ReconResult | None`; an `IbkrAccountRollup` (total_nav, n_executing, agg_divergence) on the overview model.
|
||||
|
||||
- [ ] **Step 1: Failing test** (`test_overview_exec_signal.py`): with a seeded executing `multistrat` (exec data present) and a pure-sim strat, `fleet()` marks `multistrat.exec_status == "EXEC"` with a non-None `recon`, and the sim strat has `exec_status == ""`. `overview()` yields an `IbkrAccountRollup` with `total_nav` = the latest `ibkr_account_nav` and `n_executing == 1`.
|
||||
|
||||
- [ ] **Step 2: Run it** → FAIL.
|
||||
|
||||
- [ ] **Step 3: Implement.** In `dashboard_service.fleet()` set `exec_status="EXEC"` + `recon=exec_vs_sim(sid,...)` when the strat has exec data (its `*_exec` track exists AND has records); else "". Build `IbkrAccountRollup` from `IbkrAccountRepo` (`nav_series()[-1]`, count of executing strats, aggregate divergence = mean of per-strat divergences). One computation via `exec_vs_sim` (no duplicate math).
|
||||
|
||||
- [ ] **Step 4: Run** → PASS.
|
||||
|
||||
- [ ] **Step 5: `cockpit.html`** — mark an executing strategy's row with the `REAL PAPER` badge, and render a nested **"real trades"** sub-row (reuse the existing `.exec-tag` + `tr.exrow` style) showing: what actually filled (real return, "N trades · M holdings →" linking to its holdings detail), the `plan_pill(recon)` ("0.3% behind plan"), and a plain "cost to trade X% · no fees" note. Only for `exec_status=="EXEC"`. No separate rollup card (the consolidated Paper-books page is the account home). Plain language only.
|
||||
|
||||
- [ ] **Step 6: Web test** — GET `/` renders the `REAL PAPER` badge + the "real trades" sub-row + the "behind plan" pill on the executing row, NOT on a sim-only row; assert plain language (no raw ids / `bps` / "vs sim"); no exec data anywhere → no sub-row, no 500.
|
||||
|
||||
- [ ] **Step 7: Commit** — `feat(D3): overview real-trades sub-line + behind-plan pill (plain language, REAL PAPER badge)`.
|
||||
|
||||
---
|
||||
|
||||
## Task 5: D4 — consolidated `/paper` "Paper books" page (both books) + per-strategy breakdown
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/adapters/web/templates/paper.html` (the existing `/paper` — make it the two-book landing), `src/fxhnt/adapters/web/app.py` (`/paper` route → both books), `src/fxhnt/application/dashboard_service.py` (`paper_books()` + the IBKR book detail's per-strategy breakdown)
|
||||
- Test: `tests/integration/test_paper_books_web.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `IbkrAccountRepo`, `exec_vs_sim` (per executing strat).
|
||||
- Produces: `dashboard_service.ibkr_account_view() -> IbkrAccountView(total_nav_series, all_positions, recent_fills, per_strategy: list[StratExecRow])` where `StratExecRow` = (strategy_id, display_name, executed_return, divergence, position_count, fill_count).
|
||||
|
||||
- [ ] **Step 1: Failing test** (`test_paper_books_web.py`): GET `/paper` renders BOTH book cards — the crypto book (`PAPER`) and the IBKR account (`REAL PAPER`) — each linking to its detail. `paper_books()` returns both books' summaries. The IBKR book DETAIL: `per_strategy` list with ONE row today; a **scale test** seeds a synthetic SECOND executing strat → `per_strategy` has TWO rows (proves "new row, not redesign"). Assert plain language throughout (no raw ids / `bps`).
|
||||
|
||||
- [ ] **Step 2: Run it** → FAIL.
|
||||
|
||||
- [ ] **Step 3: Implement `paper_books()` + per-strategy breakdown.** `paper_books()` returns both books' summaries (crypto: existing simulated stats; IBKR: `IbkrAccountRepo` account value + N strategies + `plan_pill` data). The `/paper` route renders the two-book `paper.html`. The IBKR book detail's `per_strategy` = for each strat with exec data, a row (human `display_name`, `exec_vs_sim` return + behind/ahead-of-plan, its position/trade counts from strategy-tagged `exec_fills`). All plain-language.
|
||||
|
||||
- [ ] **Step 4: Run** → PASS.
|
||||
|
||||
- [ ] **Step 5: `paper.html` — the consolidated "Paper books" landing.** Render BOTH books as clickable cards: the **Crypto 4-edge book** (`PAPER`, its existing simulated summary — value/since/Sharpe/worst-drop/status + mini curve) and the **IBKR paper account** (`REAL PAPER` — account value / N strategies trading / holdings count / `plan_pill` / mini curve with the plan overlaid). Each card links to that book's holdings detail (Task 3). The IBKR book's DETAIL page (Task 3's `strategy.html`) additionally shows the **per-strategy breakdown** table — one row per executing strategy (human name, real return, `plan_pill`, holdings/trade counts); renders a SET → 1 row today, N later, no layout change; a dimmed "not started yet" row shows the scale path. Plain language; empty account → clean empty state, no 500.
|
||||
|
||||
- [ ] **Step 6: Web test** — the scale test above passes (2-strat fixture → 2 rows); empty account → empty state, no 500.
|
||||
|
||||
- [ ] **Step 7: Commit** — `feat(D4): consolidated /paper Paper-books page (both books) + per-strategy breakdown (scales by row)`.
|
||||
|
||||
---
|
||||
|
||||
## Verify (whole feature)
|
||||
|
||||
- Full suite green FOREGROUND (`uv run pytest -q`) + `from fxhnt.cli import app`.
|
||||
- Deploy via `mode=deploy` (git-sync); the `ibkr_account_nav` table + the `exec_fills.strategy_id` column create/ALTER on startup (idempotent).
|
||||
- Browser-verify LOCALLY (port-forward prod pg → 15432, `create_app`): `/` shows the "real trades" sub-line + "behind plan" pill; `/paper` shows BOTH book cards (crypto `PAPER` + IBKR `REAL PAPER`); clicking the IBKR book shows the holdings + trades + value-over-time + per-strategy breakdown; the Backtests page shows the IBKR tracks' curves. Read the numbers; confirm NO dev jargon / raw ids / `bps` appear anywhere.
|
||||
- After the next `execute-multistrat` run, confirm `exec_fills` + `ibkr_account_nav` populate with real DU-account data.
|
||||
|
||||
## Self-Review
|
||||
|
||||
- **Spec coverage:** D1 (backtests incl. IBKR tracks)→Task1, D2.1→Task2, D2.2/2.3 (holdings detail + behind-plan signal)→Task3, D3 (overview real-trades line)→Task4, D4 (consolidated Paper books)→Task5. One "behind plan" signal (`exec_vs_sim`) used by Task3/4/5. Plain-language rule → `display_names.py` (Task3) + the "no raw ids/bps/jargon" assertions in the Task3/4/5 web tests. Multi-strat scaling → strategy-tagged `exec_fills` (Task2) + the scale test (Task5). ✓
|
||||
- **Placeholder scan:** none — each task has concrete code/tests.
|
||||
- **Type consistency:** `sim_returns_for`, `IbkrAccountRepo`, `capture_fills`/`persist_execution`, `exec_vs_sim`/`ReconResult`, `FleetRow.exec_status`/`recon`, `IbkrAccountView`/`StratExecRow` — names consistent across tasks.
|
||||
@@ -0,0 +1,225 @@
|
||||
# Capacity-Aware Capital Allocator — Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Upgrade the existing B2a allocation engine so it weights edges by inverse-vol (risk-parity, capped), caps each edge at its honest-reference capacity ceiling, and sizes from the honest-reference backtest (not the too-short forward record) — with the forward gate remaining the live-deploy switch.
|
||||
|
||||
**Architecture:** A focused change to `allocation_engine.compute_allocation` (equal-weight → inverse-vol capped) + a per-strategy capacity ceiling cap in `target_capital`, fed by a new provider that maps each deploy strategy to its honest-reference return series + capacity ceiling. `record_allocation` sources returns from that provider instead of `nav_history`. Everything else (forward-gating, cross-venue, funding-decay, killswitch, vol-target, $ conversion) is reused unchanged.
|
||||
|
||||
**Tech Stack:** Python 3.12, numpy (already used in `allocation_engine`), the merged honest-reference modules (`honest_reference`, `honest_gates`), pytest.
|
||||
|
||||
## Global Constraints
|
||||
|
||||
- **Upgrade, not a new system.** Reuse `AllocationPolicy`, `full_history_vol`, `equal_weight_marginal_prune`, `killswitch_active`, `target_capital`, `record_allocation`, `StrategyAllocationRow`, the rebalancers. Do not duplicate.
|
||||
- **Scope = paper.** No change to the `capital_ceiling` ($35k human gate) or going live; no new execution path. Forward-gating (deploy + promoted) stays the live-deploy switch.
|
||||
- **Sizing basis = the honest-reference backtest series**, NOT `nav_history` (8-day forward, fails `min_history_days`). The forward gate is the live-deploy switch only.
|
||||
- **Weighting = inverse-vol (risk-parity), capped at `max_strategy_weight`** (concentration prudence — a young, not-yet-forward-proven fund must not bet on one edge).
|
||||
- **Capacity ceiling per deploy strategy**, from the honest-reference: `bybit_4edge` = book-level (its sleeves, tstrend excluded — retired standalone), `positioning` = its sleeve curve, `multistrat` (IBKR) = a documented high **equity default** ($10M) — do NOT fabricate a crypto number for equity.
|
||||
- `capacity_min_sharpe` default **1.0**; bump `ALLOCATION_POLICY_VERSION` on any policy field change.
|
||||
- Tests run `pytest -o addopts=""` (repo `addopts` injects `-n`/xdist, absent locally). Use `python3`.
|
||||
|
||||
## File Structure
|
||||
|
||||
- `src/fxhnt/application/allocation_engine.py` — add `inverse_vol_weights`, `capacity_ceiling_from_curve`; change `compute_allocation`'s base weights; add the capacity cap to `target_capital`.
|
||||
- `src/fxhnt/application/allocation_honest_inputs.py` — NEW: `honest_allocation_inputs(store, spread_bps, unlock_events, *, deploy_sids, target_aum, ...) -> tuple[dict[str, dict[int,float]], dict[str,float]]` returning (per-strategy honest return series, per-strategy capacity ceiling). The deploy-strategy → honest-reference mapping lives here.
|
||||
- `src/fxhnt/application/allocation_policy.py` — add `capacity_min_sharpe`, `equity_default_ceiling`; bump version.
|
||||
- `src/fxhnt/application/allocation_ingest.py` — `record_allocation` sources returns + ceilings from the provider.
|
||||
- Tests: `tests/unit/test_allocation_engine.py` (extend), `tests/unit/test_allocation_honest_inputs.py`, `tests/integration/test_record_allocation_capacity.py`.
|
||||
|
||||
---
|
||||
|
||||
### Task 1: `inverse_vol_weights` — risk-parity base weights
|
||||
|
||||
**Files:** Modify `src/fxhnt/application/allocation_engine.py`; Test `tests/unit/test_allocation_engine.py`.
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `full_history_vol(returns) -> float` (already in the module).
|
||||
- Produces: `inverse_vol_weights(returns_by_strategy: dict[str, dict[str,float]]) -> dict[str,float]` (∝ 1/full_history_vol, summing to 1; zero-vol strategy → weight 0; all-degenerate → equal weight).
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
```python
|
||||
# tests/unit/test_allocation_engine.py
|
||||
from fxhnt.application.allocation_engine import inverse_vol_weights
|
||||
|
||||
def test_inverse_vol_weights_favour_low_vol_and_sum_to_one():
|
||||
rbs = {"lo": {"1": 0.001, "2": -0.001, "3": 0.001},
|
||||
"hi": {"1": 0.10, "2": -0.10, "3": 0.10}}
|
||||
w = inverse_vol_weights(rbs)
|
||||
assert abs(sum(w.values()) - 1.0) < 1e-9 and w["lo"] > w["hi"]
|
||||
```
|
||||
- [ ] **Step 2: Run** `python3 -m pytest tests/unit/test_allocation_engine.py::test_inverse_vol_weights_favour_low_vol_and_sum_to_one -v -o addopts=""` → FAIL (undefined).
|
||||
- [ ] **Step 3: Implement** (add near `full_history_vol`):
|
||||
```python
|
||||
def inverse_vol_weights(returns_by_strategy: dict[str, dict[str, float]]) -> dict[str, float]:
|
||||
"""Risk-parity base: weight each strategy ∝ 1/full_history_vol, normalized to sum 1. Zero/degenerate-vol
|
||||
strategy → 0; all-degenerate → equal weight. Reuses the ex-ante full-history vol (never trailing)."""
|
||||
inv = {s: (1.0 / v if (v := full_history_vol(r)) > 0.0 else 0.0) for s, r in returns_by_strategy.items()}
|
||||
total = sum(inv.values())
|
||||
if total <= 0.0:
|
||||
n = len(returns_by_strategy)
|
||||
return {s: 1.0 / n for s in returns_by_strategy} if n else {}
|
||||
return {s: v / total for s, v in inv.items()}
|
||||
```
|
||||
- [ ] **Step 4: Run** the test → PASS.
|
||||
- [ ] **Step 5: Commit** `git add -A && git commit -m "feat(alloc): inverse-vol (risk-parity) base weights"`
|
||||
|
||||
---
|
||||
|
||||
### Task 2: `capacity_ceiling_from_curve` — the ceiling from a capacity curve
|
||||
|
||||
**Files:** Modify `src/fxhnt/application/allocation_engine.py`; Test `tests/unit/test_allocation_engine.py`.
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: `capacity_ceiling_from_curve(curve: dict[float,float], min_sharpe: float) -> float | None` — the LARGEST aum whose honest Sharpe ≥ `min_sharpe`; `None` if the curve is empty or the first (smallest) AUM already fails (edge unviable even at the smallest size); the largest tested AUM if it never drops below (unbounded in-range).
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
```python
|
||||
def test_capacity_ceiling_is_last_aum_above_min_sharpe():
|
||||
from fxhnt.application.allocation_engine import capacity_ceiling_from_curve
|
||||
curve = {35_000: 3.5, 100_000: 2.5, 350_000: 0.3, 1_000_000: -2.5} # xsfunding-shaped
|
||||
assert capacity_ceiling_from_curve(curve, 1.0) == 100_000 # last AUM with Sh >= 1.0
|
||||
robust = {35_000: 2.9, 1_000_000: 2.6, 10_000_000: 1.6}
|
||||
assert capacity_ceiling_from_curve(robust, 1.0) == 10_000_000 # never drops -> top of range
|
||||
assert capacity_ceiling_from_curve({35_000: 0.2}, 1.0) is None # unviable even at smallest
|
||||
assert capacity_ceiling_from_curve({}, 1.0) is None
|
||||
```
|
||||
- [ ] **Step 2: Run** → FAIL.
|
||||
- [ ] **Step 3: Implement**
|
||||
```python
|
||||
def capacity_ceiling_from_curve(curve: dict[float, float], min_sharpe: float) -> float | None:
|
||||
"""Largest AUM whose honest Sharpe >= min_sharpe (the capacity ceiling). None if empty or the edge is
|
||||
unviable even at the smallest tested AUM. Report the ceiling next to the curve so the threshold is visible."""
|
||||
ok = [aum for aum in sorted(curve) if curve[aum] >= min_sharpe]
|
||||
return max(ok) if ok else None
|
||||
```
|
||||
- [ ] **Step 4: Run** → PASS.
|
||||
- [ ] **Step 5: Commit** `git add -A && git commit -m "feat(alloc): capacity ceiling from an honest capacity curve"`
|
||||
|
||||
---
|
||||
|
||||
### Task 3: `compute_allocation` uses inverse-vol (capped), not equal-weight
|
||||
|
||||
**Files:** Modify `src/fxhnt/application/allocation_engine.py`; Test `tests/unit/test_allocation_engine.py`.
|
||||
|
||||
**Interfaces:** Consumes `inverse_vol_weights` (Task 1). `compute_allocation` signature unchanged.
|
||||
|
||||
- [ ] **Step 1: Write the failing test** — the survivor base is inverse-vol capped, not 1/n:
|
||||
```python
|
||||
def test_compute_allocation_uses_inverse_vol_base_capped():
|
||||
from fxhnt.application.allocation_engine import compute_allocation
|
||||
from fxhnt.application.allocation_policy import AllocationPolicy
|
||||
# two survivors, very different vol; both pass the (disabled) marginal gate; enough history
|
||||
n = 400
|
||||
rbs = {"lo": {str(i): (0.002 if i % 2 else -0.002) for i in range(n)},
|
||||
"hi": {str(i): (0.05 if i % 2 else -0.05) for i in range(n)}}
|
||||
pol = AllocationPolicy(marginal_gate=False, min_history_days=60, max_strategy_weight=0.9)
|
||||
w = compute_allocation(rbs, pol)
|
||||
assert w["lo"] > w["hi"] # low-vol gets more (was equal under the old code)
|
||||
```
|
||||
- [ ] **Step 2: Run** → FAIL (old code gives equal base → `w["lo"] == w["hi"]`).
|
||||
- [ ] **Step 3: Implement** — in `compute_allocation`, replace the equal-weight `base` line
|
||||
```python
|
||||
base = {s: min(1.0 / n, policy.max_strategy_weight) for s in survivors}
|
||||
```
|
||||
with inverse-vol capped:
|
||||
```python
|
||||
rp = inverse_vol_weights({s: eligible[s] for s in survivors})
|
||||
base = {s: min(rp.get(s, 0.0), policy.max_strategy_weight) for s in survivors}
|
||||
```
|
||||
(Keep everything else — the NO-renormalise cap behavior, killswitch, vol-target K — identical.)
|
||||
- [ ] **Step 4: Run** the new test AND the full existing allocation suite `python3 -m pytest tests/unit/test_allocation_engine.py -v -o addopts=""`. Some existing equal-weight assertions may need updating to the inverse-vol expectation — update them to the correct new values, do NOT weaken. Expected: all pass.
|
||||
- [ ] **Step 5: Commit** `git add -A && git commit -m "feat(alloc): compute_allocation weights by inverse-vol (capped) not equal-weight"`
|
||||
|
||||
---
|
||||
|
||||
### Task 4: Honest-reference allocation inputs provider (series + ceilings per deploy strategy)
|
||||
|
||||
**Files:** Create `src/fxhnt/application/allocation_honest_inputs.py`; Test `tests/unit/test_allocation_honest_inputs.py`.
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `honest_sleeve_returns` (`honest_reference`), `book_returns`/`risk_parity_weights` (`honest_gates`), `sleeve_weights_and_contributions` (`bybit_paper_weights`), `capacity_curve` (`honest_reference`), `capacity_ceiling_from_curve` (Task 2).
|
||||
- Produces: `honest_allocation_inputs(store, spread_bps, unlock_events, *, deploy_sids, target_aum, aums, participation_cap=0.03, impact_k=1.0, min_sharpe=1.0, equity_default_ceiling=10_000_000.0) -> tuple[dict[str, dict[int,float]], dict[str,float]]` → (returns_by_strategy, capacity_ceiling_by_strategy). Pure (store reads only).
|
||||
|
||||
- [ ] **Step 1: Confirm the mapping (read, no code yet).** Read `honest_report.build_honest_report` for how sleeves → honest series + capacity curves are produced; confirm `bybit_4edge`'s sleeves in the registry (`["tstrend","unlock","xsfunding","positioning"]`) and that `tstrend` has no standalone store series (excluded). Note the mapping to write:
|
||||
- `positioning` → its own `honest_sleeve_returns` series + `capacity_ceiling_from_curve` of its capacity curve.
|
||||
- `bybit_4edge` → the risk-parity **book** of its available crypto sleeves (`xsfunding`, `unlock`, `positioning`; `tstrend` excluded) via `book_returns`; ceiling from the book's capacity curve.
|
||||
- `multistrat` (or any non-crypto deploy sid) → `equity_default_ceiling`, series left to the caller's fallback (out of v1 crypto scope; deploy set is currently crypto-only).
|
||||
- [ ] **Step 2: Write the failing test** with a fake `_FeatureStore` (model on `tests/integration/test_honest_reference.py`'s fake store) for `deploy_sids={"positioning","bybit_4edge"}`:
|
||||
```python
|
||||
def test_provider_returns_series_and_ceilings_per_deploy_strategy():
|
||||
from fxhnt.application.allocation_honest_inputs import honest_allocation_inputs
|
||||
store = _fake_store() # supports xsfunding + positioning sleeves
|
||||
aums = [35_000, 100_000, 350_000, 1_000_000, 10_000_000]
|
||||
series, ceilings = honest_allocation_inputs(store, {}, [], deploy_sids={"positioning", "bybit_4edge"},
|
||||
target_aum=35_000, aums=aums)
|
||||
assert set(series) == {"positioning", "bybit_4edge"} and all(len(s) > 1 for s in series.values())
|
||||
assert set(ceilings) == {"positioning", "bybit_4edge"} # each has a ceiling (float or None)
|
||||
```
|
||||
- [ ] **Step 3: Run** → FAIL (module missing).
|
||||
- [ ] **Step 4: Implement** the provider per the Step-1 mapping: for each crypto deploy sid, build the honest series at `target_aum` and the capacity curve across `aums` → ceiling; for a non-crypto sid, ceiling = `equity_default_ceiling` (series omitted, caller falls back). Return `(series, ceilings)`.
|
||||
- [ ] **Step 5: Run** the test → PASS. **Commit** `git add -A && git commit -m "feat(alloc): honest-reference series+ceiling provider per deploy strategy"`
|
||||
|
||||
---
|
||||
|
||||
### Task 5: `record_allocation` sizes from the provider + applies the capacity cap
|
||||
|
||||
**Files:** Modify `src/fxhnt/application/allocation_engine.py` (`target_capital` capacity cap) and `src/fxhnt/application/allocation_ingest.py`; Test `tests/integration/test_record_allocation_capacity.py`.
|
||||
|
||||
**Interfaces:** `target_capital(target_weights, equity, policy, *, capacity_ceiling=None)` — new optional per-strategy dollar ceiling.
|
||||
|
||||
- [ ] **Step 1: Write the failing test** for the capacity cap in `target_capital`:
|
||||
```python
|
||||
def test_target_capital_caps_at_per_strategy_capacity():
|
||||
from fxhnt.application.allocation_engine import target_capital
|
||||
from fxhnt.application.allocation_policy import AllocationPolicy
|
||||
pol = AllocationPolicy(capital_ceiling=1_000_000.0)
|
||||
w = {"cap_limited": 0.5, "robust": 0.5}
|
||||
d = target_capital(w, equity=1_000_000.0, policy=pol,
|
||||
capacity_ceiling={"cap_limited": 100_000.0}) # cap_limited breaks at $100k
|
||||
assert d["cap_limited"] <= 100_000.0 and d["robust"] > d["cap_limited"]
|
||||
```
|
||||
- [ ] **Step 2: Run** → FAIL (no `capacity_ceiling` param).
|
||||
- [ ] **Step 3: Implement** the cap in `target_capital` (apply `min(dollars, capacity_ceiling[s])` per strategy after the existing `capital_ceiling` scale; `None`/missing → no cap). Then wire `record_allocation`: build `returns` and `ceilings` from `honest_allocation_inputs(...)` (target_aum = the equity) instead of `nav_history`, and pass `capacity_ceiling=ceilings` to `target_capital`. Keep the deploy+promoted gating, funding-decay, and killswitch exactly as-is.
|
||||
- [ ] **Step 4: Run** the target_capital test AND an integration test of `record_allocation` against a fake store + a small in-memory DB (model on existing `allocation_ingest` tests if present; else assert the written `StrategyAllocationRow`s respect the ceilings). Expected: PASS.
|
||||
- [ ] **Step 5: Commit** `git add -A && git commit -m "feat(alloc): record_allocation sizes from honest-reference + caps at capacity (spec 4b/4c)"`
|
||||
|
||||
---
|
||||
|
||||
### Task 6: Policy knobs + capacity-ceiling visibility
|
||||
|
||||
**Files:** Modify `src/fxhnt/application/allocation_policy.py`; Modify the honest-report/cockpit display; Test `tests/unit/test_allocation_policy.py`.
|
||||
|
||||
**Interfaces:** `AllocationPolicy` gains `capacity_min_sharpe: float = 1.0` and `equity_default_ceiling: float = 10_000_000.0`; `ALLOCATION_POLICY_VERSION` bumped to 3.
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
```python
|
||||
def test_policy_has_capacity_knobs_and_version_bumped():
|
||||
from fxhnt.application.allocation_policy import AllocationPolicy, ALLOCATION_POLICY_VERSION
|
||||
p = AllocationPolicy()
|
||||
assert p.capacity_min_sharpe == 1.0 and p.equity_default_ceiling == 10_000_000.0
|
||||
assert ALLOCATION_POLICY_VERSION >= 3
|
||||
```
|
||||
- [ ] **Step 2: Run** → FAIL.
|
||||
- [ ] **Step 3: Implement** — add the two fields to `AllocationPolicy`; bump `ALLOCATION_POLICY_VERSION = 3`. Surface the per-strategy capacity ceiling in `build_honest_report`'s output (a `"capacity_ceiling"` per sleeve/book next to its curve) so the threshold choice is visible (spec §8).
|
||||
- [ ] **Step 4: Run** the test + the honest-report test (confirm the new key is present, existing shape intact). Expected: PASS.
|
||||
- [ ] **Step 5: Commit** `git add -A && git commit -m "feat(alloc): capacity policy knobs + ceiling shown next to the curve (spec 6/8)"`
|
||||
|
||||
---
|
||||
|
||||
## Self-Review
|
||||
|
||||
**Spec coverage:** §4a inverse-vol → Tasks 1,3. §4b capacity cap → Tasks 2,5. §4c sizing-from-backtest → Tasks 4,5. §5 granularity (book/sleeve/equity-default) → Task 4. §6 forward-gating → unchanged (reused; Task 5 keeps deploy+promoted). §7 scope (paper, no live) → all tasks read-only-ish (record_allocation writes only the allocation table it already owns; no new execution). §8 ceiling visibility → Task 6. §9 decisions → knobs Task 6.
|
||||
|
||||
**Placeholder scan:** Tasks 1-3,6 have complete code. Task 4 Step 1 is a deliberate READ (the deploy-strategy→honest mapping must be confirmed against the real registry/store before coding — the crux), then Steps 4 gives the mapping to implement; Task 5 Step 3 describes the wiring against the confirmed provider signature. Acceptable — these are integration tasks whose exact store fixtures depend on Task 4's confirmed API, not hidden work.
|
||||
|
||||
**Type consistency:** honest series are `dict[int,float]` (epoch-day keyed, matching `honest_sleeve_returns`); `full_history_vol`/`compute_allocation` take `dict[str,float]` (date-string keyed) — **the provider (Task 4) must return the series in the shape `compute_allocation` expects (str-keyed dates)** or convert at the seam; call this out in Task 4/5 and add an assertion. Weights/ceilings are `dict[str,float]`.
|
||||
|
||||
---
|
||||
|
||||
## Execution Handoff
|
||||
|
||||
Plan complete and saved to `docs/superpowers/plans/2026-07-15-capacity-aware-capital-allocator.md`. Two execution options:
|
||||
1. **Subagent-Driven (recommended)** — fresh subagent per task, review between tasks (this is what caught the honest-reference bugs).
|
||||
2. **Inline Execution** — execute in this session with checkpoints.
|
||||
|
||||
Which approach?
|
||||
@@ -0,0 +1,589 @@
|
||||
# Crypto Honest-Reference Pipeline — Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Build a coin-level capacity-, cost-, and leverage-honest backtest reference for the Bybit crypto sleeves, plus the falsification gates, risk-parity allocation, ex-ante risk layer, and a gated pump-detection candidate edge — as a pure research/report pipeline that does not touch live sizing or the cockpit.
|
||||
|
||||
**Architecture:** Reuse the existing `sleeve_weights_and_contributions(store, sleeve)` → `{day:{symbol:(weight, ret)}}` engine (no edge logic reimplemented). A new `honest_reference.py` applies a per-coin ADV participation cap (from `store.read_panel(symbols, "turnover")`) + square-root market-impact slippage (floored at the coin's `bybit_real_spread`) and re-aggregates to a NET daily return, parameterized by AUM. Gates, risk-parity weights, leverage-DD, and the pump candidate are pure functions on those return series. A CLI report assembles the numbers. Everything is read-only.
|
||||
|
||||
**Tech Stack:** Python 3.12, stdlib `statistics`/`math`, the existing `_FeatureStore` Protocol, `nav_stats.nav_backfill_stats`, pytest. No new dependencies.
|
||||
|
||||
## Global Constraints
|
||||
|
||||
- **Research/validation scope only.** No change to live sizing, the cockpit, `backtest_summary`, or the recon gate. No writes to prod tables.
|
||||
- **Reuse the real sleeve engines** via `sleeve_weights_and_contributions` — never reimplement edge logic.
|
||||
- **Fidelity is a hard stop:** with the cap disabled (AUM→small) and impact off (`impact_k=0`, no spread), the honest series MUST equal `sleeve_returns_from_store(cost_bps=0)` (== `reconciliation_series(...)`), within 1e-9. A mismatch blocks all downstream tasks.
|
||||
- `participation_cap` = **0.03**; AUM curve = **{35_000, 100_000, 350_000, 1_000_000, 10_000_000}**; impact-`k` sensitivity = **{1.0, 2.0, 4.0}**. Single global knobs, never per-sleeve tuned.
|
||||
- Weighting is **risk-parity / inverse-vol on honest vol** — never an in-sample Sharpe grid.
|
||||
- **G4 anti-reactive:** any de-risk rule must beat "do nothing" in an honest A/B or it is excluded. No reactive de-sizing (vol-target / drawdown-trigger) — A/B-rejected 3× historically.
|
||||
- Pure + network-free functions (inputs injected), matching the `bybit_*_eval.py` pattern.
|
||||
- Tests run with `pytest -o addopts=""` in the dev sandbox (the repo `addopts` injects `-n` / xdist which is absent locally).
|
||||
- `unlock` is **not hard-cut**; G1/G2 decide its weight (expected ≈ 0).
|
||||
|
||||
## File Structure
|
||||
|
||||
- `src/fxhnt/application/honest_reference.py` — core: `honest_sleeve_returns`, `capacity_curve`, `spread_floor_from_snapshot`. One responsibility: turn coin weights + turnover + spread into NET return series per AUM.
|
||||
- `src/fxhnt/application/honest_gates.py` — `risk_parity_weights`, `book_returns`, `gate_survives` (G1), `loo_marginal` (G2), `beats_concentration` (G3), `ab_beats_nothing` (G4).
|
||||
- `src/fxhnt/application/honest_leverage.py` — `max_drawdown_at_leverage`, `leverage_ceiling`.
|
||||
- `src/fxhnt/application/pump_detection.py` — `pump_signal`, `pump_sleeve_weights`, the candidate edge #4.
|
||||
- `src/fxhnt/adapters/persistence/bybit_spread.py` — `read_spread_snapshot(dsn) -> dict[str,float]` (reads `bybit_real_spread`, coin→spread_bps). (New tiny reader; `read_panel` covers `turnover` already.)
|
||||
- `bin/fxt`/CLI: `honest-reference-report` command → prints/writes the research table. (Wire into the existing typer app in `src/fxhnt/cli.py`.)
|
||||
- Tests: `tests/integration/test_honest_reference.py`, `tests/unit/test_honest_gates.py`, `tests/unit/test_honest_leverage.py`, `tests/unit/test_pump_detection.py`.
|
||||
|
||||
---
|
||||
|
||||
### Task 1: Honest return core + fidelity invariant (spec §5 0a/0d)
|
||||
|
||||
**Files:**
|
||||
- Create: `src/fxhnt/application/honest_reference.py`
|
||||
- Test: `tests/integration/test_honest_reference.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `sleeve_weights_and_contributions(store, sleeve, *, universe, unlock_events) -> dict[int, dict[str, tuple[float,float]]]` and `reconciliation_series(weights_by_day) -> dict[int,float]` (both `application/bybit_paper_weights.py`); `sleeve_returns_from_store(store, edge, cost_bps=0.0)` (`application/bybit_book_eval.py`).
|
||||
- Produces: `honest_sleeve_returns(weights_by_day, turnover_panel, spread_bps, *, aum, participation_cap=0.03, impact_k=1.0) -> dict[int,float]`.
|
||||
|
||||
- [ ] **Step 1: Write the failing fidelity test**
|
||||
|
||||
```python
|
||||
# tests/integration/test_honest_reference.py
|
||||
from fxhnt.application.honest_reference import honest_sleeve_returns
|
||||
|
||||
def test_no_cap_no_impact_reproduces_gross_reconciliation():
|
||||
# {day: {symbol: (weight, ret)}} — the exact shape sleeve_weights_and_contributions returns.
|
||||
wbd = {1: {"AAA": (0.5, 0.02), "BBB": (-0.5, -0.01)},
|
||||
2: {"AAA": (0.4, -0.03), "BBB": (-0.6, 0.05)}}
|
||||
turnover = {"AAA": {1: 1e9, 2: 1e9}, "BBB": {1: 1e9, 2: 1e9}} # huge ADV -> cap never binds
|
||||
# cap disabled via tiny AUM relative to ADV, impact off -> must equal Σ w·r per day
|
||||
out = honest_sleeve_returns(wbd, turnover, spread_bps={}, aum=1_000.0,
|
||||
participation_cap=0.03, impact_k=0.0)
|
||||
assert abs(out[1] - (0.5*0.02 + -0.5*-0.01)) < 1e-9
|
||||
assert abs(out[2] - (0.4*-0.03 + -0.6*0.05)) < 1e-9
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `pytest tests/integration/test_honest_reference.py::test_no_cap_no_impact_reproduces_gross_reconciliation -v -o addopts=""`
|
||||
Expected: FAIL — `ModuleNotFoundError: fxhnt.application.honest_reference`.
|
||||
|
||||
- [ ] **Step 3: Write minimal implementation**
|
||||
|
||||
```python
|
||||
# src/fxhnt/application/honest_reference.py
|
||||
"""Capacity- and cost-honest daily return series for a Bybit sleeve, from coin-level weights.
|
||||
|
||||
Pure + network-free: coin weights (sleeve_weights_and_contributions), the per-coin turnover (ADV) panel,
|
||||
and the per-coin quoted spread are INJECTED. With the cap non-binding (huge ADV vs AUM) and impact off
|
||||
(impact_k=0, no spread) this reduces to Σ_symbol weight·ret == reconciliation_series (the fidelity invariant)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
|
||||
|
||||
def honest_sleeve_returns(
|
||||
weights_by_day: dict[int, dict[str, tuple[float, float]]],
|
||||
turnover_panel: dict[str, dict[int, float]],
|
||||
spread_bps: dict[str, float],
|
||||
*, aum: float, participation_cap: float = 0.03, impact_k: float = 1.0,
|
||||
) -> dict[int, float]:
|
||||
out: dict[int, float] = {}
|
||||
for day, row in weights_by_day.items():
|
||||
gross = 0.0
|
||||
cost = 0.0
|
||||
for sym, (w, r) in row.items():
|
||||
adv = turnover_panel.get(sym, {}).get(day, 0.0)
|
||||
target_notional = abs(w) * aum
|
||||
# capacity: never hold more than participation_cap * ADV; scale the weight down if we would.
|
||||
if adv > 0.0 and target_notional > participation_cap * adv:
|
||||
scale = (participation_cap * adv) / target_notional
|
||||
else:
|
||||
scale = 1.0
|
||||
w_capped = w * scale
|
||||
gross += w_capped * r
|
||||
# slippage on the traded notional (approx daily full rebalance == |w_capped|·aum).
|
||||
traded_notional = abs(w_capped) * aum
|
||||
if traded_notional > 0.0 and adv > 0.0:
|
||||
impact = impact_k * math.sqrt(traded_notional / adv) * 1e4 # bps
|
||||
else:
|
||||
impact = 0.0
|
||||
slip_bps = max(impact, spread_bps.get(sym, 0.0))
|
||||
cost += abs(w_capped) * (slip_bps / 1e4)
|
||||
out[day] = gross - cost
|
||||
return out
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run test to verify it passes**
|
||||
|
||||
Run: `pytest tests/integration/test_honest_reference.py::test_no_cap_no_impact_reproduces_gross_reconciliation -v -o addopts=""`
|
||||
Expected: PASS.
|
||||
|
||||
- [ ] **Step 5: Add the REAL-engine fidelity test (the hard stop) + commit**
|
||||
|
||||
Add a test that builds a small fake `_FeatureStore` (model on `tests/integration/test_bybit_book_reconciliation.py`), runs `sleeve_weights_and_contributions(store, "xsfunding")`, and asserts `honest_sleeve_returns(..., aum=1.0, impact_k=0.0, spread_bps={})` equals `sleeve_returns_from_store(store, "xsfunding", cost_bps=0.0)` day-by-day within 1e-9. Then:
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/application/honest_reference.py tests/integration/test_honest_reference.py
|
||||
git commit -m "feat(honest-ref): coin-level honest return core + fidelity invariant (spec 0a/0d)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 2: Capacity cap parameterized by AUM (spec §5 0b)
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/application/honest_reference.py` (already caps in Task 1 — this task adds `capacity_curve`)
|
||||
- Test: `tests/integration/test_honest_reference.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: `capacity_curve(weights_by_day, turnover_panel, spread_bps, *, aums, participation_cap=0.03, impact_k=1.0) -> dict[float, dict[int,float]]` (AUM → net series).
|
||||
|
||||
- [ ] **Step 1: Write the failing test — the cap bites at high AUM, not low**
|
||||
|
||||
```python
|
||||
def test_capacity_cap_bites_only_at_high_aum():
|
||||
from fxhnt.application.honest_reference import honest_sleeve_returns
|
||||
wbd = {1: {"THIN": (1.0, 0.10)}} # +10% day on a thin coin
|
||||
turnover = {"THIN": {1: 1_000_000.0}} # $1M/day ADV
|
||||
lo = honest_sleeve_returns(wbd, turnover, {}, aum=35_000.0, participation_cap=0.03, impact_k=0.0)
|
||||
hi = honest_sleeve_returns(wbd, turnover, {}, aum=10_000_000.0, participation_cap=0.03, impact_k=0.0)
|
||||
# at $35k: target $35k > 3%*$1M=$30k -> scaled to 30/35; at $10M: target $10M -> scaled to 30k/10M (tiny)
|
||||
assert abs(lo[1] - 0.10 * (30_000/35_000)) < 1e-9
|
||||
assert hi[1] < lo[1] # capacity destroys the return at scale
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run** `pytest tests/integration/test_honest_reference.py::test_capacity_cap_bites_only_at_high_aum -v -o addopts=""` — Expected: PASS (the cap logic is already in Task 1; this locks its behavior).
|
||||
|
||||
- [ ] **Step 3: Implement `capacity_curve`**
|
||||
|
||||
```python
|
||||
def capacity_curve(weights_by_day, turnover_panel, spread_bps, *, aums,
|
||||
participation_cap=0.03, impact_k=1.0):
|
||||
return {aum: honest_sleeve_returns(weights_by_day, turnover_panel, spread_bps,
|
||||
aum=aum, participation_cap=participation_cap, impact_k=impact_k)
|
||||
for aum in aums}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Write + run a test that `capacity_curve` returns one series per AUM and is monotone-non-increasing in Sharpe** (compute `nav_backfill_stats` on the cumulative nav of each). Expected: PASS.
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/application/honest_reference.py tests/integration/test_honest_reference.py
|
||||
git commit -m "feat(honest-ref): AUM-parameterized capacity curve (spec 0b)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 3: Market-impact slippage + spread floor + k-sensitivity (spec §5 0c)
|
||||
|
||||
**Files:**
|
||||
- Create: `src/fxhnt/adapters/persistence/bybit_spread.py`
|
||||
- Modify: `src/fxhnt/application/honest_reference.py` (add `spread_floor_from_snapshot`)
|
||||
- Test: `tests/integration/test_honest_reference.py`, `tests/unit/test_bybit_spread.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: `read_spread_snapshot(dsn) -> dict[str,float]` (coin→spread_bps); `honest_sleeve_returns(..., impact_k)` already charges impact; this task locks its behavior + the spread floor.
|
||||
|
||||
- [ ] **Step 1: Write the failing test — impact scales with √(size/ADV) and floors at spread**
|
||||
|
||||
```python
|
||||
def test_impact_scales_and_floors_at_spread():
|
||||
from fxhnt.application.honest_reference import honest_sleeve_returns
|
||||
wbd = {1: {"C": (1.0, 0.0)}} # flat return -> net = -cost
|
||||
turnover = {"C": {1: 1_000_000.0}}
|
||||
# size $10k of $1M ADV -> impact = k*sqrt(0.01)*1e4 = k*1000 bps; k=1 -> 1000 bps? scale by traded frac.
|
||||
r = honest_sleeve_returns(wbd, turnover, spread_bps={}, aum=10_000.0, participation_cap=1.0, impact_k=1.0)
|
||||
# cost = |w|*(impact/1e4); impact = 1*sqrt(10000/1e6)*1e4 = 1000 bps -> cost = 1*0.10 = 0.10
|
||||
assert abs(r[1] - (-0.10)) < 1e-6
|
||||
# spread floor: with a wide spread and impact_k=0, cost == spread fraction
|
||||
r2 = honest_sleeve_returns(wbd, turnover, spread_bps={"C": 50.0}, aum=10_000.0,
|
||||
participation_cap=1.0, impact_k=0.0)
|
||||
assert abs(r2[1] - (-50.0/1e4)) < 1e-9
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run** the test — Expected: PASS (impact + floor already implemented in Task 1; this locks it).
|
||||
|
||||
- [ ] **Step 3: Implement the spread reader**
|
||||
|
||||
```python
|
||||
# src/fxhnt/adapters/persistence/bybit_spread.py
|
||||
"""Read the current bybit_real_spread snapshot (coin -> spread_bps). Read-only; a point-in-time cross-section
|
||||
(all rows share one fetched_at), used as the per-coin slippage FLOOR in the honest-reference model."""
|
||||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import create_engine, text
|
||||
|
||||
|
||||
def read_spread_snapshot(dsn: str) -> dict[str, float]:
|
||||
eng = create_engine(dsn)
|
||||
with eng.connect() as c:
|
||||
rows = c.execute(text("select coin, spread_bps from bybit_real_spread")).all()
|
||||
return {coin: float(bps) for coin, bps in rows if bps is not None}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Write `tests/unit/test_bybit_spread.py`** using an in-memory SQLite with a `bybit_real_spread(coin, spread_bps)` table + two rows; assert `read_spread_snapshot` returns the dict. Run — Expected: PASS.
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/adapters/persistence/bybit_spread.py tests/unit/test_bybit_spread.py tests/integration/test_honest_reference.py
|
||||
git commit -m "feat(honest-ref): sqrt-impact slippage + real-spread floor reader (spec 0c)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 4: Risk-parity weighting + book returns (spec §7 Phase 2)
|
||||
|
||||
**Files:**
|
||||
- Create: `src/fxhnt/application/honest_gates.py`
|
||||
- Test: `tests/unit/test_honest_gates.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: honest per-sleeve series `dict[int,float]` (Task 1).
|
||||
- Produces: `risk_parity_weights(returns_by_sleeve) -> dict[str,float]` (∝ 1/vol, sum to 1); `book_returns(returns_by_sleeve, weights) -> dict[int,float]` (weighted, over common days).
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
```python
|
||||
# tests/unit/test_honest_gates.py
|
||||
from fxhnt.application.honest_gates import risk_parity_weights, book_returns
|
||||
|
||||
def test_risk_parity_weights_inverse_vol_sum_to_one():
|
||||
rbs = {"lowvol": {1: 0.001, 2: -0.001, 3: 0.001}, # tiny vol -> big weight
|
||||
"hivol": {1: 0.10, 2: -0.10, 3: 0.10}} # big vol -> small weight
|
||||
w = risk_parity_weights(rbs)
|
||||
assert abs(sum(w.values()) - 1.0) < 1e-9
|
||||
assert w["lowvol"] > w["hivol"]
|
||||
|
||||
def test_book_returns_weighted_over_common_days():
|
||||
rbs = {"a": {1: 0.02, 2: 0.04}, "b": {2: 0.10, 3: 0.20}} # common day = 2 only
|
||||
br = book_returns(rbs, {"a": 0.5, "b": 0.5})
|
||||
assert set(br) == {2} and abs(br[2] - (0.5*0.04 + 0.5*0.10)) < 1e-9
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run** `pytest tests/unit/test_honest_gates.py -v -o addopts=""` — Expected: FAIL (module missing).
|
||||
|
||||
- [ ] **Step 3: Implement**
|
||||
|
||||
```python
|
||||
# src/fxhnt/application/honest_gates.py
|
||||
"""Falsification gates + risk-parity allocation on the honest per-sleeve return series. Pure functions."""
|
||||
from __future__ import annotations
|
||||
|
||||
import statistics as st
|
||||
|
||||
|
||||
def risk_parity_weights(returns_by_sleeve: dict[str, dict[int, float]]) -> dict[str, float]:
|
||||
inv = {}
|
||||
for s, series in returns_by_sleeve.items():
|
||||
vals = list(series.values())
|
||||
vol = st.pstdev(vals) if len(vals) > 1 else 0.0
|
||||
inv[s] = (1.0 / vol) if vol > 0.0 else 0.0
|
||||
total = sum(inv.values())
|
||||
if total <= 0.0:
|
||||
n = len(returns_by_sleeve)
|
||||
return {s: 1.0 / n for s in returns_by_sleeve} if n else {}
|
||||
return {s: v / total for s, v in inv.items()}
|
||||
|
||||
|
||||
def book_returns(returns_by_sleeve: dict[str, dict[int, float]],
|
||||
weights: dict[str, float]) -> dict[int, float]:
|
||||
days = None
|
||||
for series in returns_by_sleeve.values():
|
||||
days = set(series) if days is None else (days & set(series))
|
||||
days = sorted(days or set())
|
||||
return {d: sum(weights.get(s, 0.0) * returns_by_sleeve[s][d] for s in returns_by_sleeve) for d in days}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run** `pytest tests/unit/test_honest_gates.py -v -o addopts=""` — Expected: PASS.
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/application/honest_gates.py tests/unit/test_honest_gates.py
|
||||
git commit -m "feat(honest-ref): risk-parity weights + book aggregation (spec Phase 2)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 5: Gates G1 survives, G2 LOO, G3 beats-concentration (spec §6)
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/application/honest_gates.py`
|
||||
- Test: `tests/unit/test_honest_gates.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `risk_parity_weights`, `book_returns` (Task 4); `nav_backfill_stats(nav: list[float]) -> BackfillStats(.sharpe)` (`application/nav_stats.py`).
|
||||
- Produces: `sharpe_of(series) -> float`; `gate_survives(series, *, min_sharpe=1.0) -> bool`; `loo_marginal(returns_by_sleeve) -> dict[str,float]` (full_book_sharpe − drop-one_sharpe per sleeve); `beats_concentration(returns_by_sleeve) -> bool` (rp-book Sharpe > best single sleeve).
|
||||
|
||||
- [ ] **Step 1: Write the failing tests**
|
||||
|
||||
```python
|
||||
def test_gate_survives_threshold():
|
||||
from fxhnt.application.honest_gates import gate_survives
|
||||
strong = {i: (0.01 if i % 2 else -0.002) for i in range(400)} # positive drift, modest vol
|
||||
assert gate_survives(strong, min_sharpe=1.0) is True
|
||||
flat = {i: (0.001 if i % 2 else -0.001) for i in range(400)}
|
||||
assert gate_survives(flat, min_sharpe=1.0) is False
|
||||
|
||||
def test_loo_marginal_flags_dead_weight():
|
||||
from fxhnt.application.honest_gates import loo_marginal
|
||||
# 'dead' is pure noise; dropping it should NOT hurt (marginal ~ <= 0)
|
||||
good = {i: (0.01 if i % 2 else -0.003) for i in range(400)}
|
||||
dead = {i: (0.002 if i % 3 else -0.002) for i in range(400)}
|
||||
m = loo_marginal({"good": good, "dead": dead})
|
||||
assert m["good"] > m["dead"]
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run** — Expected: FAIL (functions missing).
|
||||
|
||||
- [ ] **Step 3: Implement**
|
||||
|
||||
```python
|
||||
# add to honest_gates.py
|
||||
import math
|
||||
from fxhnt.application.nav_stats import nav_backfill_stats
|
||||
|
||||
|
||||
def _nav(series: dict[int, float]) -> list[float]:
|
||||
nav, v = [], 1.0
|
||||
for d in sorted(series):
|
||||
v *= (1.0 + series[d])
|
||||
nav.append(v)
|
||||
return [1.0] + nav
|
||||
|
||||
|
||||
def sharpe_of(series: dict[int, float]) -> float:
|
||||
return nav_backfill_stats(_nav(series)).sharpe
|
||||
|
||||
|
||||
def gate_survives(series: dict[int, float], *, min_sharpe: float = 1.0) -> bool:
|
||||
return sharpe_of(series) > min_sharpe
|
||||
|
||||
|
||||
def loo_marginal(returns_by_sleeve: dict[str, dict[int, float]]) -> dict[str, float]:
|
||||
full = sharpe_of(book_returns(returns_by_sleeve, risk_parity_weights(returns_by_sleeve)))
|
||||
out = {}
|
||||
for drop in returns_by_sleeve:
|
||||
rest = {s: v for s, v in returns_by_sleeve.items() if s != drop}
|
||||
rest_sh = sharpe_of(book_returns(rest, risk_parity_weights(rest))) if rest else 0.0
|
||||
out[drop] = full - rest_sh
|
||||
return out
|
||||
|
||||
|
||||
def beats_concentration(returns_by_sleeve: dict[str, dict[int, float]]) -> bool:
|
||||
book_sh = sharpe_of(book_returns(returns_by_sleeve, risk_parity_weights(returns_by_sleeve)))
|
||||
best_single = max((sharpe_of(v) for v in returns_by_sleeve.values()), default=0.0)
|
||||
return book_sh > best_single
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run** `pytest tests/unit/test_honest_gates.py -v -o addopts=""` — Expected: PASS.
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/application/honest_gates.py tests/unit/test_honest_gates.py
|
||||
git commit -m "feat(honest-ref): gates G1 survives / G2 LOO / G3 beats-concentration (spec 6)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 6: Leverage ceiling vs real drawdown + G4 anti-reactive (spec §7 book layer, §6 G4)
|
||||
|
||||
**Files:**
|
||||
- Create: `src/fxhnt/application/honest_leverage.py`
|
||||
- Modify: `src/fxhnt/application/honest_gates.py` (add `ab_beats_nothing`)
|
||||
- Test: `tests/unit/test_honest_leverage.py`, `tests/unit/test_honest_gates.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: `max_drawdown_at_leverage(book_series, leverage) -> float` (negative); `leverage_ceiling(book_series, *, dd_limit=-0.35, stress=2.0, grid=(1.0,1.25,1.5,1.75,2.0)) -> float`; `ab_beats_nothing(base_series, derisked_series) -> bool`.
|
||||
|
||||
- [ ] **Step 1: Write the failing tests**
|
||||
|
||||
```python
|
||||
# tests/unit/test_honest_leverage.py
|
||||
from fxhnt.application.honest_leverage import max_drawdown_at_leverage, leverage_ceiling
|
||||
|
||||
def test_drawdown_scales_with_leverage():
|
||||
s = {1: 0.05, 2: -0.10, 3: 0.04}
|
||||
dd1 = max_drawdown_at_leverage(s, 1.0)
|
||||
dd2 = max_drawdown_at_leverage(s, 2.0)
|
||||
assert dd1 < 0.0 and dd2 < dd1 # 2x is a deeper (more negative) drawdown
|
||||
|
||||
def test_leverage_ceiling_respects_stress_buffer():
|
||||
s = {1: 0.02, 2: -0.05, 3: 0.03, 4: -0.02} # mild book
|
||||
# with a 2x stress buffer under -35%, the ceiling picks the largest grid L whose 2*maxDD >= -35%
|
||||
lev = leverage_ceiling(s, dd_limit=-0.35, stress=2.0)
|
||||
assert 1.0 <= lev <= 2.0
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run** — Expected: FAIL (module missing).
|
||||
|
||||
- [ ] **Step 3: Implement**
|
||||
|
||||
```python
|
||||
# src/fxhnt/application/honest_leverage.py
|
||||
"""Real fat-tail drawdown of the honest book at constant leverage, and the ex-ante leverage ceiling
|
||||
(K(D*)) — anchored to a REAL drawdown with a stress buffer, never to a smooth modeled curve."""
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
|
||||
|
||||
def max_drawdown_at_leverage(book_series: dict[int, float], leverage: float) -> float:
|
||||
peak, trough, v = 1.0, 0.0, 1.0
|
||||
for d in sorted(book_series):
|
||||
step = 1.0 + leverage * book_series[d]
|
||||
if step <= 0.0: # wipeout at this leverage
|
||||
return -1.0
|
||||
v *= step
|
||||
peak = max(peak, v)
|
||||
trough = min(trough, v / peak - 1.0)
|
||||
return trough
|
||||
|
||||
|
||||
def leverage_ceiling(book_series, *, dd_limit=-0.35, stress=2.0,
|
||||
grid=(1.0, 1.25, 1.5, 1.75, 2.0)) -> float:
|
||||
ok = [L for L in grid if stress * max_drawdown_at_leverage(book_series, L) >= dd_limit]
|
||||
return max(ok) if ok else 0.0
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Add `ab_beats_nothing` to `honest_gates.py` + test** (G4): a de-risk variant series only "wins" if its Sharpe exceeds the base by a margin AND it is not merely lower-vol; keep it simple — `sharpe_of(derisked) > sharpe_of(base)`. Test that an identical series does NOT beat nothing.
|
||||
|
||||
```python
|
||||
def ab_beats_nothing(base_series: dict[int, float], derisked_series: dict[int, float]) -> bool:
|
||||
return sharpe_of(derisked_series) > sharpe_of(base_series)
|
||||
```
|
||||
|
||||
- [ ] **Step 5: Run both test files + commit**
|
||||
|
||||
Run: `pytest tests/unit/test_honest_leverage.py tests/unit/test_honest_gates.py -v -o addopts=""` — Expected: PASS.
|
||||
```bash
|
||||
git add src/fxhnt/application/honest_leverage.py src/fxhnt/application/honest_gates.py tests/unit/test_honest_leverage.py tests/unit/test_honest_gates.py
|
||||
git commit -m "feat(honest-ref): leverage ceiling vs real DD + G4 anti-reactive gate (spec 7)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 7: Candidate edge #4 — pump/anomaly detection (spec §8)
|
||||
|
||||
**Files:**
|
||||
- Create: `src/fxhnt/application/pump_detection.py`
|
||||
- Test: `tests/unit/test_pump_detection.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `store.read_panel(symbols, "turnover")` and `crypto_close_panel()`; `honest_sleeve_returns` (Task 1) for the execution gate.
|
||||
- Produces: `pump_signal(close_panel, turnover_panel, *, z=3.0) -> dict[int, dict[str, float]]` (day→coin→signed target weight, ONLY on same-day-observable anomalies, no look-ahead); `evaluate_pump_edge(weights_by_day, turnover_panel, spread_bps, *, aum, impact_k) -> dict` (returns net Sharpe + the 3-gate verdict).
|
||||
|
||||
- [ ] **Step 1: Write the failing test — signal only fires on an anomaly, uses only past data**
|
||||
|
||||
```python
|
||||
# tests/unit/test_pump_detection.py
|
||||
from fxhnt.application.pump_detection import pump_signal
|
||||
|
||||
def test_signal_fires_on_volume_zspike_and_is_causal():
|
||||
# day 10 has a turnover z-spike vs the trailing window -> a signal; earlier calm days -> none.
|
||||
turnover = {"X": {d: 1_000_000.0 for d in range(1, 10)}}
|
||||
turnover["X"][10] = 50_000_000.0 # 50x spike
|
||||
close = {"X": {d: 100.0 for d in range(1, 11)}}
|
||||
sig = pump_signal(close, turnover, z=3.0)
|
||||
assert 10 in sig and "X" in sig[10] # fires on the spike day
|
||||
assert all("X" not in sig.get(d, {}) for d in range(1, 10)) # not before (causal)
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run** — Expected: FAIL (module missing).
|
||||
|
||||
- [ ] **Step 3: Implement** a causal detector: for each coin-day, z-score today's turnover vs the trailing N-day window (strictly `< day`); emit a signed target (e.g. short the pump, `-1`, capped) when `z > z`. No future data touched.
|
||||
|
||||
```python
|
||||
# src/fxhnt/application/pump_detection.py
|
||||
"""Candidate edge #4: causal pump/anomaly detection. A per-coin, same-day-observable turnover z-spike ->
|
||||
a (short-the-pump) target weight. Must be run through honest_sleeve_returns to see if it survives the real
|
||||
pump-day slippage (the killer). Pure; no look-ahead."""
|
||||
from __future__ import annotations
|
||||
|
||||
import statistics as st
|
||||
|
||||
_WINDOW = 20
|
||||
|
||||
|
||||
def pump_signal(close_panel, turnover_panel, *, z: float = 3.0) -> dict[int, dict[str, float]]:
|
||||
out: dict[int, dict[str, float]] = {}
|
||||
for sym, series in turnover_panel.items():
|
||||
days = sorted(series)
|
||||
for i, d in enumerate(days):
|
||||
hist = [series[days[j]] for j in range(max(0, i - _WINDOW), i)] # strictly past
|
||||
if len(hist) < _WINDOW:
|
||||
continue
|
||||
mu, sd = st.mean(hist), st.pstdev(hist)
|
||||
if sd > 0.0 and (series[d] - mu) / sd > z:
|
||||
out.setdefault(d, {})[sym] = -1.0 # short the pump (defined-risk sizing added downstream)
|
||||
return out
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Add `evaluate_pump_edge` + a test** that (a) builds `{day:{sym:(weight, ret)}}` from the signal × a synthetic reversal return, (b) runs `honest_sleeve_returns(..., impact_k=1.0)`, and asserts the returned dict has keys `sharpe`, `survives_execution` (net Sharpe > 0 after slippage), `marginal_vs_positioning`. Run — Expected: PASS.
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/application/pump_detection.py tests/unit/test_pump_detection.py
|
||||
git commit -m "feat(honest-ref): candidate edge #4 causal pump detector + 3-gate eval (spec 8)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 8: Report CLI — the research deliverable (spec §9)
|
||||
|
||||
**Files:**
|
||||
- Create: `src/fxhnt/application/honest_report.py`
|
||||
- Modify: `src/fxhnt/cli.py` (add `honest-reference-report` command)
|
||||
- Test: `tests/integration/test_honest_report.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: everything above + a real `_FeatureStore` (the analytical store), `read_spread_snapshot`, `read_panel(symbols, "turnover")`, `unlock_events` source.
|
||||
- Produces: `build_honest_report(store, spread_bps, unlock_events, *, sleeves, aums, ks) -> dict` (per-sleeve honest stats per (AUM,k), the risk-parity book, gate verdicts G1-G4, leverage ceiling, and the edge-#4 verdict) — pure; the CLI just prints it.
|
||||
|
||||
- [ ] **Step 1: Write the failing test** — `build_honest_report` on a fake store returns a dict with keys `sleeves`, `capacity_curve`, `gates`, `leverage_ceiling`, `edge4`; the `k`-sensitivity has 3 entries; the AUM curve has 5. Run — Expected: FAIL.
|
||||
|
||||
- [ ] **Step 2: Implement `build_honest_report`** — loop sleeves × aums × ks calling `honest_sleeve_returns` + `nav_backfill_stats`; assemble the risk-parity book at the base (aum=100k, k=2); call `gate_survives`/`loo_marginal`/`beats_concentration`/`leverage_ceiling`/`evaluate_pump_edge`. **Survivorship guard:** assert the coin universe count from `read_panel` matches the ingestion's survivorship-free count (log a warning if a survivorship flag is unavailable — do not silently assume).
|
||||
|
||||
- [ ] **Step 3: Wire the CLI command** in `src/fxhnt/cli.py`:
|
||||
|
||||
```python
|
||||
@app.command("honest-reference-report")
|
||||
def honest_reference_report() -> None:
|
||||
"""Research-only: print the capacity/cost/leverage-honest reference table + gate verdicts. Read-only."""
|
||||
from fxhnt.application.honest_report import build_honest_report
|
||||
# ... construct the analytical store + spread snapshot + unlock events, then:
|
||||
import json, typer
|
||||
typer.echo(json.dumps(build_honest_report(...), indent=2, default=str))
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run** `pytest tests/integration/test_honest_report.py -v -o addopts=""` — Expected: PASS. Then run the CLI once against the real store in-cluster (`kubectl exec <dagster-pod> -c daemon -- python -m fxhnt.cli honest-reference-report`) and eyeball the numbers vs the SQL research estimates (xsfunding Sh ≈ 3, unlock collapses, book Sh ≈ 1.7 at P98-equivalent). Discrepancy > ~0.3 Sharpe ⇒ investigate before trusting.
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/application/honest_report.py src/fxhnt/cli.py tests/integration/test_honest_report.py
|
||||
git commit -m "feat(honest-ref): honest-reference research report CLI (spec 9)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Self-Review
|
||||
|
||||
**Spec coverage:** §3 data → Tasks 1/3 (turnover via `read_panel`, spread via `bybit_spread.py`). §5 0a → Task 1 (fidelity invariant). §5 0b → Task 2 (capacity curve, AUM series). §5 0c → Task 3 (sqrt-impact + spread floor + k-sensitivity). §5 0d → Tasks 1/8 (`nav_backfill_stats`). §6 G1/G2/G3 → Task 5. §6 G4 → Task 6. §7 allocation → Task 4; leverage ceiling → Task 6; position cap → Tasks 1-2; tail-drop → G1/G2 (Task 5) surfaced in the report (Task 8). §8 edge #4 → Task 7. §9 report/scope → Task 8. §10 survivorship guard → Task 8 Step 2; slippage-sensitivity → Task 8 `ks`. No live-cockpit change anywhere — confirmed (all read-only, one CLI print). **Gap intentionally deferred:** the "hard ADV cap vs slippage-penalty" belt-and-suspenders — Task 1 implements BOTH (cap scales the weight, impact charges the residual), satisfying §11.
|
||||
|
||||
**Placeholder scan:** no TBD/TODO; every code step shows real code against the real APIs (`sleeve_weights_and_contributions`, `read_panel`, `nav_backfill_stats`, `reconciliation_series`). Task 8 Steps 1-2 describe the loop shape with exact keys rather than 60 lines of assembly — acceptable as the report is pure glue over tested units; the test pins its output keys.
|
||||
|
||||
**Type consistency:** series are `dict[int,float]` throughout; coin weights `dict[int,dict[str,tuple[float,float]]]`; turnover/close panels `dict[str,dict[int,float]]`; spread `dict[str,float]`. `sharpe_of`/`gate_survives`/`book_returns` signatures match across Tasks 4-8.
|
||||
|
||||
---
|
||||
|
||||
## Execution Handoff
|
||||
|
||||
Plan complete and saved to `docs/superpowers/plans/2026-07-15-crypto-honest-reference-pipeline.md`. Two execution options:
|
||||
|
||||
1. **Subagent-Driven (recommended)** — fresh subagent per task, review between tasks, fast iteration.
|
||||
2. **Inline Execution** — execute tasks in this session with checkpoints.
|
||||
|
||||
Which approach?
|
||||
@@ -0,0 +1,262 @@
|
||||
# Book Representation + Venue-Cap Knob Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development. Steps use checkbox (`- [ ]`) tracking.
|
||||
|
||||
**Goal:** Switch the crypto deploy target from per-sleeve (collapses to positioning, ~18% deployed) to the diversified `bybit_4edge` BOOK (measured 75% deployed at vol 5.2% / maxDD −3.2% — 4× more at lower risk), and add a `max_venue_weight` fund-config knob so per-venue concentration (75% on Bybit) is cappable at live.
|
||||
|
||||
**Architecture:** Registry re-points the deploy tier to `bybit_4edge` (positioning/xsfunding/unlock become tracked constituents); the honest-inputs provider re-adds the book series dispatch (keeping the sleeve generalization). A new `max_venue_weight` on `AllocationPolicy` + `apply_venue_caps` engine function caps each venue's total gross; it is a fund_config column set via `fxhnt fund-config`.
|
||||
|
||||
**Tech Stack:** Python 3.12, SQLAlchemy 2.0, typer, pytest. In-cluster measurement already done (book = 75% deploy; funding-stress check PASSED).
|
||||
|
||||
## Global Constraints
|
||||
|
||||
- Basis: the 2026-07-16 measurements — book (`_book_series` risk-parity of {xsfunding,unlock,positioning}) @ $100k: vol 0.0523, maxDD −0.0324, Sharpe 0.161/pd, deploys **75%** at target_vol 0.25 (K hits max_leverage 1.5). Funding-stress check PASSED (crises in-sample, smoothness legitimate).
|
||||
- `max_venue_weight` DEFAULT = **1.0** (no cap → byte-identical to today's deployment). It caps the fraction of `equity` deployable as gross to any single venue.
|
||||
- `record_allocation(policy=None)` and `apply_venue_caps(..., max_venue_weight=1.0)` MUST be byte-identical to today (no-op at the default).
|
||||
- `fund_config` read fallback + partial-upsert semantics unchanged; a missing `max_venue_weight` column/value → default 1.0 (never crash the nightly). Column added to the EXISTING prod table via an idempotent ALTER (mirror `forward_nav._ensure_summary_policy_columns`: Postgres `ADD COLUMN IF NOT EXISTS`, SQLite PRAGMA-check).
|
||||
- Bump `ALLOCATION_POLICY_VERSION` 3 → 4 (policy shape changed).
|
||||
- Marginal LOO gate, ex-ante full-history-vol K, anti-reactive invariant: UNTOUCHED.
|
||||
- Commit trailer: `Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>`.
|
||||
|
||||
---
|
||||
|
||||
### Task A: switch the crypto deploy target to the `bybit_4edge` book
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/registry.py`
|
||||
- Modify: `src/fxhnt/application/allocation_honest_inputs.py`
|
||||
- Modify: `src/fxhnt/adapters/web/templates/sim.html`
|
||||
- Test: `tests/unit/test_allocation_honest_inputs.py`, `tests/unit/test_registry_tiers.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: deploy set = `{bybit_4edge}`; provider sizes `bybit_4edge`→`_book_series`, `{positioning,xsfunding,unlock}`→`_sleeve_series`.
|
||||
|
||||
- [ ] **Step 1: Update the tier tests to the target state (write the expectation first)**
|
||||
|
||||
In `tests/unit/test_registry_tiers.py`: `_DEPLOY` back to `{"bybit_4edge"}` (positioning is now a research constituent). Update the test name/docstring accordingly (deploy tier = exactly the bybit_4edge book).
|
||||
|
||||
- [ ] **Step 2: Run — expect FAIL** (registry still has positioning deploy / bybit_4edge research)
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && .venv/bin/python3 -m pytest tests/unit/test_registry_tiers.py -q`
|
||||
Expected: FAIL.
|
||||
|
||||
- [ ] **Step 3: Registry edits** (`registry.py`):
|
||||
- `bybit_4edge`: `"tier": "research"` → **`"tier": "deploy"`**. Rewrite its leading comment: it is the fund's DEPLOYED crypto book (risk-parity of its sleeves; measured 75% deploy @ 0.25 vol, funding-stress-checked); constituents tracked separately as reference.
|
||||
- `positioning`: `"tier": "deploy"` → **`"tier": "research"`** (constituent of the book; still forward-tracked as reference).
|
||||
- `xsfunding`: **REMOVE** `"promote_on_pass": True` (it is a book constituent, not a standalone deploy edge — must not auto-promote and double-count the book). Keep `tier: research`.
|
||||
- `unlock`: **REMOVE** `"promote_on_pass": True`. Keep `tier: research`.
|
||||
|
||||
- [ ] **Step 4: Provider edits** (`allocation_honest_inputs.py`):
|
||||
- `_CRYPTO_DEPLOY_SIDS = frozenset({"positioning", "xsfunding", "unlock", "bybit_4edge"})`.
|
||||
- `_deploy_series(store, sid, ...)`:
|
||||
```python
|
||||
if sid == "bybit_4edge":
|
||||
return _book_series(store, spread_bps, unlock_events, aum=aum,
|
||||
participation_cap=participation_cap, impact_k=impact_k)
|
||||
if sid in _CRYPTO_DEPLOY_SIDS:
|
||||
return _sleeve_series(store, sid, spread_bps, unlock_events, aum=aum,
|
||||
participation_cap=participation_cap, impact_k=impact_k)
|
||||
return {}
|
||||
```
|
||||
- Update the module docstring's deploy→honest mapping: `bybit_4edge`→the risk-parity BOOK (the deploy target), the three sleeves→their own sleeve series (available if ever deployed standalone).
|
||||
|
||||
- [ ] **Step 5: sim.html** — revert the label: `bybit_4edge` span `· reference` → `· deploy`; and the two comments (lines ~9, ~20) `REFERENCE book`/`(reference, default)` → `DEPLOY book`/`(deploy, default)` (it is the deploy book again).
|
||||
|
||||
- [ ] **Step 6: Update `test_allocation_honest_inputs.py`** — the pre-existing tests from the prior task asserted `_CRYPTO_DEPLOY_SIDS == {positioning,xsfunding,unlock}` and `bybit_4edge` not a crypto deploy sid. Update to the new set (includes `bybit_4edge`) and assert `bybit_4edge` now gets the BOOK series (non-empty, ISO-keyed) via the provider. Keep the sleeve-series assertions.
|
||||
|
||||
- [ ] **Step 7: Run — expect PASS**
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && .venv/bin/python3 -m pytest tests/unit/test_allocation_honest_inputs.py tests/unit/test_registry_tiers.py -q`
|
||||
Expected: PASS.
|
||||
|
||||
- [ ] **Step 8: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/registry.py src/fxhnt/application/allocation_honest_inputs.py src/fxhnt/adapters/web/templates/sim.html tests/unit/test_allocation_honest_inputs.py tests/unit/test_registry_tiers.py
|
||||
git commit -m "feat(alloc): switch crypto deploy target to the bybit_4edge book (75% deploy vs 18% per-sleeve)
|
||||
|
||||
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task B: venue-cap in the allocation engine + policy
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/application/allocation_policy.py` (`max_venue_weight` field, version bump)
|
||||
- Modify: `src/fxhnt/application/allocation_engine.py` (`apply_venue_caps`)
|
||||
- Modify: `src/fxhnt/application/allocation_ingest.py` (wire it into `record_allocation`)
|
||||
- Test: `tests/unit/test_venue_caps.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: `AllocationPolicy.max_venue_weight: float = 1.0`; `apply_venue_caps(dollars, venue_by_sid, max_venue_weight, equity) -> dict[str,float]` (per-venue gross cap, sign-preserving, no-op at ≥1.0).
|
||||
|
||||
- [ ] **Step 1: Write failing tests**
|
||||
|
||||
```python
|
||||
# tests/unit/test_venue_caps.py
|
||||
from fxhnt.application.allocation_engine import apply_venue_caps
|
||||
|
||||
def test_no_cap_at_default_is_identity():
|
||||
d = {"a": 60000.0, "b": -30000.0}
|
||||
v = {"a": "bybit-perp", "b": "ibkr-equity"}
|
||||
assert apply_venue_caps(dict(d), v, 1.0, 100000.0) == d
|
||||
|
||||
def test_single_venue_over_cap_scales_down_sign_preserving():
|
||||
# both on bybit-perp, gross 75k, cap 0.6*100k=60k -> scale 0.8
|
||||
d = {"a": 50000.0, "b": -25000.0}
|
||||
v = {"a": "bybit-perp", "b": "bybit-perp"}
|
||||
out = apply_venue_caps(dict(d), v, 0.6, 100000.0)
|
||||
assert round(out["a"], 2) == 40000.0 and round(out["b"], 2) == -20000.0
|
||||
|
||||
def test_under_cap_venue_untouched():
|
||||
d = {"a": 40000.0, "b": 40000.0}
|
||||
v = {"a": "bybit-perp", "b": "ibkr-equity"} # each venue 40k < 60k cap
|
||||
assert apply_venue_caps(dict(d), v, 0.6, 100000.0) == d
|
||||
|
||||
def test_missing_venue_treated_as_own_bucket():
|
||||
d = {"a": 80000.0}
|
||||
v = {} # sid with no venue -> its own ("") bucket
|
||||
out = apply_venue_caps(dict(d), v, 0.6, 100000.0)
|
||||
assert round(out["a"], 2) == 60000.0
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run — expect FAIL**
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && .venv/bin/python3 -m pytest tests/unit/test_venue_caps.py -q`
|
||||
Expected: FAIL.
|
||||
|
||||
- [ ] **Step 3: Implement `apply_venue_caps`** (`allocation_engine.py`):
|
||||
|
||||
```python
|
||||
def apply_venue_caps(dollars: dict[str, float], venue_by_sid: dict[str, str],
|
||||
max_venue_weight: float, equity: float) -> dict[str, float]:
|
||||
"""Cap each venue's total deployed gross at `max_venue_weight * equity` (sign-preserving, proportional
|
||||
scale-down within the venue). `max_venue_weight >= 1.0` (or non-finite equity) -> no-op. A sid missing from
|
||||
`venue_by_sid` falls in the "" bucket (its own venue)."""
|
||||
if not dollars or max_venue_weight >= 1.0 or not math.isfinite(equity) or equity <= 0.0:
|
||||
return dollars
|
||||
cap = max_venue_weight * equity
|
||||
by_venue: dict[str, list[str]] = {}
|
||||
for sid in dollars:
|
||||
by_venue.setdefault(venue_by_sid.get(sid, ""), []).append(sid)
|
||||
for sids in by_venue.values():
|
||||
gross = sum(abs(dollars[s]) for s in sids)
|
||||
if gross > cap and gross > 0.0:
|
||||
scale = cap / gross
|
||||
for s in sids:
|
||||
dollars[s] *= scale
|
||||
return dollars
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Policy** (`allocation_policy.py`): add `max_venue_weight: float = 1.0 # cap on the fraction of equity deployed as gross to any single venue (1.0 = no cap)` after `max_strategy_weight`; bump `ALLOCATION_POLICY_VERSION = 3` → `4`.
|
||||
|
||||
- [ ] **Step 5: Wire into `record_allocation`** (`allocation_ingest.py`, after `dollars = target_capital(...)`):
|
||||
|
||||
```python
|
||||
venue_by_sid = {sid: STRATEGY_REGISTRY.get(sid, {}).get("venue", "") for sid in dollars}
|
||||
dollars = apply_venue_caps(dollars, venue_by_sid, getattr(pol, "max_venue_weight", 1.0), equity)
|
||||
```
|
||||
|
||||
(import `apply_venue_caps` from allocation_engine; `STRATEGY_REGISTRY` is already imported.)
|
||||
|
||||
- [ ] **Step 6: Run — expect PASS** (venue tests + existing allocation tests still green)
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && .venv/bin/python3 -m pytest tests/unit/test_venue_caps.py tests/unit/test_compute_allocation.py tests/unit/test_record_allocation_policy.py -q`
|
||||
Expected: PASS.
|
||||
|
||||
- [ ] **Step 7: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/application/allocation_policy.py src/fxhnt/application/allocation_engine.py src/fxhnt/application/allocation_ingest.py tests/unit/test_venue_caps.py
|
||||
git commit -m "feat(alloc): max_venue_weight per-venue gross cap (default 1.0 = no-op)
|
||||
|
||||
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task C: `max_venue_weight` as a fund_config knob (end-to-end)
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/adapters/persistence/cockpit_models.py` (`FundConfigRow.max_venue_weight`)
|
||||
- Modify: `src/fxhnt/application/fund_config.py` (field + validation + idempotent ALTER + read/write/seed)
|
||||
- Modify: `src/fxhnt/cli.py` (`--max-venue-weight` flag on `fund-config set`; show it)
|
||||
- Modify: `src/fxhnt/adapters/orchestration/assets.py` (`cockpit_forward` replace adds `max_venue_weight`)
|
||||
- Test: `tests/unit/test_fund_config.py`, `tests/unit/test_fund_config_cli.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `apply_venue_caps` (Task B), `AllocationPolicy.max_venue_weight` (Task B).
|
||||
- Produces: `FundConfig.max_venue_weight` (default 1.0); `fxhnt fund-config set --max-venue-weight`.
|
||||
|
||||
- [ ] **Step 1: Write failing tests** (extend `test_fund_config.py`):
|
||||
|
||||
```python
|
||||
def test_max_venue_weight_default_and_roundtrip():
|
||||
import os
|
||||
p = "./.pytest_fc_venue.db"
|
||||
if os.path.exists(p): os.remove(p)
|
||||
from fxhnt.application.fund_config import read_fund_config, write_fund_config, seed_fund_config
|
||||
seed_fund_config(f"sqlite:///{p}")
|
||||
assert read_fund_config(f"sqlite:///{p}").max_venue_weight == 1.0
|
||||
write_fund_config(f"sqlite:///{p}", max_venue_weight=0.6)
|
||||
cfg = read_fund_config(f"sqlite:///{p}")
|
||||
assert cfg.max_venue_weight == 0.6 and cfg.target_vol == 0.20 # partial-override: vol unchanged
|
||||
os.remove(p)
|
||||
|
||||
def test_max_venue_weight_validation():
|
||||
import pytest, os
|
||||
p = "./.pytest_fc_venue_val.db"
|
||||
if os.path.exists(p): os.remove(p)
|
||||
from fxhnt.application.fund_config import write_fund_config
|
||||
for bad in (0.0, -0.1, 1.5):
|
||||
with pytest.raises(ValueError):
|
||||
write_fund_config(f"sqlite:///{p}", max_venue_weight=bad)
|
||||
if os.path.exists(p): os.remove(p)
|
||||
```
|
||||
|
||||
Also extend the fallback test to assert `read_fund_config(absent).max_venue_weight == 1.0`.
|
||||
|
||||
- [ ] **Step 2: Run — expect FAIL**
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && .venv/bin/python3 -m pytest tests/unit/test_fund_config.py -q`
|
||||
Expected: FAIL.
|
||||
|
||||
- [ ] **Step 3: Model + module** :
|
||||
- `cockpit_models.FundConfigRow`: add `max_venue_weight: Mapped[float] = mapped_column(Float)`.
|
||||
- `fund_config.py`: add `max_venue_weight: float` to the `FundConfig` dataclass; `_defaults()` → `max_venue_weight=1.0`; `_validate` accepts `max_venue_weight` and enforces `0 < max_venue_weight <= 1.0`; `write_fund_config` gains the `max_venue_weight=None` kwarg (partial-overlay like the others — including the "at least one field" guard); `seed_fund_config` inserts `max_venue_weight=1.0`.
|
||||
- **Idempotent ALTER** for the existing prod table — add an `_ensure_fund_config_columns(engine)` helper (mirror `forward_nav._ensure_summary_policy_columns`) that adds `max_venue_weight DOUBLE PRECISION` (Postgres `ADD COLUMN IF NOT EXISTS`; SQLite PRAGMA-check) and BACKFILLS existing row(s) to 1.0 where NULL; call it in `write_fund_config`/`seed_fund_config` right after `create_all` (so an existing seeded table gains the column without losing the operator's target_vol). A fresh table gets the column from `create_all`, so the ALTER is a no-op there.
|
||||
|
||||
- [ ] **Step 4: CLI** (`cli.py`): `fund-config set` gains `max_venue_weight: float = typer.Option(None, "--max-venue-weight", help="Max fraction of equity deployed to any single venue (0,1]. 1.0 = no cap.")`; pass it to `write_fund_config`; include it in both `show` and `set` echo lines.
|
||||
|
||||
- [ ] **Step 5: Wire the nightly** (`assets.py` `cockpit_forward`): the `dataclasses.replace(...)` gains `max_venue_weight=_cfg.max_venue_weight`.
|
||||
|
||||
- [ ] **Step 6: Run — expect PASS**
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && .venv/bin/python3 -m pytest tests/unit/test_fund_config.py tests/unit/test_fund_config_cli.py -q`
|
||||
Expected: PASS.
|
||||
|
||||
- [ ] **Step 7: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/adapters/persistence/cockpit_models.py src/fxhnt/application/fund_config.py src/fxhnt/cli.py src/fxhnt/adapters/orchestration/assets.py tests/unit/test_fund_config.py tests/unit/test_fund_config_cli.py
|
||||
git commit -m "feat(fund-config): max_venue_weight knob (fxhnt fund-config set --max-venue-weight)
|
||||
|
||||
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task D: full-suite green
|
||||
|
||||
- [ ] **Step 1:** `cd /home/jgrusewski/Work/fxhnt && .venv/bin/python3 -m pytest tests/unit -q` → all pass.
|
||||
- [ ] **Step 2:** `.venv/bin/python3 -m pytest tests/integration -k "allocation or honest or funding" -q` → all pass.
|
||||
- [ ] **Step 3:** commit any fixes (else skip).
|
||||
|
||||
## Post-implementation (controller-run)
|
||||
Deploy (git-sync), `fxhnt migrate-cockpit` (adds the column, backfills 1.0), re-measure in-cluster: confirm the deploy set is `{bybit_4edge}` and it deploys ~75% at target_vol 0.25 with `max_venue_weight=1.0`; then demonstrate the knob (`fxhnt fund-config set --max-venue-weight 0.6` → book caps to 60%). Cockpit local verify.
|
||||
|
||||
## Self-Review
|
||||
- Coverage: book switch → Task A; venue-cap engine → Task B; venue-cap knob end-to-end → Task C; green → Task D.
|
||||
- Byte-identity guards: `apply_venue_caps` no-op at 1.0; policy=None unchanged; fund_config fallback/ALTER default 1.0.
|
||||
- Type consistency: `max_venue_weight`, `apply_venue_caps`, `FundConfig` fields used identically across tasks.
|
||||
538
docs/superpowers/plans/2026-07-16-deploy-fund-config-knobs.md
Normal file
538
docs/superpowers/plans/2026-07-16-deploy-fund-config-knobs.md
Normal file
@@ -0,0 +1,538 @@
|
||||
# Deploy-the-Fund Knobs Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Make the fund's capital + risk appetite CLI-configurable via a single persisted SSOT, and switch the crypto allocation from the double-counted `bybit_4edge` book to its individual capacity-bounded sleeves.
|
||||
|
||||
**Architecture:** A single-row `fund_config` ORM table (rides the existing `CockpitBase.metadata.create_all`), a `application/fund_config.py` with read (settings-fallback) / write (partial-upsert + validation) / seed, a `fxhnt fund-config show|set` typer CLI, the nightly `cockpit_forward` reads it for equity + an `AllocationPolicy` override, and the registry+provider re-point the crypto allocation to sleeves.
|
||||
|
||||
**Tech Stack:** Python 3.12, SQLAlchemy 2.0 (Mapped/mapped_column, DeclarativeBase), typer CLI, pytest. In-memory SQLite for tests (StaticPool), Postgres/Timescale in prod.
|
||||
|
||||
## Global Constraints
|
||||
|
||||
- Spec: `docs/superpowers/specs/2026-07-16-deploy-fund-config-knobs-design.md`. Copy exact values from it.
|
||||
- Initial fund-config values: **capital 100_000, target_vol 0.20, kelly_fraction 0.5**.
|
||||
- Validation (fail loud at the boundary, never silently clamp): `capital > 0`, `0 < target_vol <= 1.0`, `0 < kelly_fraction <= 1.0`.
|
||||
- **Graceful fallback**: when the `fund_config` table or row id=1 is absent, `read_fund_config` returns `FundConfig(capital=settings.paper_capital, target_vol=AllocationPolicy().target_vol, kelly_fraction=AllocationPolicy().kelly_fraction)` — byte-identical to today's nightly behaviour. A fresh DB or a failed read must NEVER change the allocation.
|
||||
- **Partial-override upsert**: `write_fund_config(capital=X)` changes only `capital`, leaving `target_vol`/`kelly_fraction` at their current effective values (classic upsert bug to avoid).
|
||||
- **Seed is insert-if-absent** (`id=1` PK collision → no-op); a re-run must never clobber an operator-set value.
|
||||
- Marginal LOO gate is KEPT (do not weaken/remove it to force diversification). The ex-ante `K` stays computed from full-history vol (never trailing) — the anti-reactive invariant.
|
||||
- No heavy DDL on the startup hot path (one tiny `create_all` table is fine; it is IF-NOT-EXISTS and a no-op after first run).
|
||||
- `record_allocation(dsn, equity, store=None, ...)` with `store=None` and `policy=None` must remain byte-identical to today.
|
||||
- Commit messages end with: `Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>`.
|
||||
|
||||
---
|
||||
|
||||
### Task 1: `fund_config` model + read/write/fallback/validation
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/adapters/persistence/cockpit_models.py` (add `FundConfigRow`)
|
||||
- Create: `src/fxhnt/application/fund_config.py`
|
||||
- Test: `tests/unit/test_fund_config.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `CockpitBase` (cockpit_models), `get_settings()` (config), `AllocationPolicy` (allocation_policy).
|
||||
- Produces:
|
||||
- `FundConfigRow` — ORM model, `__tablename__ = "fund_config"`, single-row (PK `id` fixed 1).
|
||||
- `@dataclass(frozen=True) FundConfig(capital: float, target_vol: float, kelly_fraction: float)`
|
||||
- `read_fund_config(dsn: str) -> FundConfig` (settings-fallback on absent table/row)
|
||||
- `write_fund_config(dsn: str, *, capital: float | None = None, target_vol: float | None = None, kelly_fraction: float | None = None) -> FundConfig` (partial-upsert + validation; returns new effective config)
|
||||
- `seed_fund_config(dsn: str) -> None` (insert-if-absent: 100_000 / 0.20 / 0.5)
|
||||
|
||||
- [ ] **Step 1: Write failing tests**
|
||||
|
||||
```python
|
||||
# tests/unit/test_fund_config.py
|
||||
import pytest
|
||||
from fxhnt.application.fund_config import (
|
||||
FundConfig, read_fund_config, write_fund_config, seed_fund_config,
|
||||
)
|
||||
from fxhnt.adapters.persistence.cockpit_models import CockpitBase, FundConfigRow
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.pool import StaticPool
|
||||
|
||||
def _dsn():
|
||||
return "sqlite://"
|
||||
|
||||
def _fresh_engine_dsn():
|
||||
# a dsn whose table has NOT been created yet (fallback path)
|
||||
return "sqlite://"
|
||||
|
||||
def test_read_fallback_when_table_absent(monkeypatch):
|
||||
# No migrate() called → table absent → settings/policy defaults
|
||||
from fxhnt.application.allocation_policy import AllocationPolicy
|
||||
from fxhnt.config import get_settings
|
||||
cfg = read_fund_config("sqlite://")
|
||||
assert cfg.capital == get_settings().paper_capital
|
||||
assert cfg.target_vol == AllocationPolicy().target_vol
|
||||
assert cfg.kelly_fraction == AllocationPolicy().kelly_fraction
|
||||
|
||||
def test_write_then_read_roundtrip():
|
||||
dsn = "sqlite:///./.pytest_fund_cfg_rt.db"
|
||||
import os
|
||||
if os.path.exists("./.pytest_fund_cfg_rt.db"): os.remove("./.pytest_fund_cfg_rt.db")
|
||||
write_fund_config(dsn, capital=250_000.0, target_vol=0.20, kelly_fraction=0.5)
|
||||
cfg = read_fund_config(dsn)
|
||||
assert (cfg.capital, cfg.target_vol, cfg.kelly_fraction) == (250_000.0, 0.20, 0.5)
|
||||
os.remove("./.pytest_fund_cfg_rt.db")
|
||||
|
||||
def test_partial_override_leaves_other_fields():
|
||||
dsn = "sqlite:///./.pytest_fund_cfg_po.db"
|
||||
import os
|
||||
if os.path.exists("./.pytest_fund_cfg_po.db"): os.remove("./.pytest_fund_cfg_po.db")
|
||||
write_fund_config(dsn, capital=100_000.0, target_vol=0.20, kelly_fraction=0.5)
|
||||
write_fund_config(dsn, capital=300_000.0) # only capital
|
||||
cfg = read_fund_config(dsn)
|
||||
assert cfg.capital == 300_000.0
|
||||
assert cfg.target_vol == 0.20 # unchanged
|
||||
assert cfg.kelly_fraction == 0.5 # unchanged
|
||||
os.remove("./.pytest_fund_cfg_po.db")
|
||||
|
||||
@pytest.mark.parametrize("kw", [
|
||||
{"capital": 0.0}, {"capital": -5.0},
|
||||
{"target_vol": 0.0}, {"target_vol": 1.5},
|
||||
{"kelly_fraction": 0.0}, {"kelly_fraction": 2.0},
|
||||
])
|
||||
def test_validation_rejects(kw):
|
||||
dsn = "sqlite:///./.pytest_fund_cfg_val.db"
|
||||
import os
|
||||
if os.path.exists("./.pytest_fund_cfg_val.db"): os.remove("./.pytest_fund_cfg_val.db")
|
||||
with pytest.raises(ValueError):
|
||||
write_fund_config(dsn, **kw)
|
||||
if os.path.exists("./.pytest_fund_cfg_val.db"): os.remove("./.pytest_fund_cfg_val.db")
|
||||
|
||||
def test_seed_is_insert_if_absent():
|
||||
dsn = "sqlite:///./.pytest_fund_cfg_seed.db"
|
||||
import os
|
||||
if os.path.exists("./.pytest_fund_cfg_seed.db"): os.remove("./.pytest_fund_cfg_seed.db")
|
||||
seed_fund_config(dsn)
|
||||
assert read_fund_config(dsn) == FundConfig(100_000.0, 0.20, 0.5)
|
||||
write_fund_config(dsn, capital=500_000.0)
|
||||
seed_fund_config(dsn) # must NOT clobber
|
||||
assert read_fund_config(dsn).capital == 500_000.0
|
||||
os.remove("./.pytest_fund_cfg_seed.db")
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run tests — expect ImportError/failures**
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && python3 -m pytest tests/unit/test_fund_config.py -q`
|
||||
Expected: FAIL (module/model not defined).
|
||||
|
||||
- [ ] **Step 3: Add the ORM model**
|
||||
|
||||
In `cockpit_models.py`, after the existing simple rows (match the `Mapped`/`mapped_column` style; import `Integer`, `Float`, `DateTime` are already imported — verify and add any missing):
|
||||
|
||||
```python
|
||||
class FundConfigRow(CockpitBase):
|
||||
"""Single-row (id=1) fund-level knobs — the SSOT for capital + risk appetite, set ONLY via
|
||||
`fxhnt fund-config` (never hand-edited), read by the nightly allocation. Absent row/table → the nightly
|
||||
falls back to settings/policy defaults (see application/fund_config.read_fund_config)."""
|
||||
__tablename__ = "fund_config"
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True) # always 1
|
||||
capital: Mapped[float] = mapped_column(Float)
|
||||
target_vol: Mapped[float] = mapped_column(Float)
|
||||
kelly_fraction: Mapped[float] = mapped_column(Float)
|
||||
updated_at: Mapped[dt.datetime] = mapped_column(DateTime)
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Write `application/fund_config.py`**
|
||||
|
||||
```python
|
||||
"""The fund-level knobs SSOT: capital + risk appetite (target_vol, kelly_fraction) as a single-row DB record,
|
||||
set ONLY via the `fxhnt fund-config` CLI, read by the nightly allocation. Absent table/row → settings/policy
|
||||
defaults (byte-identical to the pre-fund_config nightly). No hand-editing — the CLI is the sole writer."""
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import create_engine, select
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy.pool import StaticPool
|
||||
|
||||
from fxhnt.adapters.persistence.cockpit_models import CockpitBase, FundConfigRow
|
||||
from fxhnt.application.allocation_policy import AllocationPolicy
|
||||
from fxhnt.config import get_settings
|
||||
|
||||
_ROW_ID = 1
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FundConfig:
|
||||
capital: float
|
||||
target_vol: float
|
||||
kelly_fraction: float
|
||||
|
||||
|
||||
def _defaults() -> FundConfig:
|
||||
p = AllocationPolicy()
|
||||
return FundConfig(capital=get_settings().paper_capital,
|
||||
target_vol=p.target_vol, kelly_fraction=p.kelly_fraction)
|
||||
|
||||
|
||||
def _engine(dsn: str):
|
||||
kw: dict[str, Any] = {"future": True}
|
||||
if dsn == "sqlite://" or (dsn.startswith("sqlite://") and ":memory:" in dsn):
|
||||
kw |= {"connect_args": {"check_same_thread": False}, "poolclass": StaticPool}
|
||||
return create_engine(dsn, **kw)
|
||||
|
||||
|
||||
def _validate(capital: float | None, target_vol: float | None, kelly_fraction: float | None) -> None:
|
||||
if capital is not None and not (capital > 0):
|
||||
raise ValueError(f"capital must be > 0, got {capital}")
|
||||
if target_vol is not None and not (0 < target_vol <= 1.0):
|
||||
raise ValueError(f"target_vol must be in (0, 1.0], got {target_vol}")
|
||||
if kelly_fraction is not None and not (0 < kelly_fraction <= 1.0):
|
||||
raise ValueError(f"kelly_fraction must be in (0, 1.0], got {kelly_fraction}")
|
||||
|
||||
|
||||
def read_fund_config(dsn: str) -> FundConfig:
|
||||
"""Row id=1 as a FundConfig; settings/policy defaults if the table or row is absent (never raises on a
|
||||
fresh DB — the nightly must not break)."""
|
||||
try:
|
||||
eng = _engine(dsn)
|
||||
with Session(eng) as s:
|
||||
row = s.get(FundConfigRow, _ROW_ID)
|
||||
except Exception: # noqa: BLE001 — absent table on a fresh DB → defaults
|
||||
return _defaults()
|
||||
if row is None:
|
||||
return _defaults()
|
||||
return FundConfig(capital=row.capital, target_vol=row.target_vol, kelly_fraction=row.kelly_fraction)
|
||||
|
||||
|
||||
def write_fund_config(dsn: str, *, capital: float | None = None, target_vol: float | None = None,
|
||||
kelly_fraction: float | None = None) -> FundConfig:
|
||||
"""Partial upsert of row id=1: overlay only the provided fields on the current effective config, validate,
|
||||
write. Returns the new effective FundConfig. Creates the table if absent."""
|
||||
if capital is None and target_vol is None and kelly_fraction is None:
|
||||
raise ValueError("write_fund_config: provide at least one of capital/target_vol/kelly_fraction")
|
||||
_validate(capital, target_vol, kelly_fraction)
|
||||
eng = _engine(dsn)
|
||||
CockpitBase.metadata.create_all(eng, tables=[FundConfigRow.__table__])
|
||||
cur = read_fund_config(dsn)
|
||||
new = FundConfig(
|
||||
capital=cur.capital if capital is None else float(capital),
|
||||
target_vol=cur.target_vol if target_vol is None else float(target_vol),
|
||||
kelly_fraction=cur.kelly_fraction if kelly_fraction is None else float(kelly_fraction),
|
||||
)
|
||||
with Session(eng) as s:
|
||||
s.merge(FundConfigRow(id=_ROW_ID, capital=new.capital, target_vol=new.target_vol,
|
||||
kelly_fraction=new.kelly_fraction,
|
||||
updated_at=dt.datetime.now(dt.UTC).replace(tzinfo=None)))
|
||||
s.commit()
|
||||
return new
|
||||
|
||||
|
||||
def seed_fund_config(dsn: str) -> None:
|
||||
"""Insert row id=1 with the initial knobs (100k / 0.20 / 0.5) IF ABSENT. Never clobbers an operator-set
|
||||
value (idempotent)."""
|
||||
eng = _engine(dsn)
|
||||
CockpitBase.metadata.create_all(eng, tables=[FundConfigRow.__table__])
|
||||
with Session(eng) as s:
|
||||
if s.get(FundConfigRow, _ROW_ID) is not None:
|
||||
return
|
||||
s.add(FundConfigRow(id=_ROW_ID, capital=100_000.0, target_vol=0.20, kelly_fraction=0.5,
|
||||
updated_at=dt.datetime.now(dt.UTC).replace(tzinfo=None)))
|
||||
s.commit()
|
||||
```
|
||||
|
||||
- [ ] **Step 5: Run tests — expect PASS**
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && python3 -m pytest tests/unit/test_fund_config.py -q`
|
||||
Expected: PASS (all).
|
||||
|
||||
- [ ] **Step 6: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/adapters/persistence/cockpit_models.py src/fxhnt/application/fund_config.py tests/unit/test_fund_config.py
|
||||
git commit -m "feat(fund-config): single-row fund_config SSOT (read/write/fallback/validate/seed)
|
||||
|
||||
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 2: `fxhnt fund-config` CLI + wire seed into `migrate-cockpit`
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/cli.py` (add `fund-config` typer sub-app or two commands; wire `seed_fund_config` into `migrate_cockpit`)
|
||||
- Test: `tests/unit/test_fund_config_cli.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `read_fund_config`, `write_fund_config`, `seed_fund_config` (Task 1), `typer`, `get_settings().operational_dsn`.
|
||||
- Produces: CLI `fxhnt fund-config show` and `fxhnt fund-config set [--capital] [--target-vol] [--kelly]`.
|
||||
|
||||
- [ ] **Step 1: Write failing tests** (use typer's `CliRunner`)
|
||||
|
||||
```python
|
||||
# tests/unit/test_fund_config_cli.py
|
||||
import os
|
||||
from typer.testing import CliRunner
|
||||
from fxhnt.cli import app
|
||||
from fxhnt.application.fund_config import read_fund_config
|
||||
|
||||
runner = CliRunner()
|
||||
|
||||
def _isolate(monkeypatch, path):
|
||||
if os.path.exists(path): os.remove(path)
|
||||
monkeypatch.setattr("fxhnt.cli.get_settings", lambda: _S(f"sqlite:///{path}"))
|
||||
|
||||
class _S:
|
||||
def __init__(self, dsn): self.operational_dsn = dsn; self.paper_capital = 100_000.0
|
||||
|
||||
def test_set_then_show(monkeypatch):
|
||||
path = "./.pytest_fund_cli.db"; _isolate(monkeypatch, path)
|
||||
r = runner.invoke(app, ["fund-config", "set", "--capital", "250000", "--target-vol", "0.2", "--kelly", "0.5"])
|
||||
assert r.exit_code == 0, r.output
|
||||
assert read_fund_config(f"sqlite:///{path}").capital == 250_000.0
|
||||
r2 = runner.invoke(app, ["fund-config", "show"])
|
||||
assert r2.exit_code == 0
|
||||
assert "250000" in r2.output.replace(",", "") or "250000.0" in r2.output.replace(",", "")
|
||||
os.remove(path)
|
||||
|
||||
def test_set_rejects_bad_value(monkeypatch):
|
||||
path = "./.pytest_fund_cli_bad.db"; _isolate(monkeypatch, path)
|
||||
r = runner.invoke(app, ["fund-config", "set", "--target-vol", "1.5"])
|
||||
assert r.exit_code != 0
|
||||
if os.path.exists(path): os.remove(path)
|
||||
|
||||
def test_set_no_flags_errors(monkeypatch):
|
||||
path = "./.pytest_fund_cli_noflag.db"; _isolate(monkeypatch, path)
|
||||
r = runner.invoke(app, ["fund-config", "set"])
|
||||
assert r.exit_code != 0
|
||||
if os.path.exists(path): os.remove(path)
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run — expect FAIL** (`fund-config` command absent)
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && python3 -m pytest tests/unit/test_fund_config_cli.py -q`
|
||||
Expected: FAIL.
|
||||
|
||||
- [ ] **Step 3: Implement the CLI** (typer sub-app `fund_config_app` registered on `app`)
|
||||
|
||||
```python
|
||||
# in cli.py, near the other command groups
|
||||
fund_config_app = typer.Typer(help="Fund-level knobs (capital + risk appetite) — the allocation SSOT.")
|
||||
app.add_typer(fund_config_app, name="fund-config")
|
||||
|
||||
@fund_config_app.command("show")
|
||||
def fund_config_show() -> None:
|
||||
"""Print the effective fund knobs (capital, target_vol, kelly_fraction)."""
|
||||
from fxhnt.application.fund_config import read_fund_config
|
||||
cfg = read_fund_config(get_settings().operational_dsn)
|
||||
typer.echo(f"capital={cfg.capital:.2f} target_vol={cfg.target_vol:.4f} kelly_fraction={cfg.kelly_fraction:.4f}")
|
||||
|
||||
@fund_config_app.command("set")
|
||||
def fund_config_set(
|
||||
capital: float = typer.Option(None, help="Fund capital (equity the allocation sizes against). > 0."),
|
||||
target_vol: float = typer.Option(None, "--target-vol", help="Annualised vol target (0, 1.0]."),
|
||||
kelly: float = typer.Option(None, "--kelly", help="Kelly fraction (0, 1.0]."),
|
||||
) -> None:
|
||||
"""Set one or more fund knobs (partial override; unspecified fields unchanged)."""
|
||||
from fxhnt.application.fund_config import write_fund_config
|
||||
try:
|
||||
cfg = write_fund_config(get_settings().operational_dsn, capital=capital,
|
||||
target_vol=target_vol, kelly_fraction=kelly)
|
||||
except ValueError as e:
|
||||
typer.echo(f"error: {e}"); raise typer.Exit(code=1) from e
|
||||
typer.echo(f"set: capital={cfg.capital:.2f} target_vol={cfg.target_vol:.4f} kelly_fraction={cfg.kelly_fraction:.4f}")
|
||||
```
|
||||
|
||||
Then in `migrate_cockpit()` (after `ForwardNavRepo(dsn).migrate()`), add:
|
||||
|
||||
```python
|
||||
from fxhnt.application.fund_config import seed_fund_config
|
||||
seed_fund_config(dsn)
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run — expect PASS**
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && python3 -m pytest tests/unit/test_fund_config_cli.py -q`
|
||||
Expected: PASS.
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/cli.py tests/unit/test_fund_config_cli.py
|
||||
git commit -m "feat(fund-config): fxhnt fund-config show|set CLI + seed on migrate-cockpit
|
||||
|
||||
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 3: nightly reads `fund_config` → equity + `AllocationPolicy`
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/application/allocation_ingest.py` (optional `policy` param on `record_allocation`)
|
||||
- Modify: `src/fxhnt/adapters/orchestration/assets.py` (`cockpit_forward`: read fund_config → equity + policy)
|
||||
- Test: `tests/unit/test_record_allocation_policy.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `read_fund_config` (Task 1), `resolve_allocation_policy` / `AllocationPolicy` (allocation_policy), `dataclasses.replace`.
|
||||
- Produces: `record_allocation(dsn, equity, *, store=None, spread_bps=None, unlock_events=None, policy=None)` — `policy=None` keeps today's resolution (byte-identical); a provided `AllocationPolicy` is used verbatim for both the honest and nav paths.
|
||||
|
||||
- [ ] **Step 1: Inspect** how `record_allocation` currently builds its policy (`resolve_allocation_policy`) so the `policy=None` default is provably identical. Read `allocation_ingest.py` lines around the policy build and `allocation_policy.resolve_allocation_policy`.
|
||||
|
||||
- [ ] **Step 2: Write failing tests**
|
||||
|
||||
```python
|
||||
# tests/unit/test_record_allocation_policy.py
|
||||
import dataclasses
|
||||
from fxhnt.application.allocation_policy import AllocationPolicy, resolve_allocation_policy
|
||||
|
||||
def test_policy_none_matches_resolved_default(monkeypatch):
|
||||
# record_allocation(policy=None) must build the SAME policy resolve_allocation_policy() returns today.
|
||||
# Assert via the helper the code will use: a None arg resolves to resolve_allocation_policy().
|
||||
from fxhnt.application import allocation_ingest as ai
|
||||
resolved = ai._effective_policy(None) # helper introduced in Step 3
|
||||
assert resolved == resolve_allocation_policy()
|
||||
|
||||
def test_policy_passthrough():
|
||||
from fxhnt.application import allocation_ingest as ai
|
||||
p = dataclasses.replace(resolve_allocation_policy(), target_vol=0.20, kelly_fraction=0.5)
|
||||
assert ai._effective_policy(p) is p
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Implement.** In `allocation_ingest.py` add a tiny helper and thread it:
|
||||
|
||||
```python
|
||||
def _effective_policy(policy):
|
||||
from fxhnt.application.allocation_policy import resolve_allocation_policy
|
||||
return resolve_allocation_policy() if policy is None else policy
|
||||
```
|
||||
|
||||
Add `policy=None` to the `record_allocation` signature and replace its internal `resolve_allocation_policy()` call with `pol = _effective_policy(policy)`, using `pol` wherever the policy is consumed (compute_allocation / target_capital / hashing). Verify every prior use of the resolved policy now reads `pol`.
|
||||
|
||||
- [ ] **Step 4: Wire `cockpit_forward`** (`assets.py` ~line 1073). Replace `equity = get_settings().paper_capital` with:
|
||||
|
||||
```python
|
||||
from fxhnt.application.fund_config import read_fund_config
|
||||
import dataclasses as _dc
|
||||
from fxhnt.application.allocation_policy import resolve_allocation_policy
|
||||
_cfg = read_fund_config(dsn)
|
||||
equity = _cfg.capital
|
||||
_policy = _dc.replace(resolve_allocation_policy(), target_vol=_cfg.target_vol,
|
||||
kelly_fraction=_cfg.kelly_fraction)
|
||||
```
|
||||
|
||||
and pass `policy=_policy` to BOTH `record_allocation(...)` calls (the honest-store path and the nav-fallback path).
|
||||
|
||||
- [ ] **Step 5: Run tests — expect PASS**
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && python3 -m pytest tests/unit/test_record_allocation_policy.py tests/unit/test_compute_allocation.py -q`
|
||||
Expected: PASS (new + existing allocation tests still green).
|
||||
|
||||
- [ ] **Step 6: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/application/allocation_ingest.py src/fxhnt/adapters/orchestration/assets.py tests/unit/test_record_allocation_policy.py
|
||||
git commit -m "feat(fund-config): nightly allocation reads fund_config for equity + policy (target_vol/kelly)
|
||||
|
||||
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 4: per-sleeve allocation — registry + provider
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/registry.py` (`xsfunding`/`unlock` → `promote_on_pass: True`; `bybit_4edge` → `tier: research`)
|
||||
- Modify: `src/fxhnt/application/allocation_honest_inputs.py` (crypto-sleeve generalization)
|
||||
- Test: `tests/unit/test_allocation_honest_inputs.py` (extend existing if present, else create)
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `_sleeve_series` (existing, works for any sleeve), `honest_allocation_inputs` (existing signature unchanged).
|
||||
- Produces: `_CRYPTO_DEPLOY_SIDS = frozenset({"positioning", "xsfunding", "unlock"})`; `_deploy_series` returns the per-sleeve honest series for each of those three (no `bybit_4edge` branch in the allocation path).
|
||||
|
||||
- [ ] **Step 1: Write failing tests** (a fake store returning deterministic per-sleeve weights/turnover; assert xsfunding/unlock are sized as sleeves, and bybit_4edge yields no series in the allocation path)
|
||||
|
||||
```python
|
||||
# tests/unit/test_allocation_honest_inputs.py (extend / create)
|
||||
from fxhnt.application.allocation_honest_inputs import (
|
||||
honest_allocation_inputs, _CRYPTO_DEPLOY_SIDS,
|
||||
)
|
||||
|
||||
def test_crypto_deploy_sids_are_the_three_sleeves():
|
||||
assert _CRYPTO_DEPLOY_SIDS == frozenset({"positioning", "xsfunding", "unlock"})
|
||||
|
||||
def test_xsfunding_and_unlock_get_a_series_not_equity_default(fake_store, spread_bps, unlock_events):
|
||||
# fake_store yields non-empty weights for xsfunding & unlock
|
||||
returns, ceilings = honest_allocation_inputs(
|
||||
fake_store, spread_bps, unlock_events,
|
||||
deploy_sids={"xsfunding", "unlock"}, target_aum=35_000.0, aums=[35_000.0, 350_000.0])
|
||||
assert "xsfunding" in returns and returns["xsfunding"] # sleeve-sized, ISO-keyed
|
||||
assert "unlock" in returns and returns["unlock"]
|
||||
# ceilings present (not the equity default 10M sentinel for these crypto sleeves)
|
||||
assert ceilings["xsfunding"] != 10_000_000.0
|
||||
|
||||
def test_bybit_4edge_is_not_a_crypto_deploy_sid():
|
||||
assert "bybit_4edge" not in _CRYPTO_DEPLOY_SIDS
|
||||
```
|
||||
|
||||
(Reuse the existing test's `fake_store`/fixtures if the test file already exists; otherwise build a minimal fake exposing `read_panel` + the `sleeve_weights_and_contributions` path — mirror the pattern already used in the honest-reference tests.)
|
||||
|
||||
- [ ] **Step 2: Run — expect FAIL**
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && python3 -m pytest tests/unit/test_allocation_honest_inputs.py -q`
|
||||
Expected: FAIL.
|
||||
|
||||
- [ ] **Step 3: Registry edits** (`registry.py`):
|
||||
- `xsfunding` entry: add `"promote_on_pass": True,` (top-level key, sibling of `tier`).
|
||||
- `unlock` entry: add `"promote_on_pass": True,`.
|
||||
- `bybit_4edge` entry: change `"tier": "deploy"` → `"tier": "research"`. Leave everything else (gate_spec, definition, inception_t0) unchanged. Update the entry's leading comment to note it is now a tracked REFERENCE aggregate (allocation runs per-sleeve; see the 2026-07-16 spec).
|
||||
|
||||
- [ ] **Step 4: Provider edits** (`allocation_honest_inputs.py`):
|
||||
- `_CRYPTO_DEPLOY_SIDS = frozenset({"positioning", "xsfunding", "unlock"})`.
|
||||
- `_deploy_series`: replace the positioning/bybit_4edge branches with: `if sid in _CRYPTO_DEPLOY_SIDS: return _sleeve_series(store, sid, spread_bps, unlock_events, aum=aum, participation_cap=participation_cap, impact_k=impact_k)` then `return {}`.
|
||||
- Keep `_book_series` / `_BOOK_SLEEVES` (still used by the honest-reference REPORT), but remove the `bybit_4edge` dispatch from `_deploy_series`. Update the module docstring's deploy→honest mapping to the three sleeves + note the book is no longer an allocation target.
|
||||
|
||||
- [ ] **Step 5: Run — expect PASS** (and re-run the honest-reference tests to confirm `_book_series` still works)
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && python3 -m pytest tests/unit/test_allocation_honest_inputs.py tests/unit/ -k "honest or allocation" -q`
|
||||
Expected: PASS.
|
||||
|
||||
- [ ] **Step 6: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/registry.py src/fxhnt/application/allocation_honest_inputs.py tests/unit/test_allocation_honest_inputs.py
|
||||
git commit -m "feat(alloc): per-sleeve crypto allocation (promote xsfunding/unlock; bybit_4edge->reference)
|
||||
|
||||
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 5: full-suite green + book-family sanity
|
||||
|
||||
**Files:** none (verification task)
|
||||
|
||||
- [ ] **Step 1: Run the allocation + fund-config + honest-reference unit suites**
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && python3 -m pytest tests/unit -k "fund_config or allocation or honest or compute_allocation" -q`
|
||||
Expected: all PASS.
|
||||
|
||||
- [ ] **Step 2: Grep for any remaining hard reference to `bybit_4edge` as a deploy-tier allocation target** (e.g. code that assumed the book is allocated). Fix or note any found.
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && grep -rn "bybit_4edge" src/fxhnt/application/ src/fxhnt/adapters/ | grep -iv "exec\|levered\|ref\|comment"`
|
||||
Expected: no code that *allocates* to `bybit_4edge` (execution wiring is out of scope / suspended).
|
||||
|
||||
- [ ] **Step 3: Commit any fixes** from Step 2 (if none, skip).
|
||||
|
||||
---
|
||||
|
||||
## Post-implementation (controller-run, NOT a subagent task — in-cluster + local)
|
||||
|
||||
These are the spec's §5 falsification gate + cockpit verification. The controller runs them after the branch is green, per the user's "run queries in the cluster" and "verify cockpit locally" directives:
|
||||
|
||||
1. **Deploy** (git-sync nightly picks up master) + `fxhnt migrate-cockpit` (creates `fund_config`, seeds 100k/0.20/0.5) in-cluster.
|
||||
2. **G1/G2/G3** — in-cluster diagnostic: dump `survivors_after_marginal_prune`, the per-sleeve weights, deployed gross $ and %, and confirm `K` uses full-history vol. Report the numbers (do NOT override the gate).
|
||||
3. **Cockpit local verify** — `bybit_4edge`-as-reference renders sanely (deploy section shows the sleeves, no broken headline).
|
||||
4. Report the honest verdict (per-sleeve deploys more, or it concentrates to positioning + the vol-target is the lever).
|
||||
|
||||
## Self-Review
|
||||
|
||||
- **Spec coverage:** §4a→T1, §4b→T2, §4c→T3, §4d→T4, §4e→T1(seed)+T2(wire), §5→post-impl gate, §7 risks→Global Constraints + T4 + post-impl. Covered.
|
||||
- **Placeholders:** none — every step has concrete code/commands.
|
||||
- **Type consistency:** `FundConfig`, `read_fund_config`, `write_fund_config`, `seed_fund_config`, `_effective_policy`, `_CRYPTO_DEPLOY_SIDS` used identically across tasks.
|
||||
298
docs/superpowers/plans/2026-07-16-equity-leg-deployable.md
Normal file
298
docs/superpowers/plans/2026-07-16-equity-leg-deployable.md
Normal file
@@ -0,0 +1,298 @@
|
||||
# Equity Leg Deployable Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: superpowers:subagent-driven-development. Checkbox (`- [ ]`) steps.
|
||||
|
||||
**Goal:** Make the IBKR US-ETF `multistrat` book honest-sized + capacity-bounded (real ADV from ingested ETF volume, real trading costs), so it deploys as the growth vehicle above the crypto book's ~$350k break.
|
||||
|
||||
**Architecture:** An equity analog of the crypto honest-reference. Ingest real Yahoo ETF volume (PIT); a real cost model (sqrt-impact on dollar-ADV + per-ETF spread + IBKR commission) net of `multistrat`'s turnover; a cost-netted `book_series` sizing series + ADV capacity curve; an `equity_allocation_inputs` provider merged into `record_allocation`.
|
||||
|
||||
**Tech Stack:** Python 3.12, SQLAlchemy 2.0, typer, pytest. Spec: `docs/superpowers/specs/2026-07-16-equity-leg-deployable-design.md`.
|
||||
|
||||
## Global Constraints
|
||||
|
||||
- `FUND_INSTRUMENTS = ["SPY","IEF","GLD","PDBC","DBMF"]` (from `multistrat`).
|
||||
- **Per-ETF half-spread constants (bps), documented — the one estimated input** (no ETF bid/ask feed): `{"SPY":0.3,"IEF":1.0,"GLD":1.0,"PDBC":4.0,"DBMF":8.0}`. Default for an unlisted symbol: 5.0 bps.
|
||||
- **IBKR commission**: `commission_per_share = 0.0035` USD; `commission_bps = 100 * commission_per_share / price` (tiny for high-priced ETFs — dominant cost is spread).
|
||||
- **Dollar-ADV**: `adv_usd[sym][day] = pit_close[sym][day] * volume[sym][day]`, trailing **63-day MEDIAN** (outlier-robust; a volume spike must not inflate capacity). A day with missing volume is excluded (no fabricated ADV).
|
||||
- **PIT discipline**: volume snapshot is first-seen-frozen (a re-fetch must NOT overwrite a stored (symbol,date) — mirror `snapshot_yahoo_closes`).
|
||||
- **Byte-identity**: `record_allocation` with `store=None`, or with no equity sid in the deploy set, must be byte-identical to today. The equity provider only adds series/ceilings for `ibkr-equity` deploy sids.
|
||||
- **Cost invariant**: costs must REDUCE the equity Sharpe (net ≤ gross); the turnover charged is the ACTUAL traded weight change (two-way), from `multistrat.target_weights`, not the raw signal.
|
||||
- Reuse the crypto templates (`honest_sleeve_returns` shape, `capacity_ceiling_from_curve`, load-once/reprice). Do NOT reintroduce a flat 15bp cost.
|
||||
- Commit trailer: `Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>`.
|
||||
|
||||
---
|
||||
|
||||
### Task 1: Ingest ETF volume (fetch + PIT store + snapshot + backfill CLI)
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/adapters/data/yahoo_daily.py` (`.volumes()`)
|
||||
- Modify: `src/fxhnt/adapters/persistence/cockpit_models.py` (`YahooPitVolumeRow`)
|
||||
- Modify: `src/fxhnt/adapters/persistence/forward_nav.py` (`snapshot_yahoo_volumes`, `yahoo_pit_volumes`)
|
||||
- Modify: `src/fxhnt/adapters/orchestration/assets.py` (nightly snapshot of volumes)
|
||||
- Modify: `src/fxhnt/cli.py` (`backfill-etf-volume`)
|
||||
- Test: `tests/unit/test_yahoo_volume_pit.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: `YahooDailyClient.volumes(symbol) -> dict[str,float]` (date→raw share volume); `ForwardNavRepo.snapshot_yahoo_volumes(symbol, series, at)` / `.yahoo_pit_volumes(symbol) -> dict[str,float]`.
|
||||
|
||||
- [ ] **Step 1: Write failing tests** — a fake Yahoo response dict → `volumes()` extracts `quote[0]["volume"]` mapped to dates (skipping None); a repo roundtrip: `snapshot_yahoo_volumes` then `yahoo_pit_volumes` returns the series; first-seen-frozen (a second snapshot with a changed value for an existing date does NOT overwrite).
|
||||
|
||||
```python
|
||||
# tests/unit/test_yahoo_volume_pit.py
|
||||
import datetime as dt
|
||||
from fxhnt.adapters.persistence.forward_nav import ForwardNavRepo
|
||||
|
||||
def test_snapshot_and_read_volumes_first_seen_frozen():
|
||||
r = ForwardNavRepo("sqlite://"); r.migrate()
|
||||
at = dt.datetime(2026, 7, 16)
|
||||
r.snapshot_yahoo_volumes("SPY", {"2026-07-01": 1000.0, "2026-07-02": 2000.0}, at)
|
||||
r.snapshot_yahoo_volumes("SPY", {"2026-07-01": 9999.0, "2026-07-03": 3000.0}, at) # 07-01 must NOT change
|
||||
assert r.yahoo_pit_volumes("SPY") == {"2026-07-01": 1000.0, "2026-07-02": 2000.0, "2026-07-03": 3000.0}
|
||||
|
||||
def test_volumes_extracts_quote_volume(monkeypatch):
|
||||
from fxhnt.adapters.data.yahoo_daily import YahooDailyClient
|
||||
c = YahooDailyClient()
|
||||
fake = {"chart": {"result": [{"timestamp": [1719792000, 1719878400],
|
||||
"indicators": {"quote": [{"volume": [111.0, None]}], "adjclose": [{"adjclose": [1.0, 2.0]}]}}]}}
|
||||
monkeypatch.setattr(c, "_get", lambda url: fake)
|
||||
out = c.volumes("SPY")
|
||||
assert list(out.values()) == [111.0] # None volume day skipped
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run — expect FAIL**
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && .venv/bin/python3 -m pytest tests/unit/test_yahoo_volume_pit.py -q`
|
||||
Expected: FAIL.
|
||||
|
||||
- [ ] **Step 3: Implement.**
|
||||
- `yahoo_daily.py` — `volumes(symbol)` mirrors `adj_closes` but reads `ind["quote"][0]["volume"]`:
|
||||
```python
|
||||
def volumes(self, symbol: str) -> dict[str, float]:
|
||||
res = self._get(_BASE.format(sym=symbol, rng=self._rng))["chart"]["result"][0]
|
||||
ts = res["timestamp"]
|
||||
vol = res["indicators"]["quote"][0].get("volume") or []
|
||||
out: dict[str, float] = {}
|
||||
for t, v in zip(ts, vol, strict=False):
|
||||
if v is not None:
|
||||
out[dt.datetime.fromtimestamp(t, dt.timezone.utc).strftime("%Y-%m-%d")] = float(v)
|
||||
return out
|
||||
```
|
||||
- `cockpit_models.py` — `YahooPitVolumeRow` mirroring `YahooPitCloseRow` (`__tablename__="yahoo_pit_volume"`, PK (symbol,date), `volume: Mapped[float]`, `first_seen_at`).
|
||||
- `forward_nav.py` — `snapshot_yahoo_volumes` / `yahoo_pit_volumes` mirroring the close methods exactly (first-seen-frozen).
|
||||
- `assets.py` — in the block that calls `repo.snapshot_yahoo_closes(sym, live.adj_closes(sym), at)` (~line 421), add `repo.snapshot_yahoo_volumes(sym, live.volumes(sym), at)` in the same loop (wrap the volume fetch in try/except so a volume-fetch failure never breaks the close snapshot — WARN + continue).
|
||||
- `cli.py` — `@app.command("backfill-etf-volume")`: for each `FUND_INSTRUMENTS` sym, `YahooDailyClient(rng="25y").volumes(sym)` → `repo.snapshot_yahoo_volumes(...)`; echo counts. (Import `FUND_INSTRUMENTS` from `fxhnt.domain.strategies.multistrat`.)
|
||||
|
||||
- [ ] **Step 4: Run — expect PASS**
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && .venv/bin/python3 -m pytest tests/unit/test_yahoo_volume_pit.py -q`
|
||||
Expected: PASS.
|
||||
|
||||
- [ ] **Step 5: Commit** (`git add` the 5 files + test; message `feat(equity): ingest ETF volume (PIT) + backfill-etf-volume CLI`).
|
||||
|
||||
---
|
||||
|
||||
### Task 2: `equity_honest.py` — real cost model + capacity curve
|
||||
|
||||
**Files:**
|
||||
- Create: `src/fxhnt/application/equity_honest.py`
|
||||
- Test: `tests/unit/test_equity_honest.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `IMPACT_COEFF_BPS` (honest_reference), `sharpe_of`/`per_period_sharpe`, `capacity_ceiling_from_curve` (allocation_engine).
|
||||
- Produces:
|
||||
- `ETF_HALF_SPREAD_BPS: dict[str,float]` (the documented constants) + `_spread_bps(sym) -> float`.
|
||||
- `dollar_adv(pit_closes, pit_volumes, window=63) -> dict[str, dict[str,float]]` (per-sym trailing-median dollar volume).
|
||||
- `equity_honest_returns(book_ret_by_day, weights_by_day, adv_usd, *, aum, participation_cap=0.03, impact_k=1.0, commission_per_share=0.0035) -> dict[str,float]`
|
||||
- `equity_capacity_curve(book_ret_by_day, weights_by_day, adv_usd, *, aums, ...) -> dict[float, dict[str,float]]`
|
||||
|
||||
- [ ] **Step 1: Write failing tests.**
|
||||
- `dollar_adv`: given closes + volumes over N days, the ADV is the trailing 63d (or shorter early) median of `close*volume`; a missing-volume day is excluded.
|
||||
- `equity_honest_returns`: (a) with ZERO turnover (constant weights) → net == gross book_ret (no cost). (b) with turnover + tiny AUM (adv >> notional) → cost ≈ spread-only (impact ~0), net < gross by `Σ traded_w * spread/1e4`. (c) net Sharpe ≤ gross Sharpe. (d) a position exceeding `participation_cap * adv` is scaled down (capacity), reducing that day's gross contribution.
|
||||
- `equity_capacity_curve`: monotonic-ish — higher AUM → lower or equal net Sharpe (impact grows); returns one series per aum.
|
||||
|
||||
```python
|
||||
# tests/unit/test_equity_honest.py (sketch — fill real numbers)
|
||||
from fxhnt.application.equity_honest import dollar_adv, equity_honest_returns, equity_capacity_curve, ETF_HALF_SPREAD_BPS
|
||||
|
||||
def test_zero_turnover_no_cost():
|
||||
days = ["2026-01-01","2026-01-02","2026-01-03"]
|
||||
book = {d: 0.001 for d in days}
|
||||
w = {d: {"SPY": 0.5} for d in days} # constant weights -> no turnover after day 1
|
||||
adv = {"SPY": {d: 1e12 for d in days}} # infinite liquidity -> no impact
|
||||
net = equity_honest_returns(book, w, adv, aum=100_000.0)
|
||||
# day 1 enters the position (turnover from 0->0.5) -> small spread cost; days 2-3 no turnover -> net==book
|
||||
assert abs(net["2026-01-02"] - book["2026-01-02"]) < 1e-12
|
||||
assert abs(net["2026-01-03"] - book["2026-01-03"]) < 1e-12
|
||||
|
||||
def test_costs_reduce_sharpe():
|
||||
... # net sharpe <= gross sharpe over a turnover-y series
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run — expect FAIL**
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && .venv/bin/python3 -m pytest tests/unit/test_equity_honest.py -q`
|
||||
Expected: FAIL.
|
||||
|
||||
- [ ] **Step 3: Implement `equity_honest.py`.** Mirror `honest_sleeve_returns`'s structure (turnover-basis cost, participation cap), but with equity inputs. Reference implementation:
|
||||
|
||||
```python
|
||||
"""Equity honest-reference: the IBKR US-ETF book's cost-netted return series + capacity, the analog of the
|
||||
crypto honest_reference. Real costs: sqrt market-impact on REAL dollar-ADV (ingested Yahoo volume) + a
|
||||
documented per-ETF spread + IBKR commission, charged on the book's ACTUAL traded weight (two-way turnover).
|
||||
Pure; no I/O (callers pass preloaded closes/volumes/weights)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import statistics
|
||||
|
||||
from fxhnt.application.honest_reference import IMPACT_COEFF_BPS
|
||||
from fxhnt.domain.gauntlet import per_period_sharpe
|
||||
|
||||
ETF_HALF_SPREAD_BPS: dict[str, float] = {"SPY": 0.3, "IEF": 1.0, "GLD": 1.0, "PDBC": 4.0, "DBMF": 8.0}
|
||||
_DEFAULT_SPREAD_BPS = 5.0
|
||||
|
||||
|
||||
def _spread_bps(sym: str) -> float:
|
||||
return ETF_HALF_SPREAD_BPS.get(sym, _DEFAULT_SPREAD_BPS)
|
||||
|
||||
|
||||
def dollar_adv(pit_closes: dict[str, dict[str, float]], pit_volumes: dict[str, dict[str, float]],
|
||||
*, window: int = 63) -> dict[str, dict[str, float]]:
|
||||
"""Per-symbol trailing-`window` MEDIAN dollar volume (close*volume). Missing-volume days excluded."""
|
||||
out: dict[str, dict[str, float]] = {}
|
||||
for sym, vols in pit_volumes.items():
|
||||
closes = pit_closes.get(sym, {})
|
||||
dv = {d: closes[d] * vols[d] for d in sorted(vols) if d in closes and vols[d] is not None}
|
||||
dates = sorted(dv)
|
||||
cur: dict[str, float] = {}
|
||||
for i, d in enumerate(dates):
|
||||
lo = max(0, i - window + 1)
|
||||
cur[d] = statistics.median(dv[x] for x in dates[lo:i + 1])
|
||||
out[sym] = cur
|
||||
return out
|
||||
|
||||
|
||||
def equity_honest_returns(book_ret_by_day: dict[str, float], weights_by_day: dict[str, dict[str, float]],
|
||||
adv_usd: dict[str, dict[str, float]], *, aum: float,
|
||||
participation_cap: float = 0.03, impact_k: float = 1.0,
|
||||
commission_per_share: float = 0.0035,
|
||||
price_by_day: dict[str, dict[str, float]] | None = None) -> dict[str, float]:
|
||||
"""Cost-netted daily book return. Cost = Σ_sym traded_w * (max(impact, spread) + commission)/1e4, on the
|
||||
two-way traded weight; positions exceeding participation_cap*ADV are scaled down (capacity)."""
|
||||
out: dict[str, float] = {}
|
||||
prev: dict[str, float] = {}
|
||||
for day in sorted(book_ret_by_day):
|
||||
row = weights_by_day.get(day, {})
|
||||
cur: dict[str, float] = {}
|
||||
scale_hit = 0.0
|
||||
for sym, w in row.items():
|
||||
adv = adv_usd.get(sym, {}).get(day, 0.0)
|
||||
notional = abs(w) * aum
|
||||
if adv <= 0.0:
|
||||
cur[sym] = 0.0 # no ADV -> zero capacity (honest, not infinite)
|
||||
scale_hit += abs(w) # this weight can't be held -> its return is forgone
|
||||
elif notional > participation_cap * adv:
|
||||
cur[sym] = w * (participation_cap * adv) / notional
|
||||
scale_hit += abs(w) - abs(cur[sym])
|
||||
else:
|
||||
cur[sym] = w
|
||||
cost = 0.0
|
||||
for sym in set(cur) | set(prev):
|
||||
traded_w = abs(cur.get(sym, 0.0) - prev.get(sym, 0.0))
|
||||
if traded_w <= 0.0:
|
||||
continue
|
||||
adv = adv_usd.get(sym, {}).get(day, 0.0)
|
||||
traded_notional = traded_w * aum
|
||||
impact = impact_k * IMPACT_COEFF_BPS * math.sqrt(traded_notional / adv) if adv > 0.0 and traded_notional > 0.0 else 0.0
|
||||
price = (price_by_day or {}).get(sym, {}).get(day, 0.0)
|
||||
commission_bps = (100.0 * commission_per_share / price) if price > 0.0 else 0.0
|
||||
slip_bps = max(impact, _spread_bps(sym)) + commission_bps
|
||||
cost += traded_w * (slip_bps / 1e4)
|
||||
# capacity-forgone return: reduce the day's gross by the scaled-out fraction of the book's own return
|
||||
out[day] = book_ret_by_day[day] * (1.0 - scale_hit) - cost if scale_hit < 1.0 else -cost
|
||||
prev = cur
|
||||
return out
|
||||
|
||||
|
||||
def equity_capacity_curve(book_ret_by_day, weights_by_day, adv_usd, *, aums,
|
||||
participation_cap: float = 0.03, impact_k: float = 1.0,
|
||||
commission_per_share: float = 0.0035, price_by_day=None):
|
||||
"""{aum: equity_honest_returns(...at aum)} — load-once inputs, reprice per AUM."""
|
||||
return {aum: equity_honest_returns(book_ret_by_day, weights_by_day, adv_usd, aum=aum,
|
||||
participation_cap=participation_cap, impact_k=impact_k,
|
||||
commission_per_share=commission_per_share, price_by_day=price_by_day)
|
||||
for aum in aums}
|
||||
```
|
||||
|
||||
**Implementer note:** the `scale_hit` capacity-forgone term is an approximation (scales the whole book's return by the un-held fraction). Confirm it is monotone (higher AUM → more scale_hit → lower net) and document it. If a cleaner per-symbol attribution is available from `book_series`'s internals, prefer it — but do NOT block on it; the book-level approximation is acceptable for v1 and must be stated.
|
||||
|
||||
- [ ] **Step 4: Run — expect PASS.** Run the test file; fix until green.
|
||||
|
||||
- [ ] **Step 5: Commit** (`feat(equity): real ETF cost model + ADV capacity curve (equity_honest.py)`).
|
||||
|
||||
---
|
||||
|
||||
### Task 3: `equity_allocation_inputs` provider + `record_allocation` merge
|
||||
|
||||
**Files:**
|
||||
- Create: `src/fxhnt/application/equity_allocation_inputs.py`
|
||||
- Modify: `src/fxhnt/application/allocation_ingest.py` (merge equity series/ceilings)
|
||||
- Test: `tests/unit/test_equity_allocation_inputs.py`, `tests/integration/test_record_allocation_equity.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `MultiStratStrategy` + `book_series`/`target_weights` (multistrat), `YahooPitClient` + `yahoo_pit_volumes` (repo), `equity_honest.*`, `capacity_ceiling_from_curve`.
|
||||
- Produces: `equity_allocation_inputs(repo, deploy_sids, *, target_aum, aums, participation_cap=0.03, impact_k=1.0, min_sharpe=1.0) -> tuple[dict[str,dict[str,float]], dict[str,float|None]]`.
|
||||
|
||||
- [ ] **Step 1: Confirm the multistrat shapes.** Read `multistrat.book_series` and `multistrat.target_weights` signatures/returns and `MultiStratStrategy.advance` — determine the exact type of the daily book-return series and the weights vector (keys = ISO date? epoch? sym→weight?). The provider must feed `equity_honest_returns` `book_ret_by_day: {isodate: ret}` and `weights_by_day: {isodate: {sym: weight}}`. Adapt keys as needed (ISO strings, matching the nav record for `deploy_book_returns` alignment).
|
||||
|
||||
- [ ] **Step 2: Write failing tests.** A fake repo returning seeded `yahoo_pit_closes`/`yahoo_pit_volumes` for FUND_INSTRUMENTS over enough days; `equity_allocation_inputs(repo, {"multistrat"}, target_aum=100_000, aums=[...])` returns a non-empty ISO-keyed series for `multistrat` (len ≥ 60) + a finite ceiling. And in `record_allocation`: with an equity deploy sid present + store injected, `multistrat` gets a series (is eligible, not dropped); with `store=None` OR no equity sid, the result is byte-identical to before (add a regression assertion).
|
||||
|
||||
- [ ] **Step 3: Run — expect FAIL.**
|
||||
|
||||
- [ ] **Step 4: Implement `equity_allocation_inputs.py`:**
|
||||
- For each equity deploy sid (registry `venue == "ibkr-equity"`; today only `multistrat`):
|
||||
- `closes = {sym: repo.yahoo_pit_closes(sym) for sym in FUND_INSTRUMENTS}`; `volumes = {sym: repo.yahoo_pit_volumes(sym) for sym in FUND_INSTRUMENTS}`.
|
||||
- `book_ret = book_series(closes, ...)` → `{isodate: ret}`; `weights = target_weights(closes, ...)` → `{isodate: {sym: w}}` (adapt from Step 1).
|
||||
- `adv = dollar_adv(closes, volumes)`.
|
||||
- base series = `equity_honest_returns(book_ret, weights, adv, aum=target_aum, price_by_day=closes)`; store ISO-keyed under the sid.
|
||||
- ceiling = `capacity_ceiling_from_curve({aum: sharpe_of(equity_honest_returns(...at aum)) for aum in aums}, min_sharpe)` via `equity_capacity_curve` (load-once).
|
||||
- Absent/short PIT data → no series (caller falls back), ceiling `None`.
|
||||
- [ ] **Step 4b: Wire `record_allocation`** (`allocation_ingest.py`, in the `store is not None` branch after `honest_allocation_inputs`):
|
||||
|
||||
```python
|
||||
from fxhnt.application.equity_allocation_inputs import equity_allocation_inputs
|
||||
eq_sids = {sid for sid in funded_sids
|
||||
if STRATEGY_REGISTRY.get(sid, {}).get("venue") == "ibkr-equity"}
|
||||
if eq_sids:
|
||||
eq_returns, eq_ceilings = equity_allocation_inputs(
|
||||
repo, eq_sids, target_aum=equity,
|
||||
aums=[100_000, 1_000_000, 10_000_000, 50_000_000],
|
||||
min_sharpe=getattr(pol, "capacity_min_sharpe", 1.0))
|
||||
honest_returns = {**honest_returns, **eq_returns}
|
||||
ceilings = {**ceilings, **eq_ceilings}
|
||||
```
|
||||
|
||||
(`repo` is the `ForwardNavRepo` already constructed at the top of `record_allocation`.)
|
||||
|
||||
- [ ] **Step 5: Run — expect PASS** (unit + integration; existing allocation tests still green — byte-identity for no-equity/store=None).
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && .venv/bin/python3 -m pytest tests/unit/test_equity_allocation_inputs.py tests/integration/test_record_allocation_equity.py tests/unit -k "allocation or record" -q`
|
||||
|
||||
- [ ] **Step 6: Commit** (`feat(equity): equity_allocation_inputs provider merged into record_allocation`).
|
||||
|
||||
---
|
||||
|
||||
### Task 4: full-suite green
|
||||
|
||||
- [ ] **Step 1:** `cd /home/jgrusewski/Work/fxhnt && .venv/bin/python3 -m pytest tests/unit -q` → all pass.
|
||||
- [ ] **Step 2:** `.venv/bin/python3 -m pytest tests/integration -k "allocation or honest or equity or multistrat or yahoo" -q` → all pass.
|
||||
- [ ] **Step 3:** commit any fixes (else skip).
|
||||
|
||||
## Post-implementation (controller-run, in-cluster)
|
||||
Deploy (git-sync) + `fxhnt migrate-cockpit` (creates `yahoo_pit_volume`) + `fxhnt backfill-etf-volume`. Then the §5 gates: G1 multistrat sizes (cost-netted Sharpe < gross), G2 real ADV ceiling (finite, DBMF/PDBC-bound), G3 cost realism (SPY/IEF/GLD effective bps << 15), G4 combined marginal gate {crypto book, multistrat} — report the honest verdict + combined deployment.
|
||||
|
||||
## Self-Review
|
||||
- Coverage: §4a→T1, §4b→T2, §4c→T2, §4d→T3, gates→post-impl.
|
||||
- Byte-identity: equity merge only fires for ibkr-equity deploy sids with store injected; no-equity/store=None unchanged (T3 regression test).
|
||||
- Reuse: IMPACT_COEFF_BPS, capacity_ceiling_from_curve, load-once/reprice, first-seen PIT.
|
||||
- The estimated spread constants + the `scale_hit` capacity approximation are documented, not hidden.
|
||||
87
docs/superpowers/plans/2026-07-16-equity-ucits-rebase.md
Normal file
87
docs/superpowers/plans/2026-07-16-equity-ucits-rebase.md
Normal file
@@ -0,0 +1,87 @@
|
||||
# Equity Capacity/Cost UCITS Re-base Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: superpowers:subagent-driven-development. Checkbox (`- [ ]`) steps.
|
||||
|
||||
**Goal:** Re-base the equity leg's capacity ceiling + cost model onto the UCITS lines the EU account actually trades (CSPX/IDTM/IGLN/ICOM/DBMF-LU) instead of the US ETFs — with a regime flag so it switches back to the deep US-ETF liquidity once the fund elects MiFID-II professional status (~€500k). Returns (`book_series` on US underlyings) are unchanged — only ADV/spread inputs are re-sourced.
|
||||
|
||||
**Architecture:** The equity honest-reference already exists (`equity_honest.py`, `equity_allocation_inputs.py`) sourcing ADV from US-ETF volume + US spreads. This adds: UCITS volume ingestion (Yahoo LSE tickers), per-line UCITS spreads, and a `equity_execution_regime` config (`ucits` default | `us`). `equity_allocation_inputs` sources each US sleeve's ADV + spread from the regime's instrument (US ticker's own data, or its UCITS-mapped ticker's) while `book_series`/`target_weights` stay US.
|
||||
|
||||
**Tech Stack:** Python 3.12, SQLAlchemy 2.0, typer, pytest. Spec context: `docs/superpowers/specs/2026-07-16-equity-leg-deployable-design.md` (this refines §4a/§4b for EU/UCITS reality).
|
||||
|
||||
## Global Constraints
|
||||
|
||||
- The US→UCITS map is `settings.ucits.map` (`config.py`): `SPY→CSPX (IE00B5BMR087)`, `IEF→IDTM (IE00B1FZS798)`, `GLD→IGLN (IE00B4ND3602)`, `PDBC→ICOM (IE00BDFL4P12)`, `DBMF→DBMF (LU2951555585)`.
|
||||
- **Yahoo LSE tickers** for UCITS volume: `<ticker>.L` (CSPX.L, IDTM.L, IGLN.L, ICOM.L, DBMF.L). Per-symbol fetch failures are isolated (Task-1-of-the-prior-feature pattern) — a missing/thin line (esp. DBMF-UCITS) must NOT break ingestion.
|
||||
- **UCITS per-line half-spread (bps), documented estimates** (wider than US): `CSPX 1.5, IDTM 2.0, IGLN 2.0, ICOM 8.0, DBMF 30.0` (DBMF-UCITS is new/small/thin). Default 10.0.
|
||||
- **Regime flag** `equity_execution_regime: "ucits" | "us"`, DEFAULT `"ucits"` (the fund is retail EU today). Switching to `"us"` is a deliberate operator action AFTER electing MiFID-II professional status (~€500k portfolio + trading frequency) — NEVER auto-switched on AUM.
|
||||
- **Returns unchanged**: `book_series`/`target_weights` always use US closes (the edge proxy holds in both regimes; UCITS track the US underlyings). Only `adv_usd` + `spread_bps` fed to `equity_honest_returns` change by regime.
|
||||
- **Byte-identity of the `us` regime**: with `equity_execution_regime="us"`, the equity model is byte-identical to today (US ADV + `ETF_HALF_SPREAD_BPS`). The re-base only changes behavior under the (new default) `ucits` regime.
|
||||
- **ADV-coverage guard (from the prior fix)** still applies — if the UCITS volume is absent/thin (< coverage threshold), the sid emits NO series → nav fallback (a thin DBMF-UCITS must not fabricate capacity).
|
||||
- Commit trailer: `Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>`.
|
||||
|
||||
---
|
||||
|
||||
### Task 1: UCITS volume ingestion + config (spreads, yahoo ticker, regime flag)
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/config.py` (`UcitsListing.yahoo_ticker` + `.half_spread_bps`; `equity_execution_regime`)
|
||||
- Modify: `src/fxhnt/cli.py` (`backfill-etf-volume` also ingests the UCITS LSE tickers)
|
||||
- Modify: `src/fxhnt/adapters/orchestration/assets.py` (nightly snapshots UCITS volume too)
|
||||
- Test: `tests/unit/test_ucits_volume_config.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: `UcitsListing.yahoo_ticker` (default `f"{ticker}.L"`) + `half_spread_bps`; `settings.<...>.equity_execution_regime`.
|
||||
|
||||
- [ ] **Step 1: Failing tests** — the UCITS map exposes a `yahoo_ticker` (`CSPX.L`, `DBMF.L`, ...) and a `half_spread_bps` per line; `equity_execution_regime` defaults to `"ucits"`; an env override sets it to `"us"`.
|
||||
- [ ] **Step 2: Run — expect FAIL.**
|
||||
- [ ] **Step 3: Implement.**
|
||||
- `config.py` `UcitsListing`: add `yahoo_ticker: str = ""` (post-init/validator: default to `f"{ticker}.L"` when empty) and `half_spread_bps: float`. Populate the map's 5 entries with the Global-Constraints spread values (CSPX 1.5, IDTM 2.0, IGLN 2.0, ICOM 8.0, DBMF 30.0). DBMF's yahoo_ticker may need to be `"DBMF.L"` — set it explicitly; if Yahoo lacks it the ingestion skips gracefully.
|
||||
- Add `equity_execution_regime: str = "ucits"` to a settings group (recommend `AllocationSettings`, env `FXHNT_ALLOCATION_EQUITY_EXECUTION_REGIME`) with a comment: `ucits` (retail EU, default) | `us` (after electing MiFID-II professional, ~€500k) — sets which instruments' ADV/spread the equity capacity+cost model uses; returns stay US.
|
||||
- `cli.py` `backfill-etf-volume`: after the US `FUND_INSTRUMENTS` loop, ALSO iterate `settings.ucits.map.values()` and `YahooDailyClient(rng="25y").volumes(listing.yahoo_ticker)` → `repo.snapshot_yahoo_volumes(listing.yahoo_ticker, ...)`. Per-symbol try/except (a thin UCITS line that Yahoo lacks is logged + skipped, not fatal). Echo counts.
|
||||
- `assets.py` nightly: in the same volume-snapshot loop, also snapshot each UCITS `yahoo_ticker` (guarded like the US volume snapshot).
|
||||
- [ ] **Step 4: Run — expect PASS.**
|
||||
- [ ] **Step 5: Commit** (`feat(equity): ingest UCITS LSE volume + regime flag + UCITS spread config`).
|
||||
|
||||
---
|
||||
|
||||
### Task 2: regime-aware equity ADV/spread sourcing
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/application/equity_allocation_inputs.py`
|
||||
- Test: `tests/unit/test_equity_allocation_inputs.py` (extend)
|
||||
|
||||
**Interfaces:**
|
||||
- `equity_allocation_inputs(repo, deploy_sids, *, target_aum, aums, min_sharpe=1.0, regime="ucits", ucits_map=None, us_spread_bps=None, ucits_spread_bps=None)` — new `regime` + injectable maps (default from settings). Returns unchanged shape.
|
||||
|
||||
- [ ] **Step 1: Failing tests.**
|
||||
- regime="us": `adv_usd[sym]` sourced from `yahoo_pit_volumes(sym)` (US) + spread `ETF_HALF_SPREAD_BPS[sym]` — byte-identical to current behavior (regression assertion against the pre-regime output).
|
||||
- regime="ucits": `adv_usd[sym]` sourced from the UCITS-mapped ticker's `yahoo_pit_volumes(ucits_ticker)` (remapped to the US sleeve key so `equity_honest_returns`'s US-keyed weights line up) + spread = the per-line UCITS spread. With a seeded LOW UCITS volume and a HIGH US volume, the `ucits` ceiling must be MATERIALLY LOWER than the `us` ceiling (proves the re-base bites).
|
||||
- regime="ucits" with a thin/absent UCITS line (e.g. DBMF.L empty) → that sleeve's ADV is absent → coverage guard trips or that sleeve is capacity-scaled — verify it does NOT fabricate capacity (no series if coverage < threshold).
|
||||
- [ ] **Step 2: Run — expect FAIL.**
|
||||
- [ ] **Step 3: Implement.**
|
||||
- In `equity_allocation_inputs`, build the per-sleeve ADV + spread from the regime:
|
||||
- `us`: `adv_source_ticker[sym] = sym`; `spread[sym] = (us_spread_bps or ETF_HALF_SPREAD_BPS).get(sym, default)`.
|
||||
- `ucits`: `adv_source_ticker[sym] = ucits_map[sym].yahoo_ticker`; `spread[sym] = ucits_map[sym].half_spread_bps`.
|
||||
- Load `volumes_by_source = {sym: repo.yahoo_pit_volumes(adv_source_ticker[sym])}`; `adv = dollar_adv({sym: us_closes[sym]}, {sym: volumes_by_source[sym]})` — **KEY**: `dollar_adv` needs price×volume; use the US close as the price proxy for the USD-denominated UCITS line (the USD UCITS lines track the US underlying's USD price closely; document this — a small basis, acceptable for a capacity estimate; a follow-on could use the real UCITS price). So `adv_usd` is keyed by the US sleeve `sym` but its VOLUME comes from the UCITS line. Spread passed as the regime spread dict.
|
||||
- Pass `spread_bps=<regime spread dict>` into `equity_honest_returns`/`equity_capacity_curve` (add a `spread_bps` param there if it currently hardcodes `ETF_HALF_SPREAD_BPS` — check Task-2-of-the-prior-feature `equity_honest.py`; if `_spread_bps(sym)` reads the module constant, thread an optional `spread_bps: dict|None` override so the regime dict wins). Do NOT mutate the module constant.
|
||||
- Defaults: `regime = settings...equity_execution_regime`, `ucits_map = settings.ucits.map`, spreads from config — resolved at the call in `record_allocation` (pass them in) OR defaulted inside via `get_settings()`; keep it injectable for tests.
|
||||
- [ ] **Step 4: `record_allocation`** passes `regime=getattr(pol/settings, ...)` (thread the regime from settings; the equity block already exists). Confirm the `us`-regime path stays byte-identical.
|
||||
- [ ] **Step 5: Run — expect PASS** (unit + the byte-identity regression + full allocation tests).
|
||||
- [ ] **Step 6: Commit** (`feat(equity): regime-aware ADV/spread (ucits default | us) for the capacity+cost model`).
|
||||
|
||||
---
|
||||
|
||||
### Task 3: full-suite green
|
||||
|
||||
- [ ] **Step 1:** `.venv/bin/python3 -m pytest tests/unit -q` → all pass.
|
||||
- [ ] **Step 2:** `.venv/bin/python3 -m pytest tests/integration -k "allocation or equity or yahoo or multistrat" -q` → all pass.
|
||||
- [ ] **Step 3:** commit any fixes.
|
||||
|
||||
## Post-implementation (controller-run, in-cluster)
|
||||
Deploy (git-sync) + `fxhnt migrate-cockpit` + `fxhnt backfill-etf-volume` (now also fetches the UCITS LSE lines). Then re-run the equity gates UNDER `equity_execution_regime=ucits`: report the UCITS-real ceiling (expect MATERIALLY below the US ≥$10M — likely $0.5–3M, DBMF/ICOM-bound) + which UCITS line binds + the net Sharpe under UCITS spreads (expect below the 1.58 US estimate). Confirm DBMF.L coverage (if Yahoo lacks it, the sleeve falls to the coverage guard). Contrast with `regime=us` to show the professional-status upside. NOTE: run LIGHT (equity provider only — do NOT re-run the 2× crypto-book path that OOM-killed the daemon; keep in-cluster diagnostics to one honest path at a time).
|
||||
|
||||
## Self-Review
|
||||
- Byte-identity: `regime="us"` reproduces today's equity model (regression test).
|
||||
- Returns unchanged: only adv/spread inputs re-sourced; `book_series`/`target_weights` untouched.
|
||||
- Thin-line safety: ADV-coverage guard + per-symbol ingestion isolation protect against a missing/thin DBMF-UCITS.
|
||||
- Documented approximation: US close as the price proxy for the USD UCITS line's dollar-ADV (small basis; follow-on = real UCITS price).
|
||||
@@ -0,0 +1,173 @@
|
||||
# Honest-Inputs Load-Once/Reprice Refactor Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: superpowers:subagent-driven-development. Checkbox (`- [ ]`) steps.
|
||||
|
||||
**Goal:** Refactor `allocation_honest_inputs` so each crypto sleeve's AUM-INDEPENDENT panels (weights + turnover) are loaded ONCE and repriced across all capacity-curve AUMs, instead of a full panel reload per AUM — so the nightly can use a wider AUM grid without OOM-killing on the book deploy set.
|
||||
|
||||
**Architecture:** Split the current load+reprice `_sleeve_series`/`_book_series`/`_deploy_series` into a load step (`_sleeve_inputs` → (wbd, turnover), heavy, once) and a reprice step (reuse `honest_reference.honest_sleeve_returns` / `capacity_curve`, cheap, per AUM). `honest_allocation_inputs` loads once per sid then reprices for base + the whole curve. Byte-identical honest series/ceilings; only the compute schedule changes. Then widen the nightly AUM grid.
|
||||
|
||||
**Tech Stack:** Python 3.12, pytest. The honest-reference REPORT path (`honest_report.py`) already does load-once/reprice — this brings the allocation provider to the same pattern (and reuses `capacity_curve`).
|
||||
|
||||
## Global Constraints
|
||||
|
||||
- **Byte-identity:** for the same store/spread/unlock_events/aums, the NEW `honest_allocation_inputs` must return the SAME `returns_by_strategy` (ISO-keyed) and the SAME `ceilings` as the OLD one. The series math is unchanged — same `wbd`, same `turnover`, same `honest_sleeve_returns(...)` calls; only WHEN the panel is loaded changes.
|
||||
- **Load-once invariant:** `store.read_panel(...)` must be called ONCE per distinct sleeve (1 for a sleeve sid, 3 for the `bybit_4edge` book) regardless of how many AUMs are in the grid — NOT once per (sleeve × AUM). Enforce with a test that counts `read_panel` calls.
|
||||
- `_BOOK_SLEEVES = ("xsfunding", "unlock", "positioning")`, `_CRYPTO_DEPLOY_SIDS = {positioning, xsfunding, unlock, bybit_4edge}` — unchanged. tstrend still excluded (no standalone store series).
|
||||
- The old `_sleeve_series`/`_book_series`/`_deploy_series` become dead after the refactor → DELETE them (wire-or-delete; no dead code). They are used ONLY inside `allocation_honest_inputs.py` (verified — the report path has its own impl).
|
||||
- Reuse `honest_reference.capacity_curve` for the sleeve reprice (do not reimplement the per-AUM loop).
|
||||
- Commit trailer: `Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>`.
|
||||
|
||||
---
|
||||
|
||||
### Task 1: load-once/reprice refactor of the provider
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/application/allocation_honest_inputs.py`
|
||||
- Test: `tests/unit/test_allocation_honest_inputs.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `sleeve_weights_and_contributions` (bybit_paper_weights), `honest_sleeve_returns` + `capacity_curve` (honest_reference), `book_returns` + `risk_parity_weights` + `sharpe_of` (honest_gates), `capacity_ceiling_from_curve` (allocation_engine).
|
||||
- Produces (new internal helpers, replacing the old ones):
|
||||
- `_sleeve_inputs(store, sleeve, unlock_events) -> tuple[dict, dict] | None`
|
||||
- `_book_curve(store, spread_bps, unlock_events, aums, *, participation_cap, impact_k) -> dict[float, dict[int,float]]`
|
||||
- `_deploy_curve(store, sid, spread_bps, unlock_events, aums, *, participation_cap, impact_k) -> dict[float, dict[int,float]]`
|
||||
- `honest_allocation_inputs(...)` — SAME public signature and return type.
|
||||
|
||||
- [ ] **Step 1: Write the load-once-count test first** (extend `test_allocation_honest_inputs.py`). Wrap the fake store so `read_panel` increments a counter; call `honest_allocation_inputs(deploy_sids={"bybit_4edge"}, target_aum=100_000, aums=[35_000,100_000,350_000,1_000_000,10_000_000])` (5 AUMs) and assert `read_panel` was called exactly **3 times** (one per book sleeve), NOT 3×(1+5). Also add a sleeve case: `deploy_sids={"positioning"}` with the same 5 AUMs → `read_panel` called exactly **1 time**.
|
||||
|
||||
Look at the existing test file's fake store / `_seed(...)` pattern and mirror it — wrap or subclass the store used there to count `read_panel`.
|
||||
|
||||
- [ ] **Step 2: Run — expect FAIL** (current code reloads per AUM → 18 and 6 calls)
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && .venv/bin/python3 -m pytest tests/unit/test_allocation_honest_inputs.py -q`
|
||||
Expected: the new count tests FAIL (too many read_panel calls); the existing series/ceiling tests still PASS.
|
||||
|
||||
- [ ] **Step 3: Implement the refactor.** Replace `_sleeve_series`, `_book_series`, `_deploy_series` with:
|
||||
|
||||
```python
|
||||
def _sleeve_inputs(store: Any, sleeve: str,
|
||||
unlock_events: list[dict[str, Any]] | None) -> tuple[dict, dict] | None:
|
||||
"""AUM-INDEPENDENT heavy inputs for a sleeve: (weights_by_day, turnover_panel). Load ONCE, reprice across
|
||||
AUMs via honest_sleeve_returns/capacity_curve — the turnover panel read is the memory cost, so it must NOT
|
||||
be reloaded per AUM. None when the sleeve is uncomputable (e.g. unlock with no events). READ-ONLY."""
|
||||
wbd = sleeve_weights_and_contributions(store, sleeve, unlock_events=unlock_events)
|
||||
if not wbd:
|
||||
return None
|
||||
symbols = sorted({s for row in wbd.values() for s in row})
|
||||
turnover = store.read_panel(symbols, "turnover")
|
||||
return wbd, turnover
|
||||
|
||||
|
||||
def _book_curve(store: Any, spread_bps: dict[str, float], unlock_events: list[dict[str, Any]] | None,
|
||||
aums, *, participation_cap: float, impact_k: float) -> dict[float, dict[int, float]]:
|
||||
"""{aum: risk-parity BOOK series} across `aums`, loading each `_BOOK_SLEEVES` sleeve's inputs ONCE then
|
||||
repricing per AUM. Empty sleeves (uncomputable) are dropped; an AUM with no computable sleeve -> {}."""
|
||||
loaded = {sleeve: inp for sleeve in _BOOK_SLEEVES
|
||||
if (inp := _sleeve_inputs(store, sleeve, unlock_events)) is not None}
|
||||
out: dict[float, dict[int, float]] = {}
|
||||
for aum in aums:
|
||||
per: dict[str, dict[int, float]] = {}
|
||||
for sleeve, (wbd, turnover) in loaded.items():
|
||||
series = honest_sleeve_returns(wbd, turnover, spread_bps, aum=aum,
|
||||
participation_cap=participation_cap, impact_k=impact_k)
|
||||
if series:
|
||||
per[sleeve] = series
|
||||
out[aum] = book_returns(per, risk_parity_weights(per)) if per else {}
|
||||
return out
|
||||
|
||||
|
||||
def _deploy_curve(store: Any, sid: str, spread_bps: dict[str, float],
|
||||
unlock_events: list[dict[str, Any]] | None, aums, *,
|
||||
participation_cap: float, impact_k: float) -> dict[float, dict[int, float]]:
|
||||
"""{aum: honest series} for a crypto deploy `sid` across `aums`, loading AUM-independent inputs ONCE.
|
||||
`bybit_4edge` -> the risk-parity book; a sleeve sid -> its own series (reuses `capacity_curve`); other -> {}."""
|
||||
if sid == "bybit_4edge":
|
||||
return _book_curve(store, spread_bps, unlock_events, aums,
|
||||
participation_cap=participation_cap, impact_k=impact_k)
|
||||
if sid in _CRYPTO_DEPLOY_SIDS:
|
||||
inputs = _sleeve_inputs(store, sid, unlock_events)
|
||||
if inputs is None:
|
||||
return {aum: {} for aum in aums}
|
||||
wbd, turnover = inputs
|
||||
return capacity_curve(wbd, turnover, spread_bps, aums=aums,
|
||||
participation_cap=participation_cap, impact_k=impact_k)
|
||||
return {aum: {} for aum in aums}
|
||||
```
|
||||
|
||||
Rewrite the loop body of `honest_allocation_inputs` (keep the signature + the non-crypto branch) so it loads/reprices once:
|
||||
|
||||
```python
|
||||
for sid in deploy_sids:
|
||||
if sid not in _CRYPTO_DEPLOY_SIDS:
|
||||
ceilings[sid] = equity_default_ceiling # equity / non-crypto — caller falls back; no series
|
||||
continue
|
||||
# AUM-independent panels loaded ONCE inside _deploy_curve; base + curve are repriced from them.
|
||||
all_aums = sorted({float(target_aum), *(float(a) for a in aums)})
|
||||
curve_series = _deploy_curve(store, sid, spread_bps, unlock_events, all_aums,
|
||||
participation_cap=participation_cap, impact_k=impact_k)
|
||||
base = curve_series.get(float(target_aum), {})
|
||||
if base:
|
||||
returns_by_strategy[sid] = {_iso(day): ret for day, ret in base.items()}
|
||||
curve = {aum: sharpe_of(curve_series.get(float(aum), {})) for aum in aums}
|
||||
ceilings[sid] = capacity_ceiling_from_curve(curve, min_sharpe)
|
||||
```
|
||||
|
||||
Update imports (add `capacity_curve` from honest_reference) and the module docstring (deploy→honest mapping now describes the load-once/reprice; the report path already does this).
|
||||
|
||||
**Note on float keying:** the curve dict is keyed by the AUM values passed to `_deploy_curve`. Build `all_aums` and look up with the SAME float() cast on both sides (as shown) so `target_aum` and grid entries match by key even if one is int and one is float.
|
||||
|
||||
- [ ] **Step 4: Run — expect PASS** (count tests pass; series/ceiling tests unchanged and still pass = byte-identity held)
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && .venv/bin/python3 -m pytest tests/unit/test_allocation_honest_inputs.py -q`
|
||||
Expected: PASS (all).
|
||||
|
||||
- [ ] **Step 5: Confirm no other consumer of the deleted helpers**
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && grep -rn "_sleeve_series\|_book_series\|_deploy_series" src/ tests/ | grep -v "docstring\|# "`
|
||||
Expected: no CODE references remain (only prose/docstring mentions, which you should also update if they name the removed functions).
|
||||
|
||||
- [ ] **Step 6: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/application/allocation_honest_inputs.py tests/unit/test_allocation_honest_inputs.py
|
||||
git commit -m "refactor(alloc): load-once/reprice honest inputs (panels loaded once, repriced per AUM)
|
||||
|
||||
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 2: widen the nightly AUM grid + full suite green
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/application/allocation_ingest.py`
|
||||
|
||||
- [ ] **Step 1:** In `record_allocation`, widen the capacity-curve grid back to characterize the book's break with headroom (now cheap — one load, many reprices): `aums=[35_000, 100_000, 350_000]` → `aums=[35_000, 100_000, 350_000, 1_000_000, 10_000_000]`. Update the leading comment: the load-once/reprice provider (Task 1) makes grid length cheap (repricing, not reloading), so the wider grid no longer OOMs; it restores capacity resolution above the book's ~$350k break so growth past it is detectable.
|
||||
|
||||
- [ ] **Step 2: Full unit suite**
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && .venv/bin/python3 -m pytest tests/unit -q`
|
||||
Expected: all pass.
|
||||
|
||||
- [ ] **Step 3: Allocation/honest integration**
|
||||
|
||||
Run: `cd /home/jgrusewski/Work/fxhnt && .venv/bin/python3 -m pytest tests/integration -k "allocation or honest or funding" -q`
|
||||
Expected: all pass (2 skipped ok).
|
||||
|
||||
- [ ] **Step 4: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/application/allocation_ingest.py
|
||||
git commit -m "feat(alloc): widen nightly capacity-curve AUM grid (load-once makes it cheap)
|
||||
|
||||
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
## Post-implementation (controller-run, in-cluster)
|
||||
Deploy (git-sync), re-run `record_allocation` with the honest store: confirm (1) NO OOM with the widened 5-AUM grid on the book deploy set, (2) `read_panel` load-once holds (the whole point), (3) the book ceiling is resolved over the wider grid (still ~$350k or refined), (4) deployment still ~$75k (75%) — byte-identical sizing, just a wider capacity probe.
|
||||
|
||||
## Self-Review
|
||||
- Byte-identity: same wbd/turnover/honest_sleeve_returns → identical series+ceilings (only load schedule changed); existing series/ceiling tests are the guard.
|
||||
- Load-once: enforced by the read_panel-count test (invariant, not a comment).
|
||||
- Dead code: old three helpers deleted; grep gate in Task 1 Step 5.
|
||||
- Reuse: `capacity_curve` reused for the sleeve path (DRY).
|
||||
@@ -0,0 +1,73 @@
|
||||
# IBKR UCITS Volume Source Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: superpowers:subagent-driven-development. Checkbox (`- [ ]`) steps.
|
||||
|
||||
**Goal:** Source the UCITS regime's ADV from **IBKR real daily volume** (`reqHistoricalData TRADES` — the authoritative on-exchange volume of the exact line we execute) instead of Yahoo's coarse LSE feed. Fetch runs in the IBKR-network-reachable rebalancer context → DB; the daemon/nightly reads the DB.
|
||||
|
||||
**Architecture:** A parallel `ibkr_pit_volume` PIT table (first-seen-frozen, mirrors `yahoo_pit_volume`); an `IbkrBroker.historical_daily_volume` method (reqHistoricalData TRADES); a `backfill-ucits-volume-ibkr` CLI (rebalancer context); a `ucits_volume_source` config flag (`yahoo` default | `ibkr`) that `equity_allocation_inputs` honors; a fetch CronJob. Spread stays the estimated per-line constant (real IBKR bid/ask needs a market-data subscription — Error 354, a documented follow-on).
|
||||
|
||||
**Tech Stack:** Python 3.12, SQLAlchemy 2.0, `ib_async` (runtime-installed in the rebalancer image), typer, pytest, k8s CronJob.
|
||||
|
||||
## Global Constraints
|
||||
|
||||
- **Probe-verified (2026-07-16):** IBKR `reqHistoricalData(contract, "", "N D", "1 day", "TRADES", useRTH=True, formatDate=1)` returns real daily volume for the UCITS lines (CSPX/CBU0/ICOM/DBMF all resolved via LSEETF+ISIN; DBMF ~16 shares/day confirms genuine illiquidity). `reqMktData` bid/ask = NaN / Error 354 (no market-data subscription) → **spread stays estimated; do NOT attempt real-time spread here.**
|
||||
- **Network reality:** only pods labeled `app: multistrat-rebalancer` (or trading-service/broker-gateway) can reach `ib-gateway:4004` (NetworkPolicy). The daemon CANNOT. So the IBKR fetch MUST run in the rebalancer context (a CronJob), write to the DB; the daemon reads the DB. `ib_async` is a runtime `pip install ib_async==2.1.0` (extra `ibkr = ["ib-async>=2.0"]`), NOT in the base image.
|
||||
- **PIT discipline:** `ibkr_pit_volume` is first-seen-frozen (mirror `snapshot_yahoo_volumes` — insert only (symbol,date) not present).
|
||||
- **Byte-identity of `ucits_volume_source="yahoo"` (the default):** `equity_allocation_inputs` must be byte-identical to today when the flag is `yahoo`. The IBKR read path only activates on `ibkr`.
|
||||
- Store IBKR volume keyed by the UCITS `yahoo_ticker` string (e.g. `CSPX.L`) so the equity provider's existing per-sleeve key mapping is reused unchanged — the source flag only switches WHICH table is read.
|
||||
- Commit trailer: `Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>`.
|
||||
|
||||
---
|
||||
|
||||
### Task 1: IBKR volume storage + broker fetch method + config flag
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/adapters/persistence/cockpit_models.py` (`IbkrPitVolumeRow`)
|
||||
- Modify: `src/fxhnt/adapters/persistence/forward_nav.py` (`snapshot_ibkr_volumes` / `ibkr_pit_volumes`)
|
||||
- Modify: `src/fxhnt/adapters/broker/ibkr.py` (`historical_daily_volume`)
|
||||
- Modify: `src/fxhnt/config.py` (`AllocationSettings.ucits_volume_source`)
|
||||
- Test: `tests/unit/test_ibkr_volume_pit.py`
|
||||
|
||||
**Interfaces:**
|
||||
- `IbkrPitVolumeRow` (`ibkr_pit_volume`, PK (symbol,date), volume Float, first_seen_at) — rides `create_all`.
|
||||
- `ForwardNavRepo.snapshot_ibkr_volumes(symbol, series, at)` / `.ibkr_pit_volumes(symbol) -> dict[str,float]` (mirror the yahoo methods exactly, first-seen-frozen).
|
||||
- `IbkrBroker.historical_daily_volume(*, ticker, isin, exchange="LSEETF", currency="USD", days=90) -> dict[str,float]`: resolve via `_resolve_contract`, `reqHistoricalData(contract, "", f"{days} D", "1 day", "TRADES", useRTH=True, formatDate=1)`, return `{bar.date.isoformat() or str(bar.date): float(bar.volume)}` for bars with `volume not in (None, -1)`. Empty dict on resolve-fail/no-data (log + return {}, never raise into the caller's loop).
|
||||
- `AllocationSettings.ucits_volume_source: str = "yahoo"` (yahoo | ibkr), env `FXHNT_ALLOCATION_UCITS_VOLUME_SOURCE`, comment: which store the UCITS-regime ADV reads from; `ibkr` = the real IBKR on-exchange volume (fetched by the `ucits-volume-ibkr` CronJob), `yahoo` = the Yahoo LSE feed (default until the CronJob is running).
|
||||
|
||||
- [ ] **Step 1: Failing tests** — `ibkr_pit_volume` roundtrip + first-seen-frozen (mirror the yahoo volume test); `ucits_volume_source` default "yahoo" + env override "ibkr". (The broker method needs a live gateway — do NOT unit-test the fetch; test the storage + config only. Note in the test file why the broker fetch is integration-only.)
|
||||
- [ ] **Step 2: Run — expect FAIL.**
|
||||
- [ ] **Step 3: Implement** all four (mirror the yahoo volume + the existing `market_price` reqHistoricalData pattern for the broker method; `bar.date` from ib_async is a `datetime.date` — format to ISO `YYYY-MM-DD`).
|
||||
- [ ] **Step 4: Run — expect PASS.**
|
||||
- [ ] **Step 5: Commit** (`feat(equity): IBKR pit-volume store + historical_daily_volume + ucits_volume_source flag`).
|
||||
|
||||
---
|
||||
|
||||
### Task 2: `backfill-ucits-volume-ibkr` CLI + source-flag read
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/cli.py` (`backfill-ucits-volume-ibkr`)
|
||||
- Modify: `src/fxhnt/application/equity_allocation_inputs.py` (read ibkr vs yahoo by flag)
|
||||
- Test: `tests/unit/test_equity_allocation_inputs.py` (extend)
|
||||
|
||||
- [ ] **Step 1: Failing tests** — `equity_allocation_inputs(..., regime="ucits", volume_source="ibkr")` reads `ibkr_pit_volumes(ucits_ticker)`; `volume_source="yahoo"` (default) reads `yahoo_pit_volumes` (byte-identical to today, regression). Seed one repo with DIFFERENT ibkr vs yahoo volume for a UCITS ticker → the ceiling differs by source (proves the switch reads the right table).
|
||||
- [ ] **Step 2: Run — expect FAIL.**
|
||||
- [ ] **Step 3: Implement:**
|
||||
- `equity_allocation_inputs` gains `volume_source: str = "yahoo"` (default from `settings.allocation.ucits_volume_source` at the `record_allocation` call). In the ucits branch, `vol_read = repo.ibkr_pit_volumes if volume_source == "ibkr" else repo.yahoo_pit_volumes`; `ucits_volumes = {sym: vol_read(resolved_map[sym].yahoo_ticker) ...}`. The us branch is unchanged (always yahoo US volume — Databento is the future US source, out of scope). `record_allocation` threads `volume_source=get_settings().allocation.ucits_volume_source`.
|
||||
- `cli.py` `backfill-ucits-volume-ibkr`: `pip`-installed `ib_async` is assumed present (rebalancer context). For each `settings.ucits.map` listing, `IbkrBroker(host, port, client_id).historical_daily_volume(ticker=listing.ticker, isin=listing.isin, exchange=listing.exchange, currency=listing.currency, days=<opt, default 90>)` → `repo.snapshot_ibkr_volumes(listing.yahoo_ticker, series, at)`. Per-line try/except (a thin/unresolvable line logs + continues). Echo per-line day counts. Use `get_settings().ibkr` for host/port/client_id.
|
||||
- [ ] **Step 4: Run — expect PASS** (+ full unit suite).
|
||||
- [ ] **Step 5: Commit** (`feat(equity): backfill-ucits-volume-ibkr CLI + source-flag read in equity provider`).
|
||||
|
||||
---
|
||||
|
||||
### Task 3: fetch CronJob (infra) + full green — CONTROLLER-run (not a subagent task)
|
||||
- [ ] Add `infra/k8s/.../ucits-volume-ibkr-cronjob.yaml`: label `app: multistrat-rebalancer`, image `fxhnt-cockpit:latest`, `imagePullSecrets: scw-registry`, env (FXHNT_OPERATIONAL_DSN + PGPASSWORD from the same secret the rebalancer uses, FXHNT_IBKR_HOST=ib-gateway, PORT=4004, CLIENT_ID=27), command `pip install --quiet ib_async==2.1.0 && fxhnt backfill-ucits-volume-ibkr`. Schedule daily before the ucits-rebalancer (e.g. `30 8 * * 1-5`). Clone the ucits-rebalancer's spec for the DSN/secret wiring.
|
||||
- [ ] `.venv/bin/python3 -m pytest tests/unit -q` + `tests/integration -k "equity or allocation or ibkr or yahoo"` → green.
|
||||
|
||||
## Post-implementation (controller-run, in-cluster)
|
||||
Deploy (git-sync) + migrate-cockpit (creates `ibkr_pit_volume`). Run the fetch as a one-off Job (rebalancer context) → confirm `ibkr_pit_volume` populates with real volume (DBMF ~16 shares/day). Flip `FXHNT_ALLOCATION_UCITS_VOLUME_SOURCE=ibkr` and re-run the equity gate → the book sizes on the real IBKR ADV (partial: liquid lines + DBMF scaled to ~0). Contrast the IBKR-sourced ceiling vs the Yahoo-sourced one. NOTE: the fetch MUST run labeled `app: multistrat-rebalancer` (netpol); the daemon cannot reach the gateway.
|
||||
|
||||
## Self-Review
|
||||
- Byte-identity: `ucits_volume_source="yahoo"` (default) unchanged; ibkr path additive.
|
||||
- Network-correct: fetch in rebalancer context → DB → daemon reads DB (no IBKR in the daemon).
|
||||
- Spread deferred honestly (subscription follow-on; Error 354 documented).
|
||||
- PIT first-seen-frozen for ibkr_pit_volume.
|
||||
@@ -0,0 +1,221 @@
|
||||
# Foxhunt → Bizworx Consolidation — Implementation Plan
|
||||
|
||||
> **For the operator/agent:** execute **inline, with a checkpoint after every task** (this is live infra +
|
||||
> real-money fund + secrets — NOT a subagent-driven code plan). Steps use `- [ ]` for tracking. Spec:
|
||||
> `docs/superpowers/specs/2026-07-18-foxhunt-to-bizworx-consolidation-design.md`.
|
||||
|
||||
**Goal:** Move the fxhnt fund onto the bizworx Kapsule + consolidate to one gitea, verify it fully, THEN
|
||||
retire the foxhunt Kapsule (cost saving).
|
||||
|
||||
**Architecture:** Build the fund on bizworx **alongside** the still-live foxhunt cluster, migrate the one
|
||||
stateful thing (Postgres/Timescale), prove it works in parallel, cut over DNS, observe, and only then tear
|
||||
down. Reuse bizworx's existing argo + observability + gitea.
|
||||
|
||||
**Tech stack:** Scaleway Kapsule (2 clusters, 2 projects, same org), kubectl (contexts `bizworx-prod-…` +
|
||||
`foxhunt-…`), `scw` CLI, Terragrunt/Terraform (`infra/live/production`), argo (git-sync deploy), gitea,
|
||||
TimescaleDB/Postgres, IBKR ib-gateway, tailscale.
|
||||
|
||||
## Global Constraints (apply to EVERY task)
|
||||
|
||||
- **VERIFY-FIRST, NOTHING DESTRUCTIVE UNTIL PROVEN.** No delete / teardown / `terraform destroy` / PVC
|
||||
removal / foxhunt scale-down happens before Phase 1 is fully verified AND the observation window (Task 10)
|
||||
passes. Tasks 1–9 are strictly additive; foxhunt stays fully live through Task 8, suspended-but-intact
|
||||
through teardown.
|
||||
- **EVALUATE PER STEP.** Every task ends with an explicit `EVAL:` gate. If it fails → STOP, do not proceed.
|
||||
- **ROLLBACK is always available** until Task 12: re-point DNS + un-suspend foxhunt CronJobs (foxhunt's
|
||||
Postgres still holds pre-cutover state).
|
||||
- **Secrets NEVER through the transcript.** Create them with `! kubectl create secret …` / openbao /
|
||||
external-secrets. Never echo/commit secret values.
|
||||
- **Real-money fund:** before arming any rebalancer on bizworx, a dry-run + recon-gate check must pass.
|
||||
- **Two contexts, always explicit:** `FOX=foxhunt-34a1e3c4-ac35-48c8-ab49-5f6ec4df32c1`,
|
||||
`BIZ=bizworx-prod-kapsule-05e1dae3-8f35-4ad4-8fce-0f9a7691a760`. Never rely on current-context.
|
||||
- Cockpit changes verified LOCALLY in a real browser + on prod, reading the numbers.
|
||||
- Per-item operator sign-off before deleting any Phase-2 data.
|
||||
|
||||
---
|
||||
|
||||
## PHASE 1 — build + verify on bizworx (foxhunt stays live) — additive only
|
||||
|
||||
### Task 1: Prerequisites + target namespace
|
||||
|
||||
**Files:** none (cluster ops).
|
||||
- [ ] Confirm both kube-contexts respond: `kubectl --context $BIZ get ns` and `kubectl --context $FOX get ns`.
|
||||
- [ ] Pick + confirm a StorageClass on bizworx for the fund (foxhunt used `sbs-default`/`scw-bssd`; bizworx has
|
||||
`sbs-5k`,`sbs-default`): `kubectl --context $BIZ get storageclass`.
|
||||
- [ ] Create the target namespace: `kubectl --context $BIZ create ns foxhunt`.
|
||||
- [ ] Record the foxhunt Postgres + TimescaleDB versions to match on bizworx:
|
||||
`kubectl --context $FOX exec -n foxhunt deploy/postgres -- psql -U postgres -tAc "select version(); select extversion from pg_extension where extname='timescaledb';"`
|
||||
|
||||
**EVAL:** both contexts work, ns `foxhunt` exists on bizworx, TimescaleDB version noted. **No foxhunt change.**
|
||||
|
||||
### Task 2: Secrets on bizworx
|
||||
|
||||
**Files:** none (secrets via `!`/openbao — never committed).
|
||||
- [ ] List the secrets the fund workloads consume on foxhunt (names only, NOT values):
|
||||
`kubectl --context $FOX get secrets -n foxhunt` → expect `db-credentials`, `databento-credentials`, an
|
||||
IBKR/ib-gateway secret, `scw-registry`, `argo-git-ssh-key`/`git-ssh-key`, tailscale, cockpit TLS.
|
||||
- [ ] For each: recreate on bizworx sourced from openbao/external-secrets, or by hand with
|
||||
`! kubectl --context $BIZ create secret … -n foxhunt` (operator runs; values never in transcript).
|
||||
- [ ] Verify presence: `kubectl --context $BIZ get secrets -n foxhunt` shows every required name.
|
||||
|
||||
**EVAL:** every fund secret exists on bizworx (by name). A missing secret wedges a workload later — do not
|
||||
proceed until complete. **No foxhunt change.**
|
||||
|
||||
### Task 3: Consolidate gitea (mirror repos; DO NOT re-point DNS yet)
|
||||
|
||||
**Files:** none (gitea admin ops).
|
||||
- [ ] Inventory both giteas' repos (names/sizes) to confirm scope:
|
||||
foxhunt gitea (via `git.fxhnt.ai` today) has `gitadmin/fxhnt`, `gitadmin/foxhunt` (+ any others — list them).
|
||||
- [ ] On the **bizworx gitea**, create the `gitadmin` org/user + empty `fxhnt` and `foxhunt` repos.
|
||||
- [ ] Mirror all refs/tags/LFS from foxhunt gitea → bizworx gitea (gitea migration UI/API, or
|
||||
`git clone --mirror` from `git.fxhnt.ai` then `git push --mirror` to the bizworx gitea endpoint via a
|
||||
temporary port-forward / tailnet address).
|
||||
- [ ] Verify parity: for `fxhnt` and `foxhunt`, `git ls-remote` ref count + latest master/main sha match
|
||||
between the two giteas.
|
||||
|
||||
**EVAL:** bizworx gitea holds both repos with matching refs/tags. `git.fxhnt.ai` STILL points at foxhunt
|
||||
(unchanged) — this task is a non-destructive copy. **No foxhunt change; both giteas intact.**
|
||||
|
||||
### Task 4: Re-target the k8s manifests (branch, dry-run only)
|
||||
|
||||
**Files:** `infra/k8s/**` (on this branch `chore/foxhunt-bizworx-consolidation`).
|
||||
- [ ] Identify manifests that hardcode the foxhunt cluster/endpoints: gitea `gitea-sshd.foxhunt.svc…`
|
||||
(still valid — same ns name), image registry `rg.fr-par.scw.cloud/bizworx/…` (already bizworx — good),
|
||||
StorageClass names, any nodeSelector/affinity, the `git.fxhnt.ai` git-sync URL, ingress hosts.
|
||||
- [ ] Change only what must change for bizworx (StorageClass if different; git-sync stays on `git.fxhnt.ai`
|
||||
since DNS will re-point; ns stays `foxhunt`). Commit to the branch.
|
||||
- [ ] Dry-run against bizworx: `kubectl --context $BIZ apply -n foxhunt --dry-run=server -f <manifest>` for
|
||||
each (server-side dry-run catches admission/StorageClass errors).
|
||||
|
||||
**EVAL:** every fund manifest passes server-side dry-run on bizworx. **Nothing applied for real yet.**
|
||||
|
||||
### Task 5: Stand up Postgres/Timescale on bizworx (empty)
|
||||
|
||||
**Files:** `infra/k8s/databases/**`.
|
||||
- [ ] Apply the Postgres/Timescale manifest to bizworx ns `foxhunt` (new PVC on the chosen StorageClass).
|
||||
- [ ] Verify it comes up + the TimescaleDB extension version MATCHES foxhunt (from Task 1):
|
||||
`kubectl --context $BIZ exec -n foxhunt deploy/postgres -- psql -U postgres -tAc "select extversion from pg_extension where extname='timescaledb';"`
|
||||
|
||||
**EVAL:** bizworx Postgres is Running and its TimescaleDB version == foxhunt's. If mismatch → pin the image to
|
||||
match BEFORE any restore (a version skew breaks hypertable restore).
|
||||
|
||||
### Task 6: DB migration REHEARSAL (non-destructive dump/restore)
|
||||
|
||||
**Files:** none (a rehearsal; foxhunt keeps running/serving).
|
||||
- [ ] `pg_dump` the foxhunt `fxhnt` DB using the **hypertable-aware** path (call `timescaledb_pre_restore()`
|
||||
on the target before restore and `timescaledb_post_restore()` after; use `--no-owner` as needed). Stream
|
||||
foxhunt→bizworx (e.g. `kubectl exec … pg_dump … | kubectl exec -i … pg_restore …`) — read-only on foxhunt.
|
||||
- [ ] Verify row counts match on the load-bearing tables:
|
||||
`forward_nav`, `backtest_summary`, `bybit_sleeve_ret`, `xsp_option_bars`, the feature/`*_features` tables,
|
||||
`exec_fills`, `ibkr_account_nav`. Compare `SELECT count(*)` foxhunt vs bizworx for each.
|
||||
|
||||
**EVAL:** every checked table's row count matches. This proves the migration procedure works BEFORE cutover.
|
||||
foxhunt untouched (dump is read-only). (The real cutover in Task 8 re-runs this on the frozen DB.)
|
||||
|
||||
### Task 7: Stand up the fund on bizworx IN PARALLEL (no DNS yet) — the "alles werkt eerst" gate
|
||||
|
||||
**Files:** `infra/k8s/{jobs,gitea,monitoring→skip,...}`.
|
||||
- [ ] Apply the fund workloads to bizworx ns `foxhunt`: dagster, ib-gateway, fxhnt-dashboard (cockpit),
|
||||
pointed at the bizworx Postgres (restored in Task 6) + git-sync from the **bizworx gitea** (use its
|
||||
tailnet/ClusterIP endpoint directly for now, since `git.fxhnt.ai` still points at foxhunt).
|
||||
- [ ] Apply the CronJobs **all SUSPENDED** (`suspend: true`) initially.
|
||||
- [ ] **ib-gateway:** verify it connects to IBKR paper on bizworx (no 2FA/IP-limit) + a `whatIf` order is
|
||||
accepted: exec the gateway health / a dry-run probe.
|
||||
- [ ] **Cockpit:** port-forward the bizworx cockpit and verify in a **real browser** it renders the migrated
|
||||
data (NAVs, gates, holdings) — read the numbers; compare against the live foxhunt cockpit.
|
||||
- [ ] **Rebalancer dry-run:** run `fxhnt-ucits-rebalancer` once manually as a dry-run Job on bizworx (EU hours)
|
||||
→ it should plan/price the UCITS lines without error (or fill, on paper).
|
||||
- [ ] **Dagster:** trigger the nightly asset graph once on bizworx → green.
|
||||
|
||||
**EVAL (the big one):** the ENTIRE fund works on bizworx in parallel with foxhunt still live — cockpit correct,
|
||||
ib-gateway connected, rebalancer dry-run clean, dagster green, recon gate reconciles the migrated refs. **Only
|
||||
if ALL pass** do we proceed to cutover. Nothing on foxhunt has changed.
|
||||
|
||||
---
|
||||
|
||||
## PHASE 1 — cutover (only after Task 7 passes)
|
||||
|
||||
### Task 8: Cutover window (downtime OK, flexible)
|
||||
|
||||
**Files:** DNS/tailnet + ingress config.
|
||||
- [ ] Announce/pick a quiet window (fund is ~1×/day; any convenient time).
|
||||
- [ ] Freeze foxhunt: `kubectl --context $FOX patch cronjob <each fund cronjob> -n foxhunt -p '{"spec":{"suspend":true}}'`
|
||||
and scale down foxhunt dagster/cockpit writers so its Postgres is quiescent.
|
||||
- [ ] Final delta dump/restore foxhunt→bizworx Postgres (re-run Task 6's procedure on the now-frozen DB) so
|
||||
bizworx has the latest state. Re-verify the key row counts.
|
||||
- [ ] Copy `unlock_calendar.json` (or rely on the DB-first loader), ib-gateway settings if not re-authed.
|
||||
- [ ] Re-point DNS: `git.fxhnt.ai` and `dashboard.fxhnt.ai` → bizworx (tailnet node / ingress; ACME TLS for
|
||||
the dashboard). Fund git-sync now resolves the bizworx gitea.
|
||||
- [ ] Un-suspend the fund CronJobs on **bizworx** (start with the UCITS rebalancer; leave US/bybit suspended
|
||||
as they already are).
|
||||
|
||||
**EVAL:** on bizworx via the real hostnames — cockpit at `dashboard.fxhnt.ai` renders correctly, `git.fxhnt.ai`
|
||||
serves the repos + git-sync works, the armed rebalancer's next run succeeds. foxhunt is now suspended (NOT
|
||||
deleted). **Rollback still trivial** (re-point DNS + un-suspend foxhunt).
|
||||
|
||||
### Task 9: Cockpit + prod verification post-cutover
|
||||
|
||||
**Files:** none.
|
||||
- [ ] Verify cockpit LOCALLY in a real browser against the bizworx DB (dev-cockpit loop) AND on
|
||||
`dashboard.fxhnt.ai`; read the numbers (`feedback_verify_cockpit_locally_not_prod`).
|
||||
- [ ] Confirm the reconciliation gate + forward tracks are intact (no spurious re-inception; deterministic
|
||||
recompute from the migrated DB anchor).
|
||||
|
||||
**EVAL:** prod cockpit + gate correct on bizworx. If anything is off → rollback (foxhunt still intact).
|
||||
|
||||
---
|
||||
|
||||
## PHASE 1 — observation (keep BOTH, delete NOTHING)
|
||||
|
||||
### Task 10: Parallel observation window
|
||||
|
||||
**Files:** none.
|
||||
- [ ] Run the fund on bizworx for an agreed window (e.g. several trading days) with foxhunt suspended-but-intact.
|
||||
- [ ] Each day, evaluate: dagster nightly green, the UCITS rebalancer run clean, recon gate healthy, cockpit
|
||||
numbers sane, no stray-holdings/venue surprises.
|
||||
|
||||
**EVAL:** N consecutive clean days on bizworx. **This is the gate that unlocks any teardown.** Until it passes,
|
||||
nothing is deleted.
|
||||
|
||||
---
|
||||
|
||||
## PHASE 2 — triage the rest (per-item operator sign-off before any delete)
|
||||
|
||||
### Task 11: Keep/drop inventory of non-fund ns `foxhunt`
|
||||
|
||||
**Files:** produce `docs/superpowers/specs/…-phase2-triage.md` (the inventory).
|
||||
- [ ] For every non-fund workload (0-replica legacy: api/backtesting-service/broker-gateway/
|
||||
data-acquisition-service/ml-training-service/questdb/loki/pushgateway; other: minio/redis/kanidm/
|
||||
mattermost/stalwart/netbird) and every orphan PVC (training-data 500Gi, gitlab 50Gi, feature-cache,
|
||||
fxhnt-backtest-data, multistrat-state, test-data, tempo, …): classify DROP / MIGRATE / KEEP-EXTERNAL.
|
||||
- [ ] Operator signs off **per item** before its data is deleted (look before deleting).
|
||||
|
||||
**EVAL:** a signed-off inventory; no ambiguous items remain.
|
||||
|
||||
---
|
||||
|
||||
## PHASE 3 — teardown (ONLY after Task 10 + Task 11)
|
||||
|
||||
### Task 12: Retire the foxhunt Kapsule
|
||||
|
||||
**Files:** `infra/live/production/kapsule/**` (Terragrunt, project `c293eb98`), `infra/k8s/**`.
|
||||
- [ ] Remove the foxhunt k8s manifests / argo apps.
|
||||
- [ ] `terraform destroy` (via Terragrunt) the foxhunt Kapsule: control plane + `platform` (3× dev1_l) +
|
||||
`ci-compile-cpu` pools.
|
||||
- [ ] Explicitly delete retain-class volumes that won't auto-release (`*-retain` PVCs), the LB, private
|
||||
network `4ac46c9e…`, and other foxhunt-project scw resources; drop the confirmed-dead PVCs from Task 11.
|
||||
- [ ] Remove the temporary foxhunt API ACL entry for this box (`scw k8s acl` — the `bizworx-devbox-v4` entry),
|
||||
unless still useful.
|
||||
- [ ] Update memory: foxhunt Kapsule retired; fund lives on bizworx; single gitea = bizworx.
|
||||
|
||||
**EVAL:** foxhunt Kapsule + nodes gone; fund still healthy on bizworx; scw monthly cost dropped by the
|
||||
foxhunt control plane + 3 dev1_l nodes + its block storage. No wanted data lost.
|
||||
|
||||
---
|
||||
|
||||
## Self-review notes
|
||||
- **Verify-first honored:** Tasks 1–7 are additive (foxhunt untouched); the only foxhunt change before teardown
|
||||
is *suspend* (Task 8), fully reversible. First delete is Task 12, gated behind Tasks 10 + 11.
|
||||
- **Per-step eval honored:** every task has an `EVAL:` gate.
|
||||
- **Secrets:** Task 2 uses `!`/openbao, never the transcript.
|
||||
- **Rollback:** available through Task 11 (foxhunt intact-and-suspended).
|
||||
@@ -0,0 +1,104 @@
|
||||
# Production Scheduling — Consolidate onto Dagster (K8sRunLauncher) + build the proper image
|
||||
|
||||
> **For the operator/agent:** execute inline, checkpoint per task. Live infra + real-money fund. Operator
|
||||
> decision (2026-07-19): go ALL-DAGSTER properly — upgrade the Dagster infra so heavy compute runs in
|
||||
> per-run K8s Jobs (own memory), fold the cron precompute into the DAG, and build the image the right way
|
||||
> (podman → bizworx registry, bypassing the dead Argo/Kaniko/GitLab pipeline).
|
||||
|
||||
**Goal:** One orchestrator (Dagster) owns the whole nightly ETL + precompute as an asset graph, with per-run
|
||||
memory isolation via `K8sRunLauncher` (heavy assets get 6-8Gi Jobs, the daemon stays 4Gi). Remove the
|
||||
`compare-measured-precompute` cron (folded into the DAG). Keep EXECUTION (rebalancers) in K8s CronJobs
|
||||
(deliberate money-exec blast-radius isolation). Delete the dead VRP. Result: consistent, documented, not-confusing.
|
||||
|
||||
## Why this is the right end-state (and why it needs an infra upgrade)
|
||||
- The split felt "buggy" because `bybit_sleeve_ret` is written by a SUSPENDED own-memory cron, cross-timed to the
|
||||
Dagster 23:30 ingest. Folding it into Dagster removes the timing coupling.
|
||||
- BUT the cron exists for a HARD reason: the sleeve-ret compute needs **6-8Gi** and OOM-killed the 4Gi daemon.
|
||||
Naively making it a daemon-run asset re-breaks the nightly. The CORRECT fix = `K8sRunLauncher`: each Dagster
|
||||
run launches as its own right-sized K8s Job; the daemon only schedules/launches (stays tiny). This is the
|
||||
standard Dagster production pattern.
|
||||
|
||||
## Current-state facts (verified 2026-07-19)
|
||||
- Dagster ETL HEALTHY: 23:30 run SUCCESS, all 14 assets materialized, forward_nav fresh 07-18. (working_directory
|
||||
/app→/code/src change already applied — harmless, more-correct.)
|
||||
- `dagster_k8s` NOT in the image (orchestration extra = dagster/webserver/postgres only). RBAC: dagster SA
|
||||
`tailscale-dagster` CANNOT create jobs. Run launcher: default in-process (4Gi daemon).
|
||||
- Image `rg.fr-par.scw.cloud/bizworx/fxhnt-cockpit:latest` (633MB) built via Dockerfile
|
||||
`infra/docker/fxhnt.Dockerfile` (`pip install .[web,pg,data,factory,orchestration,poc]`). The Argo/Kaniko
|
||||
build pipeline is DEAD (ci-compile-cpu pool gone + clones the killed GitLab).
|
||||
- Build capability: devbox has **podman 4.9.3, x86_64** = cluster amd64/linux. Registry live + reachable.
|
||||
|
||||
## Task 1: Add dagster-k8s to deps + build the image RIGHT (podman → bizworx registry)
|
||||
**Files:** `pyproject.toml`, `infra/docker/fxhnt.Dockerfile` (verify), build via podman.
|
||||
- [ ] Add `dagster-k8s>=0.28,<0.29` to the `orchestration` extra in pyproject.toml (matches dagster 1.12 / the
|
||||
0.28 dagster-postgres pin). `uv lock` to update uv.lock.
|
||||
- [ ] `podman login rg.fr-par.scw.cloud` with an scw secret-key (username `nologin`, password = SCW secret key;
|
||||
NEVER print it). Verify login OK.
|
||||
- [ ] `podman build --platform linux/amd64 -f infra/docker/fxhnt.Dockerfile -t
|
||||
rg.fr-par.scw.cloud/bizworx/fxhnt-cockpit:dagster-k8s-<sha> .` (tag with the git sha, NOT :latest yet).
|
||||
- [ ] `podman push` the sha-tag. VERIFY it lands in the registry (`scw registry image list`).
|
||||
- **EVAL:** image pushed; `podman run --rm <tag> python -c "import dagster_k8s; print(dagster_k8s.__version__)"`
|
||||
succeeds. Do NOT retag :latest until the new image is proven on-cluster (Task 4). Roll back = keep :latest.
|
||||
|
||||
## Task 2: RBAC — let the dagster SA launch Jobs + a Job SA with fund access
|
||||
**Files:** `infra/k8s/orchestration/dagster.yaml` (Role/RoleBinding + a run-job SA).
|
||||
- [ ] Add a Role granting the dagster SA: create/get/list/watch/delete on `jobs` (batch) + `pods`, get on
|
||||
`pods/log`. Bind to `tailscale-dagster` (or a dedicated `dagster-runner` SA).
|
||||
- [ ] Define the SA the LAUNCHED Jobs use — needs the fund secrets (db-credentials, databento, tiingo),
|
||||
git-sync key, and the same NP egress (postgres, gitea, apiserver, public HTTPS). Mirror the cron pods'
|
||||
access. (Heavy assets that hit ib-gateway would need that NP too — but the nightly precompute does not.)
|
||||
- [ ] **VERIFY:** `kubectl auth can-i create jobs --as=system:serviceaccount:foxhunt:<sa>` = yes.
|
||||
- **EVAL:** RBAC in place, least-privilege (namespaced Role, not cluster-admin).
|
||||
|
||||
## Task 3: Configure K8sRunLauncher + per-asset memory + make sleeve-ret/backtest-refs assets
|
||||
**Files:** `infra/k8s/orchestration/dagster.yaml` (dagster.yaml instance config), `src/fxhnt/adapters/
|
||||
orchestration/{assets.py,definitions.py}`.
|
||||
- [ ] dagster.yaml: add `run_launcher: module dagster_k8s class K8sRunLauncher` with job_namespace foxhunt,
|
||||
the new image, image_pull_secrets scw-registry, the Job SA, and default resources. Add a `run_k8s_config`
|
||||
/ tag-based per-run memory (heavy assets → 6Gi req/8Gi limit, pinned to the large pool).
|
||||
- [ ] Wrap `persist_bybit_sleeve_returns` (already an import-light helper) as a Dagster `@asset(deps=[
|
||||
bybit_warehouse_refresh])` with a k8s memory tag; add a `backtest_refs` asset (deps on the sleeve-ret +
|
||||
nav assets) wrapping the `backtest-refs` logic. Add both to combined_book_job selection + defs.
|
||||
- [ ] Keep the plain helper functions intact (unit-testable); the assets just call them (no logic duplication).
|
||||
- [ ] Add/point tests: the existing sleeve-ret + backtest-refs unit tests still pass; a new test asserts the
|
||||
assets are in the job selection.
|
||||
- **EVAL:** `uv run pytest` for the touched modules green; `python -c "import ...definitions"` lists the new
|
||||
assets in combined_book_job. Do NOT deploy yet.
|
||||
|
||||
## Task 4: Deploy the new image + config, materialize the nightly once, VERIFY end-to-end
|
||||
**Files:** none (deploy).
|
||||
- [ ] Point dagster deploy at the new sha-tagged image (edit dagster.yaml image or a kustomize overlay), apply,
|
||||
rollout restart. git-sync pulls the new src (with the new assets).
|
||||
- [ ] Trigger `combined_book_forward_job` once. WATCH: the daemon LAUNCHES a K8s Job (not in-process); the Job
|
||||
gets 6-8Gi on the large pool; it materializes ALL assets incl the new sleeve-ret + backtest-refs WITHOUT
|
||||
OOM-killing the daemon.
|
||||
- [ ] **VERIFY:** `bybit_sleeve_ret` max(run_date) = today, src_updated = now; backtest_summary fresh; forward
|
||||
tracks still fresh; cockpit reflects it; daemon did NOT restart/OOM. Check `kubectl get jobs -n foxhunt`
|
||||
shows the dagster-launched run Job completed.
|
||||
- **EVAL:** full nightly runs in an isolated Job, data fresh, daemon healthy. THIS is the money shot. If OOM or
|
||||
launcher fails → debug (RBAC/image/config), do NOT retag :latest. Roll back = old image + re-arm the cron.
|
||||
- [ ] Once proven: retag the sha image as `:latest` (or update the manifest to the sha) so it's the deployed default.
|
||||
|
||||
## Task 5: Remove the now-redundant compare-measured cron + delete VRP + arm execution + document
|
||||
**Files:** `infra/k8s/jobs/*.yaml`, memory, progress anchor.
|
||||
- [ ] DELETE `fxhnt-compare-measured-precompute` cron + its NP (its work is now the Dagster assets). `git rm` the
|
||||
manifest. (Keep `factory` cron — it's migrate-cockpit + factory-cycle, separate concern; ARM it.)
|
||||
- [ ] DELETE `fxhnt-vrp-rebalancer` cron + NP + manifest (shelved/falsified; vrp_nav already de-wired). Update
|
||||
memory (project_fxhnt_diversifier_hunt_2026_07_15, project_fxhnt_vrp_cvar_crash_research): VRP exec cron DELETED.
|
||||
- [ ] ARM the EXECUTION crons that should run: `factory` (daily), `ucits-volume-ibkr` + `ucits-rebalancer`
|
||||
(weekday, ucits already armed). Keep SUSPENDED: `bybit-rebalancer` (no creds), `ibkr-rebalancer` (PRIIPs).
|
||||
Patch live + set suspend in manifests (source of truth).
|
||||
- [ ] **VERIFY:** cronjobs = {factory, ucits-volume-ibkr, ucits-rebalancer} armed; {bybit, ibkr} suspended;
|
||||
{compare-measured, vrp} GONE. Dagster owns all ETL/precompute.
|
||||
- [ ] Document the final architecture in the progress anchor + a CLAUDE-visible note: Dagster (K8sRunLauncher,
|
||||
per-run memory) = ALL nightly ETL+precompute; K8s CronJobs = execution only. Why exec stays crons
|
||||
(blast-radius). Update reference_fxhnt_prod_deploy_mechanics with the podman build path (Argo build dead).
|
||||
- **EVAL:** consistent single-orchestrator ETL, execution isolated in crons, nothing suspended-by-accident,
|
||||
VRP gone, architecture documented. Done.
|
||||
|
||||
## Global constraints
|
||||
- VERIFY-FIRST, per-task eval. REAL MONEY: only ucits-rebalancer trades (paper). Never arm bybit/ibkr exec.
|
||||
- NEVER print the SCW secret key / registry password. podman login via stdin or env, not argv.
|
||||
- Image: tag with git sha, prove on-cluster BEFORE :latest. Old :latest stays as instant rollback.
|
||||
- Manifests are source of truth: every live change also edits the committed yaml.
|
||||
- Don't duplicate logic: assets call the existing import-light helpers; keep them unit-testable.
|
||||
@@ -0,0 +1,599 @@
|
||||
# Positioning Per-Coin Gross Cap Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Add an ex-ante per-coin gross-weight cap (0.07) to the `positioning` sleeve so no single coin dominates the book, then re-inception + backfill the forward anchor so the live gate keeps running against the capped construction.
|
||||
|
||||
**Architecture:** The cap is applied to the FORMING weights in `positioning_weights` (before returns), so it is strictly causal (no lookahead). It threads through the existing seams (`_book_breakdown` → `positioning_returns_from_store` → `_positioning_series` → `sleeve_returns_from_store`) as an optional `coin_gross_cap` param that defaults to `None` (OFF → byte-identical to today). It is switched ON at 0.07 only in `bybit_book_persist.py`. The registry `positioning` + `bybit_4edge` definition versions bump to re-inception; `inception_t0` pins the existing OOS start (2026-07-06) so the forward track backfills the window with the capped construction instead of resetting the gate to 0.
|
||||
|
||||
**Tech Stack:** Python 3.12, numpy-free pure math, pytest (`-n auto`), TimescaleFeatureStore (Postgres, table `bybit_features`).
|
||||
|
||||
## Global Constraints
|
||||
|
||||
- The cap is applied to WEIGHTS (in `positioning_weights`), NOT to returns/contributions — it must be ex-ante/causal (the cap decision cannot see the coin's realized return).
|
||||
- `coin_gross_cap` default is `None` (OFF) at EVERY seam. With it OFF, every existing test and the whole backtest must be byte-identical to today. The OFF path must not add per-term division drift.
|
||||
- After clipping `|w| ≤ coin_gross_cap`, RENORMALIZE the day's weights to unit gross (`Σ|w| = 1`), matching `positioning_weights`' existing unit-gross contract.
|
||||
- The validated cap value is **0.07** (full-history sweep 1295d: Sharpe 2.60→2.69, maxDD −32.3%→−29.9%, per-year Sharpe non-decreasing every year). Do NOT use 0.04 (falsified: Sharpe drops to 2.43).
|
||||
- Re-inception uses the EXISTING pattern: bump `version` in `definition.params.version`, and set `inception_t0: {"date": "2026-07-06", "version": <new>}` (outside `params`, NOT hashed) so the anchor backdates and the forward window is recomputed, not reset. See `forward_definition.inception_override`.
|
||||
- `positioning` is a CONSTITUENT of `bybit_4edge`; both registry entries' definition versions must bump together (the book's sleeve construction changed).
|
||||
- Never touch real money / execution. This is a construction change to the modeled+forward track only.
|
||||
- Run tests with `uv run --extra dev python -m pytest ...`. The positioning eval integration tests need no extra services (they use a fake store).
|
||||
|
||||
---
|
||||
|
||||
### Task 1: Add the ex-ante `coin_gross_cap` to `positioning_weights`
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/application/bybit_positioning_eval.py:78-104` (the `positioning_weights` function)
|
||||
- Test: `tests/integration/test_bybit_positioning_eval.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: nothing new.
|
||||
- Produces: `positioning_weights(ratio_by_day, *, direction="contrarian", coin_gross_cap: float | None = None) -> dict[int, dict[str, float]]`. When `coin_gross_cap` is a float, each day's weights are clipped to `|w| ≤ coin_gross_cap` then renormalized to unit gross. When `None`, output is byte-identical to today.
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
Add to `tests/integration/test_bybit_positioning_eval.py`:
|
||||
|
||||
```python
|
||||
def test_coin_gross_cap_clips_and_renormalizes() -> None:
|
||||
# One coin dominates (retail extreme) -> uncapped weight far exceeds 0.07; cap must clip + renormalize.
|
||||
ratio = {0: {"DOMUSDT": 0.95, "AUSDT": 0.50, "BUSDT": 0.50, "CUSDT": 0.05}}
|
||||
uncapped = positioning_weights(ratio)[0]
|
||||
assert max(abs(w) for w in uncapped.values()) > 0.07 # DOM dominates uncapped
|
||||
capped = positioning_weights(ratio, coin_gross_cap=0.07)[0]
|
||||
assert max(abs(w) for w in capped.values()) <= 0.07 + 1e-9 # no coin exceeds the cap
|
||||
assert abs(sum(abs(w) for w in capped.values()) - 1.0) < 1e-9 # still unit gross
|
||||
# sign structure preserved: DOM still short (over-long crowd), C still long (over-short)
|
||||
assert capped["DOMUSDT"] < 0.0 and capped["CUSDT"] > 0.0
|
||||
|
||||
|
||||
def test_coin_gross_cap_none_is_byte_identical() -> None:
|
||||
ratio = {0: {"AAAUSDT": 0.9, "BBBUSDT": 0.5, "CCCUSDT": 0.1}}
|
||||
assert positioning_weights(ratio, coin_gross_cap=None) == positioning_weights(ratio)
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `uv run --extra dev python -m pytest tests/integration/test_bybit_positioning_eval.py::test_coin_gross_cap_clips_and_renormalizes -v`
|
||||
Expected: FAIL with `TypeError: positioning_weights() got an unexpected keyword argument 'coin_gross_cap'`
|
||||
|
||||
- [ ] **Step 3: Implement the cap in `positioning_weights`**
|
||||
|
||||
Replace the body of `positioning_weights` (lines 91-104) so the signature gains `coin_gross_cap` and, after the existing unit-gross normalization, applies the clip+renormalize when the cap is set:
|
||||
|
||||
```python
|
||||
def positioning_weights(ratio_by_day: dict[int, dict[str, float]], *,
|
||||
direction: str = "contrarian",
|
||||
coin_gross_cap: float | None = None) -> dict[int, dict[str, float]]:
|
||||
"""`{day: {coin: w}}` market-neutral cross-sectional weights from the long/short positioning signal.
|
||||
|
||||
`direction="contrarian"` (DEFAULT) SHORTs the high-buyRatio coins (retail over-long → fade) and LONGs the
|
||||
low-buyRatio coins (retail over-short). `direction="momentum"` is the EXACT SIGN-FLIP: LONG the over-long
|
||||
/ SHORT the over-short (informed-flow continuation). Construction per day:
|
||||
1. DEMEAN the per-day buyRatio across coins (so the book is dollar/market-neutral, Σw ≈ 0);
|
||||
2. apply the DIRECTION sign — contrarian = −demeaned (short the rich/over-long), momentum = +demeaned;
|
||||
3. normalise to UNIT GROSS (Σ|w| = 1);
|
||||
4. (OPTIONAL) if `coin_gross_cap` is set, CLIP each |w| to the cap and RENORMALISE to unit gross —
|
||||
an EX-ANTE per-coin concentration limit (applied to the FORMING weight, so it is strictly causal:
|
||||
it cannot see the coin's realized return). Validated at 0.07 (full-history sweep: Sharpe 2.60→2.69,
|
||||
maxDD −32.3%→−29.9%, per-year non-decreasing) — caps the LABUSDT-class implosions (a 10.3% long leg
|
||||
that did −59%/−79.5% on 2026-07-06/07) without over-throttling the edge (0.04 falsified: Sharpe→2.43).
|
||||
|
||||
Because the gross is identical for both directions, `momentum` weights are the exact per-coin negation of
|
||||
`contrarian`. A day with <2 coins, or zero gross after demeaning, yields no weights (skipped). Pure."""
|
||||
if direction not in ("contrarian", "momentum"):
|
||||
raise ValueError(f"direction must be 'contrarian' or 'momentum', got {direction!r}")
|
||||
sign = -1.0 if direction == "contrarian" else 1.0
|
||||
out: dict[int, dict[str, float]] = {}
|
||||
for day, row in ratio_by_day.items():
|
||||
if len(row) < 2:
|
||||
continue
|
||||
mean = st.mean(row.values())
|
||||
raw = {c: sign * (v - mean) for c, v in row.items()}
|
||||
gross = sum(abs(w) for w in raw.values())
|
||||
if gross <= 0.0:
|
||||
continue
|
||||
w = {c: x / gross for c, x in raw.items()}
|
||||
if coin_gross_cap is not None:
|
||||
w = {c: max(-coin_gross_cap, min(coin_gross_cap, x)) for c, x in w.items()}
|
||||
capped_gross = sum(abs(x) for x in w.values())
|
||||
if capped_gross <= 0.0:
|
||||
continue
|
||||
w = {c: x / capped_gross for c, x in w.items()}
|
||||
out[int(day)] = w
|
||||
return out
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run test to verify it passes**
|
||||
|
||||
Run: `uv run --extra dev python -m pytest tests/integration/test_bybit_positioning_eval.py -v`
|
||||
Expected: PASS (the two new tests + all existing positioning_weights tests still green)
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/application/bybit_positioning_eval.py tests/integration/test_bybit_positioning_eval.py
|
||||
git commit -m "feat(positioning): ex-ante per-coin gross cap in positioning_weights (default off)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 2: Thread `coin_gross_cap` through `_book_breakdown` and `positioning_returns_from_store`
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/application/bybit_positioning_eval.py:140-166` (`_book_breakdown` signature + the `positioning_weights` call at line 177)
|
||||
- Modify: `src/fxhnt/application/bybit_positioning_eval.py:287-303` (`positioning_returns_from_store` signature + its `_book_breakdown` call)
|
||||
- Test: `tests/integration/test_bybit_positioning_eval.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `positioning_weights(..., coin_gross_cap=...)` from Task 1.
|
||||
- Produces:
|
||||
- `_book_breakdown(store, *, universe, cost_bps, direction, capacity_capital=None, participation=0.10, turnover_window=30, tail_cap_k=None, book_vol_window=60, coin_gross_cap=None) -> dict`
|
||||
- `positioning_returns_from_store(store, *, universe=None, cost_bps=5.5, direction="contrarian", capacity_capital=None, tail_cap_k=None, coin_gross_cap=None) -> dict[int, float]`
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
Add to `tests/integration/test_bybit_positioning_eval.py` (reuse the module's existing fake store fixture — grep the file for the store used by `test_capacity_haircut_caps_gain_only_at_scale`, e.g. a `_FakeStore`/`_ReadOnlyStore` builder, and construct one where a single coin has an extreme long_ratio + a large adverse next-day return):
|
||||
|
||||
```python
|
||||
def test_coin_gross_cap_reduces_single_coin_loss_in_book(positioning_store_with_shock):
|
||||
# positioning_store_with_shock: one coin (retail over-short) craters next day; uncapped it dominates the
|
||||
# book loss, capped its weight (and thus its loss contribution) is bounded.
|
||||
store = positioning_store_with_shock
|
||||
uncapped = positioning_returns_from_store(store, cost_bps=0.0)
|
||||
capped = positioning_returns_from_store(store, cost_bps=0.0, coin_gross_cap=0.07)
|
||||
shock_day = min(uncapped, key=lambda d: uncapped[d]) # the worst day uncapped
|
||||
assert capped[shock_day] > uncapped[shock_day] # the cap made the worst day less bad
|
||||
```
|
||||
|
||||
If the module has no reusable multi-coin shock fixture, build the store inline in the test from a `long_ratio` panel (one coin at ratio 0.95, three near 0.5) + a `close` panel where the over-short coin drops ~50% the next day, mirroring the `test_capacity_haircut_*` store construction in the same file.
|
||||
|
||||
- [ ] **Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `uv run --extra dev python -m pytest tests/integration/test_bybit_positioning_eval.py::test_coin_gross_cap_reduces_single_coin_loss_in_book -v`
|
||||
Expected: FAIL with `TypeError: positioning_returns_from_store() got an unexpected keyword argument 'coin_gross_cap'`
|
||||
|
||||
- [ ] **Step 3: Thread the param through both functions**
|
||||
|
||||
In `_book_breakdown`, add `coin_gross_cap: float | None = None` to the signature (after `book_vol_window: int = 60`), and change the weight-forming call (currently line 177):
|
||||
|
||||
```python
|
||||
weights_by_day = positioning_weights(_ratio_by_day(long_ratio), direction=direction,
|
||||
coin_gross_cap=coin_gross_cap)
|
||||
```
|
||||
|
||||
In `positioning_returns_from_store`, add `coin_gross_cap: float | None = None` to the signature (after `tail_cap_k`) and pass it through:
|
||||
|
||||
```python
|
||||
return _book_breakdown(store, universe=universe, cost_bps=cost_bps, direction=direction,
|
||||
capacity_capital=capacity_capital, tail_cap_k=tail_cap_k,
|
||||
coin_gross_cap=coin_gross_cap)["net"]
|
||||
```
|
||||
|
||||
Update the `positioning_returns_from_store` docstring's param note to add: `coin_gross_cap` (None default) applies the ex-ante per-coin concentration cap (see `positioning_weights`).
|
||||
|
||||
- [ ] **Step 4: Run test to verify it passes**
|
||||
|
||||
Run: `uv run --extra dev python -m pytest tests/integration/test_bybit_positioning_eval.py -v`
|
||||
Expected: PASS
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/application/bybit_positioning_eval.py tests/integration/test_bybit_positioning_eval.py
|
||||
git commit -m "feat(positioning): thread coin_gross_cap through book_breakdown + returns_from_store"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 3: Thread `coin_gross_cap` through the sleeve seam (`_positioning_series` + `sleeve_returns_from_store`)
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/application/bybit_book_eval.py:226-240` (`_positioning_series`)
|
||||
- Modify: `src/fxhnt/application/bybit_book_eval.py:243-278` (`sleeve_returns_from_store` signature + the positioning branch)
|
||||
- Test: `tests/integration/test_bybit_positioning_eval.py` (or `tests/integration/test_web_new_edges.py` if that is where sleeve_returns_from_store is exercised — grep first)
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `positioning_returns_from_store(..., coin_gross_cap=...)` from Task 2.
|
||||
- Produces:
|
||||
- `_positioning_series(store, *, universe, cost_bps, direction="contrarian", capacity_capital=None, tail_cap_k=None, coin_gross_cap=None) -> dict[int, float]`
|
||||
- `sleeve_returns_from_store(store, edge, *, universe=None, cost_bps=0.0, unlock_events=None, stablecoin_spot_panel=None, capacity_capital=None, tail_cap_k=None, coin_gross_cap=None) -> dict[int, float]`. The `coin_gross_cap` is forwarded ONLY to the `positioning` branch (other edges ignore it).
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
Add (same test file, reuse the store from Task 2's fixture/inline construction):
|
||||
|
||||
```python
|
||||
def test_sleeve_returns_positioning_forwards_coin_gross_cap(positioning_store_with_shock):
|
||||
from fxhnt.application.bybit_book_eval import sleeve_returns_from_store
|
||||
store = positioning_store_with_shock
|
||||
uncapped = sleeve_returns_from_store(store, "positioning", cost_bps=0.0)
|
||||
capped = sleeve_returns_from_store(store, "positioning", cost_bps=0.0, coin_gross_cap=0.07)
|
||||
shock_day = min(uncapped, key=lambda d: uncapped[d])
|
||||
assert capped[shock_day] > uncapped[shock_day] # cap forwarded -> worst day improved
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `uv run --extra dev python -m pytest tests/integration/test_bybit_positioning_eval.py::test_sleeve_returns_positioning_forwards_coin_gross_cap -v`
|
||||
Expected: FAIL with `TypeError: sleeve_returns_from_store() got an unexpected keyword argument 'coin_gross_cap'`
|
||||
|
||||
- [ ] **Step 3: Thread through both functions**
|
||||
|
||||
In `_positioning_series` (line 226), add `coin_gross_cap: float | None = None` to the signature (after `tail_cap_k`) and forward it:
|
||||
|
||||
```python
|
||||
return positioning_returns_from_store(store, universe=universe, cost_bps=cost_bps, direction=direction,
|
||||
capacity_capital=capacity_capital, tail_cap_k=tail_cap_k,
|
||||
coin_gross_cap=coin_gross_cap)
|
||||
```
|
||||
|
||||
In `sleeve_returns_from_store` (line 243), add `coin_gross_cap: float | None = None` to the signature (after `tail_cap_k`), and change ONLY the positioning branch (line 276) to forward it:
|
||||
|
||||
```python
|
||||
if edge == "positioning":
|
||||
# ... existing comment about raw store + universe unchanged ...
|
||||
return _positioning_series(store, universe=universe, cost_bps=cost_bps,
|
||||
capacity_capital=capacity_capital, tail_cap_k=tail_cap_k,
|
||||
coin_gross_cap=coin_gross_cap)
|
||||
```
|
||||
|
||||
Leave the tstrend / unlock / stablecoin / xsfunding branches unchanged (they never receive `coin_gross_cap`).
|
||||
|
||||
- [ ] **Step 4: Run test to verify it passes**
|
||||
|
||||
Run: `uv run --extra dev python -m pytest tests/integration/test_bybit_positioning_eval.py -v`
|
||||
Expected: PASS
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/application/bybit_book_eval.py tests/integration/test_bybit_positioning_eval.py
|
||||
git commit -m "feat(positioning): forward coin_gross_cap through sleeve_returns_from_store"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 4: Turn the cap ON at 0.07 in the live precompute + add the constant
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/application/bybit_forward_track.py:47` (add `POSITIONING_COIN_GROSS_CAP` next to `POSITIONING_TAIL_CAP_K`)
|
||||
- Modify: `src/fxhnt/application/bybit_book_persist.py:77-84` (pass the cap into `sleeve_returns_from_store`)
|
||||
- Test: `tests/integration/test_bybit_paper_reconciliation.py` (or the persist test — grep for a test that calls `_persist_bybit_book` / `persist_bybit_sleeve_returns`; if none exists at unit level, assert the constant is wired via a focused test in `test_bybit_positioning_eval.py`)
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `sleeve_returns_from_store(..., coin_gross_cap=...)` from Task 3.
|
||||
- Produces: `POSITIONING_COIN_GROSS_CAP = 0.07` in `bybit_forward_track.py`; the live `_persist_bybit_book` sleeve computation applies it to the positioning sleeve.
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
Add to `tests/integration/test_bybit_positioning_eval.py`:
|
||||
|
||||
```python
|
||||
def test_positioning_coin_gross_cap_constant_is_007() -> None:
|
||||
from fxhnt.application.bybit_forward_track import POSITIONING_COIN_GROSS_CAP
|
||||
assert POSITIONING_COIN_GROSS_CAP == 0.07
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `uv run --extra dev python -m pytest tests/integration/test_bybit_positioning_eval.py::test_positioning_coin_gross_cap_constant_is_007 -v`
|
||||
Expected: FAIL with `ImportError: cannot import name 'POSITIONING_COIN_GROSS_CAP'`
|
||||
|
||||
- [ ] **Step 3: Add the constant + wire it into the precompute**
|
||||
|
||||
In `src/fxhnt/application/bybit_forward_track.py`, directly after line 47 (`POSITIONING_TAIL_CAP_K = 4.0`):
|
||||
|
||||
```python
|
||||
# Ex-ante per-coin gross-weight cap for the positioning sleeve — no single coin's |weight| may exceed this
|
||||
# fraction of the book's unit gross. Validated 2026-07-20 over the full 1295-day history (Sharpe 2.60→2.69,
|
||||
# maxDD −32.3%→−29.9%, per-year Sharpe non-decreasing every year); caps LABUSDT-class single-coin implosions
|
||||
# (a 10.3% long leg that did −59%/−79.5% on 2026-07-06/07) without over-throttling (0.04 falsified → 2.43).
|
||||
POSITIONING_COIN_GROSS_CAP = 0.07
|
||||
```
|
||||
|
||||
In `src/fxhnt/application/bybit_book_persist.py`, update the import (line 78) and the sleeve computation (lines 80-83):
|
||||
|
||||
```python
|
||||
from fxhnt.application.bybit_forward_track import POSITIONING_COIN_GROSS_CAP, POSITIONING_TAIL_CAP_K
|
||||
sleeve_rets = {
|
||||
s: sleeve_returns_from_store(store, s, universe=universe, cost_bps=cost_bps,
|
||||
unlock_events=unlock_events,
|
||||
capacity_capital=settings.paper_capital,
|
||||
tail_cap_k=POSITIONING_TAIL_CAP_K,
|
||||
coin_gross_cap=POSITIONING_COIN_GROSS_CAP)
|
||||
for s in _DEFAULT_BYBIT_SLEEVES}
|
||||
```
|
||||
|
||||
(`coin_gross_cap` is ignored by every sleeve except positioning — Task 3 only forwards it in the positioning branch — so passing it for all sleeves in the dict comprehension is correct and harmless.)
|
||||
|
||||
- [ ] **Step 4: Run test to verify it passes**
|
||||
|
||||
Run: `uv run --extra dev python -m pytest tests/integration/test_bybit_positioning_eval.py -v`
|
||||
Expected: PASS
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/application/bybit_forward_track.py src/fxhnt/application/bybit_book_persist.py tests/integration/test_bybit_positioning_eval.py
|
||||
git commit -m "feat(positioning): enable coin_gross_cap=0.07 in the live bybit book precompute"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 5: Bump definition versions to re-inception positioning + bybit_4edge (backdated anchor, no gate reset)
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/registry.py` (the `positioning` entry ~line 249-259 and the `bybit_4edge` entry ~line 198-211; also `bybit_4edge_levered` ~line 229-241 which shares the sleeve construction)
|
||||
- Test: `tests/` — grep for `test_forward_definition` / a registry-consistency test; add an assertion that the bumped versions + inception_t0 are consistent
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: the capped construction from Task 4 (the version bump signals the construction changed).
|
||||
- Produces: `positioning`, `bybit_4edge`, `bybit_4edge_levered` definitions at `version: 3` with `inception_t0: {"date": "2026-07-06", "version": 3}`.
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
Add `tests/unit/test_positioning_cap_reinception.py`:
|
||||
|
||||
```python
|
||||
from fxhnt.registry import STRATEGY_REGISTRY
|
||||
|
||||
|
||||
def test_positioning_and_book_bumped_to_v3_backdated():
|
||||
for sid in ("positioning", "bybit_4edge", "bybit_4edge_levered"):
|
||||
d = STRATEGY_REGISTRY[sid]["definition"]
|
||||
assert d["params"]["version"] == 3, f"{sid} version not bumped to 3"
|
||||
ov = d.get("inception_t0")
|
||||
assert ov and ov["version"] == 3, f"{sid} inception_t0 not re-pinned to v3"
|
||||
# backdated to the existing OOS start, NOT reset to today -> the forward window is recomputed, not lost
|
||||
assert ov["date"] == "2026-07-06"
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `uv run --extra dev python -m pytest tests/unit/test_positioning_cap_reinception.py -v`
|
||||
Expected: FAIL (versions are currently 2)
|
||||
|
||||
- [ ] **Step 3: Bump the three registry entries**
|
||||
|
||||
In each of the `positioning`, `bybit_4edge`, and `bybit_4edge_levered` entries in `src/fxhnt/registry.py`, change `"version": 2` to `"version": 3` inside `definition.params`, and change the `inception_t0` to `{"date": "2026-07-06", "version": 3}`. Note: `positioning`'s existing `inception_t0` date is `2026-07-06`; `bybit_4edge` / `bybit_4edge_levered` currently use `2026-07-07` — for this backfill set ALL three to `2026-07-06` (the positioning shock's first day) so the whole book re-warms over the same window. Add a comment on each:
|
||||
|
||||
```python
|
||||
"inception_t0": {"date": "2026-07-06", "version": 3}, # v3: positioning coin_gross_cap=0.07 (2026-07-20 backfill)
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run test to verify it passes**
|
||||
|
||||
Run: `uv run --extra dev python -m pytest tests/unit/test_positioning_cap_reinception.py tests/ -k "forward_definition or registry" -v`
|
||||
Expected: PASS (the new test + any existing registry/definition consistency tests)
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/registry.py tests/unit/test_positioning_cap_reinception.py
|
||||
git commit -m "feat(positioning): re-inception positioning+bybit_4edge v3 (coin_gross_cap, backdated 2026-07-06)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 6: Full-suite verification + lint/type gate
|
||||
|
||||
**Files:** none (verification task)
|
||||
|
||||
**Interfaces:** consumes all prior tasks.
|
||||
|
||||
- [ ] **Step 1: Run the full positioning + book + forward suite**
|
||||
|
||||
Run: `uv run --extra dev python -m pytest tests/ -k "positioning or book or forward or reconcil or bybit" -q`
|
||||
Expected: all pass (no regression; the OFF-path byte-identity tests from Tasks 1-3 guarantee unchanged behavior everywhere the cap is not enabled).
|
||||
|
||||
- [ ] **Step 2: Lint + type-check the changed files**
|
||||
|
||||
Run: `uv run --extra dev ruff check src/fxhnt/application/bybit_positioning_eval.py src/fxhnt/application/bybit_book_eval.py src/fxhnt/application/bybit_book_persist.py src/fxhnt/application/bybit_forward_track.py src/fxhnt/registry.py`
|
||||
Then: `uv run --extra dev --extra orchestration mypy src/fxhnt/application/bybit_positioning_eval.py src/fxhnt/application/bybit_book_eval.py`
|
||||
Expected: no NEW errors on changed lines (pre-existing whole-file findings that are not in the diff are acceptable — confirm via `git diff --unified=0 ... | grep '^+' | awk 'length>120'` returning nothing).
|
||||
|
||||
- [ ] **Step 3: Commit any lint fixes** (only if Step 2 surfaced issues on changed lines)
|
||||
|
||||
```bash
|
||||
git add -A
|
||||
git commit -m "chore(positioning): lint/type fixes for coin_gross_cap"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Post-merge deploy + backfill (OPERATOR steps — outside the coding plan, run after merge)
|
||||
|
||||
These are runtime/prod steps, NOT code, listed so the operator has the full picture. Do them after the branch is merged to master and pushed:
|
||||
|
||||
1. **Deploy the code:** `git push origin master` (the Dagster K8sRunLauncher git-syncs master HEAD; a code-only change needs no image rebuild). Rollout-restart the dagster daemon if it caches code: `kubectl --context=$BIZ -n foxhunt rollout restart deploy/dagster-daemon`.
|
||||
2. **Backfill the anchor:** the re-inception is AUTOMATIC on the next `combined_book_daily` run (23:30 UTC) — the bumped version + backdated `inception_t0` cause `forward_engine` to reset the anchor to `t0=2026-07-06` and recompute the forward window with the capped construction. Do NOT run `migrate-forward-anchors` (re-inception is automatic — see `reference_fxhnt_prod_deploy_mechanics`).
|
||||
3. **Refresh the backtest ref:** the `bybit_book_precompute` asset re-persists the `positioning` / `bybit_4edge` / `bybit_4edge_levered` `backtest_summary` from the capped `_persist_bybit_book` in the same nightly run, so the gate reconciles the forward track against the capped (capturable) reference.
|
||||
4. **Verify next morning:** query the prod DB (port-forward svc/postgres) — `positioning` forward should re-show ~13 days (recomputed, capped), the 2026-07-06/07 daily returns should be materially less negative than −7.32%/−6.89%, and the gate should read WAIT "building" (not reset to 0) or its real verdict.
|
||||
|
||||
---
|
||||
|
||||
## Self-Review
|
||||
|
||||
**1. Spec coverage:** cap in weights (T1) → threaded through book (T2) → sleeve seam (T3) → enabled live (T4) → re-inception+backfill (T5) → verified (T6). The "backfill not reset" constraint is implemented via `inception_t0` backdating in T5 + operator step 2. Covered.
|
||||
|
||||
**2. Placeholder scan:** every code step shows the exact code; the only deferred item is the shock-store fixture in T2/T3 which is explicitly instructed to reuse the existing `test_capacity_haircut_*` store construction in the same file (a real, named pattern, not a placeholder). Acceptable — the implementer has a concrete reference.
|
||||
|
||||
**3. Type consistency:** `coin_gross_cap: float | None = None` is identical across `positioning_weights`, `_book_breakdown`, `positioning_returns_from_store`, `_positioning_series`, `sleeve_returns_from_store`. Constant `POSITIONING_COIN_GROSS_CAP = 0.07` in `bybit_forward_track.py`, imported in `bybit_book_persist.py`. Versions all → 3, `inception_t0.date` all → "2026-07-06". Consistent.
|
||||
|
||||
---
|
||||
|
||||
### Task 7: Thread `coin_gross_cap` into the LIVE forward-nav path (Critical fix from final review)
|
||||
|
||||
**Why:** The final whole-branch review found the cap threads to the reconciliation REF (via `_persist_bybit_book`) but NOT to the live FORWARD NAV (built by `BybitFourEdgeStrategy`), so the gate would compare a capped ref against an uncapped forward and read a healthy book as "diverging below backtest" (reproduced: forward −25% vs ref −12.5% on a LABUSDT shock). Fail-safe (WAIT, never false-PASS) but defeats the branch's purpose. This threads the cap the same way `positioning_tail_cap_k` is already threaded.
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/application/bybit_forward_track.py` (`BybitFourEdgeStrategy.__init__` ~135-157, `advance` ~168-172)
|
||||
- Modify: `src/fxhnt/adapters/orchestration/assets.py` (builders at 105-122, 128-143, 149-158, 164-169; nightly call sites 868-876, 916-924, 957-965)
|
||||
- Modify: `src/fxhnt/adapters/orchestration/migration_builders.py` (builders at 106-117, 127-141, 148-156)
|
||||
- Modify: `src/fxhnt/application/bybit_positioning_eval.py:280` (`positioning_weights_and_contributions` — the Important SSOT-attribution fix)
|
||||
- Test: `tests/integration/test_positioning_forward_track.py` (+ the attribution invariant in `tests/integration/test_bybit_paper_reconciliation.py`)
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `POSITIONING_COIN_GROSS_CAP = 0.07` (Task 4), `sleeve_returns_from_store(..., coin_gross_cap=...)` (Task 3), `positioning_weights(..., coin_gross_cap=...)` (Task 1).
|
||||
- Produces: `BybitFourEdgeStrategy(..., positioning_coin_gross_cap: float | None = None, ...)`; the four `assets.py` builders + three `migration_builders.py` builders gain a `positioning_coin_gross_cap` param forwarded to the strategy; the three nightly call sites pass `POSITIONING_COIN_GROSS_CAP`.
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
Add to `tests/integration/test_positioning_forward_track.py` — a shock store where the forward NAV's worst day is materially less negative when the cap is threaded (mirror the shock-store construction already used in the positioning tests):
|
||||
|
||||
```python
|
||||
def test_forward_nav_applies_coin_gross_cap() -> None:
|
||||
"""The forward NAV (BybitFourEdgeStrategy store-read path) must apply the per-coin cap when given, so it
|
||||
reconciles against the capped ref instead of diverging. Shock store: one over-short coin craters ~50%."""
|
||||
from fxhnt.adapters.warehouse.timescale_feature_store import TimescaleFeatureStore
|
||||
from fxhnt.application.bybit_forward_track import BybitFourEdgeStrategy
|
||||
|
||||
_DAY = 86_400
|
||||
store = TimescaleFeatureStore("sqlite://", table="bybit_features")
|
||||
px = {"LABUSDT": 100.0, "AAAUSDT": 100.0, "BBBUSDT": 100.0, "CCCUSDT": 100.0}
|
||||
for d in range(4):
|
||||
store.write_features("LABUSDT", [(d * _DAY, {"long_ratio": 0.05, "close": px["LABUSDT"]})])
|
||||
store.write_features("AAAUSDT", [(d * _DAY, {"long_ratio": 0.50, "close": px["AAAUSDT"]})])
|
||||
store.write_features("BBBUSDT", [(d * _DAY, {"long_ratio": 0.48, "close": px["BBBUSDT"]})])
|
||||
store.write_features("CCCUSDT", [(d * _DAY, {"long_ratio": 0.52, "close": px["CCCUSDT"]})])
|
||||
if d == 2:
|
||||
px["LABUSDT"] *= 0.50
|
||||
uncapped_rows, _ = BybitFourEdgeStrategy(store, sleeves=["positioning"]).advance(None, {})
|
||||
capped_rows, _ = BybitFourEdgeStrategy(
|
||||
store, sleeves=["positioning"], positioning_coin_gross_cap=0.07).advance(None, {})
|
||||
store.close()
|
||||
assert uncapped_rows and capped_rows, "forward strategy produced no rows — fixture broken"
|
||||
u = dict(uncapped_rows); c = dict(capped_rows)
|
||||
worst = min(u, key=lambda k: u[k])
|
||||
assert c[worst] > u[worst], f"cap must reduce the forward worst day: uncapped={u[worst]}, capped={c[worst]}"
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `uv run --extra dev python -m pytest tests/integration/test_positioning_forward_track.py::test_forward_nav_applies_coin_gross_cap -v`
|
||||
Expected: FAIL with `TypeError: BybitFourEdgeStrategy.__init__() got an unexpected keyword argument 'positioning_coin_gross_cap'`
|
||||
|
||||
- [ ] **Step 3: Add the param to `BybitFourEdgeStrategy` and thread it in `advance`**
|
||||
|
||||
In `__init__` (after `positioning_tail_cap_k: float | None = None,` on line 142) add:
|
||||
```python
|
||||
positioning_coin_gross_cap: float | None = None,
|
||||
```
|
||||
And in the body (after `self._positioning_tail_cap_k = positioning_tail_cap_k` line 154):
|
||||
```python
|
||||
self._positioning_coin_gross_cap = positioning_coin_gross_cap
|
||||
```
|
||||
In `advance`'s store-read `sleeve_returns_from_store(...)` call (lines 168-172), add the kwarg:
|
||||
```python
|
||||
series = sleeve_returns_from_store(
|
||||
self._store, sleeve, universe=self._universe, cost_bps=self._cost_bps,
|
||||
unlock_events=self._unlock_events, stablecoin_spot_panel=self._stablecoin_spot_panel,
|
||||
capacity_capital=self._positioning_capacity_capital,
|
||||
tail_cap_k=self._positioning_tail_cap_k,
|
||||
coin_gross_cap=self._positioning_coin_gross_cap)
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Thread through every builder (parallel to `positioning_tail_cap_k`)**
|
||||
|
||||
In `src/fxhnt/adapters/orchestration/assets.py`, for EACH of the four builders (`build_bybit_4edge_forward_nav`, `build_bybit_4edge_levered_forward_nav`, and the two single-sleeve builders at 149 and 164): add a `positioning_coin_gross_cap=None` param to the signature (next to `positioning_tail_cap_k=None`) and forward `positioning_coin_gross_cap=positioning_coin_gross_cap` into the `BybitFourEdgeStrategy(...)` call. For the THREE nightly call sites (868-876, 916-924, 957-965) that already pass `positioning_tail_cap_k=POSITIONING_TAIL_CAP_K`: add `positioning_coin_gross_cap=POSITIONING_COIN_GROSS_CAP` (import it alongside `POSITIONING_TAIL_CAP_K`).
|
||||
|
||||
Do the identical change in `src/fxhnt/adapters/orchestration/migration_builders.py` for its three builders that pass `positioning_tail_cap_k=POSITIONING_TAIL_CAP_K` (106-117, 127-141, 148-156): add `positioning_coin_gross_cap=POSITIONING_COIN_GROSS_CAP` to each `BybitFourEdgeStrategy(...)` call, importing the constant.
|
||||
|
||||
- [ ] **Step 5: Fix the attribution SSOT (the Important finding)**
|
||||
|
||||
In `src/fxhnt/application/bybit_positioning_eval.py`, `positioning_weights_and_contributions` (line 263): add `coin_gross_cap: float | None = None` to its signature and forward it in the `positioning_weights(...)` call at line 280:
|
||||
```python
|
||||
weights_by_day = positioning_weights(_ratio_by_day(long_ratio), direction=direction,
|
||||
coin_gross_cap=coin_gross_cap)
|
||||
```
|
||||
Then thread it through `sleeve_weights_and_contributions` in `src/fxhnt/application/bybit_paper_weights.py` (the caller at ~80-84) so the paper-book per-symbol weights match the capped sleeve return (preserving the `Σ weight·return == sleeve_returns_from_store` invariant documented at `bybit_paper_weights.py:9`). Default None everywhere.
|
||||
|
||||
- [ ] **Step 6: Run the tests + verify no regression**
|
||||
|
||||
Run: `uv run --extra dev python -m pytest tests/integration/test_positioning_forward_track.py tests/integration/test_bybit_paper_reconciliation.py tests/integration/test_bybit_positioning_eval.py -v`
|
||||
Expected: all pass (new forward-nav cap test passes; existing reconciliation invariant test still passes with default-None on both sides).
|
||||
|
||||
- [ ] **Step 7: Commit**
|
||||
|
||||
```bash
|
||||
git add -A
|
||||
git commit -m "fix(positioning): thread coin_gross_cap into the LIVE forward-nav path + attribution (final-review Critical)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 8: Cap the LIVE paper-book POSITIONS + the report-ref path (Critical + Important from Task-7 review)
|
||||
|
||||
**Why:** The Task-7 review traced two more uncapped paths:
|
||||
- **Critical:** the live paper-book position sizing (`combined_symbol_weights*` / `latest_raw_sleeve_weights` → `derive_and_persist_bybit_paper_book` → the nightly `bybit_paper_book` asset) never passes the positioning haircuts, so the fund's ACTUAL persisted positions for the positioning sleeve are sized uncapped even though the reconciled return is capped. It turns out capacity + tail_cap were ALSO never threaded here (pre-existing gap). Operator decision (2026-07-20): thread ALL THREE haircuts so executed positions match the reconciled returns (full SSOT).
|
||||
- **Important:** `bybit_4edge_backtest_summary` + its caller `report_backtest_summary` pass `positioning_tail_cap_k` but no `positioning_coin_gross_cap` — a real, independently-callable uncapped ref path (masked in the scheduled flow only by the `_BYBIT_BOOK_SIDS` special-case).
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/application/bybit_paper_book.py` — `combined_symbol_weights` (~48), `combined_symbol_weights_and_returns` (~105), `latest_raw_sleeve_weights` (~80), and `derive_and_persist_bybit_paper_book` (the persist driver): add `positioning_capacity_capital`, `positioning_tail_cap_k`, `positioning_coin_gross_cap` (all `float | None = None`) and forward into every `sleeve_weights_and_contributions(...)` call.
|
||||
- Modify: `src/fxhnt/adapters/orchestration/assets.py` — `build_bybit_paper_book` (~180): add the 3 params, forward into `derive_and_persist_bybit_paper_book`; nightly call site (~1105): pass `positioning_capacity_capital=s.paper_capital, positioning_tail_cap_k=POSITIONING_TAIL_CAP_K, positioning_coin_gross_cap=POSITIONING_COIN_GROSS_CAP` (import the two constants).
|
||||
- Modify: `src/fxhnt/application/bybit_forward_track.py` — `bybit_4edge_backtest_summary` (~210): add `positioning_coin_gross_cap: float | None = None` next to `positioning_tail_cap_k`, forward into the `BybitFourEdgeStrategy(...)` call.
|
||||
- Modify: `src/fxhnt/application/backtest_refs_report.py` — `report_backtest_summary` (~42): import `POSITIONING_COIN_GROSS_CAP`, pass `positioning_coin_gross_cap=POSITIONING_COIN_GROSS_CAP` into `bybit_4edge_backtest_summary(...)`.
|
||||
- Test: `tests/integration/test_bybit_paper_reconciliation.py` (or the paper-book test file — grep for a test that calls `combined_symbol_weights_and_returns` / `derive_and_persist_bybit_paper_book`).
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `sleeve_weights_and_contributions(..., positioning_capacity_capital=None, positioning_tail_cap_k=None, positioning_coin_gross_cap=None)` (Task 7 added coin_gross_cap; VERIFY it also has capacity/tail params — if `sleeve_weights_and_contributions` does NOT yet accept capacity_capital/tail_cap_k, add them there too, forwarding into its `positioning_weights`/underlying call, same as coin_gross_cap).
|
||||
- Produces: the 3 haircut params on `combined_symbol_weights`, `combined_symbol_weights_and_returns`, `latest_raw_sleeve_weights`, `derive_and_persist_bybit_paper_book`, `build_bybit_paper_book`; `positioning_coin_gross_cap` on `bybit_4edge_backtest_summary` + `report_backtest_summary`. All default None.
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
Add a paper-book position test: on a LABUSDT-shock store, the positioning sleeve's persisted per-symbol weight for the crashing coin is smaller when `positioning_coin_gross_cap=0.07` is threaded through `combined_symbol_weights_and_returns` than without.
|
||||
|
||||
```python
|
||||
def test_paper_book_positions_capped_by_coin_gross_cap() -> None:
|
||||
"""The live paper-book per-symbol weights must honor the per-coin cap so executed positions match the
|
||||
capped reconciled return (SSOT). On a store with one dominant positioning coin, the capped weight for that
|
||||
coin is strictly smaller than uncapped."""
|
||||
from fxhnt.adapters.warehouse.timescale_feature_store import TimescaleFeatureStore
|
||||
from fxhnt.application.bybit_paper_book import combined_symbol_weights_and_returns
|
||||
_DAY = 86_400
|
||||
store = TimescaleFeatureStore("sqlite://", table="bybit_features")
|
||||
for d in range(4):
|
||||
store.write_features("LABUSDT", [(d * _DAY, {"long_ratio": 0.02, "close": 100.0})])
|
||||
for i, sym in enumerate(("AAAUSDT", "BBBUSDT", "CCCUSDT", "DDDUSDT")):
|
||||
store.write_features(sym, [(d * _DAY, {"long_ratio": 0.45 + 0.02 * i, "close": 100.0})])
|
||||
w_unc, _r, _d = combined_symbol_weights_and_returns(store, sleeves=("positioning",))
|
||||
w_cap, _r2, _d2 = combined_symbol_weights_and_returns(
|
||||
store, sleeves=("positioning",), positioning_coin_gross_cap=0.07)
|
||||
store.close()
|
||||
assert w_unc.get("positioning") and w_cap.get("positioning"), "no positioning weights — fixture broken"
|
||||
lab_unc = abs(w_unc["positioning"].get("LABUSDT", 0.0))
|
||||
lab_cap = abs(w_cap["positioning"].get("LABUSDT", 0.0))
|
||||
assert lab_unc > 0.0
|
||||
assert lab_cap < lab_unc, f"cap must shrink the dominant coin's position: unc={lab_unc}, cap={lab_cap}"
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run to verify it fails**
|
||||
|
||||
Run: `uv run --extra dev python -m pytest tests/integration/test_bybit_paper_reconciliation.py::test_paper_book_positions_capped_by_coin_gross_cap -v`
|
||||
Expected: FAIL with `TypeError: combined_symbol_weights_and_returns() got an unexpected keyword argument 'positioning_coin_gross_cap'`
|
||||
|
||||
- [ ] **Step 3: Thread the 3 haircuts through the paper-book weight functions**
|
||||
|
||||
For `combined_symbol_weights`, `combined_symbol_weights_and_returns`, and `latest_raw_sleeve_weights` in `bybit_paper_book.py`: add `positioning_capacity_capital: float | None = None`, `positioning_tail_cap_k: float | None = None`, `positioning_coin_gross_cap: float | None = None` to each signature, and forward all three into their `sleeve_weights_and_contributions(...)` call(s) (as `capacity_capital=`/`tail_cap_k=`/`positioning_coin_gross_cap=` per that function's actual param names — grep `def sleeve_weights_and_contributions` for the exact names). If `sleeve_weights_and_contributions` lacks capacity_capital/tail_cap_k params, add them there first (forwarding into its positioning path), then here.
|
||||
|
||||
- [ ] **Step 4: Thread through the persist driver + builder**
|
||||
|
||||
`derive_and_persist_bybit_paper_book` (bybit_paper_book.py): add the 3 params, forward into its `combined_symbol_weights_and_returns(...)` call. `build_bybit_paper_book` (assets.py:180): add the 3 params, forward into `derive_and_persist_bybit_paper_book(...)`. Nightly call site (assets.py:~1105): pass `positioning_capacity_capital=s.paper_capital, positioning_tail_cap_k=POSITIONING_TAIL_CAP_K, positioning_coin_gross_cap=POSITIONING_COIN_GROSS_CAP` (import both constants from `bybit_forward_track`).
|
||||
|
||||
- [ ] **Step 5: Fix the report-ref path (Important)**
|
||||
|
||||
`bybit_4edge_backtest_summary` (bybit_forward_track.py:~210): add `positioning_coin_gross_cap: float | None = None` next to `positioning_tail_cap_k`, forward into `BybitFourEdgeStrategy(..., positioning_coin_gross_cap=positioning_coin_gross_cap)`. `report_backtest_summary` (backtest_refs_report.py:~42): import `POSITIONING_COIN_GROSS_CAP`, pass `positioning_coin_gross_cap=POSITIONING_COIN_GROSS_CAP` into the `bybit_4edge_backtest_summary(...)` call.
|
||||
|
||||
- [ ] **Step 6: Run tests + no regression**
|
||||
|
||||
Run: `uv run --extra dev python -m pytest tests/integration/test_bybit_paper_reconciliation.py tests/integration/test_positioning_forward_track.py tests/unit/test_backtest_refs_report.py tests/integration/test_bybit_positioning_eval.py -v`
|
||||
Expected: all pass. Then confirm no site was missed: `grep -c positioning_tail_cap_k` vs `grep -c positioning_coin_gross_cap` should MATCH in bybit_paper_book.py, assets.py, bybit_forward_track.py, backtest_refs_report.py.
|
||||
|
||||
- [ ] **Step 7: Commit**
|
||||
|
||||
```bash
|
||||
git add -A
|
||||
git commit -m "fix(positioning): cap live paper-book positions + report-ref path (Task-7-review Critical+Important)"
|
||||
```
|
||||
474
docs/superpowers/plans/2026-07-20-vrp-code-removal.md
Normal file
474
docs/superpowers/plans/2026-07-20-vrp-code-removal.md
Normal file
@@ -0,0 +1,474 @@
|
||||
# VRP Strategy Full Removal — Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development to implement this plan task-by-task. Steps use checkbox tracking; delete leaves before callers so imports never break mid-removal.
|
||||
|
||||
**Derived from:** a discovery workflow (24 file classifications + 3 deep-traces) — `wf_ffbdc8b5-c02`. The
|
||||
tricky shared-vs-VRP calls were resolved by trace: **DVOL pipeline = delete** (only VRP reads it),
|
||||
**black_scholes.py = delete** (VRP-only), **surfer vendor = untouched** (docstring mention only),
|
||||
**xsp_option_bars TABLE/MODEL/data = KEEP** (only the VRP-only `XspOptionBarsRepo` *class* is deleted).
|
||||
|
||||
## Goal
|
||||
|
||||
Fully delete the VRP (variance-risk-premium) strategy — the shelved/FALSIFIED equity-VRP
|
||||
sleeve and its executed twin `vrp_exec` — from the fxhnt repo, leaving **zero orphaned code,
|
||||
zero dead registry keys, and zero dangling imports**. The nightly Dagster pipeline, the cockpit
|
||||
web app, live IBKR/UCITS execution, the Bybit paper book, and all four kept crypto edges must
|
||||
remain fully functional and the pytest suite must be green at the end.
|
||||
|
||||
The removal spans four coupled clusters, all of which are VRP-exclusive per the deep-traces:
|
||||
|
||||
1. **VRP application modules**: `vrp_book`, `vrp_eval`, `vrp_defined_risk_eval`,
|
||||
`vrp_exec_record`, `vrp_marking`, `vrp_pricing`, `vrp_research`.
|
||||
2. **VRP option-math + IV plumbing**: `black_scholes.py` (VRP-only), the entire DVOL pipeline
|
||||
(`deribit_dvol.py`, `dvol_ingest.py`, the `dvol` feature panel, the unused
|
||||
`VolIndexClient.dvol()` port).
|
||||
3. **VRP persistence/orchestration**: `xsp_option_bars.py` repo class, the `vrp_nav` Dagster
|
||||
asset + its ~14 private OPRA-freeze/backfill helpers, the `backfill-xsp-opra` ingest path,
|
||||
the `VrpStrategy` builder, and the `vrp`/`vrp_exec` registry entries.
|
||||
4. **Comment/registry-string scrubs** across shared cockpit infrastructure that merely *names*
|
||||
VRP as an illustrative example.
|
||||
|
||||
## Architecture
|
||||
|
||||
VRP was ARCHIVED 2026-07-15 (fails the LOO marginal-Sharpe gate; see
|
||||
`pearl_fxhnt_diversifier_hunt_2026_07_15`). The two registry entries already carry
|
||||
`archived: True`, and `vrp_nav` is already **de-wired** from `definitions.py`'s nightly asset
|
||||
graph — so no live nightly asset materializes VRP today. What remains is dead weight plus two
|
||||
active-but-purposeless CLI/CronJob ingest paths (`ingest-dvol`, `backfill-xsp-opra`) and a set
|
||||
of stale registry keys/comment references.
|
||||
|
||||
Dependency direction (leaf → root), which dictates task order so imports never break mid-removal:
|
||||
|
||||
```
|
||||
black_scholes.py ────────────► vrp_defined_risk_eval.py ──┐
|
||||
deribit_dvol.py ─► dvol_ingest.py ─► (dvol panel) ────────┤
|
||||
xsp_option_bars.py (repo class) ──► vrp_book.py ──────────┤──► cli.py commands
|
||||
vrp_pricing.py / vrp_marking.py ──► vrp_book/eval/exec ───┤ migration_builders (_build_vrp)
|
||||
└──► assets.py vrp_nav + helpers
|
||||
registry.py vrp / vrp_exec
|
||||
forward_ingest _REF_ALIAS
|
||||
sim_curves / dashboard_service /
|
||||
app.py / definitions.py (scrub)
|
||||
```
|
||||
|
||||
Rule: delete leaf modules **and their tests together** first, then the callers that referenced
|
||||
them (asset graph, registry, CLI, migration builder), then the pure comment-scrub edits last.
|
||||
|
||||
## DVOL and black_scholes — resolved decisions (per deep-traces)
|
||||
|
||||
- **DVOL pipeline → DELETE (do not keep).** Exhaustive grep confirms the only readers of the
|
||||
`dvol` feature panel (`store.read_panel(..., "dvol")`) are `vrp_eval.py` and
|
||||
`vrp_defined_risk_eval.py`. `VolIndexClient.dvol()` (`ports/market_data.py:22`) has **zero**
|
||||
callers. DVOL is not a Dagster asset. DVOL is the Deribit IV index — a data source entirely
|
||||
distinct from the OPRA `xsp_option_bars` data being kept; there is no keep-carve-out for it.
|
||||
Delete `deribit_dvol.py`, `dvol_ingest.py`, the `ingest-dvol` CLI command, the
|
||||
`VolIndexClient.dvol()` port method, and their tests. The `dvol` warehouse rows in
|
||||
`bybit_features` are inert once nothing reads them — no migration needed; they can age out.
|
||||
|
||||
- **black_scholes.py → DELETE (VRP-only, not shared math).** The only importers are
|
||||
`vrp_defined_risk_eval.py` and its own unit test `tests/unit/test_black_scholes.py`.
|
||||
`vrp_pricing.py` defines a *separate* `bs_put` (different signature) and does **not** import
|
||||
from `black_scholes.py`, so there is no hidden cross-file sharing. Delete both files.
|
||||
|
||||
- **`xsp_option_bars` TABLE + `XspOptionBarRow` model + OPRA `.dbn` → KEEP.** Only the
|
||||
`XspOptionBarsRepo` *repo class* (`adapters/persistence/xsp_option_bars.py`) is VRP-only and
|
||||
gets deleted. The ORM row model `XspOptionBarRow` in `cockpit_models.py`, the physical table,
|
||||
the 13-year OPRA data, and `ForwardNavRepo.xsp_freeze_max_date()` (raw `SELECT max(date)`)
|
||||
all stay untouched.
|
||||
|
||||
## Global Constraints
|
||||
|
||||
1. **KEEP the OPRA data.** Do **not** drop or migrate the `xsp_option_bars` table, do **not**
|
||||
remove `XspOptionBarRow` from `cockpit_models.py`, do **not** delete OPRA `.dbn` files, and
|
||||
do **not** touch `ForwardNavRepo.xsp_freeze_max_date()`. Only the VRP *ingest/consumer*
|
||||
machinery (`XspOptionBarsRepo` class, `_backfill_month`/`_freeze_*` helpers, `backfill-xsp-opra`)
|
||||
is removed. The freeze-staleness health axis (`_FREEZE_SOURCES = {"opra-pit"}`) stays generic.
|
||||
2. **Real-money paths must err false-WAIT, never false-EXECUTE.** VRP execution
|
||||
(`execute-vrp`/`vrp_exec_record`) is being deleted, not disabled — but during the removal no
|
||||
edit may cause any *surviving* real-money path (multistrat/UCITS IBKR, Bybit paper) to
|
||||
silently promote/execute on missing state. After removing the `vrp`/`vrp_exec` registry keys,
|
||||
confirm the cockpit gate and `forward_ingest` promotion loop still default to WAIT for every
|
||||
surviving strategy and never KeyError into a false-PASS.
|
||||
3. **Full pytest suite green before finishing.** Run `uv run pytest -n auto` (or
|
||||
`python -m pytest -n auto` per `pyproject.toml addopts = "-n auto"`) from the repo root and
|
||||
confirm 0 failures / 0 errors before the final commit. Per-task commands below are the fast
|
||||
local gate; the full-suite run in the final task is the release gate.
|
||||
4. **The false VRP STALE health alarm must be gone.** After removal, verify **no `vrp` key
|
||||
remains in `STRATEGY_REGISTRY`** (`vrp` and `vrp_exec` both removed) so the forward-health
|
||||
freeze/staleness axis and `dashboard_service._active_ibkr_account_sids()` can no longer
|
||||
surface a ghost VRP STALE row. Verification command in the final task.
|
||||
|
||||
### Pre-flight note (already-broken test)
|
||||
|
||||
`tests/unit/test_vrp_rebalancer_yaml.py` asserts the existence of
|
||||
`infra/k8s/jobs/fxhnt-vrp-rebalancer.yaml`, which **no longer exists** in the tree. This test is
|
||||
already failing/erroring at HEAD; it is deleted in Task 1 alongside the other VRP tests, which
|
||||
also removes that pre-existing red.
|
||||
|
||||
---
|
||||
|
||||
## Task 1 — Delete VRP application modules + their tests (leaves first)
|
||||
|
||||
**Files to delete (whole file):**
|
||||
- `src/fxhnt/application/vrp_book.py`
|
||||
- `src/fxhnt/application/vrp_eval.py`
|
||||
- `src/fxhnt/application/vrp_defined_risk_eval.py`
|
||||
- `src/fxhnt/application/vrp_exec_record.py`
|
||||
- `src/fxhnt/application/vrp_marking.py`
|
||||
- `src/fxhnt/application/vrp_pricing.py`
|
||||
- `src/fxhnt/application/vrp_research.py`
|
||||
- `src/fxhnt/application/black_scholes.py`
|
||||
- `tests/unit/test_vrp_book.py`
|
||||
- `tests/unit/test_vrp_pricing.py`
|
||||
- `tests/unit/test_vrp_exec_helpers.py`
|
||||
- `tests/unit/test_vrp_exec_ladder_cap.py`
|
||||
- `tests/unit/test_vrp_exec_record.py`
|
||||
- `tests/unit/test_vrp_rebalancer_yaml.py` (also clears the pre-existing red — see pre-flight note)
|
||||
- `tests/unit/test_registry_vrp.py`
|
||||
- `tests/unit/test_black_scholes.py`
|
||||
- `tests/integration/test_vrp_eval.py`
|
||||
- `tests/integration/test_vrp_defined_risk_eval.py`
|
||||
- `tests/integration/test_vrp_nav_asset.py`
|
||||
- `tests/integration/test_cockpit_vrp_render.py`
|
||||
|
||||
**Rationale:** every one of these modules is imported only by other `vrp_*` modules or by the
|
||||
`cli.py`/`assets.py`/`migration_builders.py` callers handled in later tasks. Deleting the tests
|
||||
in the *same* task prevents collection-time `ImportError` on the deleted modules. Do NOT yet
|
||||
touch `assets.py`, `cli.py`, `registry.py`, or `migration_builders.py` — those still reference
|
||||
these modules only via **lazy (in-function) imports**, so the module tree still imports cleanly
|
||||
after this task (the broken imports are inside function bodies not executed at import time). They
|
||||
are removed in Tasks 3–5.
|
||||
|
||||
**Covering test command** (must import-clean and not collect the deleted files):
|
||||
```
|
||||
uv run pytest tests/unit tests/integration -q --co 2>&1 | tail -20
|
||||
```
|
||||
Expect: collection succeeds with the deleted VRP tests absent and no `ImportError`.
|
||||
|
||||
**Commit:** `refactor(vrp): delete VRP application modules + black_scholes + their tests`
|
||||
|
||||
---
|
||||
|
||||
## Task 2 — Delete the DVOL pipeline + its tests
|
||||
|
||||
**Files to delete (whole file):**
|
||||
- `src/fxhnt/adapters/data/deribit_dvol.py`
|
||||
- `src/fxhnt/application/dvol_ingest.py`
|
||||
- `tests/integration/test_deribit_dvol.py`
|
||||
- `tests/integration/test_dvol_ingest.py`
|
||||
|
||||
**Edit `src/fxhnt/ports/market_data.py`:**
|
||||
- Remove the `VolIndexClient` Protocol (the `dvol()` method block, ~lines 21–24). Confirmed
|
||||
zero implementers and zero callers repo-wide; it is dead scaffolding.
|
||||
|
||||
**Do NOT touch:** `deribit_funding.py` and the `ingest-deribit-funding` CronJob/command —
|
||||
that is the *funding* (cross-venue carry) edge, a separate kept data source that reads the
|
||||
`funding`/`deribit_funding` features, never `dvol`.
|
||||
|
||||
**Rationale:** `ingest_dvol` and `DeribitDVOL` are consumed only by the `ingest-dvol` CLI
|
||||
command (removed in Task 3) which fed only the two VRP evaluators (deleted in Task 1). The
|
||||
`dvol` feature panel now has no readers.
|
||||
|
||||
**Covering test command:**
|
||||
```
|
||||
uv run pytest tests/integration -q --co 2>&1 | tail -20
|
||||
grep -rn "VolIndexClient\|import.*deribit_dvol\|import.*dvol_ingest" src/ | grep -v "/cli.py"
|
||||
```
|
||||
Expect: no collection errors; the grep returns only the (still-present, removed next task)
|
||||
`cli.py` lazy imports.
|
||||
|
||||
**Commit:** `refactor(vrp): delete DVOL pipeline (deribit_dvol, dvol_ingest, VolIndexClient port) + tests`
|
||||
|
||||
---
|
||||
|
||||
## Task 3 — Delete VRP + DVOL commands from `cli.py`
|
||||
|
||||
**File to edit:** `src/fxhnt/cli.py`
|
||||
|
||||
**Exact edits (delete each command block in full, including its in-function lazy imports):**
|
||||
- Delete `@app.command("execute-vrp")` `execute_vrp` (lines ~439–499), including the lazy
|
||||
`from fxhnt.application.vrp_exec_record import plan_and_record_vrp`. **Keep** the module-level
|
||||
and other commands' use of `ForwardNavRepo` — only remove this command's local import line if
|
||||
it has one; the `ForwardNavRepo` module itself is reused by `execute-multistrat` and others.
|
||||
- Delete `@app.command("ingest-dvol")` `ingest_dvol_cmd` (lines ~1217–1266), including the lazy
|
||||
`from fxhnt.adapters.data.deribit_dvol import DeribitDVOL` and
|
||||
`from fxhnt.application.dvol_ingest import ingest_dvol`.
|
||||
- Delete `@app.command("vrp-eval")` `vrp_eval` (lines ~2193–2288), including the lazy
|
||||
`from fxhnt.application.vrp_eval import (verify_vrp_edge, vrp_metrics_from_store, walk_forward_vrp)`.
|
||||
- Delete `@app.command("vrp-defined-risk-eval")` `vrp_defined_risk_eval` (lines ~2291–2403),
|
||||
including the lazy `from fxhnt.application.vrp_defined_risk_eval import (...)`.
|
||||
- Delete the private helpers `_print_vrp_verify` (~2406–2446) and `_print_vrp_walk_forward`
|
||||
(~2449–2469) — called only by the two eval commands above.
|
||||
|
||||
**Decision — `backfill-xsp-opra` (lines ~578–623): DELETE this command too.** It calls
|
||||
`_backfill_month` and `XspOptionBarsRepo`, both of which are deleted in Tasks 4/5, and its sole
|
||||
purpose is freezing OPRA marks into `xsp_option_bars` **for VRP recompute**. Keeping the OPRA
|
||||
*data* (Constraint 1) does not require keeping the *ingest path* — the data already exists and
|
||||
the only consumer of freshly-frozen marks was VRP. Delete the `@app.command("backfill-xsp-opra")`
|
||||
`backfill_xsp_opra` function and its `from fxhnt.adapters.orchestration.assets import _backfill_month`
|
||||
+ `from fxhnt.adapters.persistence.xsp_option_bars import XspOptionBarsRepo` local imports. (If
|
||||
the executor finds a documented non-VRP consumer of freshly-frozen OPRA marks, stop and flag —
|
||||
none was found in the trace.)
|
||||
|
||||
**Covering test command:**
|
||||
```
|
||||
uv run python -c "import fxhnt.cli" # module must import clean
|
||||
uv run fxhnt --help >/dev/null # Typer app builds with commands gone
|
||||
grep -n "vrp\|dvol\|backfill-xsp\|VrpStrategy\|black_scholes\|XspOptionBarsRepo" src/fxhnt/cli.py
|
||||
```
|
||||
Expect: import + `--help` succeed; grep returns no hits.
|
||||
|
||||
**Commit:** `refactor(vrp): remove execute-vrp/ingest-dvol/vrp-eval/vrp-defined-risk-eval/backfill-xsp-opra CLI commands`
|
||||
|
||||
---
|
||||
|
||||
## Task 4 — Remove the `vrp_nav` asset + OPRA-freeze helpers from `assets.py`
|
||||
|
||||
**File to edit:** `src/fxhnt/adapters/orchestration/assets.py`
|
||||
|
||||
**Exact edits (delete code):**
|
||||
- Delete the `@asset`-decorated `vrp_nav` function (lines ~686–718), including its inline
|
||||
imports of `DatabentoDataProvider`, `XspOptionBarsRepo`, `VrpStrategy` (~line 694).
|
||||
- Delete the exclusive private helper chain (lines ~492–683), all VRP-only and now callerless
|
||||
after Tasks 1/3: `_iso_date`, `_forward_from_marks` (+ its inline
|
||||
`fxhnt.application.vrp_pricing.parity_forward` import ~502), `_band_contracts`,
|
||||
`_band_rows_from_marks`, `_day_marks`, `_xsp_forward_estimate`, `_freeze_xsp_slice`,
|
||||
`_freeze_day`, `_prev_trading_day`, `_DEGENERATE_FORWARD_MARKERS`,
|
||||
`_is_degenerate_forward_error`, `_freeze_latest_session` (+ its inline
|
||||
`DatabentoRangeUnavailable` import ~614), `_rough_forward`, `_backfill_month`.
|
||||
- Delete the trailing `# vrp_nav ARCHIVED 2026-07-15 — de-wired (fn kept below)` comment
|
||||
(~line 1131) — the function no longer exists below.
|
||||
- Reword the module docstring (~line 4): drop the trailing `/vrp` from the forward-track asset
|
||||
enumeration.
|
||||
|
||||
**Do NOT touch:** `deribit_funding_bars` (~825–856, unrelated Deribit perp *funding*, kept for
|
||||
cross-venue-carry), and all other assets (`futures_bars`, `bybit_*`, `multistrat_*`,
|
||||
`sixtyforty_nav`, `positioning_nav`, `xsfunding_nav`, `unlock_nav`, `cockpit_forward`, …). The
|
||||
RETIRED-track comments at ~721 and ~748 do not mention vrp — leave as-is.
|
||||
|
||||
**Covering test command:**
|
||||
```
|
||||
uv run python -c "import fxhnt.adapters.orchestration.assets"
|
||||
uv run python -c "import fxhnt.adapters.orchestration.definitions" # Dagster code-location loads
|
||||
grep -n "vrp\|Vrp\|_backfill_month\|_freeze_\|XspOptionBarsRepo\|vrp_pricing" src/fxhnt/adapters/orchestration/assets.py
|
||||
uv run pytest tests/integration/test_orchestration_definitions.py -q
|
||||
```
|
||||
Expect: both modules import clean, grep returns no hits, definitions test passes.
|
||||
|
||||
**Commit:** `refactor(vrp): delete vrp_nav Dagster asset + OPRA-freeze/backfill helper chain`
|
||||
|
||||
---
|
||||
|
||||
## Task 5 — Remove `XspOptionBarsRepo` class + the `_build_vrp` migration builder
|
||||
|
||||
**File to delete (whole file):**
|
||||
- `src/fxhnt/adapters/persistence/xsp_option_bars.py` (the `XspOptionBarsRepo` repo class).
|
||||
All callers (`vrp_book.py`, `vrp_exec_record.py`, `vrp_marking.py`, the `vrp_nav` asset, the
|
||||
`backfill-xsp-opra` command) are gone after Tasks 1/3/4.
|
||||
- **VERIFIED 2026-07-20: `tests/integration/test_xsp_option_bars_repo.py` does NOT exist in the tree** —
|
||||
do not attempt to delete it. If a repo-class test surfaces under another name during removal, delete it;
|
||||
otherwise there is no test file for this class.
|
||||
|
||||
**KEEP (Constraint 1):** `cockpit_models.XspOptionBarRow`, the `xsp_option_bars` table, and
|
||||
`ForwardNavRepo.xsp_freeze_max_date()` — none of these import or depend on the repo class.
|
||||
|
||||
**File to edit:** `src/fxhnt/adapters/orchestration/migration_builders.py`
|
||||
- Delete the `_build_vrp` closure (lines ~71–76) and its `builders["vrp"] = ("vrp_state",
|
||||
_build_vrp)` registration (~78), plus the preceding `# ---- vrp_nav: ...` comment banner (~71).
|
||||
|
||||
**Covering test command:**
|
||||
```
|
||||
uv run python -c "import fxhnt.adapters.orchestration.migration_builders"
|
||||
grep -rn "XspOptionBarsRepo\|_build_vrp\|builders\[.vrp" src/
|
||||
uv run pytest tests/ -q -k "migration or migrate" 2>&1 | tail -20
|
||||
```
|
||||
Expect: import clean, grep returns no hits (the `XspOptionBarRow` model in `cockpit_models.py`
|
||||
is a different symbol and must still be present — verify separately:
|
||||
`grep -n "class XspOptionBarRow" src/fxhnt/adapters/persistence/cockpit_models.py` returns 1 hit).
|
||||
|
||||
**Commit:** `refactor(vrp): delete XspOptionBarsRepo class + _build_vrp migration builder (keep table + XspOptionBarRow)`
|
||||
|
||||
---
|
||||
|
||||
## Task 6 — Remove `vrp`/`vrp_exec` registry keys + `forward_ingest` alias
|
||||
|
||||
**File to edit:** `src/fxhnt/registry.py`
|
||||
- Delete the `"vrp"` `STRATEGY_REGISTRY` entry (lines ~161–172) and its preceding comment
|
||||
block (~157).
|
||||
- Delete the `"vrp_exec"` `STRATEGY_REGISTRY` entry (lines ~177–186) and its preceding comment
|
||||
block (~173–176).
|
||||
- Keep the dict itself and all 9 non-VRP entries and the `FUND_INSTRUMENTS` import/re-export.
|
||||
|
||||
**File to edit:** `src/fxhnt/application/forward_ingest.py`
|
||||
- Remove the `"vrp_exec": "vrp"` pair from `_REF_ALIAS` (line ~45), leaving
|
||||
`{"multistrat_exec": "multistrat", "bybit_4edge_exec": "bybit_4edge"}`.
|
||||
|
||||
**Rationale + Constraint 4:** removing the `vrp` key is what actually kills the false VRP STALE
|
||||
health alarm — with no `vrp` in `STRATEGY_REGISTRY`, the forward-health freeze/staleness loop
|
||||
and `dashboard_service._active_ibkr_account_sids()` can no longer iterate a ghost VRP row.
|
||||
(Note: `dashboard_service._IBKR_ACCOUNT_SIDS` hard-codes `"vrp"`; that literal is fixed in
|
||||
Task 7 in the *same PR* so the window where the key is gone but the literal remains never ships.)
|
||||
|
||||
**Covering test command:**
|
||||
```
|
||||
uv run python -c "from fxhnt.registry import STRATEGY_REGISTRY; assert 'vrp' not in STRATEGY_REGISTRY and 'vrp_exec' not in STRATEGY_REGISTRY, STRATEGY_REGISTRY.keys(); print('registry clean')"
|
||||
uv run pytest tests/unit/test_forward_definition.py tests/integration/test_forward_health.py tests/integration/test_dashboard_service.py -q 2>&1 | tail -20
|
||||
```
|
||||
Expect: assertion passes; forward/dashboard tests green (these confirm surviving strategies
|
||||
still WAIT-by-default — Constraint 2).
|
||||
|
||||
**Commit:** `refactor(vrp): remove vrp/vrp_exec registry entries + forward_ingest _REF_ALIAS`
|
||||
|
||||
---
|
||||
|
||||
## Task 7 — Fix the one behavioral literal + `sim_curves` book pill (data edits)
|
||||
|
||||
These are the two non-comment edits that gate on `vrp` being a *present* registry key and would
|
||||
misbehave once it is gone (Task 6). They ship together with — or immediately after — Task 6.
|
||||
|
||||
**File to edit:** `src/fxhnt/application/dashboard_service.py`
|
||||
- Line ~30: change `_IBKR_ACCOUNT_SIDS = frozenset({"multistrat", "vrp"})` to
|
||||
`frozenset({"multistrat"})`. **Why this is behavioral, not cosmetic:** `_active_ibkr_account_sids()`
|
||||
computes `not STRATEGY_REGISTRY.get("vrp", {}).get("archived")`; once the `vrp` key is gone
|
||||
this becomes `not None` = `True`, wrongly treating `vrp` as an ACTIVE ibkr-account sid and
|
||||
rendering a ghost/broken VRP row on the IBKR account page. Removing the literal eliminates it.
|
||||
|
||||
**File to edit:** `src/fxhnt/application/sim_curves.py`
|
||||
- Line ~15: remove the `"vrp"` string from the `SIM_BOOKS` tuple, leaving
|
||||
`("bybit_4edge", "bybit_4edge_levered", "multistrat", "multistrat_levered")`. **Why:** once
|
||||
the `vrp` registry entry is gone, a `vrp` Backtest-page pill would `KeyError`/500 at request
|
||||
time in `app.py` (`registry_backtest_kind("vrp")`). The dispatch logic in `sim_returns_for`
|
||||
is generic and unchanged.
|
||||
|
||||
**Covering test command:**
|
||||
```
|
||||
uv run pytest tests/unit/test_sim_curves.py tests/integration/test_dashboard_service.py tests/integration/test_ibkr_account_detail_web.py tests/integration/test_dashboard_ibkr_holdings.py tests/integration/test_paper_books_web.py tests/integration/test_ibkr_sim_web.py -q 2>&1 | tail -20
|
||||
```
|
||||
Expect: green; no VRP row/pill; surviving books render.
|
||||
|
||||
**Commit:** `fix(vrp): drop vrp from _IBKR_ACCOUNT_SIDS and SIM_BOOKS so ghost VRP row/pill can't render`
|
||||
|
||||
---
|
||||
|
||||
## Task 8 — Delete the now-purposeless VRP/OPRA CronJob manifest
|
||||
|
||||
**File to delete:**
|
||||
- `infra/k8s/jobs/fxhnt-opra-backfill.yaml` — invokes `fxhnt backfill-xsp-opra` (deleted in
|
||||
Task 3). With the command gone this CronJob would crash-loop; its only job was freezing OPRA
|
||||
marks for VRP recompute. (The `fxhnt-vrp-rebalancer.yaml` manifest is already absent from the
|
||||
tree — nothing to delete there; its dangling test is removed in Task 1.)
|
||||
|
||||
**Do NOT touch:** `fxhnt-deribit-carry-job.yaml` (funding/carry, not DVOL),
|
||||
`fxhnt-bybit-*`, `fxhnt-ucits-rebalancer.yaml`, `fxhnt-ibkr-rebalancer.yaml`, and all other
|
||||
manifests.
|
||||
|
||||
**Covering check:**
|
||||
```
|
||||
grep -rln "vrp\|ingest-dvol\|execute-vrp\|backfill-xsp-opra" infra/ # expect: no hits
|
||||
```
|
||||
|
||||
**Commit:** `chore(vrp): remove fxhnt-opra-backfill CronJob (backfill-xsp-opra command deleted)`
|
||||
|
||||
---
|
||||
|
||||
## Task 9 — Comment/docstring scrubs (no logic change) — LAST
|
||||
|
||||
Pure prose edits that dangling-reference the deleted `vrp` strategy id. No imports, logic,
|
||||
registry entries, or types change. Batched last because none affect importability.
|
||||
|
||||
**Files + edits (reword to drop the `vrp` example, keep meaning):**
|
||||
- `src/fxhnt/adapters/orchestration/definitions.py` — lines ~10–12: drop the
|
||||
"vrp_nav ARCHIVED … KEPT in assets.py but DE-WIRED" paragraph (the "KEPT" claim is now false);
|
||||
replace with a one-line "VRP removed entirely 2026-07-20" note or delete outright.
|
||||
- `src/fxhnt/adapters/persistence/cockpit_models.py` — line ~46 (drop the `(… that would have
|
||||
caught vrp)` parenthetical or generalize to "a dead strategy"); line ~228 (drop `/vrp` from
|
||||
`(multistrat/vrp …)`).
|
||||
- `src/fxhnt/adapters/persistence/forward_nav.py` — lines ~146–147 (generalize "the vrp freeze
|
||||
table" / "a vrp recompute is bounded by this max date" to opra-pit-anchored/strategy-generic
|
||||
wording; the `xsp_freeze_max_date` query stays); line ~199 (drop the "shelved/FALSIFIED
|
||||
vrp/vrp_exec" example from the `archived: True` registry note).
|
||||
- `src/fxhnt/adapters/persistence/paper_repo.py` — line ~519 (drop `vrp` from the
|
||||
`(multistrat/vrp …)` recompute-replay parenthetical).
|
||||
- `src/fxhnt/adapters/web/app.py` — lines ~449, 453, 473, 482, 551, 567, 728, 752, 810: drop the
|
||||
`/vrp` example from each `multistrat/vrp` phrase. No code change (the archived-filter
|
||||
comprehension already yields one fewer element once `vrp` left `SIM_BOOKS`/registry).
|
||||
- `src/fxhnt/adapters/web/templates/_macros.html` — line ~61 ("2-week vrp rot"): generalize.
|
||||
- `src/fxhnt/adapters/web/templates/cockpit.html` — line ~78: drop `/vrp` from
|
||||
"two IBKR real-account books (multistrat/vrp)".
|
||||
- `src/fxhnt/application/dashboard_service.py` — comment-only lines ~27, 55, 89, 390, 416, 417:
|
||||
drop/generalize the `vrp` examples (the code literal was already fixed in Task 7).
|
||||
- `src/fxhnt/application/sim_curves.py` — lines ~2, 12, 22: drop `vrp` from the
|
||||
`multistrat/vrp` recompute-replay examples (the tuple was already fixed in Task 7).
|
||||
- `src/fxhnt/application/forward_health.py` — docstring lines ~3, 11, 13, 14, 15, 24, 31 and the
|
||||
inline comment at ~147 ("for the vrp freeze axis"): generalize away from naming `vrp` as the
|
||||
mechanism's reason for existing; the `_FREEZE_SOURCES = {"opra-pit"}` logic and
|
||||
`xsp_freeze_max_date()` call are unchanged.
|
||||
- `src/fxhnt/application/display_names.py` — lines ~2 and ~51: drop the `vrp` example id /
|
||||
option-contract example from the illustrative prose.
|
||||
- `src/fxhnt/application/em_asia_marginal_eval.py` — lines ~6, 19, 71: replace "VRP" in the
|
||||
Stage-1 kill-gate precedent comments with a VRP-agnostic phrase ("prior failed diversifier
|
||||
candidates"). No logic change.
|
||||
- `src/fxhnt/config.py` — line ~215: drop `(… that would have caught vrp)` from the
|
||||
`stale_after_business_days` comment; keep the setting + default.
|
||||
- `src/fxhnt/ports/dashboard.py` — lines ~72 and ~190: drop the `vrp` example from the
|
||||
`multistrat/vrp` IBKR-real-account docstrings. (Line ~34 is unrelated carry — leave.)
|
||||
- `src/fxhnt/vendor/surfer/__init__.py` — lines ~11–13: compress the `surfer_poc`/
|
||||
`MomentumVrpStrategy` retirement note to a plain historical line without the VRP signpost.
|
||||
|
||||
**Do NOT edit** as part of scrubbing (already covered / must not change): the `xsp_option_bars`
|
||||
query, `_FREEZE_SOURCES`, `XspOptionBarRow`, any settings field, or any Protocol field.
|
||||
|
||||
**Covering test command** (broad — scrubs must not have touched logic):
|
||||
```
|
||||
uv run python -c "import fxhnt.adapters.web.app, fxhnt.application.dashboard_service, fxhnt.application.forward_health, fxhnt.config, fxhnt.ports.dashboard, fxhnt.application.display_names, fxhnt.application.em_asia_marginal_eval, fxhnt.application.sim_curves"
|
||||
uv run pytest tests/unit/test_display_names.py tests/integration/test_dashboard_service.py -q 2>&1 | tail -20
|
||||
```
|
||||
Expect: all import clean; tests green.
|
||||
|
||||
**Commit:** `docs(vrp): scrub stale VRP comment/docstring references across cockpit + shared infra`
|
||||
|
||||
---
|
||||
|
||||
## Task 10 — Full-suite green gate + removal verification (release gate)
|
||||
|
||||
**No file edits.** Final verification against all Global Constraints.
|
||||
|
||||
**Commands (all must pass):**
|
||||
```
|
||||
# (3) full suite green
|
||||
uv run pytest -n auto 2>&1 | tail -30 # 0 failed, 0 errors
|
||||
|
||||
# (4) no vrp in registry — false STALE alarm gone
|
||||
uv run python -c "from fxhnt.registry import STRATEGY_REGISTRY; assert not any(k.startswith('vrp') for k in STRATEGY_REGISTRY), list(STRATEGY_REGISTRY); print('no vrp in registry')"
|
||||
|
||||
# no orphaned VRP code / imports anywhere in src
|
||||
grep -rn "import.*vrp_\|VrpStrategy\|deribit_dvol\|dvol_ingest\|black_scholes\|XspOptionBarsRepo\|VolIndexClient\|_backfill_month\|plan_and_record_vrp" src/ # expect: no hits
|
||||
ls src/fxhnt/application/vrp_*.py 2>&1 # expect: No such file
|
||||
ls src/fxhnt/application/black_scholes.py src/fxhnt/application/dvol_ingest.py src/fxhnt/adapters/data/deribit_dvol.py src/fxhnt/adapters/persistence/xsp_option_bars.py 2>&1 # expect: No such file (all four)
|
||||
|
||||
# (1) OPRA data kept — table model + freeze query still present
|
||||
grep -n "class XspOptionBarRow" src/fxhnt/adapters/persistence/cockpit_models.py # expect: 1 hit
|
||||
grep -n "def xsp_freeze_max_date" src/fxhnt/adapters/persistence/forward_nav.py # expect: 1 hit
|
||||
|
||||
# (2) Dagster code-location + cockpit app both import clean (no false-execute / no outage)
|
||||
uv run python -c "import fxhnt.adapters.orchestration.definitions; import fxhnt.adapters.web.app; print('definitions + app import clean')"
|
||||
|
||||
# no residual vrp mentions in infra
|
||||
grep -rln "vrp\|ingest-dvol\|execute-vrp\|backfill-xsp-opra" infra/ # expect: no hits
|
||||
```
|
||||
|
||||
**Constraint 2 spot-check:** confirm `forward_ingest`'s promotion loop and the cockpit gate
|
||||
default to WAIT for every surviving strategy (the recon-gate never false-PASSes) — covered by
|
||||
`test_forward_health.py` / `test_dashboard_service.py` in the full run; if either regresses, the
|
||||
removal touched a shared gate path and must be reverted to the last green task.
|
||||
|
||||
**Commit (only after everything above is green):** `chore(vrp): verify full VRP removal — suite green, registry clean, OPRA data intact`
|
||||
|
||||
---
|
||||
|
||||
## Post-removal note
|
||||
|
||||
After merge, update `docs/superpowers/VRP-REMOVAL-TODO.md` and project memory
|
||||
(`pearl_fxhnt_diversifier_hunt_2026_07_15.md` already says "VRP now fully DELETED (shelved)") to
|
||||
reflect that the code deletion — not just shelving — is complete, and that the OPRA
|
||||
`xsp_option_bars` data was retained per Constraint 1.
|
||||
269
docs/superpowers/plans/2026-07-21-fxhnt-cicd-pipeline.md
Normal file
269
docs/superpowers/plans/2026-07-21-fxhnt-cicd-pipeline.md
Normal file
@@ -0,0 +1,269 @@
|
||||
# fxhnt CI/CD Pipeline — Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development to implement this
|
||||
> plan task-by-task. Steps use checkbox tracking. This plan writes/edits YAML manifests and applies them to a
|
||||
> LIVE cluster — every apply is gated behind a dry-run + explicit verify; a broken sensor/workflow must never
|
||||
> silently swallow a push or false-deploy.
|
||||
|
||||
**Goal:** A push to `master` on `gitadmin/fxhnt` auto-builds (only when deps/Dockerfile change) and auto-deploys
|
||||
the fxhnt cockpit+dagster daemons — no manual `kubectl rollout restart`.
|
||||
|
||||
**Architecture:** git-sync-first, image-on-demand. A minimal Sensor in `argo-events` hears the shared Gitea
|
||||
webhook, path-filters the push to pick `mode=build` (Kaniko rebuild) vs `mode=deploy` (rollout only), and submits
|
||||
the fxhnt-owned `fxhnt-cockpit` WorkflowTemplate into `foxhunt`.
|
||||
|
||||
**Tech Stack:** Argo Events v1.9.10 (Sensor), Argo Workflows (Kaniko build + kubectl deploy), Gitea webhook,
|
||||
Scaleway registry, K8s (bizworx Kapsule).
|
||||
|
||||
## Global Constraints
|
||||
|
||||
- **Project boundary:** fxhnt-owned objects (WorkflowTemplate, build/deploy pods, secrets, RBAC) live in
|
||||
`foxhunt`. ONLY the Sensor lives in `argo-events` (unavoidable). No fxhnt secrets in `argo`/`argo-events`; no
|
||||
cross-namespace RBAC.
|
||||
- **Node pool:** all build/deploy pods target `nodeSelector: k8s.scaleway.com/pool-name: large` (the
|
||||
`ci-compile-cpu` pool is DECOMMISSIONED — only `large`/`sfs` exist). A pod pinned to `ci-compile-cpu` never
|
||||
schedules.
|
||||
- **Registry:** `rg.fr-par.scw.cloud/bizworx/fxhnt-cockpit`, tagged `:<sha7>` AND `:latest`. Secret
|
||||
`scw-registry` (already in `foxhunt`).
|
||||
- **Real-money safety:** the pipeline only builds/deploys the cockpit+dagster DAEMONS. It must NEVER touch the
|
||||
execution CronJobs (ucits/bybit/ibkr rebalancers) or arm any executor.
|
||||
- **No silent failures:** a Sensor that can't match, or a Workflow that errors, must surface loudly (a failed
|
||||
push = a visible failed Workflow, never a swallowed no-op). Every manifest is `kubectl apply --dry-run=server`
|
||||
validated before a real apply.
|
||||
- **Reproducibility:** the deploy step annotates deployed Deployments with `fxhnt.io/deployed-sha=<sha7>`.
|
||||
|
||||
## File Structure
|
||||
|
||||
- `infra/argo/fxhnt-cockpit-workflowtemplate.yaml` — the reanimated WorkflowTemplate (ns foxhunt). Supersedes
|
||||
the old `infra/argo/cockpit-build-deploy.yaml` (which is renamed/removed in the final task).
|
||||
- `infra/argo/fxhnt-build-sensor.yaml` — the new Sensor (ns argo-events).
|
||||
- Gitea webhook on `gitadmin/fxhnt` — created via Gitea API/UI (not a repo file; documented in the plan).
|
||||
|
||||
---
|
||||
|
||||
## Task 1 — Reanimate + fix the WorkflowTemplate (build side)
|
||||
|
||||
**Files:**
|
||||
- Create: `infra/argo/fxhnt-cockpit-workflowtemplate.yaml` (start from `infra/argo/cockpit-build-deploy.yaml`)
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: a `WorkflowTemplate` named `fxhnt-cockpit` in ns `foxhunt` with params `commit-sha` (default HEAD)
|
||||
and `mode` (default `build`; values `build`|`deploy`). The Sensor (Task 3) submits Workflows referencing it.
|
||||
|
||||
- [ ] **Step 1: Copy the old template as the new file, fix the node pool**
|
||||
|
||||
Copy `infra/argo/cockpit-build-deploy.yaml` → `infra/argo/fxhnt-cockpit-workflowtemplate.yaml`. In BOTH the
|
||||
`kaniko-build` and `kubectl-deploy` templates, change every
|
||||
`nodeSelector: { k8s.scaleway.com/pool-name: ci-compile-cpu, ... }` to
|
||||
`nodeSelector: { k8s.scaleway.com/pool-name: large, topology.kubernetes.io/zone: fr-par-2 }`.
|
||||
Grep to confirm no `ci-compile-cpu` remains: `grep -n ci-compile-cpu infra/argo/fxhnt-cockpit-workflowtemplate.yaml` → no hits.
|
||||
|
||||
- [ ] **Step 2: Add the immutable `:<sha7>` tag to the Kaniko destination**
|
||||
|
||||
In the `kaniko-build` container args, alongside the existing
|
||||
`--destination=rg.fr-par.scw.cloud/bizworx/fxhnt-cockpit:latest`, ADD a second line:
|
||||
`--destination=rg.fr-par.scw.cloud/bizworx/fxhnt-cockpit:{{=sprig.trunc(7, workflow.parameters.commit-sha)}}`
|
||||
(the `sprig.trunc` expression matches how `build-image` truncates the sha). Keep `--cache=true` and the
|
||||
`--cache-repo`.
|
||||
|
||||
- [ ] **Step 3: Server-side dry-run validate the template**
|
||||
|
||||
Run: `kubectl apply -f infra/argo/fxhnt-cockpit-workflowtemplate.yaml --dry-run=server -n foxhunt`
|
||||
Expected: `workflowtemplate.argoproj.io/fxhnt-cockpit created (server dry run)` with no schema errors.
|
||||
(If it errors on an unknown field, the Argo CRD version differs — STOP and report; do not force-apply.)
|
||||
|
||||
- [ ] **Step 4: Commit**
|
||||
|
||||
```bash
|
||||
git add infra/argo/fxhnt-cockpit-workflowtemplate.yaml
|
||||
git commit -m "ci(fxhnt): reanimate cockpit WorkflowTemplate (node pool large, sha7 tag)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 2 — Fix the deploy step (verify the dashboard external route first)
|
||||
|
||||
**Files:**
|
||||
- Modify: `infra/argo/fxhnt-cockpit-workflowtemplate.yaml` (the `kubectl-deploy` template's script)
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: the WorkflowTemplate from Task 1.
|
||||
- Produces: a deploy step that refreshes the daemons correctly for whichever external route is live.
|
||||
|
||||
- [ ] **Step 1: Determine the live dashboard external route (decides restart vs scale-0→1)**
|
||||
|
||||
Run these and read the result:
|
||||
```bash
|
||||
kubectl get deploy fxhnt-dashboard -n foxhunt -o jsonpath='{range .spec.template.spec.containers[*]}{.name}{"\n"}{end}'
|
||||
kubectl get ingress -n foxhunt 2>/dev/null
|
||||
kubectl -n foxhunt describe svc fxhnt-ingress 2>/dev/null | grep -iE 'endpoints|selector'
|
||||
```
|
||||
Decide:
|
||||
- If `dashboard.fxhnt.ai` is served THROUGH the tailscale sidecar (sidecar present AND no ingress route to the
|
||||
dashboard) → KEEP the old `scale 0→1` dance (the tailscale session race is real).
|
||||
- If it is served through `fxhnt-ingress` (an ingress/route points at the dashboard svc; the tailscale sidecar
|
||||
is vestigial) → REPLACE the scale-dance with a plain `kubectl -n foxhunt rollout restart deploy/fxhnt-dashboard`.
|
||||
|
||||
Record the decision in the plan's progress ledger with the evidence. If ambiguous, KEEP `scale 0→1` (the
|
||||
conservative choice that cannot cause the stale-proxy "down").
|
||||
|
||||
- [ ] **Step 2: Apply the deploy-step decision + add the deployed-sha annotation**
|
||||
|
||||
Edit the `kubectl-deploy` script. Regardless of the restart mechanism chosen, add BEFORE the rollout:
|
||||
```
|
||||
kubectl -n foxhunt annotate deploy/fxhnt-dashboard deploy/dagster \
|
||||
fxhnt.io/deployed-sha="$(cd /workspace/src && git rev-parse --short HEAD)" --overwrite
|
||||
```
|
||||
Keep the existing `kubectl apply -f` of the three manifests (fxhnt-dashboard.yaml, dagster.yaml,
|
||||
network-policies/fxhnt-cockpit.yaml) and the `rollout status` waits. Keep `rollout restart deploy/dagster`
|
||||
(dagster has no proxy race).
|
||||
|
||||
- [ ] **Step 3: Dry-run validate again**
|
||||
|
||||
Run: `kubectl apply -f infra/argo/fxhnt-cockpit-workflowtemplate.yaml --dry-run=server -n foxhunt`
|
||||
Expected: server dry-run OK.
|
||||
|
||||
- [ ] **Step 4: Commit**
|
||||
|
||||
```bash
|
||||
git add infra/argo/fxhnt-cockpit-workflowtemplate.yaml
|
||||
git commit -m "ci(fxhnt): deploy step — <restart-choice> + deployed-sha annotation"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 3 — The Sensor (simple, single-trigger) — mode decided IN the workflow
|
||||
|
||||
**Design note (verified 2026-07-21):** the Argo Events Sensor CRD here has an OPEN schema (no strict property
|
||||
validation — `kubectl explain sensor.spec.triggers.template.conditions` errors, and the only `conditions:` in
|
||||
live sensors are in `status`, not trigger-level). Rather than gamble on a fragile sensor-side path-negation
|
||||
filter to pick build-vs-deploy, we keep the SENSOR SIMPLE (one trigger, exactly like `build-msp-portal`) and
|
||||
move the build-vs-deploy decision INTO the workflow, where a robust `git diff --name-only` between the previous
|
||||
and pushed commit is trivial and testable. This is the more robust design, not a fallback.
|
||||
|
||||
**Files:**
|
||||
- Create: `infra/argo/fxhnt-build-sensor.yaml` (ns argo-events)
|
||||
- Modify: `infra/argo/fxhnt-cockpit-workflowtemplate.yaml` (add a `decide-mode` step that git-diffs)
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: the `gitea-webhook` eventsource (shared), the `fxhnt-cockpit` WorkflowTemplate (Task 1/2).
|
||||
- Produces: a `Sensor` `build-fxhnt` (argo-events) that submits ONE Workflow per master push; the workflow
|
||||
self-selects build vs deploy.
|
||||
|
||||
- [ ] **Step 1: Write the single-trigger Sensor (copy build-msp-portal structure exactly)**
|
||||
|
||||
`kubectl get sensor build-msp-portal -n argo-events -o yaml` and mirror it. ONE dependency `push-to-master`:
|
||||
- `body.ref` == `refs/heads/master`
|
||||
- `body.pusher.username` != `argocd-image-updater`
|
||||
(NO `body.commits.#.modified` filter — the mode decision moved to the workflow.)
|
||||
ONE trigger `submit`: submits a Workflow `generateName: build-fxhnt-`, `namespace: foxhunt`,
|
||||
`workflowTemplateRef: fxhnt-cockpit`, injecting `commit-sha`=`body.after`. Do NOT set `mode` (the workflow
|
||||
computes it). This is structurally identical to the proven msp sensors → lowest risk.
|
||||
|
||||
- [ ] **Step 2: Add a `decide-mode` step to the WorkflowTemplate**
|
||||
|
||||
In `fxhnt-cockpit-workflowtemplate.yaml`, change the DAG so the FIRST step (after git-clone) is `decide-mode`:
|
||||
a small `alpine/git` script that runs, in the cloned repo at `commit-sha`:
|
||||
```
|
||||
CHANGED=$(git diff --name-only "${SHA}~1" "${SHA}" 2>/dev/null || echo "ALL")
|
||||
if echo "$CHANGED" | grep -qE '^pyproject\.toml$|^infra/docker/fxhnt\.Dockerfile$'; then
|
||||
echo build; else echo deploy; fi
|
||||
```
|
||||
Output the result (`build`/`deploy`) as an Argo output parameter `mode`. Then gate the `kaniko-build` task on
|
||||
`{{tasks.decide-mode.outputs.parameters.mode}} == build`, and `kubectl-deploy` depends on
|
||||
`build.Succeeded || build.Skipped` (unchanged). Edge case: if `${SHA}~1` doesn't exist (first commit / shallow),
|
||||
default to `build` (safe — a rebuild is never wrong, only slower). The explicit `mode` PARAM stays on the
|
||||
template (default `auto`) so a human can still force `mode=build`/`mode=deploy` on a manual `argo submit`;
|
||||
`decide-mode` only runs when `mode=auto`.
|
||||
|
||||
- [ ] **Step 3: Dry-run validate both**
|
||||
|
||||
```bash
|
||||
kubectl apply -f infra/argo/fxhnt-build-sensor.yaml --dry-run=server -n argo-events
|
||||
kubectl apply -f infra/argo/fxhnt-cockpit-workflowtemplate.yaml --dry-run=server -n foxhunt
|
||||
```
|
||||
Expected: both server-dry-run OK.
|
||||
|
||||
- [ ] **Step 4: Commit**
|
||||
|
||||
```bash
|
||||
git add infra/argo/fxhnt-build-sensor.yaml infra/argo/fxhnt-cockpit-workflowtemplate.yaml
|
||||
git commit -m "ci(fxhnt): single-trigger sensor + in-workflow git-diff mode decision"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 4 — Apply to the cluster + wire the Gitea webhook
|
||||
|
||||
**Files:** none new — this task applies the manifests and creates the webhook.
|
||||
|
||||
- [ ] **Step 1: Apply the WorkflowTemplate + Sensor for real**
|
||||
|
||||
```bash
|
||||
kubectl apply -f infra/argo/fxhnt-cockpit-workflowtemplate.yaml -n foxhunt
|
||||
kubectl apply -f infra/argo/fxhnt-build-sensor.yaml -n argo-events
|
||||
kubectl get workflowtemplate fxhnt-cockpit -n foxhunt
|
||||
kubectl get sensor build-fxhnt -n argo-events
|
||||
```
|
||||
Expected: both present. The sensor pod (`kubectl get pods -n argo-events | grep build-fxhnt`) reaches Running.
|
||||
|
||||
- [ ] **Step 2: Create the Gitea webhook on `gitadmin/fxhnt`**
|
||||
|
||||
The `gitea-webhook` eventsource endpoint is the same target the `bizworx-ops/infrastructure` repo already POSTs
|
||||
to. Find it: `kubectl get eventsource gitea-webhook -n argo-events -o yaml` (the webhook `port`/`endpoint`) and
|
||||
how the existing infra-repo webhook is configured (inspect via Gitea admin UI or API on an existing repo).
|
||||
Create the equivalent webhook on `gitadmin/fxhnt`: push events, active, POST JSON to the eventsource URL. Record
|
||||
the exact URL used in the ledger. Do NOT paste any webhook secret into the transcript.
|
||||
|
||||
- [ ] **Step 3: Verify the webhook reaches the sensor (dry, safe probe)**
|
||||
|
||||
Trigger a harmless test push (or use Gitea's "Test Delivery" webhook button) and confirm the sensor received it:
|
||||
```bash
|
||||
kubectl logs -n argo-events -l sensor-name=build-fxhnt --tail=50 | grep -iE 'triggered|push-to-master|dispatch'
|
||||
```
|
||||
Expected: the sensor logs the received push. If Gitea's test-delivery uses a non-master ref, the sensor
|
||||
correctly does NOT fire — confirm it's reachable via the eventsource logs instead.
|
||||
|
||||
---
|
||||
|
||||
## Task 5 — End-to-end verification (both modes) + retire the old file
|
||||
|
||||
**Files:**
|
||||
- Remove: `infra/argo/cockpit-build-deploy.yaml` (superseded)
|
||||
|
||||
- [ ] **Step 1: Verify mode=deploy end-to-end (code-only push)**
|
||||
|
||||
Make a trivial code-only commit on `master` (e.g. touch a comment in `src/`), push, and watch:
|
||||
```bash
|
||||
kubectl get wf -n foxhunt --sort-by=.metadata.creationTimestamp | tail -3
|
||||
```
|
||||
Expected: a `build-fxhnt-*` Workflow runs with `mode=deploy`, SKIPS the kaniko-build step, deploys, and the
|
||||
dashboard stays UP (check `dashboard.fxhnt.ai` reachable post-deploy, and
|
||||
`kubectl get deploy fxhnt-dashboard -n foxhunt -o jsonpath='{.metadata.annotations.fxhnt\.io/deployed-sha}'`
|
||||
shows the new sha). Total time ~1-2 min, no Kaniko build.
|
||||
|
||||
- [ ] **Step 2: Verify mode=build end-to-end (dep push)**
|
||||
|
||||
Make a no-op-but-real change to `pyproject.toml` (e.g. reorder a dependency, or bump a harmless comment), push,
|
||||
and watch: the Workflow runs `mode=build`, Kaniko produces `fxhnt-cockpit:<sha7>`+`:latest`
|
||||
(`kubectl run` a crane/skopeo check OR verify the workflow's kaniko step succeeded), then deploys. Confirm the
|
||||
`:<sha7>` tag exists in the registry.
|
||||
|
||||
- [ ] **Step 3: Confirm no false trigger + real-money safety**
|
||||
|
||||
- Push a change touching ONLY `docs/` → confirm a `mode=deploy` Workflow runs (harmless) OR, if docs are
|
||||
excluded, no Workflow — either is fine, but it must NOT trigger a build.
|
||||
- Confirm NO execution CronJob was modified/triggered by any pipeline run:
|
||||
`kubectl get cronjob -n foxhunt` (ucits/bybit/ibkr rebalancers unchanged, bybit/ibkr still Suspend=True).
|
||||
|
||||
- [ ] **Step 4: Retire the old file + commit**
|
||||
|
||||
```bash
|
||||
git rm infra/argo/cockpit-build-deploy.yaml
|
||||
git commit -m "ci(fxhnt): retire old cockpit-build-deploy (superseded by workflowtemplate + sensor)"
|
||||
```
|
||||
|
||||
- [ ] **Step 5: Final verification summary**
|
||||
|
||||
Confirm all Global Constraints hold: sensor in argo-events only, workflow+pods in foxhunt, node pool `large`,
|
||||
no cross-ns RBAC, no fxhnt secrets in platform ns, real-money CronJobs untouched. Record in the ledger.
|
||||
750
docs/superpowers/plans/2026-07-21-portfolio-backtester.md
Normal file
750
docs/superpowers/plans/2026-07-21-portfolio-backtester.md
Normal file
@@ -0,0 +1,750 @@
|
||||
# Portfolio Backtester Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** A new `/paper/portfolio` cockpit surface where the operator selects an arbitrary set of edges, a weighting method, and a date range, and gets a portfolio equity curve + Sharpe/CAGR/maxDD **plus** the full/left-tail correlation matrix and each edge's marginal-Sharpe contribution — reusing the existing combiner/metrics/diversification engine end-to-end.
|
||||
|
||||
**Architecture:** Three thin layers on top of a production-tested engine. (1) A cross-store **series aggregator** unions `bybit_sleeve_returns()` + `sim_curve_returns()` into the one `series_by_edge: dict[str, dict[int,float]]` shape the engine already consumes. (2) A pure **report bundler** slices to a date range, runs `combine_book` (default) or `combine_returns` (explicit-weights what-if), then `compute_stats`, `full_correlation`/`left_tail_correlation`, and `marginal_sharpe` on the aligned arrays, returning one report dict. (3) A FastAPI **route + template** (`/paper/portfolio` page + `/paper/portfolio/run` HTMX partial) rendering the curve + heatmap via Plotly (CDN) and the marginal-Sharpe table as HTML. Pure additive + read-only.
|
||||
|
||||
**Tech Stack:** Python 3.12, FastAPI + Jinja2 + HTMX, NumPy, existing `fxhnt` domain/application engine, Plotly (browser-side, CDN).
|
||||
|
||||
## Global Constraints
|
||||
|
||||
- **Read-only + additive.** No task may touch any live-fund path, the execution CLI, `/paper/sim`, or any existing route/template/service method. Only NEW files + NEW route handlers + NEW nav links + one NEW `<script>` in `base.html`.
|
||||
- **No recompute on the request path.** The route reads ONLY the precomputed stored series (`bybit_sleeve_returns`, `sim_curve_returns`) — never triggers an edge/replay computation. Mirrors the `/paper/sim` "reads the honest precomputed curve" rule.
|
||||
- **Engine is reused verbatim — do NOT rebuild.** `align_edges`/`combine_book` (`src/fxhnt/application/book_allocator.py`), `combine_returns` (`src/fxhnt/domain/portfolio/combine.py`), `compute_stats` (`src/fxhnt/domain/backtest.py`), `marginal_sharpe` (`src/fxhnt/domain/diversification/marginal.py`), `full_correlation`/`left_tail_correlation` (`src/fxhnt/domain/diversification/correlation.py`). No copies, no re-derivations.
|
||||
- **Exact epoch-day axis.** All stored series are keyed by `epoch_day = date.toordinal()` (NOT Unix days). `combine_book` returns `{epoch_day: ret}`. Convert to ISO with `datetime.date.fromordinal(epoch_day).isoformat()`. `combine_returns` uses a DIFFERENT axis (date-STRING tuples) — the bundler bridges the two explicitly (Task 2).
|
||||
- **Graceful, never 500.** A missing/empty series, an empty selection, or an unparseable weight must degrade to an empty/pending report, never a 500 — matching the `# a missing/empty nav table must never 500` guard style in `app.py`.
|
||||
- **Available edges (v1 fixed set):** `crypto_tstrend`, `unlock`, `xsfunding`, `positioning` (from `bybit_sleeve_returns()`), and `multistrat` (from `sim_curve_returns("multistrat")`). `multistrat_levered` is available from `sim_curve_returns` but is observe-only — include it as a selectable but not default-on.
|
||||
- **Plotly loads from CDN in the browser** (`https://cdn.plot.ly/plotly-2.35.2.min.js`). This is a BROWSER fetch over the tailscale-served page, NOT pod egress — the vendored-htmx "no CDN / tailnet egress" comment in `base.html` is about the POD's network policy and does NOT apply to the operator's browser. Add a one-line comment saying so next to the new `<script>`.
|
||||
- **Tests:** `pytest -n auto` is the default (`pyproject.toml`). Run individual tests with `-p no:xdist` implicitly disabled by naming the node — use `pytest tests/path::test_name -p no:cacheprovider` is NOT needed; plain `pytest tests/path::test_name` works. Web tests build the app in-memory via `create_app(...)` with SQLite `PaperRepo`/`ForwardNavRepo` + `fastapi.testclient.TestClient`.
|
||||
|
||||
---
|
||||
|
||||
## File Structure
|
||||
|
||||
- **Create** `src/fxhnt/application/portfolio_backtest.py` — the aggregator + report bundler (pure application logic, no web, no I/O beyond the two repo accessors passed in). Two public functions: `aggregate_edge_series(repo, edges)` and `build_portfolio_report(repo, *, edges, method, weights, start, end)`.
|
||||
- **Create** `tests/application/test_portfolio_backtest.py` — unit tests for both functions.
|
||||
- **Modify** `src/fxhnt/adapters/web/app.py` — add `GET /paper/portfolio` (form page) + `GET /paper/portfolio/run` (HTMX partial) handlers, and the module-level list of selectable edges. NEW handlers only; no existing handler touched.
|
||||
- **Create** `src/fxhnt/adapters/web/templates/portfolio.html` — the page: edge multiselect + method toggle + date range + `hx-get` trigger.
|
||||
- **Create** `src/fxhnt/adapters/web/templates/_portfolio_result.html` — the HTMX partial: Plotly equity curve `<div>` + Plotly heatmap `<div>` + marginal-Sharpe HTML table + headline metrics.
|
||||
- **Modify** `src/fxhnt/adapters/web/templates/base.html` — add the Plotly `<script>` (CDN) in `<head>`, and a "Portfolio" nav link to BOTH the desktop `.bar` nav and the mobile `.tabs` nav.
|
||||
- **Create** `tests/integration/test_portfolio_web.py` — web smoke: page renders, run partial returns curve+metrics+heatmap+marginal table, empty selection degrades gracefully, explicit-weights mode works.
|
||||
|
||||
---
|
||||
|
||||
## Task 1: Cross-store series aggregator
|
||||
|
||||
**Files:**
|
||||
- Create: `src/fxhnt/application/portfolio_backtest.py`
|
||||
- Test: `tests/application/test_portfolio_backtest.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `PaperRepo.bybit_sleeve_returns() -> dict[str, dict[int,float]]` and `PaperRepo.sim_curve_returns(strategy_id: str) -> dict[int,float]` (both in `src/fxhnt/adapters/persistence/paper_repo.py`, already exist).
|
||||
- Produces: `aggregate_edge_series(repo, edges: list[str]) -> dict[str, dict[int,float]]` — the `series_by_edge` shape `align_edges`/`combine_book` consume. Also produces the module constants `BYBIT_EDGES` and `SIM_EDGES` used by later tasks.
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
```python
|
||||
# tests/application/test_portfolio_backtest.py
|
||||
"""Unit tests for the portfolio-backtest aggregator + report bundler (read-only, pure)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
|
||||
from fxhnt.application.portfolio_backtest import aggregate_edge_series
|
||||
|
||||
|
||||
class _FakeRepo:
|
||||
"""Minimal stand-in exposing only the two accessors the aggregator reads."""
|
||||
|
||||
def __init__(self, bybit: dict[str, dict[int, float]], sim: dict[str, dict[int, float]]):
|
||||
self._bybit = bybit
|
||||
self._sim = sim
|
||||
|
||||
def bybit_sleeve_returns(self) -> dict[str, dict[int, float]]:
|
||||
return self._bybit
|
||||
|
||||
def sim_curve_returns(self, strategy_id: str) -> dict[int, float]:
|
||||
return self._sim.get(strategy_id, {})
|
||||
|
||||
|
||||
def _d(y: int, m: int, day: int) -> int:
|
||||
return dt.date(y, m, day).toordinal()
|
||||
|
||||
|
||||
def test_aggregate_unions_bybit_and_sim_sources():
|
||||
repo = _FakeRepo(
|
||||
bybit={
|
||||
"crypto_tstrend": {_d(2026, 1, 1): 0.01, _d(2026, 1, 2): -0.02},
|
||||
"unlock": {_d(2026, 1, 1): 0.003},
|
||||
},
|
||||
sim={"multistrat": {_d(2026, 1, 1): 0.005, _d(2026, 1, 2): 0.004}},
|
||||
)
|
||||
out = aggregate_edge_series(repo, ["crypto_tstrend", "multistrat"])
|
||||
assert set(out) == {"crypto_tstrend", "multistrat"}
|
||||
assert out["crypto_tstrend"][_d(2026, 1, 2)] == -0.02
|
||||
assert out["multistrat"][_d(2026, 1, 1)] == 0.005
|
||||
|
||||
|
||||
def test_aggregate_skips_unknown_and_empty_edges():
|
||||
repo = _FakeRepo(bybit={"unlock": {_d(2026, 1, 1): 0.003}}, sim={})
|
||||
out = aggregate_edge_series(repo, ["unlock", "does_not_exist", "multistrat"])
|
||||
assert set(out) == {"unlock"} # unknown + empty-series edges dropped
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `pytest tests/application/test_portfolio_backtest.py -v`
|
||||
Expected: FAIL — `ModuleNotFoundError: No module named 'fxhnt.application.portfolio_backtest'`
|
||||
|
||||
- [ ] **Step 3: Write minimal implementation**
|
||||
|
||||
```python
|
||||
# src/fxhnt/application/portfolio_backtest.py
|
||||
"""Portfolio backtester (cockpit slice 1): assemble an arbitrary set of edges from the PRECOMPUTED stored
|
||||
return series, run them through the EXISTING book combiner / metrics / diversification engine, and return one
|
||||
report the /paper/portfolio surface renders. Pure application logic — reads only the two repo accessors, never
|
||||
recomputes an edge/replay on the request path. Read-only; cannot touch the live fund."""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Protocol
|
||||
|
||||
# The selectable edges, grouped by their stored-series source (v1 fixed set).
|
||||
BYBIT_EDGES: tuple[str, ...] = ("crypto_tstrend", "unlock", "xsfunding", "positioning")
|
||||
SIM_EDGES: tuple[str, ...] = ("multistrat", "multistrat_levered")
|
||||
ALL_EDGES: tuple[str, ...] = BYBIT_EDGES + SIM_EDGES
|
||||
|
||||
|
||||
class _SeriesRepo(Protocol):
|
||||
def bybit_sleeve_returns(self) -> dict[str, dict[int, float]]: ...
|
||||
def sim_curve_returns(self, strategy_id: str) -> dict[int, float]: ...
|
||||
|
||||
|
||||
def aggregate_edge_series(repo: _SeriesRepo, edges: list[str]) -> dict[str, dict[int, float]]:
|
||||
"""Union the selected edges' PRECOMPUTED per-day series into one `{edge: {epoch_day: ret}}` map — the
|
||||
exact shape `book_allocator.align_edges`/`combine_book` consume. Bybit sleeves come from the one
|
||||
`bybit_sleeve_returns()` read; each IBKR recompute-replay book from its own `sim_curve_returns(id)` read.
|
||||
Unknown edges and edges whose stored series is empty are silently dropped (a selection referencing a
|
||||
not-yet-precomputed book must degrade, never error)."""
|
||||
bybit = repo.bybit_sleeve_returns()
|
||||
out: dict[str, dict[int, float]] = {}
|
||||
for e in edges:
|
||||
if e in BYBIT_EDGES:
|
||||
series = bybit.get(e) or {}
|
||||
elif e in SIM_EDGES:
|
||||
series = repo.sim_curve_returns(e)
|
||||
else:
|
||||
continue # unknown edge name — drop
|
||||
if series:
|
||||
out[e] = dict(series)
|
||||
return out
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run test to verify it passes**
|
||||
|
||||
Run: `pytest tests/application/test_portfolio_backtest.py -v`
|
||||
Expected: PASS (2 passed)
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/application/portfolio_backtest.py tests/application/test_portfolio_backtest.py
|
||||
git commit -m "feat(portfolio): cross-store edge-series aggregator"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 2: Report bundler
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/application/portfolio_backtest.py`
|
||||
- Test: `tests/application/test_portfolio_backtest.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `aggregate_edge_series` (Task 1); `align_edges`/`combine_book` from `fxhnt.application.book_allocator`; `combine_returns` from `fxhnt.domain.portfolio.combine`; `compute_stats` from `fxhnt.domain.backtest`; `marginal_sharpe` from `fxhnt.domain.diversification.marginal`; `full_correlation`/`left_tail_correlation` from `fxhnt.domain.diversification.correlation`.
|
||||
- Engine facts (verbatim, do not re-derive): `combine_book(series_by_edge, *, vol_lookback=60, target_vol=0.12, kelly_fraction=0.5, max_sleeve_weight=0.4, periods_per_year=365, rebalance_every=21, min_sleeve_sharpe=None, marginal_gate=False) -> dict[int,float]`. `align_edges(series_by_edge) -> (dates: list[int], arrs: dict[str, np.ndarray])`. `combine_returns(sleeves: list[tuple[float, tuple[str,...], np.ndarray]]) -> (dates: tuple[str,...], np.ndarray)`. `compute_stats(returns: np.ndarray, periods_per_year: int = 252) -> BacktestStats` (dataclass; access its fields — inspect the dataclass in `src/fxhnt/domain/backtest.py` and surface CAGR, Sharpe, max drawdown, vol). `marginal_sharpe(book: np.ndarray, sleeve: np.ndarray, *, weight=0.2) -> float`. `full_correlation(a, b) -> float`; `left_tail_correlation(sleeve, book, *, tail_frac=0.10) -> float`.
|
||||
- Produces: `build_portfolio_report(repo, *, edges: list[str], method: str = "book", weights: dict[str,float] | None = None, start: str | None = None, end: str | None = None) -> dict` — the report dict later tasks render. Keys: `edges` (list[str], the ones actually used), `method`, `dates_iso` (list[str]), `equity` (list[float], compounded from $1.0), `returns` (list[float]), `metrics` (dict: `cagr`,`sharpe`,`max_drawdown`,`vol`,`n_days`), `correlation` (list of `{a,b,full,tail}` for each unordered pair), `marginal` (list of `{edge, marginal_sharpe}`), `empty` (bool — True when no usable edge/overlap).
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
```python
|
||||
# append to tests/application/test_portfolio_backtest.py
|
||||
import math
|
||||
|
||||
from fxhnt.application.portfolio_backtest import build_portfolio_report
|
||||
|
||||
|
||||
def _series(start_ord: int, rets: list[float]) -> dict[int, float]:
|
||||
return {start_ord + i: r for i, r in enumerate(rets)}
|
||||
|
||||
|
||||
def _repo_with_overlap() -> "_FakeRepo":
|
||||
s = _d(2026, 1, 1)
|
||||
# 40 days so trailing windows in combine_book/left-tail have room.
|
||||
a = [0.01 if i % 2 else -0.008 for i in range(40)]
|
||||
b = [0.006 if i % 3 else -0.004 for i in range(40)]
|
||||
c = [0.004 for _ in range(40)]
|
||||
return _FakeRepo(
|
||||
bybit={"crypto_tstrend": _series(s, a), "unlock": _series(s, b)},
|
||||
sim={"multistrat": _series(s, c)},
|
||||
)
|
||||
|
||||
|
||||
def test_report_book_method_has_curve_metrics_corr_marginal():
|
||||
rep = build_portfolio_report(
|
||||
_repo_with_overlap(), edges=["crypto_tstrend", "unlock", "multistrat"], method="book"
|
||||
)
|
||||
assert rep["empty"] is False
|
||||
assert rep["method"] == "book"
|
||||
assert set(rep["edges"]) == {"crypto_tstrend", "unlock", "multistrat"}
|
||||
assert len(rep["dates_iso"]) == len(rep["equity"]) == len(rep["returns"])
|
||||
assert rep["equity"][0] > 0.0 # compounded from a positive base
|
||||
for k in ("cagr", "sharpe", "max_drawdown", "vol", "n_days"):
|
||||
assert k in rep["metrics"]
|
||||
# 3 edges -> 3 unordered pairs.
|
||||
assert len(rep["correlation"]) == 3
|
||||
for pair in rep["correlation"]:
|
||||
assert set(pair) == {"a", "b", "full", "tail"}
|
||||
assert len(rep["marginal"]) == 3
|
||||
assert all(set(m) == {"edge", "marginal_sharpe"} for m in rep["marginal"])
|
||||
|
||||
|
||||
def test_report_explicit_weights_method_uses_supplied_weights():
|
||||
rep = build_portfolio_report(
|
||||
_repo_with_overlap(),
|
||||
edges=["crypto_tstrend", "unlock"],
|
||||
method="explicit",
|
||||
weights={"crypto_tstrend": 0.7, "unlock": 0.3},
|
||||
)
|
||||
assert rep["empty"] is False
|
||||
assert rep["method"] == "explicit"
|
||||
assert len(rep["returns"]) > 0
|
||||
|
||||
|
||||
def test_report_empty_when_no_edges_selected():
|
||||
rep = build_portfolio_report(_repo_with_overlap(), edges=[], method="book")
|
||||
assert rep["empty"] is True
|
||||
assert rep["equity"] == []
|
||||
assert rep["correlation"] == []
|
||||
|
||||
|
||||
def test_report_date_range_windows_the_series():
|
||||
repo = _repo_with_overlap()
|
||||
full = build_portfolio_report(repo, edges=["crypto_tstrend", "unlock"], method="book")
|
||||
windowed = build_portfolio_report(
|
||||
repo, edges=["crypto_tstrend", "unlock"], method="book",
|
||||
start="2026-01-10", end="2026-01-20",
|
||||
)
|
||||
assert len(windowed["dates_iso"]) < len(full["dates_iso"])
|
||||
assert windowed["dates_iso"][0] >= "2026-01-10"
|
||||
assert windowed["dates_iso"][-1] <= "2026-01-20"
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `pytest tests/application/test_portfolio_backtest.py -v`
|
||||
Expected: FAIL — `ImportError: cannot import name 'build_portfolio_report'`
|
||||
|
||||
- [ ] **Step 3: Write minimal implementation**
|
||||
|
||||
Append to `src/fxhnt/application/portfolio_backtest.py`. First inspect `compute_stats` / `BacktestStats` field names in `src/fxhnt/domain/backtest.py` and map them to `cagr`/`sharpe`/`max_drawdown`/`vol` (use the actual dataclass attribute names — do not invent). The code below assumes `BacktestStats` exposes `cagr`, `sharpe`, `max_drawdown`, `vol`; adjust the four attribute reads in `_metrics_from_stats` if the real names differ.
|
||||
|
||||
```python
|
||||
import datetime as dt
|
||||
from itertools import combinations
|
||||
|
||||
import numpy as np
|
||||
|
||||
from fxhnt.application.book_allocator import align_edges, combine_book
|
||||
from fxhnt.domain.backtest import compute_stats
|
||||
from fxhnt.domain.diversification.correlation import full_correlation, left_tail_correlation
|
||||
from fxhnt.domain.diversification.marginal import marginal_sharpe
|
||||
from fxhnt.domain.portfolio.combine import combine_returns
|
||||
|
||||
_PERIODS_PER_YEAR = 365 # stored series are calendar-daily (matches combine_book's default)
|
||||
|
||||
|
||||
def _window(series_by_edge: dict[str, dict[int, float]], start: str | None, end: str | None
|
||||
) -> dict[str, dict[int, float]]:
|
||||
"""Clip every edge's {epoch_day: ret} to the inclusive [start, end] ISO date range (either bound optional)."""
|
||||
lo = dt.date.fromisoformat(start).toordinal() if start else None
|
||||
hi = dt.date.fromisoformat(end).toordinal() if end else None
|
||||
out: dict[str, dict[int, float]] = {}
|
||||
for e, s in series_by_edge.items():
|
||||
clipped = {d: r for d, r in s.items()
|
||||
if (lo is None or d >= lo) and (hi is None or d <= hi)}
|
||||
if clipped:
|
||||
out[e] = clipped
|
||||
return out
|
||||
|
||||
|
||||
def _empty_report(method: str) -> dict:
|
||||
return {"edges": [], "method": method, "dates_iso": [], "equity": [], "returns": [],
|
||||
"metrics": {"cagr": 0.0, "sharpe": 0.0, "max_drawdown": 0.0, "vol": 0.0, "n_days": 0},
|
||||
"correlation": [], "marginal": [], "empty": True}
|
||||
|
||||
|
||||
def _metrics_from_stats(returns: np.ndarray) -> dict:
|
||||
if returns.size == 0:
|
||||
return {"cagr": 0.0, "sharpe": 0.0, "max_drawdown": 0.0, "vol": 0.0, "n_days": 0}
|
||||
st = compute_stats(returns, periods_per_year=_PERIODS_PER_YEAR)
|
||||
return {"cagr": float(st.cagr), "sharpe": float(st.sharpe),
|
||||
"max_drawdown": float(st.max_drawdown), "vol": float(st.vol), "n_days": int(returns.size)}
|
||||
|
||||
|
||||
def _equity_curve(returns: np.ndarray) -> list[float]:
|
||||
"""Compound daily returns from a $1.0 base (the surface scales by capital client-side / in the template)."""
|
||||
eq, curve = 1.0, []
|
||||
for r in returns:
|
||||
eq *= (1.0 + float(r))
|
||||
curve.append(eq)
|
||||
return curve
|
||||
|
||||
|
||||
def build_portfolio_report(repo: _SeriesRepo, *, edges: list[str], method: str = "book",
|
||||
weights: dict[str, float] | None = None,
|
||||
start: str | None = None, end: str | None = None) -> dict:
|
||||
"""Assemble the full portfolio report for the selected edges + weighting method + date range.
|
||||
|
||||
method="book" -> the live inverse-vol/cap/vol-target+Kelly allocator (`combine_book`) — "what does my
|
||||
real allocation do with these edges?".
|
||||
method="explicit" -> a plain capital-weighted blend of the supplied per-edge `weights` (`combine_returns`) —
|
||||
"what if positioning were 40%?".
|
||||
|
||||
The equity/metrics reflect the chosen weighting; the correlation matrix (full AND left-tail) and per-edge
|
||||
marginal-Sharpe are computed on the RAW aligned edge arrays (properties of the edges, weight-independent).
|
||||
Read-only; never recomputes an edge."""
|
||||
series = _window(aggregate_edge_series(repo, edges), start, end)
|
||||
if not series:
|
||||
return _empty_report(method)
|
||||
|
||||
# RAW aligned per-edge arrays on the shared epoch-day axis (0.0-fill on flat days) — the basis for the
|
||||
# book curve's diagnostics (correlation + marginal-Sharpe).
|
||||
epoch_days, arrs = align_edges(series)
|
||||
if not epoch_days:
|
||||
return _empty_report(method)
|
||||
used = list(arrs)
|
||||
|
||||
if method == "explicit" and weights:
|
||||
# combine_returns aligns on the INTERSECTION of date-STRING axes; build one Sleeve per used edge.
|
||||
iso = [dt.date.fromordinal(d).isoformat() for d in epoch_days]
|
||||
sleeves = [(float(weights.get(e, 0.0)), tuple(iso), arrs[e]) for e in used]
|
||||
_cdates, book_ret = combine_returns(sleeves)
|
||||
# combine_returns may drop to the intersection; re-derive the ISO axis from its returned dates.
|
||||
dates_iso = list(_cdates)
|
||||
returns = np.asarray(book_ret, dtype=float)
|
||||
else:
|
||||
method = "book"
|
||||
book_map = combine_book(series) # {epoch_day: book_return}
|
||||
book_days = sorted(book_map)
|
||||
dates_iso = [dt.date.fromordinal(d).isoformat() for d in book_days]
|
||||
returns = np.array([book_map[d] for d in book_days], dtype=float)
|
||||
|
||||
if returns.size == 0:
|
||||
return _empty_report(method)
|
||||
|
||||
# Diagnostics on the raw edge arrays vs the book array (aligned to min length inside the domain fns).
|
||||
book_arr = returns
|
||||
correlation = [{"a": a, "b": b,
|
||||
"full": _nan_to_none(full_correlation(arrs[a], arrs[b])),
|
||||
"tail": _nan_to_none(left_tail_correlation(arrs[a], book_arr) if a == b else
|
||||
left_tail_correlation(arrs[b], book_arr))}
|
||||
for a, b in combinations(used, 2)]
|
||||
marginal = [{"edge": e, "marginal_sharpe": _nan_to_none(marginal_sharpe(book_arr, arrs[e]))}
|
||||
for e in used]
|
||||
|
||||
return {"edges": used, "method": method, "dates_iso": dates_iso,
|
||||
"equity": _equity_curve(returns), "returns": [float(r) for r in returns],
|
||||
"metrics": _metrics_from_stats(returns), "correlation": correlation,
|
||||
"marginal": marginal, "empty": False}
|
||||
|
||||
|
||||
def _nan_to_none(x: float) -> float | None:
|
||||
"""JSON/template-safe: the domain fns return NaN when a series is too short to estimate; surface as None."""
|
||||
return None if x is None or (isinstance(x, float) and np.isnan(x)) else float(x)
|
||||
```
|
||||
|
||||
NOTE on the `correlation` pair `tail` field: `left_tail_correlation(sleeve, book)` measures a sleeve vs the book on the book's worst days — it is naturally per-edge, not per-pair. For the pairwise heatmap, the honest per-PAIR tail measure is the left-tail correlation of the two edges conditioned on one being in its worst tail. Use the pair's tail co-movement: replace the `tail` computation with `left_tail_correlation(arrs[a], arrs[b])` (edge-a's tail vs edge-b) so each cell is a true pairwise number. Fix the `_window`/report code accordingly:
|
||||
|
||||
```python
|
||||
correlation = [{"a": a, "b": b,
|
||||
"full": _nan_to_none(full_correlation(arrs[a], arrs[b])),
|
||||
"tail": _nan_to_none(left_tail_correlation(arrs[a], arrs[b]))}
|
||||
for a, b in combinations(used, 2)]
|
||||
```
|
||||
|
||||
(Use this pairwise form — drop the `a == b` branch above.)
|
||||
|
||||
- [ ] **Step 4: Run test to verify it passes**
|
||||
|
||||
Run: `pytest tests/application/test_portfolio_backtest.py -v`
|
||||
Expected: PASS (6 passed). If `BacktestStats` field names differ, the failure will be an `AttributeError` in `_metrics_from_stats` — read the dataclass and fix the four reads, then re-run.
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/application/portfolio_backtest.py tests/application/test_portfolio_backtest.py
|
||||
git commit -m "feat(portfolio): report bundler (book + explicit weights, corr + marginal-Sharpe)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 3: Route + template + Plotly rendering
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/fxhnt/adapters/web/app.py` (add two handlers + one module-level edge list; NO existing handler touched)
|
||||
- Create: `src/fxhnt/adapters/web/templates/portfolio.html`
|
||||
- Create: `src/fxhnt/adapters/web/templates/_portfolio_result.html`
|
||||
- Modify: `src/fxhnt/adapters/web/templates/base.html` (Plotly `<script>` + "Portfolio" nav link in both navs)
|
||||
- Test: `tests/integration/test_portfolio_web.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Consumes: `build_portfolio_report` + `ALL_EDGES`/`BYBIT_EDGES`/`SIM_EDGES` (Tasks 1–2); `create_app(...)` construction + `PaperRepo`/`ForwardNavRepo` + `TestClient` (pattern from `tests/integration/test_bybit_sim_web.py`); the existing `_paper_repo()` helper inside `app.py` (returns the injected/settings `PaperRepo`); `templates.TemplateResponse` (already bound in `app.py`).
|
||||
- Produces: `GET /paper/portfolio` (page) + `GET /paper/portfolio/run?edges=...&method=...&start=...&end=...&w_<edge>=...` (HTMX partial). Query contract: `edges` repeated (`?edges=crypto_tstrend&edges=unlock`), `method` in `{book, explicit}`, `w_<edge>` float for explicit mode.
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
```python
|
||||
# tests/integration/test_portfolio_web.py
|
||||
"""Web smoke for the portfolio backtester (/paper/portfolio): assemble edges from the PRECOMPUTED stored
|
||||
series and render a portfolio curve + metrics + full/left-tail correlation heatmap + marginal-Sharpe table,
|
||||
WITHOUT recomputing any edge on the request path. Read-only + additive: no live-fund path touched.
|
||||
NO network — in-memory SQLite repos, seeded directly."""
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from fxhnt.adapters.persistence.forward_nav import ForwardNavRepo
|
||||
from fxhnt.adapters.persistence.paper_repo import PaperRepo
|
||||
from fxhnt.adapters.web.app import create_app
|
||||
|
||||
|
||||
def _seed(repo: PaperRepo) -> None:
|
||||
at = dt.datetime(2026, 1, 1, tzinfo=dt.timezone.utc)
|
||||
start = dt.date(2026, 1, 1)
|
||||
for i in range(40):
|
||||
d = (start + dt.timedelta(days=i)).isoformat()
|
||||
repo.upsert_bybit_sleeve_ret("crypto_tstrend", d, 0.01 if i % 2 else -0.008, at=at)
|
||||
repo.upsert_bybit_sleeve_ret("unlock", d, 0.006 if i % 3 else -0.004, at=at)
|
||||
repo.upsert_sim_curve_ret("multistrat", d, 0.004, at=at)
|
||||
|
||||
|
||||
def _client() -> TestClient:
|
||||
paper = PaperRepo("sqlite://")
|
||||
paper.migrate()
|
||||
_seed(paper)
|
||||
fwd = ForwardNavRepo("sqlite://")
|
||||
fwd.migrate()
|
||||
app = create_app(repo=fwd, bybit_paper_repo=paper, paper_repo=paper)
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
def test_portfolio_page_renders_form():
|
||||
r = _client().get("/paper/portfolio")
|
||||
assert r.status_code == 200
|
||||
body = r.text
|
||||
assert "crypto_tstrend" in body and "multistrat" in body
|
||||
assert "/paper/portfolio/run" in body # the HTMX target
|
||||
|
||||
|
||||
def test_portfolio_run_book_method_returns_curve_metrics_corr_marginal():
|
||||
r = _client().get("/paper/portfolio/run",
|
||||
params=[("edges", "crypto_tstrend"), ("edges", "unlock"),
|
||||
("edges", "multistrat"), ("method", "book")])
|
||||
assert r.status_code == 200
|
||||
body = r.text
|
||||
assert "Plotly" in body # the curve/heatmap render scripts
|
||||
assert "Sharpe" in body # headline metrics table
|
||||
assert "Marginal" in body # marginal-Sharpe table heading
|
||||
|
||||
|
||||
def test_portfolio_run_empty_selection_degrades_gracefully():
|
||||
r = _client().get("/paper/portfolio/run", params={"method": "book"})
|
||||
assert r.status_code == 200 # never a 500
|
||||
assert "Select at least one edge" in r.text or "no edges" in r.text.lower()
|
||||
|
||||
|
||||
def test_portfolio_run_explicit_weights_mode():
|
||||
r = _client().get("/paper/portfolio/run",
|
||||
params=[("edges", "crypto_tstrend"), ("edges", "unlock"),
|
||||
("method", "explicit"), ("w_crypto_tstrend", "0.7"),
|
||||
("w_unlock", "0.3")])
|
||||
assert r.status_code == 200
|
||||
assert "Plotly" in r.text
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `pytest tests/integration/test_portfolio_web.py -v`
|
||||
Expected: FAIL — `/paper/portfolio` returns 404 (route not registered).
|
||||
|
||||
First confirm the `create_app` signature accepts `paper_repo=` — check with:
|
||||
Run: `grep -n "def create_app" src/fxhnt/adapters/web/app.py`
|
||||
If the kwarg differs (e.g. no separate `paper_repo`), adjust the test's `create_app(...)` call to the real signature (the Bybit repo is passed as `bybit_paper_repo=`; some apps read the sim series off the same repo). Use whatever `_paper_repo()` inside `app.py` resolves to as the report's repo.
|
||||
|
||||
- [ ] **Step 3: Add the module-level edge list + two route handlers**
|
||||
|
||||
In `src/fxhnt/adapters/web/app.py`, near the other `/paper/...` handlers (after `paper_sim_run`), add:
|
||||
|
||||
```python
|
||||
@app.get("/paper/portfolio", response_class=HTMLResponse)
|
||||
def paper_portfolio(request: Request) -> HTMLResponse:
|
||||
"""Render the edge-multiselect + method toggle + date-range form INSTANTLY; the three-part report
|
||||
(curve + full/left-tail correlation heatmap + marginal-Sharpe table) lazy-loads from
|
||||
/paper/portfolio/run. Read-only: reads ONLY the precomputed stored series (no edge recompute)."""
|
||||
from fxhnt.application.portfolio_backtest import BYBIT_EDGES, SIM_EDGES
|
||||
# Pre-fill the date bounds from the union range of the available stored series so the pickers are
|
||||
# never blank and can't go off-data.
|
||||
repo = _paper_repo()
|
||||
bybit = {}
|
||||
try:
|
||||
bybit = repo.bybit_sleeve_returns()
|
||||
except Exception: # noqa: BLE001 — a missing table must never 500 the form
|
||||
bybit = {}
|
||||
all_days: set[int] = set()
|
||||
for s in bybit.values():
|
||||
all_days |= set(s)
|
||||
try:
|
||||
all_days |= set(repo.sim_curve_returns("multistrat"))
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
lo = dt.date.fromordinal(min(all_days)).isoformat() if all_days else ""
|
||||
hi = dt.date.fromordinal(max(all_days)).isoformat() if all_days else ""
|
||||
ctx = {"bybit_edges": BYBIT_EDGES, "sim_edges": SIM_EDGES,
|
||||
"range_min": lo, "range_max": hi, "now": _now(),
|
||||
"labels": {e: display_name(e) for e in (*BYBIT_EDGES, *SIM_EDGES)}}
|
||||
return templates.TemplateResponse(request, "portfolio.html", ctx)
|
||||
|
||||
@app.get("/paper/portfolio/run", response_class=HTMLResponse)
|
||||
def paper_portfolio_run(request: Request) -> HTMLResponse:
|
||||
"""Build the report for the selected edges + method + range and render the three-part partial.
|
||||
Empty selection / missing series degrade to a 'select an edge' / 'precomputing' note, never a 500."""
|
||||
from fxhnt.application.portfolio_backtest import ALL_EDGES, build_portfolio_report
|
||||
qp = request.query_params
|
||||
edges = [e for e in qp.getlist("edges") if e in ALL_EDGES]
|
||||
method = qp.get("method") if qp.get("method") in ("book", "explicit") else "book"
|
||||
start = qp.get("start") or None
|
||||
end = qp.get("end") or None
|
||||
weights: dict[str, float] = {}
|
||||
for e in edges:
|
||||
raw = qp.get(f"w_{e}")
|
||||
if raw:
|
||||
try:
|
||||
weights[e] = float(raw)
|
||||
except ValueError:
|
||||
pass
|
||||
try:
|
||||
report = build_portfolio_report(_paper_repo(), edges=edges, method=method,
|
||||
weights=weights or None, start=start, end=end)
|
||||
except Exception: # noqa: BLE001 — degrade to an empty report, never 500
|
||||
report = {"empty": True, "edges": [], "method": method, "dates_iso": [], "equity": [],
|
||||
"returns": [], "metrics": {}, "correlation": [], "marginal": []}
|
||||
return templates.TemplateResponse(request, "_portfolio_result.html",
|
||||
{"r": report, "selected": edges})
|
||||
```
|
||||
|
||||
Verify `display_name`, `_now`, `_paper_repo`, `HTMLResponse`, `Request`, `dt` are all already imported/defined in `app.py` (they are used by neighboring handlers). If `display_name` isn't in scope at that point, import it the same way `_SIM_BOOK_LABELS` does.
|
||||
|
||||
- [ ] **Step 4: Create the page template**
|
||||
|
||||
```html
|
||||
{# src/fxhnt/adapters/web/templates/portfolio.html #}
|
||||
{% extends "base.html" %}
|
||||
{% block title %}Portfolio · fxhnt{% endblock %}
|
||||
{% block content %}
|
||||
<section class="card">
|
||||
<h2>Portfolio Backtest</h2>
|
||||
<p class="muted">Assemble edges, pick a weighting method + date range, and see the portfolio-level
|
||||
Sharpe/CAGR/maxDD plus the correlation (full & left-tail) and marginal-Sharpe diagnostics —
|
||||
all from the precomputed series (no recompute).</p>
|
||||
|
||||
<form hx-get="/paper/portfolio/run" hx-target="#portfolio-result" hx-trigger="submit"
|
||||
hx-indicator="#portfolio-loading">
|
||||
<fieldset>
|
||||
<legend>Edges</legend>
|
||||
{% for e in bybit_edges %}
|
||||
<label class="chk"><input type="checkbox" name="edges" value="{{ e }}"> {{ labels[e] }}</label>
|
||||
{% endfor %}
|
||||
{% for e in sim_edges %}
|
||||
<label class="chk"><input type="checkbox" name="edges" value="{{ e }}"> {{ labels[e] }}</label>
|
||||
{% endfor %}
|
||||
</fieldset>
|
||||
|
||||
<fieldset>
|
||||
<legend>Method</legend>
|
||||
<label class="chk"><input type="radio" name="method" value="book" checked> Book (live allocator)</label>
|
||||
<label class="chk"><input type="radio" name="method" value="explicit"> Explicit weights</label>
|
||||
</fieldset>
|
||||
|
||||
<fieldset id="explicit-weights">
|
||||
<legend>Explicit weights (used only in “explicit” mode)</legend>
|
||||
{% for e in bybit_edges %}
|
||||
<label>{{ labels[e] }} <input type="number" step="0.05" min="0" max="1" name="w_{{ e }}"></label>
|
||||
{% endfor %}
|
||||
{% for e in sim_edges %}
|
||||
<label>{{ labels[e] }} <input type="number" step="0.05" min="0" max="1" name="w_{{ e }}"></label>
|
||||
{% endfor %}
|
||||
</fieldset>
|
||||
|
||||
<fieldset>
|
||||
<legend>Date range</legend>
|
||||
<label>Start <input type="date" name="start" value="{{ range_min }}"
|
||||
min="{{ range_min }}" max="{{ range_max }}"></label>
|
||||
<label>End <input type="date" name="end" value="{{ range_max }}"
|
||||
min="{{ range_min }}" max="{{ range_max }}"></label>
|
||||
</fieldset>
|
||||
|
||||
<button type="submit">Run backtest</button>
|
||||
<span id="portfolio-loading" class="htmx-indicator muted">running…</span>
|
||||
</form>
|
||||
|
||||
<div id="portfolio-result"></div>
|
||||
</section>
|
||||
{% endblock %}
|
||||
```
|
||||
|
||||
- [ ] **Step 5: Create the result partial (Plotly curve + heatmap + tables)**
|
||||
|
||||
```html
|
||||
{# src/fxhnt/adapters/web/templates/_portfolio_result.html #}
|
||||
{% if r.empty %}
|
||||
<p class="muted">Select at least one edge and run the backtest.</p>
|
||||
{% else %}
|
||||
<div class="metrics-row">
|
||||
<div class="metric"><span class="k">CAGR</span><span class="v">{{ '%.1f'|format(r.metrics.cagr * 100) }}%</span></div>
|
||||
<div class="metric"><span class="k">Sharpe</span><span class="v">{{ '%.2f'|format(r.metrics.sharpe) }}</span></div>
|
||||
<div class="metric"><span class="k">Max DD</span><span class="v">{{ '%.1f'|format(r.metrics.max_drawdown * 100) }}%</span></div>
|
||||
<div class="metric"><span class="k">Vol</span><span class="v">{{ '%.1f'|format(r.metrics.vol * 100) }}%</span></div>
|
||||
<div class="metric"><span class="k">Days</span><span class="v">{{ r.metrics.n_days }}</span></div>
|
||||
</div>
|
||||
|
||||
<div id="pf-equity" class="chart"></div>
|
||||
|
||||
<h3>Correlation (full / left-tail)</h3>
|
||||
<div id="pf-heatmap" class="chart"></div>
|
||||
|
||||
<h3>Marginal Sharpe (leave-one-out)</h3>
|
||||
<table class="data">
|
||||
<thead><tr><th>Edge</th><th>Marginal Sharpe</th></tr></thead>
|
||||
<tbody>
|
||||
{% for m in r.marginal %}
|
||||
<tr><td data-label="Edge">{{ m.edge }}</td>
|
||||
<td data-label="Marginal Sharpe">{{ '—' if m.marginal_sharpe is none else '%.3f'|format(m.marginal_sharpe) }}</td></tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
{# Two server-rendered matrices (full + left-tail) → one heatmap with a Full/Left-tail toggle. Jinja renders
|
||||
the matrices server-side (a JS closure can't parameterize a Jinja loop), so the JS is pure Plotly. #}
|
||||
<script>
|
||||
(function () {
|
||||
var dates = {{ r.dates_iso|tojson }};
|
||||
var equity = {{ r.equity|tojson }};
|
||||
var edges = {{ r.marginal|map(attribute='edge')|list|tojson }};
|
||||
Plotly.newPlot('pf-equity',
|
||||
[{x: dates, y: equity, type: 'scatter', mode: 'lines', line: {color: '#4c9aff'}}],
|
||||
{margin: {t: 10, r: 10, b: 30, l: 40}, paper_bgcolor: 'transparent',
|
||||
plot_bgcolor: 'transparent', font: {color: '#c9d1d9'}, yaxis: {title: 'Equity (×)'}},
|
||||
{displayModeBar: false, responsive: true});
|
||||
|
||||
var idx = {}; edges.forEach(function (e, i) { idx[e] = i; });
|
||||
function blank() { return edges.map(function () { return edges.map(function () { return null; }); }); }
|
||||
var full = blank(), tail = blank();
|
||||
{% for p in r.correlation %}
|
||||
full[idx[{{ p.a|tojson }}]][idx[{{ p.b|tojson }}]] = full[idx[{{ p.b|tojson }}]][idx[{{ p.a|tojson }}]] = {{ 'null' if p.full is none else p.full }};
|
||||
tail[idx[{{ p.a|tojson }}]][idx[{{ p.b|tojson }}]] = tail[idx[{{ p.b|tojson }}]][idx[{{ p.a|tojson }}]] = {{ 'null' if p.tail is none else p.tail }};
|
||||
{% endfor %}
|
||||
for (var i = 0; i < edges.length; i++) { full[i][i] = 1.0; tail[i][i] = 1.0; }
|
||||
Plotly.newPlot('pf-heatmap',
|
||||
[{z: full, x: edges, y: edges, type: 'heatmap', colorscale: 'RdBu', zmid: 0, zmin: -1, zmax: 1,
|
||||
name: 'full', colorbar: {title: 'ρ'}}],
|
||||
{margin: {t: 10, r: 10, b: 60, l: 80}, paper_bgcolor: 'transparent', font: {color: '#c9d1d9'},
|
||||
updatemenus: [{buttons: [
|
||||
{label: 'Full', method: 'restyle', args: ['z', [full]]},
|
||||
{label: 'Left-tail', method: 'restyle', args: ['z', [tail]]}], x: 0, y: 1.15}]},
|
||||
{displayModeBar: false, responsive: true});
|
||||
})();
|
||||
</script>
|
||||
{% endif %}
|
||||
```
|
||||
|
||||
- [ ] **Step 6: Add Plotly script + nav links to base.html**
|
||||
|
||||
In `src/fxhnt/adapters/web/templates/base.html`, directly after the vendored htmx `<script>` line (line ~7), add:
|
||||
|
||||
```html
|
||||
{# Plotly loads from CDN in the operator's BROWSER over the tailscale-served page — this is NOT pod egress,
|
||||
so the vendored-htmx "no CDN / tailnet egress" note above does not apply here. Charting layer for
|
||||
/paper/portfolio (existing server-SVG charts migrate later). SRI-pinned + crossorigin so a compromised
|
||||
CDN can't inject script (the tag is refused if the bytes don't match the hash). #}
|
||||
<script src="https://cdn.plot.ly/plotly-2.35.2.min.js"
|
||||
integrity="sha384-<FILL-FROM-CDN>" crossorigin="anonymous" charset="utf-8"></script>
|
||||
```
|
||||
|
||||
**SRI hash (required — do not ship a bare CDN tag).** Compute the `sha384` for the exact pinned version before adding the tag:
|
||||
|
||||
```bash
|
||||
curl -sS https://cdn.plot.ly/plotly-2.35.2.min.js | openssl dgst -sha384 -binary | openssl base64 -A
|
||||
```
|
||||
|
||||
Paste the result into `integrity="sha384-<...>"`. Without a matching hash the browser refuses to run the script, so this is verified the moment the chart renders (Task 4). If the operator prefers zero external dependency later, vendoring Plotly to `/static` (like `htmx.min.js`) is the drop-in alternative — out of scope for this slice, noted for the charting-migration slice.
|
||||
|
||||
In the desktop `.bar` nav (after the Backtest link, ~line 165):
|
||||
|
||||
```html
|
||||
<a href="/paper/portfolio" class="{{ 'on' if path.startswith('/paper/portfolio') }}">Portfolio</a>
|
||||
```
|
||||
|
||||
In the mobile `.tabs` nav (after the Backtest tab, ~line 173):
|
||||
|
||||
```html
|
||||
<a href="/paper/portfolio" class="tab {{ 'on' if path.startswith('/paper/portfolio') }}"><span class="ic" aria-hidden="true">◪</span>Portfolio</a>
|
||||
```
|
||||
|
||||
- [ ] **Step 7: Run the web tests**
|
||||
|
||||
Run: `pytest tests/integration/test_portfolio_web.py -v`
|
||||
Expected: PASS (4 passed). If `create_app`'s kwargs differ, fix the test's construction (Step 2 note). If `display_name` or `_now` aren't in scope in the new handlers, import/reference them as the neighboring handlers do.
|
||||
|
||||
- [ ] **Step 8: Run the full suite to confirm nothing else broke**
|
||||
|
||||
Run: `pytest -q`
|
||||
Expected: all green (the change is purely additive; no existing test should change).
|
||||
|
||||
- [ ] **Step 9: Commit**
|
||||
|
||||
```bash
|
||||
git add src/fxhnt/adapters/web/app.py src/fxhnt/adapters/web/templates/portfolio.html \
|
||||
src/fxhnt/adapters/web/templates/_portfolio_result.html \
|
||||
src/fxhnt/adapters/web/templates/base.html tests/integration/test_portfolio_web.py
|
||||
git commit -m "feat(portfolio): /paper/portfolio route + Plotly curve/heatmap + marginal-Sharpe table"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 4: Local browser verification
|
||||
|
||||
**Files:** none (verification only — per `feedback_verify_cockpit_locally_not_prod`).
|
||||
|
||||
- [ ] **Step 1: Run the cockpit locally** against a DB that has the precomputed series (or the seeded in-memory fixture), following `reference_fxhnt_local_cockpit_dev_loop`. Open `/paper/portfolio` in a real browser.
|
||||
|
||||
- [ ] **Step 2: Verify by reading the numbers** (not just "it renders"):
|
||||
- Select `crypto_tstrend` + `unlock` + `multistrat`, method = Book, full range → the equity curve draws, the metrics row shows plausible Sharpe/CAGR/maxDD, the heatmap shows a 3×3 with the Full/Left-tail toggle working, the marginal-Sharpe table lists all three.
|
||||
- Switch to Explicit weights, set 0.7/0.3 on two edges → curve/metrics change.
|
||||
- Narrow the date range → fewer days, metrics update.
|
||||
- Deselect everything and run → the "select at least one edge" note, NOT a 500.
|
||||
- Check mobile width (≤640px): the form + tables reflow (the `data-label` stacked-card pattern), charts stay within the viewport (no horizontal body scroll).
|
||||
|
||||
- [ ] **Step 3:** If any number looks wrong, STOP and use `superpowers:systematic-debugging` (root cause before fix). Do NOT patch the template to hide a bad number.
|
||||
|
||||
---
|
||||
|
||||
## Self-Review Notes (for the executor)
|
||||
|
||||
- **Spec coverage:** Success criteria 1–4 map to Tasks 1–3 (aggregator + book/explicit report + corr/marginal + read-only). Criterion 5 (Plotly renders over tailscale) maps to Task 4.
|
||||
- **`BacktestStats` fields:** Task 2 assumes `cagr`/`sharpe`/`max_drawdown`/`vol`. The executor MUST read `src/fxhnt/domain/backtest.py` and use the real attribute names — this is the one place the plan cannot verify without the file open. This is called out explicitly in Task 2 Step 3/4.
|
||||
- **`create_app` kwargs:** Task 3 assumes `create_app(repo=, bybit_paper_repo=, paper_repo=)`. The executor confirms the real signature (Task 3 Step 2 note) and adjusts the test; the `_paper_repo()` helper inside `app.py` is the source of truth for which repo the report reads.
|
||||
- **Two correlation axes:** `combine_book` → epoch-day dict; `combine_returns` → date-string tuples. The bundler bridges them (Task 2). Diagnostics (correlation, marginal) always run on the epoch-day-aligned `arrs` from `align_edges`, independent of the chosen method — matching the spec ("weight-independent properties of the edges").
|
||||
382
docs/superpowers/plans/CONSOLIDATION-PROGRESS.md
Normal file
382
docs/superpowers/plans/CONSOLIDATION-PROGRESS.md
Normal file
@@ -0,0 +1,382 @@
|
||||
# Foxhunt→Bizworx Consolidation — Live Progress (resume anchor)
|
||||
|
||||
## 🚦 T8 CUTOVER EXECUTED (2026-07-18 ~20:15 UTC) — fxhnt.ai NOW SERVES FROM BIZWORX ✅
|
||||
**The fund has cut over. foxhunt is FROZEN but INTACT (rollback available). DELETE NOTHING until T10 observation passes.**
|
||||
- **C1 (freeze foxhunt):** suspended fund cronjobs (ucits-rebalancer, ucits-volume-ibkr, compare-measured-precompute)
|
||||
+ scaled foxhunt dagster/fxhnt-dashboard/ib-gateway → 0/0. Infra cronjobs (postgres-backup/pvc-autoscaler/
|
||||
billing-scraper) LEFT RUNNING (they die at teardown, not now). PASS.
|
||||
- **C2 (delta DB migrate):** pg_dump -Fc fxhnt (47.8M) from frozen foxhunt → restore into bizworx. GOTCHA:
|
||||
`pg_restore --clean --if-exists` over the existing rehearsal copy FAILED (dropped _timescaledb_internal schema
|
||||
mid-restore → 1136 errors, DB stuck restoring:on). FIX (rehearsal-proven path): DROP+CREATE empty fxhnt DB,
|
||||
then pre_restore → pg_restore (NO --clean) → post_restore = exit 0, 0 errors. EVAL C2 PASS: all counts match
|
||||
frozen source (forward_nav 106, backtest_summary 6, bybit_sleeve_ret 7023, xsp_option_bars 814373, exec_fills 9,
|
||||
ibkr_account_nav 4, forward_anchor 20; max_anchor_t0 2026-07-14, max_fwd_nav 2026-07-17; 37 tables/3 hypertables).
|
||||
NOTE: counts were IDENTICAL to the rehearsal (fund already suspended during parallel work → zero real delta).
|
||||
LESSON: NEVER pg_restore --clean over a live timescaledb DB; always restore into a fresh empty DB.
|
||||
- **C3 (bizworx fund live):** scaled dagster/cockpit/ib-gateway→1 + rollout-restart cockpit&dagster (fresh DB
|
||||
conns). EVAL C3 PASS: cockpit serves real migrated data over the proxy (/ + /strategy/multistrat + /xsfunding
|
||||
= 200, shows PASS/WAIT/Sharpe/multistrat/xsfunding); ib-gateway 1/1 STABLE (~12min, foxhunt released IBKR
|
||||
session); dagster 3/3, ingress 3/3, postgres 1/1.
|
||||
- **C4 (DNS flip):** Scaleway fxhnt.ai records dashboard/cockpit/dagster/git A → **100.93.141.11** (bizworx
|
||||
fxhnt-ingress), TTL 60 (was 100.95.225.27 foxhunt proxy). SYNTAX: `scw dns record set dns-zone=fxhnt.ai
|
||||
name=<n> type=A ttl=60 values.0=100.93.141.11` (values.0=, NOT data=). EVAL C4 PASS over REAL public DNS:
|
||||
dashboard/git/dagster.fxhnt.ai → 100.93.141.11, HTTP 200, tls_verify=0. Content confirmed: git=bizworx Gitea,
|
||||
dashboard=cockpit "Overview · fxhnt" w/ real data, SSH `git ls-remote ssh://git@git.fxhnt.ai/gitadmin/fxhnt.git`
|
||||
returns real refs (HEAD d66d1132) via socat path. (SSH hostkey-changed warning = correct, it's bizworx now.)
|
||||
- **C5 (arm fund):** un-suspended ONLY fxhnt-ucits-rebalancer on bizworx (schedule 0 9 * * 1-5 = 09:00 UTC wkdays;
|
||||
next run tomorrow AM). US(ibkr)/bybit/vrp/factory/ucits-volume all stay SUSPENDED. EVAL C5 (first armed run
|
||||
green) is OBSERVABLE at next 09:00 UTC — deps verified (DB+ib-gateway+git-sync all live).
|
||||
- **ROLLBACK (still trivial until T11):** `scw dns record set ... values.0=100.95.225.27` (back to foxhunt proxy)
|
||||
+ `kubectl --context $FOX -n foxhunt scale deploy/dagster deploy/fxhnt-dashboard deploy/ib-gateway --replicas=1`
|
||||
+ un-suspend foxhunt fund cronjobs. foxhunt Postgres still holds pre-cutover state.
|
||||
- **NEXT:** T9 verify (cockpit LOCAL browser + prod, gate/forward-track intact) → T10 observation window (watch
|
||||
the first ucits run + a day) → T11 Phase-2 triage → T12 teardown. **ROTATE the pasted tailscale key.** DELETE NOTHING.
|
||||
|
||||
|
||||
Branch `chore/foxhunt-bizworx-consolidation`. Real date 2026-07-18. Plan:
|
||||
`docs/superpowers/plans/2026-07-18-foxhunt-to-bizworx-consolidation.md`. Contexts:
|
||||
`FOX=foxhunt-34a1e3c4-…`, `BIZ=bizworx-prod-kapsule-05e1dae3-…` (both in ~/.kube/config).
|
||||
|
||||
## Done + verified
|
||||
- **Access restored:** appended this box's egress `51.158.111.225/32` to the foxhunt Kapsule API ACL
|
||||
(`scw k8s acl`, entry `bizworx-devbox-v4`) + `scw k8s kubeconfig install` → foxhunt context works.
|
||||
- **T1** ns `foxhunt` created on bizworx; StorageClass `sbs-default` matches; source = PG16.13/tsdb2.25.1.
|
||||
- **T2** 6 fund secrets copied FOX→BIZ ns foxhunt (db-credentials, databento, tiingo-api, ibkr-credentials,
|
||||
argo-git-ssh-key, scw-registry). NOT copied: bybit-testnet (absent/suspended), tailscale-auth (deferred).
|
||||
- **T3** gitea consolidated → bizworx gitea (ns gitea): `gitadmin/fxhnt` (8 heads/0 tags, master d66d113)
|
||||
+ `gitadmin/foxhunt` (13 heads/105 tags, main 8213f29a5), refs verified identical. foxhunt (625MB) done
|
||||
via direct pack-copy (HTTP push + server-side index-pack both failed on size/OOM). `git.fxhnt.ai` STILL
|
||||
points at foxhunt (DNS re-point is T8). Caveat: foxhunt archive's gitea DB metadata may lag (disk-correct).
|
||||
- **T4** manifests re-targeted (committed): git-sync `gitea-sshd.foxhunt`→`gitea-ssh.gitea.svc.cluster.local:22`;
|
||||
nodeSelector→`large`; ib-gateway pull-secret removed (public ghcr image). Server-side dry-run clean.
|
||||
- **T5/T6** Postgres on bizworx PINNED to `timescale/timescaledb:2.25.1-pg16` (latest=2.28.3 broke catalog
|
||||
restore — REHEARSAL caught it). Rehearsal restore of `fxhnt` DB: exit 0, 0 errors, all row counts match
|
||||
(forward_nav 106, backtest_summary 6, bybit_sleeve_ret 7023, xsp_option_bars 814373, exec_fills 9,
|
||||
ibkr_account_nav 4), 37/37 tables, 3/3 hypertables. bizworx fxhnt DB currently holds this rehearsal copy.
|
||||
|
||||
## T7 IN PROGRESS (parallel verify — foxhunt still fully live)
|
||||
- **7.1 DONE** — argo-git-ssh-key deploy-key registered read-only on bizworx gitadmin/fxhnt (id 2);
|
||||
verified `git ls-remote` via the key = 8 refs.
|
||||
- **cockpit DONE** — `fxhnt-dashboard` deployed on bizworx; git-sync OK (cloned d66d113 from bizworx gitea);
|
||||
cockpit container Running+ready, renders migrated data (overview + /strategy/xsfunding + /strategy/multistrat
|
||||
all HTTP OK with real WAIT/PASS gates). Verified from INSIDE the container (port-forward blocked — see below).
|
||||
- **BLOCKER / decision pending — tailscale:** dagster + cockpit pods have a tailscale sidecar needing
|
||||
`tailscale-auth` (not copied) → sidecar CreateContainerConfigError → pod phase Pending → port-forward
|
||||
refuses. Cockpit itself is fine. DECIDE: **A** fresh reusable tailscale auth-key (bizworx pods join as
|
||||
extra nodes alongside live foxhunt ones; removed at cutover) OR **B** disable sidecar for the parallel
|
||||
phase, verify via port-forward, enable tailscale at T8 cutover with real hostnames.
|
||||
|
||||
## T7 REMAINING
|
||||
- ib-gateway → apply + verify IBKR paper connect (no 2FA/IP-limit per operator) + whatIf.
|
||||
- dagster → apply (check its NetworkPolicy allows egress to ns gitea for git-sync — cross-ns) + trigger
|
||||
nightly asset graph green.
|
||||
- CronJobs → apply all SUSPENDED; then one `fxhnt-ucits-rebalancer` dry-run (EU hours).
|
||||
- NP note: fund NPs (dagster, cronjobs) allow egress to gitea in-ns; on bizworx gitea is ns `gitea` →
|
||||
may need namespaceSelector. (cockpit had no NP → git-sync worked.)
|
||||
|
||||
## HARD RULE (operator): verify-first, per-step eval, DELETE NOTHING until Phase-1 verified + observation
|
||||
window (T10) passes. foxhunt stays live/intact through T8, suspended-but-intact until T12 teardown.
|
||||
Rollback until T11 = re-point DNS + un-suspend foxhunt.
|
||||
|
||||
## T7 update (2026-07-18, later)
|
||||
- tailscale-auth secret created on bizworx (operator gave a fresh reusable key — OPTION A; key was pasted in
|
||||
transcript, ROTATE after cutover). cockpit pod now 2/2 Running (cockpit + tailscale sidecar joined).
|
||||
- ib-gateway deployed: IBC logs in to DU9600528 (Simulated Trading) — IBKR paper connect WORKS (no 2FA/IP).
|
||||
⚠️ pod had 2 restarts during IBC startup churn — WATCH it stabilises to 1/1 ready; verify API port 4003/4004
|
||||
+ a whatIf before arming any rebalancer.
|
||||
- dagster was Pending on missing PVC `fxhnt-surfer-data` (dropped from migration; manifest still mounts it) →
|
||||
created empty 5Gi PVC (sbs-default); dagster now scheduling (git-sync init). unlock_calendar.json rebuilds.
|
||||
- REMAINING T7: confirm ib-gateway 1/1 + whatIf; confirm dagster git-sync (cross-ns NP to gitea) + trigger
|
||||
nightly asset graph green; apply CronJobs SUSPENDED; one fxhnt-ucits-rebalancer dry-run (EU hours).
|
||||
- Then T8 cutover (freeze foxhunt, delta restore, DNS re-point git.fxhnt.ai + dashboard.fxhnt.ai) → T9 verify
|
||||
→ T10 observe → T11 triage → T12 teardown. foxhunt still fully live; delete NOTHING yet.
|
||||
|
||||
## T7 update 2 (2026-07-18)
|
||||
- dagster FIXED + 3/3 Running: (1) created `dagster` DB on bizworx postgres (fresh dagster, not migrated);
|
||||
(2) dagster NP gitea-egress needed a cross-ns `namespaceSelector {kubernetes.io/metadata.name: gitea}`
|
||||
(committed in dagster.yaml) → git-sync from bizworx gitea now OK. webserver+daemon+tailscale all ready.
|
||||
- ib-gateway: connects to DU9600528 (IBC login OK) but exits 0 after ~6min = IBKR single-session conflict with
|
||||
the STILL-LIVE foxhunt ib-gateway. Connect verified; stable run only after foxhunt-suspend (T8). NOT a bug.
|
||||
- T7 CORE VERIFIED: postgres+cockpit+dagster all up on bizworx against migrated data.
|
||||
- REMAINING T7: (a) apply the SAME cross-ns gitea-NP fix to every cronjob NP (jobs/fxhnt-*.yaml) BEFORE they
|
||||
run (else git-sync blocked like dagster was); (b) apply CronJobs SUSPENDED; (c) one ucits dry-run (needs
|
||||
ib-gateway sole-session → do at/after cutover, or briefly scale foxhunt ib-gateway to 0); (d) optional dagster
|
||||
asset-graph run. Then T8 cutover.
|
||||
- ALSO for cutover: the tailscale bizworx pods joined as extra nodes (fresh key); at T8 suspend foxhunt so the
|
||||
real hostnames + IBKR session move cleanly. ROTATE the pasted tailscale key post-cutover.
|
||||
|
||||
## P1 EXECUTION (2026-07-18) — wildcard cert + the cross-project DNS-01 IAM fix
|
||||
- Applied `infra/k8s/services/fxhnt-wildcard-cert.yaml` (Certificate fxhnt.ai + *.fxhnt.ai, ClusterIssuer
|
||||
letsencrypt-prod, secret fxhnt-wildcard-tls, ns foxhunt) on BIZ.
|
||||
- **DNS-01 FAILED: `403 Forbidden: domain not found`.** Root cause: `fxhnt.ai` DNS zone lives in the **foxhunt
|
||||
project c293eb98**; the bizworx cert-manager key (`SCWW9261…`, app `bizworx-prod-runtime` 48418a71) has
|
||||
`DomainsDNSFullAccess` but SCOPED to the **bizworx project 96d0717d** only → can't touch fxhnt.ai's zone.
|
||||
- **DEAD END (operator's first pick "move the domain to bizworx project"): Scaleway BLOCKS it.** fxhnt.ai is an
|
||||
EXTERNAL domain (`scw domain domain list` → IS EXTERNAL=true, REGISTRAR=EXTERNAL). `scw dns zone update
|
||||
… project-id=…` on the apex → **`Root zone can't be updated`**; on a scratch sub-zone → **`Subdomain not
|
||||
allowed`**; `scw domain domain` has NO move-project subcommand. Verified via a throwaway `_cutovertest.fxhnt.ai`
|
||||
sub-zone (created+move-attempt+deleted). So the domain CANNOT be reassigned to the bizworx project via API.
|
||||
- **ACTUAL FIX (operator's 2nd pick, applied): add an IAM rule.** `scw iam rule create
|
||||
policy-id=8573ce25-… permission-set-names.0=DomainsDNSFullAccess project-ids.0=c293eb98-…` → policy
|
||||
`bizworx-prod-runtime-scoped` now has 2 rules (orig 6-perms@96d0717d untouched + new DNS@c293eb98). This lets
|
||||
the existing bizworx cert-manager key write the DNS-01 TXT on fxhnt.ai. Domain STAYS external in c293eb98;
|
||||
only IAM changed. NOTE for teardown: this rule references the foxhunt project — harmless to keep, or tighten
|
||||
later; if the foxhunt PROJECT (not just cluster) is ever deleted, the fxhnt.ai domain/zone must first move
|
||||
registrar-side or certs break. (Cluster teardown ≠ project deletion; the DNS zone + domain survive T12.)
|
||||
- Forced cert-manager retry (deleted order+CR) after the IAM change → 403 GONE, challenges progressing normally
|
||||
(apex fxhnt.ai challenge = `valid`/"Successfully authorized domain"; wildcard `*.fxhnt.ai` challenge finishing
|
||||
its DNS-01 propagation). TXT is being written to the fxhnt.ai zone by cert-manager (verified via
|
||||
`scw dns record list dns-zone=fxhnt.ai | grep acme`). P1 is SUCCEEDING — cert will go READY shortly.
|
||||
If resuming and cert not yet READY: `kubectl --context $BIZ -n foxhunt get certificate fxhnt-wildcard -o wide`;
|
||||
if stuck >15min, `kubectl delete order,challenge,certificaterequest --all -n foxhunt` to force a clean retry
|
||||
(IAM rule is permanent so it'll succeed).
|
||||
|
||||
## P2 DRAFTED (2026-07-18) — fxhnt-ingress proxy (NOT applied yet; waiting on P1 cert READY)
|
||||
- `infra/k8s/services/fxhnt-ingress-proxy.yaml` (committed ce1c978, draft): nginx (wildcard TLS via
|
||||
fxhnt-wildcard-tls) + socat (git SSH→gitea-ssh.gitea:22) + tailscale sidecar (TS_HOSTNAME=fxhnt-ingress).
|
||||
Fronts 3 fund vhosts BY CLUSTERIP: git→gitea-http.gitea:3000, dashboard/cockpit→fxhnt-dashboard.foxhunt:80,
|
||||
dagster→dagster.foxhunt:80. Has its own NP (egress to those svcs + apiserver 172.16.0.11:6443 for TS + DERP).
|
||||
Deps satisfied: tailscale-auth secret present; dagster NP ingress already allows fxhnt-ingress label.
|
||||
## P1+P2 COMPLETE (2026-07-18) — bizworx fxhnt.ai front verified, NO DNS change yet ✅
|
||||
- **P1 DONE:** fxhnt-wildcard cert READY (LE, SAN=*.fxhnt.ai + fxhnt.ai). IAM cross-project fix held.
|
||||
- **P2 DONE:** fxhnt-ingress proxy applied + 3/3 Running. Tailnet identity = **`fxhnt-ingress` @ `100.93.141.11`**
|
||||
(clean, no -1 suffix). THIS IS THE C4 DNS TARGET.
|
||||
- **EVAL P2 PASS (from devbox over the real tailnet → 100.93.141.11, the exact browser path post-flip):**
|
||||
dashboard / cockpit / dagster / git .fxhnt.ai ALL → **HTTP 200, tls=0 (cert chain valid)**. foxhunt still live.
|
||||
- Fixes needed to get P2 green (all committed):
|
||||
1. **RBAC:** TS sidecar needs SA+Role+RoleBinding for get/create/update/patch on its ts-state Secret (added
|
||||
`tailscale-fxhnt-ingress` SA, mirrors tailscale-dagster). Without it: "missing get/update permission".
|
||||
2. **dagster NP ingress was stale on-cluster:** the 9396f53 edit (foxhunt→fxhnt-ingress label) was committed
|
||||
but the cluster still had `tailscale-gitlab-proxy` → dagster vhost hung. Re-applied dagster.yaml → fixed.
|
||||
LESSON: `git commit` ≠ `kubectl apply`; always re-apply after editing a live manifest.
|
||||
3. **gitea headless-service / CGNAT trap:** gitea-http & gitea-ssh are HEADLESS (ClusterIP:None) → DNS returns
|
||||
the gitea POD IP in 100.64.0.0/15, which the proxy's KERNEL-MODE tailscale sidecar (TS_USERSPACE=false)
|
||||
CANNOT reach (CGNAT overlap). FIX: added a non-headless **`gitea-clusterip`** Service (ns gitea, 10.32.x,
|
||||
selector app.kubernetes.io/instance=gitea+name=gitea, ports 3000+22); nginx+socat now target it. cockpit &
|
||||
dagster were fine because they have real 10.32.x ClusterIPs. (Kernel-mode TS reaches 10.32.x ClusterIPs but
|
||||
NOT 100.64.x pod IPs — verified both ways.)
|
||||
- **Verify artifact:** `curl --resolve <host>:443:100.93.141.11 https://<host>/` from any tailnet node.
|
||||
- PHASE-1 PREP COMPLETE. Ready to schedule the C1–C5 cutover window (see T8-CUTOVER-RUNBOOK.md). C4 DNS target
|
||||
= 100.93.141.11 for records dashboard/cockpit/dagster/git. foxhunt fully live; delete NOTHING.
|
||||
|
||||
## T8 PREP (2026-07-18) — DNS/ingress map + fixes (foxhunt still fully live)
|
||||
|
||||
### Gotchas hit this session
|
||||
- **dagster tailscale sidecar CrashLoop (context deadline exceeded reaching kubernetes.default.svc).** TWO-STAGE
|
||||
root cause — the FIRST fix (06d12fd, `10.32.0.1/32:443`) was WRONG and did not hold (regressed to 2/3):
|
||||
* `kubernetes.default.svc` → ClusterIP `10.32.0.1:443`, BUT **kube-proxy DNATs it to the REAL apiserver
|
||||
endpoint `172.16.0.11:6443` BEFORE the NetworkPolicy is evaluated.** The NP sees dest `172.16.0.11:6443`,
|
||||
which is inside the NP's OTHER except entry `172.16.0.0/16` → blocked. So allowing the ClusterIP never matched.
|
||||
* CORRECT FIX: `ipBlock 172.16.0.11/32` port **6443** (the real endpoint, post-DNAT). apiserver endpoint is
|
||||
`kubectl -n default get endpoints kubernetes` = 172.16.0.11 (Kapsule fixed control-plane).
|
||||
* SIDE EFFECT this exposed: when the tailscale sidecar is NotReady, the whole dagster pod is NotReady →
|
||||
**dagster Service has NO endpoints → dagster.fxhnt.ai vhost would 502**. So this MUST be green before C4 DNS flip.
|
||||
* daemon+webserver themselves were always healthy. Cockpit's sidecar works only because cockpit has NO NP.
|
||||
* GENERAL LESSON for any pod with a tailscale sidecar + a restrictive egress NP on bizworx: must allow
|
||||
egress to 172.16.0.11:6443 (apiserver). The cronjob/rebalancer NPs do NOT have tailscale sidecars so they're
|
||||
unaffected — but if any future fund pod gets a TS sidecar under an NP, add this rule.
|
||||
- **foxhunt API ACL lapsed AGAIN**: devbox egress IP drifted `51.158.111.225`→`51.158.125.1`. Foxhunt Kapsule
|
||||
admin API (`*.api.k8s.fr-par.scw.cloud:6443`) is IP-allowlisted; bizworx keeps working (tailscale-cgnat range).
|
||||
FIX: `scw k8s acl add cluster-id=34a1e3c4… acls.0.ip=<newIP>/32 acls.0.description=bizworx-devbox-vN` (APPENDS;
|
||||
verified all entries survive). Re-check `curl ifconfig.me` vs `scw k8s acl list` whenever foxhunt 6443 resets.
|
||||
|
||||
### THE fxhnt.ai DNS/ingress architecture (drives T8 DNS cutover)
|
||||
- DNS zone = **Scaleway `fxhnt.ai`** (project c293eb98=foxhunt). ALL 13 A-records → **`100.95.225.27`** = tailnet
|
||||
node **`foxhunt-gitlab`** = pod **`tailscale-gitlab-proxy` (ns foxhunt, 4/4)** = nginx(vhosts)+2×socat+tailscale.
|
||||
**This pod DIES at foxhunt teardown (T12) → DNS MUST move off 100.95.225.27 regardless.**
|
||||
- nginx vhost map (what each fxhnt.ai host fronts):
|
||||
- `git.fxhnt.ai` → gitea-web.foxhunt:3000 ✅FUND → **bizworx gitea (ns gitea), already mirrored**
|
||||
- `dashboard/cockpit.fxhnt.ai` → 100.81.150.18:80 (fxhnt-dashboard node→cockpit:8080) ✅FUND → **bizworx cockpit**
|
||||
- `dagster.fxhnt.ai` → dagster.foxhunt:80 ✅FUND → **bizworx dagster**
|
||||
- grafana ❌ (operator: "grafana can go too" — drop, bizworx has own obs) · api(50051 legacy Rust gRPC) ❌
|
||||
· mail(stalwart)/chat(mattermost)/minio ❌legacy → all Phase-2 drop. ONLY git+dashboard/cockpit+dagster move.
|
||||
- **CRITICAL SAFETY FACT:** the FUND PIPELINE never uses fxhnt.ai — git-sync=`gitea-ssh.gitea.svc.cluster.local:22`
|
||||
(in-cluster), cockpit/dagster serve over ClusterIP+own tailnet IP. fxhnt.ai is BROWSER/human access only.
|
||||
→ the fund keeps running through the DNS flip; DNS cutover only affects operator browser access. Low-risk.
|
||||
- Tailnet identities: live foxhunt `fxhnt-dashboard`=100.81.150.18, `fxhnt-dagster`=100.116.123.29. New bizworx
|
||||
cockpit joined as **`fxhnt-dashboard-1`=100.107.99.35** (`-1` because live one holds the name). Each bizworx pod
|
||||
serves TLS DIRECTLY via `tailscale serve` on its own MagicDNS cert (verified: cockpit `tailscale serve status`
|
||||
shows https://fxhnt-dashboard-1.tailff20e.ts.net → 127.0.0.1:8080). No bizworx proxy pod needed.
|
||||
|
||||
### DNS cutover DECISION (operator): "RENAME bizworx pods to take over the identity"
|
||||
- At cutover: suspend/scale foxhunt fund pods → they RELEASE `fxhnt-dashboard`/`fxhnt-dagster` tailnet names →
|
||||
restart bizworx pods so they re-register WITHOUT the `-1` suffix (reclaim the names + get their IPs) → then
|
||||
repoint the Scaleway `fxhnt.ai` A-records (dashboard,cockpit,dagster,git) from the dying 100.95.225.27 to the
|
||||
reclaimed bizworx pod tailnet IPs. Records to change: `scw dns record` set on zone fxhnt.ai.
|
||||
- TLS RESOLVED: old proxy served a cert-manager wildcard `*.fxhnt.ai` (ClusterIssuer letsencrypt-prod, Scaleway
|
||||
DNS-01 webhook). **bizworx already has the SAME stack** (cert-manager + scaleway-certmanager-webhook 1/1 +
|
||||
scaleway-api creds + letsencrypt-prod ClusterIssuer, already issuing LE certs in ~10 ns incl grafana-tls).
|
||||
→ issuing `*.fxhnt.ai` on bizworx = just a Certificate resource. NO blocker.
|
||||
- CHOSEN CUTOVER SHAPE (supersedes the naive pod-IP flip): build a **stripped fund-only nginx+tailscale proxy on
|
||||
bizworx** (`fxhnt-ingress`) fronting the 3 fund vhosts via ClusterIP (gitea-http.gitea:3000, fxhnt-dashboard
|
||||
.foxhunt:80, dagster.foxhunt:80) + socat SSH gitea-ssh.gitea:22, terminating the wildcard `*.fxhnt.ai` cert.
|
||||
Point the 4 fxhnt.ai A-records at THIS proxy's tailnet IP. Because it uses ClusterIP not pod tailnet names,
|
||||
the "reclaim fxhnt-dashboard identity" rename is NO LONGER on the critical path (optional cosmetic).
|
||||
- **FULL EXECUTABLE RUNBOOK written: `docs/superpowers/plans/T8-CUTOVER-RUNBOOK.md`** (P1 wildcard cert →
|
||||
P2 build+verify proxy pre-flip → C1 freeze foxhunt → C2 delta dump/restore → C3 bizworx live+ib-gw sole session
|
||||
→ C4 DNS flip → C5 arm ucits. Every step has VERIFY + ROLLBACK. Nothing destructive; foxhunt suspended-not-deleted.)
|
||||
- Confirmed svc names for the proxy: gitea-http.gitea:3000 (headless), gitea-ssh.gitea:22 (headless),
|
||||
fxhnt-dashboard.foxhunt:80, dagster.foxhunt:80, ib-gateway.foxhunt:4002/4004.
|
||||
|
||||
## T7 update 3 (2026-07-18) — cronjobs
|
||||
- Cross-ns gitea-NP fix applied to ALL 15 jobs/fxhnt-*.yaml (namespaceSelector kubernetes.io/metadata.name=gitea).
|
||||
- 7 CronJobs applied to bizworx ns foxhunt, ALL suspend=true (verified): bybit-rebalancer,
|
||||
compare-measured-precompute, factory, ibkr-rebalancer, ucits-rebalancer, ucits-volume-ibkr, vrp-rebalancer.
|
||||
(real-money safety: none run parallel to the live foxhunt ucits-rebalancer.)
|
||||
- The 9 one-time Jobs (opra-backfill/warehouse-migrate/paper-backfill/bybit-*-job/deribit) NOT applied (they
|
||||
run immediately); apply on demand post-cutover with the NP fix already in place.
|
||||
- REMAINING before T8: one ucits dry-run (needs SOLE IBKR session — briefly scale foxhunt ib-gateway to 0,
|
||||
or do it as the first act of cutover) + optional dagster asset-graph run. Then T8 cutover.
|
||||
|
||||
## T9 VERIFY (2026-07-18) — cockpit + gate intact against MIGRATED bizworx DB ✅
|
||||
- **Local cockpit** (`./scripts/dev-cockpit-local.sh`, current-context=bizworx → serves the MIGRATED DB read-only
|
||||
on http://127.0.0.1:8801). ENV GOTCHA fixed: the script's `uv run` needed `--extra web --extra pg` (uvicorn in
|
||||
`web`, psycopg in `pg`); a bare `uv run` fails ModuleNotFoundError. Patched dev-cockpit-local.sh.
|
||||
- **NUMBERS CROSS-CHECKED (read the numbers, per feedback_verify_cockpit_locally_not_prod):**
|
||||
- /paper IBKR card = **$1,029,015** == ibkr_account_nav.nlv latest (2026-07-17 = 1029014.63). "not connected"
|
||||
ABSENT (the card that silently degraded in a prior session renders the REAL DU account here).
|
||||
- bybit Sh 2.11 == backtest_summary bybit_4edge 2.1067; unlock Sh 1.03 == 1.0258. All real, DB-backed.
|
||||
- /, /strategy/multistrat, /strategy/xsfunding, /paper all HTTP 200 with real content.
|
||||
- CAVEAT: Playwright MCP browser not connected this session → verified by rendering real HTML + reading
|
||||
numbers (not pixels). Local cockpit left running on :8801 for the operator to click through.
|
||||
- **T9b forward tracks INTACT (no spurious re-inception from migration):** forward_anchor t0 dates are ORIGINAL
|
||||
historical inceptions (sixtyforty 2026-06-15, unlock 2026-06-20, positioning 2026-07-06, bybit_4edge/multistrat/
|
||||
xsfunding 2026-07-07, vrp/multistrat_exec 2026-07-14) — NOT today (2026-07-18). definition_hash/version chains +
|
||||
archived_at + record_mode all preserved. DB-anchor-SSOT survived byte-for-byte; recon gate will recompute correctly.
|
||||
- **STILL PENDING:** T9 real-money confirm = watch the first armed fxhnt-ucits-rebalancer run at 09:00 UTC (tomorrow).
|
||||
Then T10 observe → T11 triage → T12 teardown. ROTATE the pasted tailscale key. DELETE NOTHING.
|
||||
|
||||
## T9 VISUAL BROWSER PASS (2026-07-18) — real Chrome via Playwright MCP ✅
|
||||
- Playwright MCP now available (plugins reloaded); installed the `chrome` channel (Google Chrome 150 at
|
||||
/opt/google/chrome/chrome — the MCP hardcodes that path, bundled chromium alone isn't enough).
|
||||
- Drove the LOCAL cockpit (http://127.0.0.1:8801, bizworx migrated DB) in a REAL browser — full render, all data:
|
||||
- Overview: Bybit 4-edge $832,534 +738% Sh 1.74 maxDD -13%, deploy $75k/75%, gate 11/21 WAIT; 4 edges
|
||||
(trend Sh0.72 / unlock Sh1.03 / xsfunding Sh3.62 / positioning Sh2.55); forward-tracks table w/ correct
|
||||
gates: sixtyforty 23/20 GO, unlock 26/14 PASS, multistrat 9/20 WAIT + real IBKR subtrack "real -0.77% ·
|
||||
9 trades · 5 holdings".
|
||||
- /paper: IBKR paper account **$1,029,015 · 1 strategy trading · 5 holdings · 0.9% behind plan** (the real
|
||||
DU account — NOT "not connected"). Bybit book $832,534.
|
||||
- Full-page screenshot saved (.playwright-mcp/cockpit-overview-bizworx.png). 1 non-material console error.
|
||||
- Matches DB cross-checks exactly. T9 cockpit verification COMPLETE (visual + numeric).
|
||||
|
||||
## FOXHUNT SHUTDOWN (2026-07-18 ~20:55 UTC) — compute stopped, cluster+PVCs INTACT (not destroyed)
|
||||
Operator decision after verifying migration complete: "shut down everything now, we can spin up if we need to
|
||||
reboot." Verified first: (1) bizworx has ZERO runtime dep on foxhunt (its postgres/gitea/git-sync/registry are
|
||||
all bizworx-local — the `postgres.foxhunt.svc` refs are bizworx's OWN ns foxhunt, different cluster); (2) the
|
||||
pre-cutover state is durably backed up OFF-cluster: `~/Work/fxhnt-cutover-backup/fxhnt_precutover_2026-07-18.dump`
|
||||
(48M, 805 objects, all tables verified via pg_restore --list in the bizworx pod); (3) fund live+serving on
|
||||
bizworx (cockpit 2/2, dagster 3/3, ingress 3/3, ib-gateway 1/1; dashboard.fxhnt.ai HTTP 200).
|
||||
- Scaled ALL foxhunt-ns deploys + statefulsets to 0 (incl postgres/gitea/tailscale-gitlab-proxy = the
|
||||
instant-rollback path — operator accepted dropping it). Also scaled cert-manager + tailscale-subnet-router to 0.
|
||||
- Suspended ALL foxhunt cronjobs (fund + infra: postgres-backup/pvc-autoscaler/billing-scraper). Deleted leftover
|
||||
Completed/Error Job pods so they don't pin nodes.
|
||||
- Pool `platform` (8b520ebb…): autoscaling=true, **min-size=0** (was 1). Can't force size=0 while autoscaling on;
|
||||
the Kapsule autoscaler drains nodes to 0 once only DaemonSets+kube-system remain (~10-15min scale-down delay).
|
||||
Monitoring the 3 dev1_l nodes → 0. `ci-compile-cpu` pool already 0.
|
||||
- **ROLLBACK now = restore the .dump into a fresh pg (few min), NOT an instant DNS flip.** foxhunt cluster + ALL
|
||||
PVCs still EXIST (training-data 500Gi, minio 150Gi, postgres 100Gi, gitlab-gitaly 50Gi, gitea valkey, etc.) —
|
||||
scaling back up reschedules pods on data intact. terraform destroy (T12, irreversible) still DEFERRED.
|
||||
- REVERSIBILITY: `scw k8s pool update 8b520ebb… min-size=1 size=1` + scale deploys back up → foxhunt returns.
|
||||
- STILL PENDING: tomorrow 09:00 UTC bizworx ucits-rebalancer run (real-money confirm); then T12 terraform destroy
|
||||
(with operator confirm). ROTATE the pasted tailscale key.
|
||||
|
||||
## DATABENTO / .dbn DATA MIGRATION (2026-07-18 ~21:10 UTC) — operator caught this before teardown
|
||||
Found databento .dbn data on foxhunt PVCs NOT covered by the Postgres migration. Split cleanly:
|
||||
- **Fund-relevant (migrated):** `/data/surfer/*.dbn` — 22 futures .dbn (ES/NQ/GC/CL/NG/6E/ZC/ZS... universe,
|
||||
updated Jul 12-13) + *_state.json. READ BY the fund's tsmom/carry RESEARCH dagster assets
|
||||
(tsmom_research.py, carry_research.py, assets.py:247 read_front_arrays `$FXHNT_SURFER_DATA_DIR/<sym>.dbn`).
|
||||
bizworx had only an EMPTY fxhnt-surfer-data PVC → research would've failed. **MIGRATED**: tar (33M gz) foxhunt
|
||||
inspector → devbox → bizworx writer pod → extracted into fxhnt-surfer-data. Verified from the dagster pod's
|
||||
OWN view: 22 .dbn at /data/surfer/*.dbn + state JSONs. (Layout note: PVC mounted at /data on dagster,
|
||||
FXHNT_SURFER_DATA_DIR=/data/surfer → files live at PVC-root/surfer/.)
|
||||
- **Archived Rust-ML (NOT fund-used): ~171GB .dbn** on training-data-pvc (59 files, 151GB: ES.FUT MBP10/trades
|
||||
quarterly) + test-data-pvc (21 files, 20GB). Operator decision (option 3): push to PRIVATE Scaleway Object
|
||||
Storage, NOT a bizworx PVC, and "cannot move 175GB over a tunnel" → **upload INTRA-SCALEWAY** (foxhunt pod →
|
||||
s3.fr-par.scw.cloud, both fr-par, no tunnel/devbox hop).
|
||||
- Created private bucket **`fxhnt-dbn-archive`** (fr-par, bizworx project 96d0717d, versioning off, ACL empty).
|
||||
- S3 creds → foxhunt secret `dbn-archive-s3` (from scw config, never printed).
|
||||
- `dbn-uploader` pod (rclone/rclone image) mounts training+test PVCs RO; `rclone copy --include "*.dbn*"
|
||||
--transfers 8` → scw:fxhnt-dbn-archive/{training,test}/. Connectivity tested (rclone lsd rc=0). 171.4GB / 80 files.
|
||||
- Upload running in background in-pod (logs /tmp/rclone-*.log, /tmp/rclone-done sentinel). Monitored.
|
||||
- To restore archived data later: `rclone copy scw:fxhnt-dbn-archive/... <dest>` (creds in bizworx-prod-runtime
|
||||
key or scw config). Kept for the archived Rust codebase per [[project_rust_infra_decommissioned_phase1]].
|
||||
- NOTE: I bumped platform pool min-size back to 1 (from 0) so the inspector/uploader pods can schedule; after the
|
||||
upload completes + surfer verified, set min-size=0 again to resume the drain. Cleanup pods: data-inspector, dbn-uploader.
|
||||
|
||||
## .dbn ARCHIVE — PIVOTED to block snapshots (rclone upload was a dead end) ✅
|
||||
- The rclone S3 upload from the STARVED foxhunt cluster STALLED: dev1_l node egress + slow scw-bssd PVC read
|
||||
(20MB/s) → 76KiB/s, ETA 4 DAYS, byte count frozen. Killed it.
|
||||
- **RIGHT SOLUTION: Scaleway block-volume SNAPSHOTS** (instant, storage-layer, no tunnel, restorable, cheap).
|
||||
The .dbn PVCs are scw block volumes → snapshotted directly:
|
||||
- `fxhnt-training-dbn-archive-2026-07-18` (snap 564a4ccb-ee69-4514-8a64-9932222a0334, 537GB vol = the 151GB
|
||||
training .dbn) — ParentVol cb327a05 (training-data-pvc).
|
||||
- `fxhnt-test-dbn-archive-2026-07-18` (snap 5ba0049f-799a-4694-b7a0-0192cf0cda11, 50GB = the 20GB test .dbn)
|
||||
— ParentVol dc144e62 (test-data-pvc).
|
||||
- BOTH `available` in ~20s, in the **bizworx project 96d0717d** (survive foxhunt cluster teardown), zone fr-par-2.
|
||||
- **RESTORE:** `scw block volume create from-snapshot-id=<snap> zone=fr-par-2` → attach to a pod → read .dbn.
|
||||
- Cleaned up: dbn-uploader + data-inspector pods deleted, dbn-archive-s3 secret deleted.
|
||||
- CORRECTION (2026-07-18 later): there is NO stray junk. The earlier "132MB / 23 objects" were the FIRST partial
|
||||
`.dbn` transfers under `training/` — the resumed upload continues them (rclone skips already-present). Verified:
|
||||
bucket holds ONLY legit `training/*.dbn*` (singletest/ + speedtest/ test files were deleted during testing = 0
|
||||
objects). **DO NOT wipe/delete `fxhnt-dbn-archive` — it IS the active archive we're building.**
|
||||
- Node drain RESUMED (platform pool min-size 1->0; no blocking pods). Autoscaler drops the last node to 0.
|
||||
|
||||
## STATE SUMMARY (2026-07-18 ~21:15 UTC)
|
||||
- fxhnt.ai fund LIVE on bizworx (T8+T9 done, cockpit verified in real Chrome). foxhunt compute → 0 (draining).
|
||||
- Data fully preserved: pg dump (~/Work/fxhnt-cutover-backup), surfer .dbn (migrated to bizworx PVC + verified
|
||||
from dagster pod), archived 171GB .dbn (2 block snapshots in bizworx project). foxhunt PVCs still exist too.
|
||||
- PENDING: tomorrow 09:00 UTC ucits run (real-money confirm); then T12 terraform destroy (operator confirm) —
|
||||
which also removes the source PVCs, but snapshots + dump + bizworx-live cover everything. ROTATE tailscale key.
|
||||
Cleanup the stray fxhnt-dbn-archive bucket.
|
||||
|
||||
## .dbn S3 COPY (2026-07-18 ~21:53 UTC) — operator: also copy compressed .dbn.zst to S3, KEEP snapshots
|
||||
The .dbn.zst are already compressed → S3 copy is a good browsable archive alongside the snapshots (belt+braces).
|
||||
- Measured real S3 throughput: ~10-17 MB/s from these small VMs (dev1_l/gp1_xs) — NOT compression-limited, it's
|
||||
node network egress. Single-file test: 537MiB in 52s = 10MB/s, PVC read 217MB/s (fine). The earlier "0 B/s
|
||||
stall" was rclone stats not updating until a big multipart completed + too many concurrent 64M-chunk streams
|
||||
choking the node. FIX: --transfers 2 --s3-chunk-size 32M --s3-upload-concurrency 2 (node-friendly) → moving.
|
||||
- Restore-snapshot-to-volume path abandoned (Kapsule CSI import of a raw scw volume = rabbit hole). Instead
|
||||
uploading directly from a foxhunt pod (dbn-uploader) that mounts training+test PVCs RO — data still live there.
|
||||
- Upload RUNNING in background: rclone copy .dbn* → scw:fxhnt-dbn-archive/{training,test}/. 137GiB, ETA ~6-7h at
|
||||
~6MB/s. Persistent Monitor watching for completion. Sentinel /tmp/rclone-done in the pod.
|
||||
- KEEP THE SNAPSHOTS (operator) — validated safety net; only after S3 upload verified (object count + sizes) do
|
||||
we consider anything. Both archives coexist.
|
||||
- foxhunt platform pool held at min-size=1 for the uploader; set min-size=0 to resume drain AFTER upload done.
|
||||
- RESUME/CHECK: `kubectl --context $FOX -n foxhunt exec dbn-uploader -- sh -c 'cat /tmp/rclone-done; tail -1 /tmp/rclone-training.log'`
|
||||
and `... rclone size scw:fxhnt-dbn-archive`. If pod died: re-create it (manifest in git history commit 2438294-ish) — rclone copy is resumable/idempotent.
|
||||
|
||||
## .dbn S3 COPY COMPLETE + VERIFIED (2026-07-19 ~02:41 UTC) ✅
|
||||
- Upload finished: **80 objects / 171.393 GiB** in `fxhnt-dbn-archive` (private, fr-par, bizworx project).
|
||||
- VERIFIED byte-for-byte vs source PVCs: `rclone check --size-only --one-way`:
|
||||
- training: **0 differences, 59 matching files** ✅
|
||||
- test: **0 differences, 21 matching files** ✅
|
||||
- source .dbn* count = 80 = bucket object count = exact.
|
||||
- **TWO independent verified archives now exist** (operator: keep snapshots): (1) S3 object storage
|
||||
`s3://fxhnt-dbn-archive/{training,test}/*.dbn.zst` (browsable, per-file restore), (2) block snapshots
|
||||
fxhnt-training/test-dbn-archive-2026-07-18 (564a4ccb / 5ba0049f, bizworx project). Both safe post-teardown.
|
||||
- Cleaned up: dbn-uploader + s3-speedtest pods, dbn-archive-s3 secrets (both clusters). Node drain RESUMED
|
||||
(platform pool min-size 1->0; no blocking pods) → autoscaler drops the last dev1_l to 0. foxhunt compute → ~$0.
|
||||
- REMAINING (next session / operator): tomorrow 09:00 UTC first armed ucits run (real-money confirm); T12
|
||||
terraform destroy foxhunt Kapsule (removes source PVCs — both .dbn archives + pg dump + bizworx-live cover it);
|
||||
rotate the pasted tailscale key. Restore .dbn: `rclone copy scw:fxhnt-dbn-archive/... <dest>` OR
|
||||
`scw block volume create from-snapshot.snapshot-id=<snap> perf-iops=5000 zone=fr-par-2`.
|
||||
|
||||
## SCHEDULING CONSISTENCY — ALL-DAGSTER K8sRunLauncher UPGRADE DONE (2026-07-19) ✅
|
||||
Operator: "go all-Dagster properly + build the image right." Executed the 5-task plan
|
||||
(docs/superpowers/plans/2026-07-19-production-scheduling-consistency.md). Result = consistent single-orchestrator
|
||||
ETL, execution isolated in crons, nothing suspended-by-accident, VRP gone, documented.
|
||||
- DIAGNOSIS (corrected): Dagster ETL was NEVER broken (23:30 ran SUCCESS, forward_nav fresh). The stale
|
||||
bybit_sleeve_ret was the SUSPENDED compare-measured cron (exists because the compute needs 6-8Gi, OOM-killed
|
||||
the 4Gi daemon). Fix = K8sRunLauncher, not naive daemon assets.
|
||||
- BUILT: dagster-k8s dep added; image built via in-cluster **Kaniko** (devbox podman blocked by sandbox mount
|
||||
perms) → `fxhnt-cockpit:dagster-k8s-25e3d7c`, verified + retagged `:latest` (skopeo Job).
|
||||
- RBAC: tailscale-dagster SA can create Jobs/pods/events. K8sRunLauncher configured (launched Jobs git-sync
|
||||
master, no PVC, fund secrets). New asset `bybit_book_precompute` (op_tags 6-8Gi) wraps `_persist_bybit_book`
|
||||
(moved to application/bybit_book_persist.py). MERGED branch→master (git-sync pulls master).
|
||||
- VERIFIED (the money shot): triggered combined_book_forward_job → launched Job dagster-run-* → materialized all
|
||||
14 assets incl the heavy precompute → **daemon 0 restarts** → bybit_sleeve_ret + backtest_summary + forward_nav
|
||||
ALL fresh 2026-07-19. Run SUCCESS 6m54s.
|
||||
- CRONS (final): DELETED compare-measured (→ asset) + vrp (shelved). ARMED factory + ucits-rebalancer +
|
||||
ucits-volume-ibkr. SUSPENDED bybit-rebalancer (no creds) + ibkr-rebalancer (PRIIPs). Manifests updated.
|
||||
- Memory: [[reference_fxhnt_dagster_k8srunlauncher_scheduling]] created; VRP files + MEMORY.md updated.
|
||||
- NOTE: the fund now runs off MASTER (28-commit consolidation branch merged 06156bd; +cron changes 1ecf398).
|
||||
127
docs/superpowers/plans/T8-CUTOVER-RUNBOOK.md
Normal file
127
docs/superpowers/plans/T8-CUTOVER-RUNBOOK.md
Normal file
@@ -0,0 +1,127 @@
|
||||
# T8 — Cutover Runbook (foxhunt → bizworx)
|
||||
|
||||
> Executable, verify-first, reversible. Expands Task 8 of the consolidation plan with the exact values
|
||||
> discovered in T8-prep. **foxhunt stays intact (suspended, NOT deleted) — rollback = re-point DNS +
|
||||
> un-suspend foxhunt.** Fund is ~1×/day; downtime OK. Run in a quiet window.
|
||||
|
||||
## Contexts / constants (verified this session)
|
||||
```
|
||||
FOX=foxhunt-34a1e3c4-ac35-48c8-ab49-5f6ec4df32c1 # foxhunt Kapsule (retiring)
|
||||
BIZ=bizworx-prod-kapsule-05e1dae3-8f35-4ad4-8fce-0f9a7691a760 # survivor
|
||||
```
|
||||
- **foxhunt API is IP-allowlisted.** If `6443` resets: `curl -s ifconfig.me` → compare to
|
||||
`scw k8s acl list cluster-id=34a1e3c4-…`; if drifted, `scw k8s acl add cluster-id=34a1e3c4-…
|
||||
acls.0.ip=<newIP>/32 acls.0.description=bizworx-devbox-vN` (APPENDS — verify all entries survive).
|
||||
bizworx never hits this (tailscale-cgnat range).
|
||||
- **DNS zone:** Scaleway `fxhnt.ai` (project c293eb98). Records set with `scw dns record set dns-zone=fxhnt.ai ...`.
|
||||
|
||||
## The single most important safety fact
|
||||
The **fund pipeline never uses `fxhnt.ai`** — git-sync = `gitea-ssh.gitea.svc.cluster.local:22` (in-cluster),
|
||||
cockpit/dagster serve over ClusterIP + their own tailnet IP. `fxhnt.ai` is **browser/human access only**.
|
||||
→ The fund keeps running through the DNS flip; the DNS cutover only affects the operator's browser access.
|
||||
|
||||
## Current fxhnt.ai ingress (all → 100.95.225.27 = `tailscale-gitlab-proxy` pod, ns foxhunt, DIES at teardown)
|
||||
| host | fronts | cutover |
|
||||
|---|---|---|
|
||||
| git.fxhnt.ai | gitea-web.foxhunt:3000 | → **bizworx gitea** (ns gitea, mirrored) |
|
||||
| dashboard/cockpit.fxhnt.ai | fxhnt-dashboard node → cockpit:8080 | → **bizworx cockpit** |
|
||||
| dagster.fxhnt.ai | dagster.foxhunt:80 | → **bizworx dagster** |
|
||||
| grafana / api / mail / chat / minio | legacy | **DROP** (operator confirmed grafana can go too) |
|
||||
|
||||
Only **git + dashboard/cockpit + dagster** move. Everything else is dropped at Phase-2/teardown.
|
||||
|
||||
## TLS: solved-path (no blocker)
|
||||
Old proxy served a cert-manager wildcard `*.fxhnt.ai` (ClusterIssuer `letsencrypt-prod`, Scaleway DNS-01).
|
||||
**bizworx already has the identical stack** — cert-manager + `scaleway-certmanager-webhook` (1/1) + `scaleway-api`
|
||||
creds + `letsencrypt-prod` ClusterIssuer, already issuing real LE certs in ~10 ns. → Issuing `*.fxhnt.ai` on
|
||||
bizworx is just a `Certificate` resource.
|
||||
|
||||
---
|
||||
|
||||
## PRE-CUTOVER (do these BEFORE the window; all non-destructive, foxhunt untouched)
|
||||
|
||||
### P1. Issue the `*.fxhnt.ai` wildcard cert on bizworx (ns foxhunt) — needs the DNS-01 solver, so do it early
|
||||
- [ ] Create `infra/k8s/services/fxhnt-wildcard-cert.yaml`: a cert-manager `Certificate` (dnsNames
|
||||
`fxhnt.ai`,`*.fxhnt.ai`; issuerRef ClusterIssuer `letsencrypt-prod`; secretName `fxhnt-wildcard-tls`;
|
||||
namespace `foxhunt`). Apply on `$BIZ`.
|
||||
- [ ] **VERIFY:** `kubectl --context $BIZ -n foxhunt get certificate fxhnt-wildcard -o wide` → READY=True.
|
||||
(DNS-01 provisions a TXT on the fxhnt.ai zone; the same zone we control. Wait until True — can take minutes.)
|
||||
- **EVAL P1:** wildcard cert READY on bizworx. foxhunt untouched. If it fails → debug the webhook, do NOT proceed.
|
||||
|
||||
### P2. Decide + build the fxhnt.ai front on bizworx (the thing DNS will point at)
|
||||
Two acceptable shapes — **default = a stripped fund-only nginx proxy** mirroring the foxhunt one (faithful,
|
||||
single tailnet node for DNS, wildcard TLS termination):
|
||||
- [ ] Create `infra/k8s/services/fxhnt-ingress-proxy.yaml` on `$BIZ` ns foxhunt: nginx + tailscale sidecar
|
||||
(`TS_HOSTNAME=fxhnt-ingress`), wildcard `fxhnt-wildcard-tls` mounted at `/etc/nginx/certs`, vhosts:
|
||||
- `git.fxhnt.ai` → `http://gitea-http.gitea.svc.cluster.local:3000` (CONFIRMED svc; headless/ClusterIP:None
|
||||
→ use nginx `resolver` + variable upstream, or proxy_pass by name works via cluster DNS round-robin)
|
||||
- `dashboard.fxhnt.ai cockpit.fxhnt.ai` → `http://fxhnt-dashboard.foxhunt.svc.cluster.local:80` (CONFIRMED)
|
||||
- `dagster.fxhnt.ai` → `http://dagster.foxhunt.svc.cluster.local:80` (CONFIRMED)
|
||||
+ socat sidecar for `git.fxhnt.ai:22` SSH → `gitea-ssh.gitea.svc.cluster.local:22` (CONFIRMED; human git over SSH).
|
||||
- [ ] **VERIFY** (still pre-flip, over tailnet directly to the proxy's tailnet IP, Host header spoofed):
|
||||
`curl -k --resolve dashboard.fxhnt.ai:443:<proxy-tailnet-ip> https://dashboard.fxhnt.ai/ -I` → 200,
|
||||
and cert CN/SAN covers `*.fxhnt.ai`. Repeat for git + dagster.
|
||||
- **EVAL P2:** all 3 fund vhosts serve 200 with a valid `*.fxhnt.ai` cert via the bizworx proxy — WITHOUT any
|
||||
DNS change yet. Record the proxy's tailnet IP for step C4. If any vhost fails → fix before the window.
|
||||
- NOTE: this proxy points at the in-cluster **Services** (`fxhnt-dashboard.foxhunt`, `dagster.foxhunt`), so the
|
||||
"reclaim tailnet identity" rename of the pods is **not required for browser access** — the proxy uses
|
||||
ClusterIP, not the pod's tailnet name. (Identity rename becomes optional cosmetic; keep it out of the
|
||||
critical path.)
|
||||
|
||||
---
|
||||
|
||||
## CUTOVER WINDOW (the only steps that touch live foxhunt — each is reversible)
|
||||
|
||||
### C1. Freeze foxhunt writers (make its Postgres quiescent)
|
||||
- [ ] Suspend every foxhunt fund CronJob:
|
||||
`for cj in $(kubectl --context $FOX -n foxhunt get cronjobs -o name); do kubectl --context $FOX -n foxhunt patch $cj -p '{"spec":{"suspend":true}}'; done`
|
||||
- [ ] Scale down foxhunt writers so nothing writes to its DB during the delta dump:
|
||||
`kubectl --context $FOX -n foxhunt scale deploy/dagster deploy/fxhnt-dashboard --replicas=0`
|
||||
(cockpit is read-only but scale it too to release the `fxhnt-dashboard` tailnet name; dagster daemon writes.)
|
||||
- [ ] Scale foxhunt `ib-gateway` to 0 → **releases the sole IBKR session** so bizworx ib-gateway can hold it:
|
||||
`kubectl --context $FOX -n foxhunt scale deploy/ib-gateway --replicas=0`
|
||||
- **VERIFY C1:** `kubectl --context $FOX -n foxhunt get deploy` all fund writers at 0/0; cronjobs SUSPEND=true.
|
||||
- **ROLLBACK:** scale back up + un-suspend — foxhunt fully restored.
|
||||
|
||||
### C2. Final delta Postgres dump/restore (frozen source → bizworx)
|
||||
Re-run the proven Task-6 procedure on the now-frozen `fxhnt` DB (version-pinned tsdb 2.25.1 both sides).
|
||||
- [ ] `pg_dump -Fc` the `fxhnt` DB from foxhunt postgres (port-forward or exec), `pg_restore --no-owner
|
||||
--no-acl --clean --if-exists` into the bizworx `fxhnt` DB (which currently holds the rehearsal copy →
|
||||
`--clean` refreshes it), wrapped in `timescaledb_pre_restore()` / `post_restore()`.
|
||||
- [ ] **VERIFY C2:** re-check the canonical row counts on bizworx match the (now-frozen) foxhunt source:
|
||||
`forward_nav, backtest_summary, bybit_sleeve_ret, xsp_option_bars, exec_fills, ibkr_account_nav`
|
||||
+ `SELECT max(t0) FROM forward_anchor` matches. 37 tables / 3 hypertables.
|
||||
- **EVAL C2:** every checked count matches the frozen source. If not → STOP, do NOT flip DNS (rollback = C1 up).
|
||||
|
||||
### C3. Bring the bizworx fund fully live (still no DNS)
|
||||
- [ ] `kubectl --context $BIZ -n foxhunt scale deploy/dagster deploy/fxhnt-dashboard deploy/ib-gateway --replicas=1`
|
||||
(they were already deployed; ensure 1 replica). ib-gateway should now stay 1/1 (sole IBKR session).
|
||||
- [ ] **VERIFY C3:** on bizworx — postgres 1/1, cockpit 2/2, dagster 3/3, **ib-gateway 1/1 stable** (no more
|
||||
crashloop now that foxhunt released the session). Confirm ib-gateway API port + one **whatIf** dry-run.
|
||||
- **EVAL C3:** all bizworx fund pods healthy incl. ib-gateway stable. If ib-gateway still crashes → confirm
|
||||
foxhunt ib-gateway is really at 0; do NOT arm any rebalancer.
|
||||
|
||||
### C4. Flip DNS (Scaleway fxhnt.ai zone → bizworx proxy tailnet IP)
|
||||
**TARGET IP (verified in EVAL P2): `100.93.141.11` (tailnet node `fxhnt-ingress`).**
|
||||
- [ ] `scw dns record set dns-zone=fxhnt.ai name=dashboard type=A data=100.93.141.11 ttl=60`
|
||||
- [ ] same for `name=cockpit`, `name=dagster`, `name=git`. (TTL was 300; lower to 60 during cutover.)
|
||||
(Records currently point at 100.95.225.27 = the dying foxhunt tailscale-gitlab-proxy.)
|
||||
- [ ] Leave grafana/api/mail/chat/minio records as-is for now (dropped at Phase-2; harmless dangling until then).
|
||||
- **VERIFY C4:** from an external network (phone off-wifi is fine): `dig +short dashboard.fxhnt.ai` = new IP;
|
||||
open `https://dashboard.fxhnt.ai` in a browser → cockpit renders migrated data, **valid TLS** (LE `*.fxhnt.ai`).
|
||||
`https://git.fxhnt.ai` serves the bizworx gitea; `git ls-remote ssh://git@git.fxhnt.ai/...` works.
|
||||
- **ROLLBACK:** `scw dns record set ... data=100.95.225.27` (back to the still-live foxhunt proxy) + C1-up +
|
||||
un-suspend foxhunt cronjobs. Fully reversible until Phase-2 teardown.
|
||||
|
||||
### C5. Arm the fund on bizworx (start conservative)
|
||||
- [ ] Un-suspend ONLY the UCITS rebalancer on bizworx (already the armed one; US + bybit stay suspended):
|
||||
`kubectl --context $BIZ -n foxhunt patch cronjob fxhnt-ucits-rebalancer -p '{"spec":{"suspend":false}}'`
|
||||
- [ ] **VERIFY C5:** its next scheduled run (09:00 UTC EU-hours) succeeds — a real fill or a clean gate WAIT,
|
||||
not a crash. Watch the Job pod: git-sync OK, DB reachable, ib-gateway reachable, recon-gate evaluated.
|
||||
- **EVAL C5:** first armed run green. If it errors → re-suspend, investigate; foxhunt still intact for rollback.
|
||||
|
||||
---
|
||||
|
||||
## POST-CUTOVER = Task 9 (verify) → Task 10 (observation window) → Task 11 (triage) → Task 12 (teardown)
|
||||
- **Rotate the pasted tailscale auth-key** (was in transcript) once the bizworx pods are settled.
|
||||
- **DELETE NOTHING** on foxhunt until T10 observation passes. foxhunt = suspended-but-intact through T12.
|
||||
@@ -0,0 +1,171 @@
|
||||
# Paper book: ISV-adaptive sleeve weighting — Design (Phase 2F-A3c)
|
||||
|
||||
**Date:** 2026-06-21
|
||||
**Status:** Design (approved)
|
||||
**Repo:** fxhnt
|
||||
**Follows:** A3 replay + A3b equal-weight. Replaces naive equal-weight with ISV-adaptive risk weighting.
|
||||
|
||||
## Motivation
|
||||
|
||||
A3b sizes the paper book by **equal-weight** (1/N active sleeves). Equal-weight ignores risk: a high-vol
|
||||
sleeve (ts-trend) gets the same book share as a low-vol one (stablecoin), so the blended curve is dominated
|
||||
by whichever sleeve is most volatile. We want the replayable sleeves sized by **inverse-vol** (risk-parity),
|
||||
and — per the user's call — with the weighting **meta-parameters themselves state-derived (ISV-adaptive)**
|
||||
rather than hardcoded, applying the foxhunt ISV discipline (first-observation bootstrap, hard floors,
|
||||
state-derived bounds, max-with-floor not blend). We want to *see the effect* in both live `/paper` and
|
||||
`/paper/replay`.
|
||||
|
||||
## Key finding (drove the design)
|
||||
|
||||
The obvious return source — each sleeve's `forward_nav` series — **does not exist** for the replayable
|
||||
sleeves. `forward_nav` (the live gate record) only has rows for `combined` (15d), `sixtyforty` (3d),
|
||||
`xsfunding` (1d). `crypto_tstrend` / `stablecoin_rotation` / `unlock` have **zero** rows. So the return
|
||||
series must come from the paper book's **own simulated per-sleeve daily returns**, computed causally as the
|
||||
backfill walks dates chronologically — not from `forward_nav`.
|
||||
|
||||
## Decisions (from brainstorming)
|
||||
|
||||
- **Returns are the sleeves' own simulated daily P&L.** On date *d*, sleeve *s*'s return =
|
||||
Σ qty·(close@*d* − close@*d−1*) over *s*'s positions carried from *d−1*, ÷ *s*'s notional that day. The
|
||||
backfill walks dates chronologically and accumulates `returns_by_sleeve[s]` causally (each date weights on
|
||||
returns strictly *before* it → no look-ahead).
|
||||
- **ISV-adaptive meta-params** (user chose "full ISV-adaptive" over mirroring the live `book_allocator`). The
|
||||
base inverse-vol math mirrors `book_allocator._inverse_vol_weights` (raw `1/std` normalize), but the cap is
|
||||
replaced by a two-sided water-fill so floor+cap resolve together; the three meta-params are state-derived
|
||||
with bootstrap + floors (detailed below).
|
||||
- **Persist per-sleeve daily returns** in a new `paper_sleeve_ret(run_date, sleeve, ret)` table, written by
|
||||
BOTH backfill and nightly snapshot, so the snapshot (which sees only one new date/run) builds the identical
|
||||
trailing window the backfill does → live `/paper` and `/paper/replay` use the **same** ISV-adaptive logic.
|
||||
("do for both — we see the effect.")
|
||||
- **Cost accepted:** the adaptive vol path diverges from the live edge-book `book_allocator` (which uses
|
||||
fixed-window `np.std`). Live and replay stay consistent with *each other* (same helper).
|
||||
|
||||
## ISV-adaptive weighting (the three state-derived meta-params)
|
||||
|
||||
Given `returns_by_sleeve` (`{sleeve: {epoch_day: ret}}`, causal) and the day's `active_sleeves`. (Review
|
||||
fixes #2/#4/#6 from the critical-review pass are folded in below.)
|
||||
|
||||
**A. Adaptive lookback `L_t`** (replaces fixed `VOL_LOOKBACK=60`) — **#2 disjoint-window signal**
|
||||
- Regime signal compares **two DISJOINT windows** (not nested) so it actually moves under stationarity:
|
||||
pool the active sleeves' most-recent returns into `recent = last L_min` and `prior = the L_min before
|
||||
those`. `ρ = std(recent) / std(prior)` (if `prior` has < `L_min` points or `std(prior) ≤ 1e-12` →
|
||||
`ρ = 1.0`, i.e. hold). `ρ>1` = vol rising vs the preceding block → shorten the window to react.
|
||||
- `L_t = round(clip(L_max / max(1.0, ρ), L_min, L_max))`. **Floor `L_min=10`**, **anchor `L_max=60`**
|
||||
(= live book). Until `≥ 2·L_min` obs exist (can't form disjoint windows), `ρ=1` → `L_t=L_max` → use all
|
||||
available (bootstrap). Vol-of-vol is the more general signal but needs more history than this book has;
|
||||
the disjoint recent-vs-prior ratio is the robust minimum that still deviates from 1 under a regime shift.
|
||||
|
||||
**B. Adaptive cold-start + floor** (replaces `MIN_OBS=20` cliff — ISV first-obs bootstrap + weight floor)
|
||||
- **First-observation bootstrap:** a sleeve with `≥2` returns contributes an inverse-vol estimate; a sleeve
|
||||
with `<2` returns can't form a std → see #6 (mixed-history) below. No cliff, no NaN.
|
||||
- **Weight floor is a bounded water-fill, NOT naive floor-then-renormalize (#4).** Naive renormalize can
|
||||
push a just-floored weight back below the floor, so we use the **same iterative clamp-and-renormalize loop**
|
||||
`_inverse_vol_weights` uses for the cap, but two-sided: each pass clamps every weight into
|
||||
`[w_floor, cap_t]`, redistributes the freed/needed mass across the unclamped sleeves, repeats until stable
|
||||
(≤ `N` passes). Feasible because `w_floor·N = φ = 0.5 ≤ 1 ≤ cap_t·N` (cap_t ≥ 1/N). `w_floor = φ/N_active`
|
||||
(`φ=0.5`, floor below equal-weight so the inverse-vol tilt survives).
|
||||
|
||||
**C. Adaptive concentration cap `cap_t`** (replaces `MAX_SLEEVE_WEIGHT=0.5`)
|
||||
- Derived from the **effective number of bets** of the raw (pre-cap) inverse-vol weights:
|
||||
`N_eff = (Σw)² / Σw²` (participation ratio — a pure state measure of concentration).
|
||||
- `cap_t = clip(κ / N_eff, 1.0/N_active, 1.0)`, `κ=2`. **Floored at `1/N_active`**, ceiled `1.0`. Fed as the
|
||||
upper bound of the two-sided water-fill in B (so cap + floor are resolved *together*, defined precedence:
|
||||
one joint clamp range `[w_floor, cap_t]`, not cap-then-floor). NOTE: at `N_active ≤ 3` the cap rarely binds
|
||||
(documented in the critical review #3); it's retained for when the book grows.
|
||||
|
||||
**#6 — mixed-history rule (some active sleeves have returns, some don't):** every active sleeve is in the
|
||||
output. Sleeves with `≥2` returns are weighted by inverse-vol over the last `L_t` obs; active sleeves with
|
||||
`<2` returns get **exactly `w_floor`** (max-uncertainty → minimum conviction, the ISV max-with-floor form),
|
||||
and the remaining mass is water-filled across the has-history sleeves by inverse-vol. If **no** active sleeve
|
||||
has `≥2` returns → equal-weight (1/N) over active.
|
||||
|
||||
**Structural constants** (shape anchors, each with a floor — not fit to data): `L_min=10`, `L_max=60` (live
|
||||
anchor), `φ=0.5`, `κ=2`. The per-date `L_t`, `cap_t`, and weights all move with observed state.
|
||||
|
||||
**Order of operations** (per date): restrict to active → if no active sleeve has `≥2` returns, equal-weight
|
||||
(1/N) → else compute `ρ→L_t` (disjoint windows), per has-history sleeve take last `L_t` obs and its inverse
|
||||
std; compute `N_eff→cap_t` from the raw inverse-vol weights; assign `<2`-history active sleeves `w_floor`;
|
||||
run the two-sided water-fill (clamp to `[w_floor, cap_t]` + renormalize, iterate) → final weights sum 1.
|
||||
|
||||
## Components
|
||||
|
||||
1. **Shared helper** `isv_adaptive_weights(returns_by_sleeve, active_sleeves, *, l_min=10, l_max=60,
|
||||
phi=0.5, kappa=2.0) -> {sleeve: float}` in `application/paper_book.py`. Pure function (no I/O). Computes
|
||||
the **raw inverse-vol weights** (`∝ 1/std` over the last `L_t` obs, normalized to sum 1 — same formula as
|
||||
`_inverse_vol_weights`' first stage, used to also derive `N_eff→cap_t`), then applies the **two-sided
|
||||
water-fill** for cap+floor (B/C) — it does NOT call `_inverse_vol_weights` with an internal cap (that
|
||||
would double-cap); it owns the single joint clamp. Adds A (`L_t`). Returns `{}` for no active sleeves;
|
||||
equal-weight (1/N) when no active sleeve has `≥2` returns.
|
||||
1b. **Shared per-sleeve return fn (#5 — one definition, no drift)** `sleeve_daily_return(prev_positions,
|
||||
close_by_symbol) -> float | None` in `application/paper_book.py` (pure). The prior-date close is carried
|
||||
by `Position.entry_price` (positions are re-derived each date at that date's close), so no separate
|
||||
prev_close arg is needed. Returns `Σ qty·(close − entry_price) / Σ|qty·entry_price|` over the sleeve's
|
||||
prior-date positions, or `None` when there is no prior position / notional is 0 (unknown symbol →
|
||||
`close.get(sym, entry_price)` = 0 contribution). BOTH `backfill_paper_book` and `PaperSnapshotService`
|
||||
call this — they must NOT each re-derive the formula. Causality: a date's return uses that date's close,
|
||||
so it is computed AFTER sizing the date and used only to size the NEXT date.
|
||||
2. **`paper_sleeve_ret` table** (`adapters/persistence/cockpit_models.py`): `(run_date, sleeve)` PK,
|
||||
`ret float`, `src_updated_at`. Timescale-friendly like `paper_nav`.
|
||||
3. **PaperRepo** (`adapters/persistence/paper_repo.py`): `upsert_sleeve_ret(run_date, sleeve, ret)`
|
||||
(idempotent per (run_date, sleeve)) and `sleeve_returns(before=run_date) -> {sleeve: {epoch_day: ret}}`
|
||||
(trailing series strictly before a date, for the snapshot path).
|
||||
4. **`backfill_paper_book`** (`application/paper_backfill.py`): walking dates in order, after pricing each
|
||||
date compute each active sleeve's daily return via `sleeve_daily_return(prior positions, prior closes,
|
||||
today closes)`, append (skip `None`) to in-memory `returns_by_sleeve`, and use
|
||||
`isv_adaptive_weights(returns_by_sleeve, active)` for that date's sizing (instead of `equal_weights`).
|
||||
Persist each non-`None` return via `upsert_sleeve_ret`. Day-0 of a sleeve has no prior position →
|
||||
`None` → no return (equal-weight cold-start covers it).
|
||||
5. **`PaperSnapshotService`** (`application/paper_snapshot.py`): load the trailing series with
|
||||
`paper_repo.sleeve_returns(before=run_date)`, compute the run_date's own per-sleeve return via the SAME
|
||||
`sleeve_daily_return` (prior persisted positions + today's closes), append, then
|
||||
`isv_adaptive_weights(...)`; persist via `upsert_sleeve_ret`. Identical logic path to the backfill (#5).
|
||||
|
||||
## Data flow
|
||||
|
||||
Backfill (chronological) → per date: active sleeves' `current_weights` → prior-date per-sleeve return →
|
||||
append to causal window → `isv_adaptive_weights` (ρ→L_t, N_eff→cap_t, first-obs bootstrap, w_floor) →
|
||||
`persist_paper_book` + `upsert_sleeve_ret`. Nightly snapshot → read trailing returns from `paper_sleeve_ret`
|
||||
+ compute today's → `isv_adaptive_weights` → persist. `/paper` and `/paper/replay` both reflect the weights.
|
||||
|
||||
## Risks / safety
|
||||
|
||||
- **Read-only/additive.** No change to sleeve `run()`/`advance()`, `combine_book`, or the gate record. Only
|
||||
the paper sizing + the new returns table/writes.
|
||||
- **Look-ahead:** each date weights on returns strictly *before* it (causal). Bootstrap/short-history uses
|
||||
equal-weight; inverse-vol engages once a sleeve has a usable vol estimate.
|
||||
- **Degenerate vol:** `_inverse_vol_weights` already returns equal-weight when total inverse-vol ≤ 0; the
|
||||
two-sided water-fill (clamp `[w_floor, cap_t]` + iterate) guarantees no sleeve is starved AND the floor
|
||||
actually holds post-renormalize (#4); `cap_t` floored at `1/N_active` so the range is always feasible.
|
||||
- **Overfitting on the tiny 3-sleeve sample** (the caution raised when choosing full-ISV): contained by the
|
||||
hard floors (`L_min`, `w_floor`, `cap_t≥1/N`) and the equal-weight bootstrap — degrades to equal-weight,
|
||||
never to a degenerate/flat book.
|
||||
- **Consistency:** the same `isv_adaptive_weights` helper + the same `paper_sleeve_ret` history feed both the
|
||||
backfill and the nightly snapshot.
|
||||
- **Idempotent:** `upsert_sleeve_ret` keyed by (run_date, sleeve); re-run backfill replaces, no dup.
|
||||
|
||||
## Out of scope
|
||||
|
||||
`_marginal_prune` correlation pruning (the paper book uses the raw inverse-vol path); fractional-Kelly
|
||||
leverage / vol-targeting the paper book (sleeve weights only, sum 1); EWMA-vol rewrite of the live
|
||||
`book_allocator`; B alerts panel; back-computing xsfunding; the `_live_sleeves`↔backfill dedup (tech debt).
|
||||
|
||||
## Acceptance criteria
|
||||
|
||||
- `isv_adaptive_weights`: active-filter, equal-weight bootstrap (no `≥2`-history sleeve), adaptive `L_t` from
|
||||
the **disjoint** regime ratio, adaptive `cap_t` from `N_eff`, mixed-history rule, two-sided water-fill;
|
||||
unit-tested — (i) empty→`{}`, (ii) no-history→1/N, (iii) low-vol sleeve gets a higher weight than a
|
||||
high-vol one, (iv) rising-vol regime (disjoint `ρ>1`) shrinks `L_t` below `L_max` while a flat-vol series
|
||||
keeps `L_t=L_max`, (v) `cap_t` never below `1/N`, (vi) **after the water-fill every active weight is in
|
||||
`[w_floor, cap_t]` and the book sums to 1** (the iterative loop, not naive renormalize), (vii) a mixed
|
||||
book (one sleeve with history, one with `<2` returns) gives the history-less sleeve exactly `w_floor`.
|
||||
- `sleeve_daily_return(prev_positions, close_by_symbol)` pure fn unit-tested (long & short positions,
|
||||
unknown-symbol→0 contribution, no-prior→`None`, zero-notional→`None`); both backfill + snapshot import it
|
||||
(no duplicated formula).
|
||||
- `paper_sleeve_ret` table + `PaperRepo.upsert_sleeve_ret`/`sleeve_returns`; idempotent; tested.
|
||||
- `backfill_paper_book` builds the causal per-sleeve return window, uses `isv_adaptive_weights`, persists
|
||||
returns; integration-tested with a fake warehouse over multiple dates (weights move off 1/N once history
|
||||
accrues; low-vol sleeve > high-vol sleeve weight).
|
||||
- `PaperSnapshotService` reads the trailing returns + uses `isv_adaptive_weights`; tested.
|
||||
- Re-run backfill in-cluster → `paper_sleeve_ret` populated; `/paper/replay` curve NON-flat and now
|
||||
risk-weighted; `/paper` live book uses the same weights; verified post-deploy.
|
||||
- Full suite green; read-only/additive; no `*.dbn` staged.
|
||||
@@ -0,0 +1,73 @@
|
||||
# Paper book: equal-weight replayable sleeves + wire unlock to warehouse — Design (Phase 2F-A3b)
|
||||
|
||||
**Date:** 2026-06-21
|
||||
**Status:** Design (approved)
|
||||
**Repo:** fxhnt
|
||||
**Follows:** 2F MVP + 2F-2 snapshot + A3 replay. Fixes the flat/empty paper book.
|
||||
|
||||
## Motivation
|
||||
|
||||
Diagnosis: `/paper` + `/paper/replay` render but the book is **empty/flat** — `paper_positions` has 0 rows
|
||||
across all 365 backfilled dates. Two root causes:
|
||||
1. **Sleeve weights are `{}`.** The backfill + nightly snapshot size positions by `current_sleeve_weights`
|
||||
(the live `book_allocator`), but the allocator weights a DIFFERENT edge set (crypto-momentum /
|
||||
futures-trend), not the replayable paper sleeves (ts-trend / stablecoin / unlock). So every replayable
|
||||
sleeve gets weight 0 → notional 0 → no positions. (Verified: ts-trend `current_weights` returns 70 names
|
||||
on the latest date, but `_paper_backfill_sleeve_weights()` returns `{}`.)
|
||||
2. **unlock builds from the absent `crypto_pit_dir`** (`/root/.fxhnt/crypto_pit`, not in-cluster) for its
|
||||
vol panel → it raises and is skipped. The warehouse (197 USDT symbols × 365 closes) holds those vols.
|
||||
|
||||
## Decisions (from brainstorming)
|
||||
|
||||
- **Equal-weight the active replayable sleeves**: per date, `sleeve_weight = 1/N` where N = the sleeves
|
||||
that produced a non-empty `current_weights(as_of)` that day. Simple, transparent, no chicken-and-egg.
|
||||
(Inverse-vol/fixed weights are possible refinements — out of scope.)
|
||||
- **Shared logic**: the equal-weight sizing must be used by BOTH the backfill AND the nightly
|
||||
`paper_book_snapshot` asset, so the live `/paper` view and the replay agree and neither is flat.
|
||||
- **unlock** built from the **warehouse** vol panel (same `{sym:{epoch_day:close}}` the ts-trend backfill
|
||||
uses, augmented to the (close, qvol, funding) tuple the unlock runner needs where available) +
|
||||
`UnlockCalendarLive` (3,185 events incl. historical cliffs covering the window). NOT `crypto_pit_dir`.
|
||||
|
||||
## Scope
|
||||
|
||||
1. **Equal-weight sizing** (`application/paper_book.py` or `paper_snapshot.py`): a helper
|
||||
`equal_weights(symbol_weights_by_sleeve) -> {sleeve: 1/N}` over the non-empty sleeves; used by
|
||||
`backfill_paper_book` (per date) and the nightly snapshot path. Drop the dependency on
|
||||
`current_sleeve_weights`/`_paper_backfill_sleeve_weights` for the paper book (the allocator covers a
|
||||
different set). `backfill_paper_book` computes `sleeve_weights` per date from the day's active sleeves.
|
||||
2. **unlock construction** (`cli.py _paper_backfill_sleeves` + the snapshot's `_live_sleeves`): build the
|
||||
unlock sleeve's vol panel from the warehouse (`crypto_close_panel`, adapted to what `UnlockShortRunner`
|
||||
needs) + `UnlockCalendarLive(<data_dir>/unlock_calendar.json)`. Remove the `crypto_pit_dir` dependency.
|
||||
If qvol/funding aren't in the warehouse, pass closes with neutral qvol/funding so the vol-based weights
|
||||
still compute (document the approximation), or skip names lacking the needed fields — never crash.
|
||||
3. Keep stablecoin as-is (BinanceSpotHistory; it legitimately produces positions only on depeg days — rare
|
||||
but real). ts-trend already works. xsfunding excluded.
|
||||
|
||||
## Data flow
|
||||
|
||||
Backfill / nightly snapshot → each replayable sleeve `current_weights(as_of)` → keep the non-empty ones →
|
||||
`sleeve_weight = 1/N` each → `persist_paper_book(... sleeve_weights, symbol_weights_by_sleeve, prices ...)`
|
||||
→ positions sized at `(1/N)·$100k·w / close`. `/paper` (live) + `/paper/replay` (history) both populate.
|
||||
|
||||
## Risks / safety
|
||||
|
||||
- **Read-only/additive**: no change to sleeve `run()`/`advance()` hot paths, `combine_book`, or the gate
|
||||
record. Only the paper sizing + unlock loader wiring change.
|
||||
- **stablecoin MTM**: its stable-pair symbols aren't in the warehouse close panel used for MTM → those
|
||||
positions mark at entry (~0 PnL, correct for pegs). Documented.
|
||||
- **unlock vol approximation**: if the warehouse lacks qvol/funding, the unlock weights use a documented
|
||||
fallback; never crash (per-name skip).
|
||||
- **Consistency**: equal-weight shared by backfill + nightly snapshot so live + replay agree.
|
||||
|
||||
## Out of scope
|
||||
Inverse-vol / fixed sleeve weights; B alerts panel; back-computing xsfunding; the `_live_sleeves`↔backfill
|
||||
duplication (the proper shared-composition refactor — noted as tech debt).
|
||||
|
||||
## Acceptance criteria
|
||||
- A shared equal-weight helper sizes the active replayable sleeves (1/N); unit-tested.
|
||||
- `backfill_paper_book` + the nightly snapshot use it (no more `{}` sleeve weights); positions persist.
|
||||
- unlock builds from the warehouse panel + calendar (no `crypto_pit_dir`); produces positions in pre-cliff
|
||||
windows; tested with a fake warehouse + historical-cliff calendar.
|
||||
- Re-run backfill → `paper_positions` non-empty across many dates; `/paper/replay` equity curve is NON-flat
|
||||
(ts-trend + unlock contribute; stablecoin on depeg days); verified post-deploy.
|
||||
- Nightly snapshot populates the live `/paper` book too (not flat).
|
||||
77
docs/superpowers/specs/2026-06-21-paper-replay-design.md
Normal file
77
docs/superpowers/specs/2026-06-21-paper-replay-design.md
Normal file
@@ -0,0 +1,77 @@
|
||||
# Paper Replay (history scrubber) — Design (Phase 2F-A3)
|
||||
|
||||
**Date:** 2026-06-21
|
||||
**Status:** Design (approved)
|
||||
**Repo:** fxhnt
|
||||
**Follows:** 2F MVP (paper view) + 2F-2 (nightly snapshot). Adds a scrubbable history.
|
||||
|
||||
## Motivation
|
||||
|
||||
`/paper` shows the live paper book (current positions, live MTM). The user wants to also *replay* — scrub
|
||||
through history and watch the paper equity curve + book evolve. The snapshot persists only forward, per
|
||||
run_date, with no equity-per-date series. This adds a backfill (recompute the paper book over past dates)
|
||||
+ a `paper_nav` equity series + a `/paper/replay` scrubber.
|
||||
|
||||
## Decisions (from brainstorming)
|
||||
|
||||
- **Backfill from warehouse history** (capped ~1yr / warehouse availability) for the **as_of-aware** sleeves
|
||||
(stablecoin, unlock, ts-trend). **xsfunding is EXCLUDED** (its `current_weights` reads a live funding
|
||||
snapshot with no historical replay — back-computing it with today's funding would be false). Flagged in UI.
|
||||
- **Persist a `paper_nav` equity-per-date table** (forward snapshot + backfill both write it).
|
||||
- **Replay = the snapshot derivation pointed at the past**: reuse `current_weights(as_of)` + warehouse
|
||||
closes per date; mark each date's book at *that date's* close.
|
||||
|
||||
## Components
|
||||
|
||||
1. **`paper_nav` table** (`adapters/persistence/cockpit_models.py`): `(run_date PK, capital, realized,
|
||||
unrealized, equity, src_updated_at)`. Timescale-friendly like `forward_nav`.
|
||||
2. **Equity-per-date write**: extend `persist_paper_book` (used by both nightly snapshot + backfill) to also
|
||||
compute + upsert the `paper_nav` row for `run_date`: `unrealized = Σ qty·(close@run_date − entry)` over
|
||||
that date's positions, `realized = cumulative realized ≤ run_date` (already tracked), `equity = capital +
|
||||
realized + unrealized`. Idempotent per run_date.
|
||||
3. **Backfill** (`application/paper_backfill.py` + `fxt paper-backfill [--start] [--days N]` CLI in the
|
||||
existing CLI, and a manually-triggerable Dagster op): for each date in the range, build the as_of-aware
|
||||
sleeves' `current_weights(as_of=date)` + allocator weights + that date's warehouse closes → `snapshot(
|
||||
as_of=date, prices=closes@date)`. Per-date/per-sleeve try-except (skip missing data); **skip xsfunding**.
|
||||
Cap to `min(warehouse_start, today − cap_days)`. Re-runnable (idempotent per run_date).
|
||||
4. **PaperRepo reads**: `nav_history() -> [(date, equity, realized, unrealized)]`, `positions_at(run_date)`,
|
||||
`trades_on(run_date)`.
|
||||
5. **`/paper/replay` view** (`adapters/web/app.py` + `templates/replay.html` + `_replay_at.html`):
|
||||
- a **date scrubber** (`<input type=range>` over the `paper_nav` dates) + a **Play/Pause** control,
|
||||
- the **equity curve** (SVG of `paper_nav` up to the selected date — reuse/extend `nav_sparkline`),
|
||||
- the **book@date**: positions (`positions_at`) marked at that date's close + the day's trades
|
||||
(`trades_on`).
|
||||
- Scrub/play fires `GET /paper/replay/at?date=<d>` (HTMX) returning the `_replay_at.html` partial.
|
||||
- Empty history → "no replay history yet — run `fxt paper-backfill`". A small note: "funding sleeve is
|
||||
live-only, excluded from replay."
|
||||
- Nav link from `/paper` ↔ `/paper/replay`.
|
||||
|
||||
## Data flow
|
||||
|
||||
Backfill (one-shot CLI/op) recomputes positions + `paper_nav` per historical date → `/paper/replay` reads
|
||||
`nav_history` (curve) + `positions_at`/`trades_on` (book@scrubbed-date). The nightly snapshot keeps
|
||||
appending `paper_nav` so replay extends forward automatically.
|
||||
|
||||
## Risks / safety
|
||||
|
||||
- **Read-only**: backfill recomputes from warehouse + domain weight fns; no broker, no live-trading change.
|
||||
- **xsfunding fidelity**: excluded from backfill (documented + UI-flagged) rather than back-computed wrongly.
|
||||
- **Missing warehouse data** for a date/symbol → that date/sleeve skipped (no nav row), logged; replay just
|
||||
shows the dates that exist.
|
||||
- **Idempotent**: backfill re-run replaces per run_date (positions/trades/nav all keyed by run_date).
|
||||
- **Performance**: backfill over ~1yr × few sleeves is a one-shot batch; `/paper/replay` reads are indexed
|
||||
by run_date. paper_nav keeps the curve cheap (no per-render recompute).
|
||||
|
||||
## Out of scope
|
||||
- B alerts panel (separate). Live *equity* feed for equities (marks at close, as MVP). Back-computing
|
||||
xsfunding history.
|
||||
|
||||
## Acceptance criteria
|
||||
- `paper_nav` table; `persist_paper_book` writes the equity row per run_date; unit-tested.
|
||||
- `fxt paper-backfill` recomputes positions + paper_nav over a date range from the warehouse for the
|
||||
as_of-aware sleeves, skips xsfunding + missing data; integration-tested (fake warehouse → multiple dates).
|
||||
- `PaperRepo.nav_history/positions_at/trades_on` tested.
|
||||
- `/paper/replay` renders the curve + scrubber + book@date; `/paper/replay/at?date=` returns the right
|
||||
book; empty-history + xsfunding-excluded notes shown; web-tested with seeded data.
|
||||
- Re-running backfill is idempotent (no dup rows).
|
||||
- Nightly snapshot unchanged except it now also writes paper_nav.
|
||||
79
docs/superpowers/specs/2026-06-21-paper-snapshot-design.md
Normal file
79
docs/superpowers/specs/2026-06-21-paper-snapshot-design.md
Normal file
@@ -0,0 +1,79 @@
|
||||
# Paper-Snapshot Job — Design (Phase 2F-2)
|
||||
|
||||
**Date:** 2026-06-21
|
||||
**Status:** Design (approved)
|
||||
**Repo:** fxhnt
|
||||
**Follows:** 2F MVP (paper-cockpit backend + `/paper` view, merged a4b806b). This feeds it real data.
|
||||
|
||||
## Motivation
|
||||
|
||||
The `/paper` cockpit view + `PaperBookService`/`PaperRepo`/`persist_paper_book` exist and are tested, but
|
||||
nothing populates them: the nightly forward/book pipeline carries only daily *returns* — each sleeve
|
||||
computes its positions/weights then discards them. This job re-exposes those weights (read-only) and
|
||||
persists a daily paper book so `/paper` shows the **real** positions/trades/PnL of the live sleeves.
|
||||
|
||||
## Decision (from brainstorming)
|
||||
|
||||
Add a uniform **`current_weights(as_of) -> {symbol: weight}`** accessor to each live sleeve, reusing its
|
||||
existing domain weight logic (NOT a standalone re-derivation — that would duplicate + drift). A new
|
||||
`PaperSnapshotService` combines: allocator **sleeve** weights × each sleeve's **symbol** weights ×
|
||||
`paper_capital` ÷ latest close → `persist_paper_book(...)`. Wired as a Dagster asset in the nightly run.
|
||||
Purely additive/read-only — the return-producing hot path is untouched (the gate record is unaffected).
|
||||
|
||||
## Components
|
||||
|
||||
1. **`PositionSource` protocol** (`src/fxhnt/ports/position_source.py`):
|
||||
`current_weights(as_of: str) -> dict[str, float]` — signed per-symbol target weights for the sleeve as
|
||||
of `as_of` (the rebalance date). Sum of |weights| ≤ the sleeve's gross.
|
||||
2. **Per-sleeve `current_weights()`** — implement on each live sleeve, reusing domain fns + the same
|
||||
warehouse reads the runner does:
|
||||
- stablecoin_rotation → `short_rich_weights(dev, thresh)`
|
||||
- unlock shorts → `inverse_vol_capped_weights(vols, max)` (negated — shorts)
|
||||
- funding carry (xsfunding) → `construction_weights(scores, mode, …)`
|
||||
- crypto ts-trend → `ts_trend_signal(closes, lookbacks)` → sign × vol-target weight per symbol
|
||||
- equity/multistrat sleeves (sixtyforty, growth_discipline, multistrat) → their fixed/!computed weight lists
|
||||
A sleeve that genuinely can't surface weights (or has no data yet) returns `{}` (skipped, logged) —
|
||||
never raises.
|
||||
3. **Allocator sleeve-weight accessor** (`book_allocator`): expose the per-sleeve inverse-vol weights it
|
||||
computes for the current trailing window as `current_sleeve_weights(series_by_edge) -> {sleeve: weight}`
|
||||
(a thin public wrapper over the existing internal `_inverse_vol_weights`/`combine_book` logic).
|
||||
4. **`PaperSnapshotService.snapshot(as_of, at)`** (`src/fxhnt/application/paper_snapshot.py`):
|
||||
- sleeve_weights = allocator current weights; for each sleeve `current_weights(as_of)`; prices = latest
|
||||
closes from the warehouse `price_provider` for all referenced symbols.
|
||||
- call `persist_paper_book(paper_repo, capital=settings.paper_capital, run_date=as_of,
|
||||
sleeve_weights=…, symbol_weights_by_sleeve=…, prices=…, at=…, enabled=settings.paper_enabled)`.
|
||||
- per-sleeve try/except: log + skip a failing sleeve; the snapshot still persists the rest.
|
||||
5. **Dagster asset `paper_book_snapshot`** (`adapters/orchestration/assets.py` + `definitions.py`):
|
||||
depends on `combined_forward_nav`; runs `PaperSnapshotService.snapshot(...)`; added to
|
||||
`daily_combined_book_schedule` (23:30). Idempotent (persist_paper_book replaces per run_date).
|
||||
|
||||
## Data flow
|
||||
|
||||
23:30 Dagster → `combined_forward_nav` (existing, unchanged) → `paper_book_snapshot` (new): allocator
|
||||
sleeve weights × per-sleeve `current_weights` × warehouse closes → `persist_paper_book` → `paper_positions`
|
||||
/`paper_trades`/realized. `/paper` reads them + marks crypto to live Binance prices (MVP view).
|
||||
|
||||
## Risks / safety
|
||||
|
||||
- **Read-only / additive:** no change to the return-producing advance path; the forward gate record is
|
||||
untouched. A snapshot failure must NOT fail the nightly book — the asset catches per-sleeve + overall and
|
||||
logs (the combined_forward_nav asset is the source of truth; snapshot is observability).
|
||||
- **Weight fidelity:** `current_weights` must reuse the SAME domain fn + inputs the runner uses, so the
|
||||
paper book matches the live strategy. Tested per sleeve against the domain fixtures.
|
||||
- **Symbol/price coverage:** a symbol with no warehouse close is skipped (no position) + logged.
|
||||
- **Capital/sizing:** mirrors `book_allocator`; paper only (no fills/slippage guarantee — by design).
|
||||
|
||||
## Out of scope
|
||||
- A3 replay scrubber; B alerts panel (separate specs).
|
||||
- Live equity price feed (equities still mark at last close in the view).
|
||||
- Changing the live trading/gate logic.
|
||||
|
||||
## Acceptance criteria
|
||||
- Each live sleeve has `current_weights(as_of)` reusing its domain logic; unit-tested per sleeve.
|
||||
- `book_allocator` exposes current per-sleeve weights; unit-tested.
|
||||
- `PaperSnapshotService.snapshot()` persists positions/trades for a multi-sleeve book from fake
|
||||
sleeves+warehouse; integration-tested; a failing sleeve is skipped (others persist).
|
||||
- `paper_book_snapshot` Dagster asset wired into the daily schedule, depends on `combined_forward_nav`;
|
||||
wiring-tested.
|
||||
- After a snapshot run, `/paper` renders real positions/trades/PnL (verified post-deploy).
|
||||
- Nightly book (combined_forward_nav) behavior + gate record unchanged.
|
||||
@@ -0,0 +1,86 @@
|
||||
# Paper-Trading Cockpit (live MTM) — Design (Phase 2F, MVP = A1+A2)
|
||||
|
||||
**Date:** 2026-06-21
|
||||
**Status:** Design (approved)
|
||||
**Repo:** fxhnt
|
||||
|
||||
## Motivation
|
||||
|
||||
The forward-track sleeves are paper-trading on daily NAV (gate=WAIT, building a live record), but the
|
||||
cockpit only shows NAV/return/gate per sleeve. The user wants to *watch the strategies trade with paper
|
||||
money in (near) real-time* — see positions, trades, and live mark-to-market PnL — to build intuition
|
||||
before micro-live. The engine already computes the trades (`book_allocator` weights + strategies'
|
||||
`positions()` + `ExecutionService`); the forward track discards everything but the daily NAV.
|
||||
|
||||
## Decisions (from brainstorming)
|
||||
|
||||
- **Capital model:** ONE paper book, configurable `paper_capital` (default **$100,000**), allocated across
|
||||
sleeves by the existing `book_allocator` (inverse-vol + fractional-Kelly). Combined equity + per-sleeve PnL.
|
||||
- **"Real-time" = live mark-to-market** of daily positions (prices poll ~45s; trades appear at the daily
|
||||
rebalance) — faithful to the daily edges (intraday trading them is a known fee-trap). A **replay** mode is
|
||||
a later sub-project (A3).
|
||||
- **Scope = MVP (A1 persist + A2 live view).** Replay (A3) and an Alerts panel (B) are separate specs.
|
||||
- **Read-only / paper-only:** no real orders. The persisted positions come from the rebalance computation
|
||||
on the `execute=False` path; existing broker gates already refuse live money.
|
||||
|
||||
## Components — A1: persist positions & trades
|
||||
|
||||
New persistence (mirrors `forward_nav.py`; Postgres TimescaleDB hypertable where applicable, SQLite for tests):
|
||||
- **`paper_positions`** — `(run_date, sleeve, symbol, qty, entry_price, entry_date, weight, src_updated_at)`;
|
||||
the open book after each daily rebalance (latest run_date = current holdings).
|
||||
- **`paper_trades`** — `(ts, sleeve, symbol, side {BUY|SELL}, qty, price, reason)`; one row per change
|
||||
(entry / exit / resize) derived by diffing today's target positions vs the prior held positions.
|
||||
|
||||
Derivation (in the daily rebalance path, alongside the existing NAV write):
|
||||
1. `book_allocator` → per-sleeve weight; `× paper_capital` → sleeve notional.
|
||||
2. Each sleeve's domain weights/`positions()` → per-symbol target weight; `× sleeve_notional / close` → target qty.
|
||||
3. Diff target vs prior held qty per (sleeve, symbol) → BUY/SELL trades; persist trades + the new positions.
|
||||
4. Realized PnL accrues on SELLs (vs entry_price); carried in the positions/trade rows.
|
||||
|
||||
## Components — A2: live view (`/paper`)
|
||||
|
||||
- **Endpoint `/paper`** (server-rendered page) + **`/paper/live`** (HTMX partial, polled ~45s).
|
||||
- Loads current open positions (latest run_date), fetches **live prices** for held symbols via
|
||||
`binance_ccxt` (crypto); equities (SPY/IEF/QQQ…) marked at their **last daily close** for the MVP (no
|
||||
live equity feed). Resilient: cache last-good price; flag stale.
|
||||
- Computes per-position unrealized PnL (`qty·(last−entry)`), per-sleeve PnL, and **combined equity =
|
||||
paper_capital + realized + unrealized**.
|
||||
- **Renders:** combined live equity (number + sparkline), per-sleeve cards (alloc %, # positions, day/total
|
||||
PnL), open-positions table (sleeve, sym, qty, entry, last, PnL, %), recent-trades log (last ~25).
|
||||
|
||||
## Data flow
|
||||
|
||||
Daily cron rebalance → derive + persist positions/trades (A1). Browser → `/paper` → read latest positions
|
||||
+ live prices → MTM → render; HTMX `/paper/live` poll refreshes the numbers. No writes from the web layer
|
||||
(read-only cockpit, consistent with the existing dashboard).
|
||||
|
||||
## Architecture notes
|
||||
|
||||
- Follows the existing hexagonal layout: domain (pure position/trade diff + MTM math), application
|
||||
(`paper_book` service deriving + persisting), adapters (`paper_repo` persistence; `binance_ccxt` price
|
||||
port; web routes in `adapters/web/app.py` + a `paper.html` template), ports (a `LivePrice` port so tests
|
||||
inject a fake).
|
||||
- Reuse `nav_sparkline` for the equity chart; reuse the `DashboardService`/templates patterns.
|
||||
|
||||
## Risks / safety
|
||||
|
||||
- **No real orders:** derivation runs on computed targets only; nothing touches the broker execute path.
|
||||
- **Live price failures:** wrap the price fetch; on error show last-good + a "stale" badge (never crash the page).
|
||||
- **Symbol mapping:** sleeves use various symbols (crypto pairs vs equity tickers); map to the right price
|
||||
source (crypto→binance_ccxt, equity→last close). Unknown symbol → mark at entry (0 PnL) + flag.
|
||||
- **Capital/sizing fidelity:** the MVP mirrors `book_allocator`'s weights; it is a faithful *paper* view,
|
||||
not a guarantee of live fills/slippage (paper, by design).
|
||||
|
||||
## Out of scope (separate specs)
|
||||
- **A3 — Replay scrubber** (animate persisted history).
|
||||
- **B — Alerts panel** (surface Alertmanager alerts in the cockpit).
|
||||
- Live *equity* price feed (MVP marks equities at last close); real order execution.
|
||||
|
||||
## Acceptance criteria
|
||||
- `paper_positions` + `paper_trades` tables created + populated by a rebalance run (derived from
|
||||
book_allocator weights × $100k); a unit test covers the target-qty + trade-diff math.
|
||||
- `/paper` renders combined equity, per-sleeve cards, open positions, and a trade log from seeded data
|
||||
(integration test with mocked live prices, mirroring `test_dashboard_service.py`).
|
||||
- `/paper/live` returns updated MTM on poll; live crypto prices via `binance_ccxt` (equities last-close).
|
||||
- Price-fetch failure degrades gracefully (last-good + stale flag), page never 500s.
|
||||
- Read-only: no broker/order side effects.
|
||||
@@ -0,0 +1,93 @@
|
||||
# Funding-carry tail: data-acquisition & ingestion design (research output)
|
||||
|
||||
**Date:** 2026-06-22
|
||||
**Status:** Design / feasibility (from the `carry-data-acquisition-research` workflow: 5 parallel research agents
|
||||
→ adversarial fact-check → synthesis). NOT yet built. Gated on a decision to bring carry into the replay.
|
||||
**Problem:** xsfunding (delta-neutral funding carry) shows a fake Sharpe (15 on 1yr, 1.80 on 6yr) because its
|
||||
real tail is invisible to our data: daily close marks + survivorship-biased (only currently-listed Binance
|
||||
USDT-perps). Deep 2022 losses were intraday squeezes, DELISTED coins (LUNA), ADL, and counterparty (FTX).
|
||||
|
||||
## 1. Source: free `data.binance.vision` covers almost everything
|
||||
- **Public, unauthenticated, LISTABLE S3 bucket** (XML listing: `https://s3-ap-northeast-1.amazonaws.com/
|
||||
data.binance.vision?delimiter=/&prefix=<P>&marker=<K>`; download `https://data.binance.vision/<KEY>`;
|
||||
`.CHECKSUM` siblings). **CONFIRMED survivorship-free**: delisted symbols retained permanently — LUNAUSDT /
|
||||
FTTUSDT / SRMUSDT klines + fundingRate present (LUNA death-week 1m + funding confirmed). **Never use REST
|
||||
`exchangeInfo`** (not survivorship-free; LUNA fully purged).
|
||||
- Date floors (CONFIRMED): UM klines/funding from 2020-01, UM metrics from 2020-09, spot from 2017-08 → the
|
||||
~6.4yr window covers 2022-05 LUNA + 2022-11 FTX.
|
||||
- **Liquidation tape is the ONLY gap** → free UM `liquidationSnapshot` was **DELETED by Binance** (404 now).
|
||||
Only source is a paid third-party capture (Tardis.dev, from 2020-01) of the websocket `forceOrder` stream —
|
||||
and it's **fundamentally lossy** (one largest liq/symbol/1000ms → under-reports cascades = the tail). Treat
|
||||
as a lower bound; do NOT buy a vendor for funding/klines/universe (all free).
|
||||
|
||||
## 2. Survivorship-free universe (the highest-ROI piece)
|
||||
1. Enumerate the **full ever-listed set from the klines prefix** (price = SSOT), not exchangeInfo: list
|
||||
`data/futures/um/monthly/klines/` CommonPrefixes (~**876** symbols today, single page; code for
|
||||
`IsTruncated`/`marker` anyway).
|
||||
2. Per symbol, derive `[first_trade_ts, last_trade_ts]` from file presence; PIT universe on D = `{sym: first ≤
|
||||
D ≤ last}`.
|
||||
3. Four traps that silently re-introduce survivorship bias: **rename/redenomination** (LUNAUSDT→1000LUNCUSDT,
|
||||
1000BONK/PEPE/SHIB/FLOKI — chain-link returns across the rename or you drop the 99.9% crash), **BUSD→USDT/
|
||||
USDC quote migration** (41 dead *BUSD + 39 *USDC overlaps), **re-listings** (disjoint intervals — emit
|
||||
multiple per symbol), **spot-leg survivorship** (apply to `data/spot/` too; reconcile funding vs klines
|
||||
symbol sets — carry needs both legs).
|
||||
4. Cross-check interval ends vs the Binance delisting-announcement feed (audit layer only) to tag voluntary
|
||||
delist / forced settlement / rename + anchor the qualitative tail events.
|
||||
|
||||
## 3. Minimal-sufficient feature set (a daily book stores daily *descriptors* of the intraday path)
|
||||
Already have: `close` (perp), `spot_close`, `funding`. **Add (load-bearing in bold):**
|
||||
- **`worst_basis`** = min over the day of cumulative `(spot_1m_ret − perp_1m_ret)` (basis MAE) — the single
|
||||
number that converts "always-holding fiction" → "would I survive the day" (a delta-neutral pos is
|
||||
liquidated by the *spread* under finite margin). From 1m spot + perp klines. **Free.**
|
||||
- **delisted-coin inclusion + `delisted` marker** (ticker_membership `end_date` at delist) + **funding on
|
||||
delisted symbols**. **Free.** (Phase-0 alone collapses most of the fake Sharpe — LUNA-class losses are
|
||||
currently DELETED from the panel.)
|
||||
- `basis_range`, `oi_eod`/`premium_eod` (free diagnostics); `liq_intensity`/`liq_flag` (**paid, lossy**).
|
||||
|
||||
## 4. Volume + effort
|
||||
- Raw 1m (throwaway): **tens of GB zipped / ~100–180 GB unzipped** (perp-1m + spot-1m, ever-listed incl.
|
||||
delisted; prefer monthly files, ~30× fewer requests). funding: **<100 MB total** (acquire first).
|
||||
- Reduced (stored in DuckDB): **~0.1 GB** net (~1000:1 compression — one daily MAE replaces 1440 bars × 2).
|
||||
- Effort ≈ **5–6 engineering days** (aggregator + delisted-aware S3 walker + single-writer Dagster asset +
|
||||
backtest consumption). Pilot ~30 symbols incl. LUNA/2022-Q2 before the full backfill.
|
||||
|
||||
## 5. Storage + single-writer-safe ingestion (zero migration)
|
||||
New features are just new `feature` values in the existing long-format store; delist via existing
|
||||
`upsert_membership`. **Fan-out compute, fan-in write:** N parallel workers (NO DuckDB handle) download +
|
||||
aggregate per symbol-shard → emit feature rows → a SINGLE serialized writer commits to DuckDB. (This is the
|
||||
workflow-shaped parallel work.)
|
||||
|
||||
## 6. Modeling the carry tail in the (still daily) book
|
||||
- **worst-basis marks:** when the day's `worst_basis` breached an intraday margin/liq threshold for a held
|
||||
symbol, realize the basis loss / margin-stop haircut instead of the benign close.
|
||||
- **liq/ADL haircut:** on `liq_flag` (or free fallback: `metrics` OI-collapse + one-sided taker), apply a
|
||||
parametric slippage/ADL haircut to the short-perp leg (× a stress multiplier, since liq data is a lower bound).
|
||||
- **delisted-coin loss:** PIT membership keeps a dying coin in-universe through its death → it pays its carry
|
||||
loss then exits (force-settle at last/settlement mark; respect the reduce-only pre-suspension window).
|
||||
|
||||
## HONEST residual limit (never ingested)
|
||||
Counterparty/exchange failure (FTX), withdrawal/settlement freezes, ADL per-position socialization, collateral
|
||||
de-peg = credit/operational events with **no market-data artifact at any vendor**. → a **scenario haircut**
|
||||
(probabilistic venue-collateral loss, position-locked-N-days), anchored to the announcement audit — never
|
||||
claimed as backtested. The improved backtest **narrows but never closes** the gap; **live `xsfunding_nav`
|
||||
stays ground truth.**
|
||||
|
||||
## Phased plan (cheapest meaningful improvement first)
|
||||
- **Phase 0 — survivorship-free universe** (~free, ~1–2d, highest ROI): enumerate 876 prefixes → intervals →
|
||||
rename-stitch + relisting-gap map → backfill delisted close+funding (+spot leg) → set membership end_date.
|
||||
*Falsification gate: re-run xsfunding on the survivorship-free universe; confirm SR drops.*
|
||||
- **Phase 1 — intraday `worst_basis` marks** (free, ~2–3d): 1m→MAE aggregator (pure + tests), pilot 30 sym
|
||||
incl. LUNA 2022-Q2, confirm the intraday tail reappears, then full backfill via fan-out/single-writer.
|
||||
- **Phase 2 — free liq/ADL proxy** (free, ~1–2d): `metrics` OI-collapse + one-sided-taker → parametric haircut.
|
||||
- **Phase 3 — paid Tardis liq tape** (only if Phases 0–2 leave a material gap): validate free first-of-month
|
||||
LUNA-era samples; yearly/Business billing needed for 2020–2026 (monthly billing only grants a few months).
|
||||
- **Phase 4 — scenario layer** (always, never ingested): counterparty/de-peg/ADL haircuts; live nav = truth.
|
||||
|
||||
## UNCERTAIN (flagged by the adversarial verifier)
|
||||
- Symbol count **876** (live single page) supersedes 922/815; volume keyed to <876 → revise upward.
|
||||
- Tardis "~$599/mo" is a single third-party blog (order-of-magnitude only); CONFIRMED from Tardis: $300 min
|
||||
order, survivorship guarantee, forceOrder from 2020-01-07, billing-interval = history-depth. Funding start
|
||||
likely ~2020-05 (not 2020-02). Kaiko/Amberdata/CoinAPI/CCData pricing = indicative only.
|
||||
|
||||
Files to extend if built: `adapters/warehouse/duckdb_feature_store.py`, `ports/warehouse.py`,
|
||||
`application/{funding_ingest,spot_ingest,xsfunding_strategy,xsfunding_replay}.py`, `adapters/orchestration/assets.py`.
|
||||
@@ -0,0 +1,84 @@
|
||||
# Converge to a trustworthy 2-edge book — Design (Phase 2F-A6)
|
||||
|
||||
**Date:** 2026-06-22
|
||||
**Status:** Design (approved — implements the foundation audit's decisions)
|
||||
**Repo:** fxhnt
|
||||
**Follows:** the foundation audit (`2026-06-22-paper-book-foundation-audit.md`). User chose D1=a (carry live-only).
|
||||
|
||||
## Motivation
|
||||
|
||||
The audit confirmed: (F1) carry P&L isn't in equity, (F6) carry tail is unmodellable from daily/survivor data,
|
||||
(F2) leverage caps the vol-target multiplier not gross (→6× via carry gross-2), (F3) no warmup (cold-start
|
||||
over-levers), (F4) overlay overfit to a 1-yr bull (6-yr maxDD −98.9%), (F5) offline sweep under-models risk.
|
||||
**Converge** to one trustworthy book per the audit's recommendation: **2 directional edges (ts-trend +
|
||||
unlock), price-MTM (signal==equity basis), gross-capped + warmed-up overlay, re-derived/validated on the real
|
||||
6-yr cycle.** xsfunding → **live-only** (kept on the `xsfunding_nav` forward-track; funding/spot data + carry
|
||||
helpers retained, just not in the replay).
|
||||
|
||||
## Decisions (from the audit)
|
||||
|
||||
- **D1 — carry live-only:** remove `xsfunding` from the replay sleeve set (`_REPLAYABLE_SLEEVES`). Keep the
|
||||
ingested funding/spot data, the `xsfunding_nav` live asset, and the carry helpers (`XsFundingReplay`,
|
||||
`sleeve_daily_carry`, `delta_neutral_return_panel`) in the tree (tested; documented as not-in-replay by
|
||||
design). Removing carry also resolves F2's gross-2 stacking (directional sleeves are ~gross-1).
|
||||
- **D3 — warmup:** `vol_target_leverage` returns `1.0` (unlevered) until ≥ `vol_lookback` (60) book-return
|
||||
observations — no maximal leverage on cold-start thin history.
|
||||
- **D2 — gross cap (belt-and-suspenders):** cap realized **gross exposure** ≤ `max_gross` (3.0) × equity in
|
||||
`derive_and_persist` (scale all positions down if exceeded). Mostly moot once carry is gone, but makes the
|
||||
invariant explicit and safe if a market-neutral sleeve is ever re-added.
|
||||
- **D4 — validate on the REAL backfill, not the sweep:** after the above, re-backfill the full 6-yr cycle
|
||||
in-cluster and MEASURE the real maxDD. If still too hot (the 30% full-Kelly adaptive budget likely is for 3×
|
||||
directional through 2022), **re-derive the vol budget by running a few REAL backfills** at lower budgets
|
||||
(e.g. 10/15/20% , ½ vs full Kelly) and pick the one with a survivable maxDD at acceptable return. The
|
||||
offline sweep is unreliable (F5) — decide on real-backfill numbers.
|
||||
- **D5 — one coherent change, then verify.** No new layering beyond D1–D3 + the D4 budget pick.
|
||||
|
||||
## Components
|
||||
|
||||
1. **`vol_target_leverage` warmup** (`paper_book.py`): add `warmup_obs: int = _VOL_LOOKBACK`; if the usable
|
||||
return window has `< warmup_obs` points, return `1.0`. (Currently returns 1.0 only for `<2` obs / `pv≤1e-12`.)
|
||||
2. **Gross cap** (`PaperBookService.derive_and_persist` + `persist_paper_book`): add `max_gross: float = 3.0`;
|
||||
after building `target`, `gross = Σ|p.qty·p.entry_price|`; if `gross > max_gross·capital_base`, scale every
|
||||
`qty` by `max_gross·capital_base / gross`. Threaded from `persist_paper_book(..., max_gross=3.0)`.
|
||||
3. **Revert carry from replay** (`cli.py`): drop `"xsfunding"` from `_REPLAYABLE_SLEEVES` (so
|
||||
`_paper_backfill_sleeves` no longer builds it). The `dn_by_date` construction may stay (inert without a
|
||||
carry sleeve) or be removed for cleanliness — implementer's call, keep tests green. Update the command echo
|
||||
("xsfunding excluded — live-only" again, now for the audited reason). Snapshot path unaffected (carry was
|
||||
never wired into the live snapshot).
|
||||
4. **D4 re-derivation (operational, in T-deploy):** real 6-yr backfills at candidate budgets → pick survivable.
|
||||
|
||||
## Data flow (after)
|
||||
|
||||
2 directional sleeves → `current_weights` → unit shadow → `sleeve_daily_return` (price MTM) →
|
||||
`paper_sleeve_ret`. Overlay: ISV weights + warmed-up vol-target (adaptive budget, to be re-derived) + kill →
|
||||
gross-capped sizing. Equity: compounding price-MTM (A3f) — **signal basis == equity basis** (both price MTM)
|
||||
→ internally consistent, every displayed number trustworthy.
|
||||
|
||||
## Risks / safety
|
||||
|
||||
- **Read-only/additive to mechanics:** warmup + gross-cap are guards (only ever REDUCE leverage/exposure);
|
||||
default `max_gross=3.0` + `warmup_obs=60`. Removing a sleeve from the set is subtractive. A3f accounting,
|
||||
shadow signal, ISV weights, killswitch, live `xsfunding_nav` untouched.
|
||||
- **Existing tests:** the `*_equal_weights_*`/leverage tests use ≤1-date or explicit params; the warmup
|
||||
(≥60 obs) won't fire on tiny test books unless they exceed 60 days — check + update any that do. The carry
|
||||
unit tests (sleeve_daily_carry, XsFundingReplay, delta_neutral_return_panel) stay (helpers retained).
|
||||
- **D4 honesty:** the 2-edge directional risk IS captured by daily price-MTM (the −98.9% was real), so the
|
||||
re-derived budget reflects real cross-cycle risk. Pick for *survivability*, accept lower bull-year return.
|
||||
- **Convergence:** this is the LAST structural change (D5); subsequent work is parameter choice on real data.
|
||||
|
||||
## Out of scope
|
||||
|
||||
Wiring carry P&L into equity (carry is live-only); the carry deep-tail data project (F6, unmodellable here);
|
||||
the live-snapshot xsfunding wiring (carry not in replay or snapshot now).
|
||||
|
||||
## Acceptance criteria
|
||||
|
||||
- `vol_target_leverage` returns 1.0 until ≥`warmup_obs` obs; unit-tested (thin window → 1.0; ≥60 → levers).
|
||||
- gross cap: a book whose raw gross > `max_gross·equity` is scaled to exactly `max_gross·equity`; equity base
|
||||
unchanged; default 3.0 leaves normal directional books (gross≈leverage≤3) untouched; tested.
|
||||
- `xsfunding` not in the replay sleeve set; backfill builds only directional (+ stablecoin) sleeves; tested.
|
||||
- full suite green.
|
||||
- in-cluster: re-backfill 6-yr → **real maxDD measured**; vol budget re-derived on real backfills to a
|
||||
survivable level; final 2-edge curve reported (return / Sharpe / **maxDD across the 2022 bear**) +
|
||||
H1/H2-style cross-regime check; `/paper` + `/paper/replay` render the trustworthy curve; verified.
|
||||
- read-only/additive (guards only); carry data + live track retained; no `*.dbn` staged.
|
||||
@@ -0,0 +1,78 @@
|
||||
# Drawdown controller v1 — Design (Phase 2F-A9)
|
||||
|
||||
**Date:** 2026-06-22
|
||||
**Status:** Design (approved — build v1 core)
|
||||
**Repo:** fxhnt
|
||||
|
||||
## Motivation / reframe
|
||||
We proved (Kelly sweep): **drawdown scales with how much the book buys; Sharpe is ~leverage-invariant**
|
||||
(kelly 0.25→−24%DD/Sh1.6, 1.0→−33%/2.2, 1.5→−39%/2.1). So DD is a SIZING CHOICE, not an edge risk.
|
||||
The current `DrawdownKillSwitch` is the wrong tool: it's binary and fires on PRICE drawdown (which includes
|
||||
recoverable market moves) → **whipsaw** (it made the −98.9% bug WORSE). An excellent controller is ~90%
|
||||
ex-ante sizing + a surgical reaction that fires only on **edge decay**, never on raw price-DD.
|
||||
|
||||
Two different questions, two different mechanisms:
|
||||
1. "How much DD do I budget?" → set by LEVERAGE ex-ante (Sharpe-invariant).
|
||||
2. "Is this DD recoverable noise or is the edge dying?" → the only thing worth reacting to → react to
|
||||
EDGE-HEALTH (Page-Hinkley on realized PnL drift), per sleeve, continuously.
|
||||
|
||||
## Architecture — multiplicative, per-sleeve sizing stack
|
||||
For each sleeve `s`, day `t`: `L[s,t] = K(D*) · V[s,t] · H[s,t]` (then a no-trade hysteresis band — v2).
|
||||
Replaces the portfolio `DrawdownKillSwitch`. Floored (never to cash; `H ≥ θ_min > 0`) + the existing
|
||||
kill-proof unit shadow book keeps the signal alive. Lives in BOTH `paper_risk_overlay` (snapshot) and
|
||||
`IncrementalRiskOverlay` (backfill) so live==replay; selectable via a `controller` flag for A/B.
|
||||
|
||||
## Components (v1 core)
|
||||
1. **`K(D*)` — ex-ante DD budget → base Kelly** (`kelly_for_dd_budget(dd_budget)` in paper_book.py):
|
||||
invert the empirical curve `DD ≈ −21% − 12%·kelly` → `kelly = clip((|D*|−21)/12, kelly_min, max)`.
|
||||
Calibrate the slope/intercept from a backfill leverage-sweep (don't hardcode blindly — fit on the data).
|
||||
Default budget `D* = −25%`. This is the PRIMARY control (~80% of the effect).
|
||||
|
||||
2. **`V[s,t]` — asymmetric vol-target ramp** (the "stop buying more when calm" fix):
|
||||
keep `vol_target_leverage` (inverse-vol), but in the STATEFUL overlay cap the UP-move per step:
|
||||
`L = min(L_raw, L_prev·(1+up_rate))` for increases; decreases apply immediately (`L = L_raw`).
|
||||
`up_rate` small (e.g. 0.15/day). Down-fast/up-slow prevents the pro-cyclical over-buy that creates the
|
||||
spike-drawdown. (Up-ramp state is per-sleeve in the overlay.)
|
||||
|
||||
3. **`H[s,t]` — per-sleeve continuous edge-health** (the anti-whipsaw layer; reuse `PageHinkleyDecay`):
|
||||
feed each sleeve's realized SHADOW daily return into a per-sleeve Page-Hinkley; derive a CONTINUOUS
|
||||
health `θ = clip(1 − max(0, (m_max − m_t))/lam, θ_min, 1)` (1 = healthy; → θ_min as PH approaches its
|
||||
kill threshold `lam`; recovers via the existing reenter logic). Multiply leverage by `θ`. This de-sizes
|
||||
on EDGE DRIFT (sustained negative PnL), not on a single down-day → no whipsaw; per-sleeve so a decaying
|
||||
sleeve is cut without touching a healthy one. `θ_min` e.g. 0.0–0.1 (floored, not hard zero).
|
||||
- Extend `PageHinkleyDecay` with a pure `health() -> float` (the continuous θ from m_max/m_t/lam),
|
||||
leaving its existing binary `update()` intact (ts_trend/stablecoin still use it).
|
||||
|
||||
## Integration + A/B
|
||||
- Add `controller: str = "killswitch"` (default keeps current behavior) selectable to `"edge_health"`
|
||||
on `paper_risk_overlay` + `IncrementalRiskOverlay`. `edge_health` replaces the `killed` cut with the
|
||||
`H` (and `V`) stack; `K(D*)` sets `kelly_fraction`. Both code paths identical → live==replay.
|
||||
- The snapshot + backfill take the controller choice from one place (a setting / the `REPLAYABLE` config)
|
||||
so live and replay use the SAME controller.
|
||||
|
||||
## Validation (the judge is the forward gate; controllers overfit)
|
||||
- **Backfill A/B**: run the 2-edge book under `killswitch` vs `edge_health` (same data, NET). Report for
|
||||
each: return, Sharpe, maxDD, **#kill/de-size episodes + their avg length** (whipsaw proxy), time-in-cash.
|
||||
PASS for edge_health: maxDD ≈ the chosen `D*` budget, Sharpe ≥ killswitch's, FEWER/shorter whipsaws.
|
||||
- **Ablation**: K(D*) alone vs +V vs +H — each layer's marginal effect.
|
||||
- **Forward gate**: run the live forward under `edge_health`; the OOS record is the real judge. Don't add
|
||||
v2 layers unless the forward shows a need.
|
||||
|
||||
## The honest limit (documented, not solved here)
|
||||
Sizing controls DD ABOVE the directional floor (~−24%): a long-crypto trend book in a −70% bear loses
|
||||
regardless of size until the signal flips. Going below the floor needs an EXPOSURE change (faster signal
|
||||
flip / tail hedge) = an EDGE change, OUT OF SCOPE. The controller targets `D*` above the floor.
|
||||
|
||||
## Out of scope (v2, only if the forward gate shows a need)
|
||||
- `G[t]` Grossman-Zhou hard-floor guardrail (`clip(1−DD/D*_hard, g_min, 1)`).
|
||||
- Hysteresis / no-trade band on `L` changes (turnover/cost-churn control — the "must buy first" cost).
|
||||
- Full ISV-adaptive params (up_rate, lam, θ_min, lookbacks derived from data variance).
|
||||
- Forward-looking vol forecast (GARCH/EWMA) for `V`.
|
||||
|
||||
## Acceptance criteria
|
||||
- `kelly_for_dd_budget`, asymmetric-ramp, and per-sleeve `PageHinkleyDecay.health()` are pure + unit-tested.
|
||||
- `controller="killswitch"` is **bit-identical** to current behavior (backward-compat; existing tests pass).
|
||||
- `controller="edge_health"` wired in BOTH paper_risk_overlay + IncrementalRiskOverlay (live==replay; a
|
||||
golden test asserts they agree per date).
|
||||
- Backfill A/B table produced (killswitch vs edge_health: return/Sharpe/maxDD/whipsaw-count).
|
||||
- Full suite green; no `*.dbn` staged.
|
||||
@@ -0,0 +1,98 @@
|
||||
# Faithful delta-neutral carry (spot ingest + basis) — Design (Phase 2F-A5)
|
||||
|
||||
**Date:** 2026-06-22
|
||||
**Status:** Design (approved)
|
||||
**Repo:** fxhnt
|
||||
**Follows:** A4. Fixes the xsfunding carry artifact (Sharpe 30.9 / book +4320%) by modeling the REAL
|
||||
delta-neutral return instead of funding-only.
|
||||
|
||||
## Motivation (root cause, verified)
|
||||
|
||||
A4 added xsfunding as a carry sleeve but modeled **funding-only** (`Σ w·funding`) because the warehouse had
|
||||
no spot prices. Funding-only is near-deterministic (sort by persistent funding → collect it → ~zero daily
|
||||
variance → **Sharpe 30.9**), and the ISV-adaptive overlay then crowds + levers it → **+4320% fantasy**. The
|
||||
LIVE strategy books the true **delta-neutral** return `w·((spot_ret − perp_ret) + funding)` — the **basis
|
||||
term `(spot_ret − perp_ret)` is the missing daily variance** (and it's capturable from price data). Restoring
|
||||
it gives the carry its real risk, so the existing system sizes it correctly. User decisions: **"just build the
|
||||
correct model"** (no interim 2-edge revert) and **"it should match the system"** (size via the existing
|
||||
ISV-adaptive overlay — no bespoke carry allocation/floor/cap; the faithful return is the fix).
|
||||
|
||||
## Key facts (grounded)
|
||||
|
||||
- `BinanceSpotHistory.daily_close(symbol) -> {epoch_day: close}` already exists (used by stablecoin) — reuse
|
||||
for spot ingest.
|
||||
- Live formula (`xsfunding_strategy.py:50`): `realized += w*((spot_ret−perp_ret)+tf_today)`; `realized -=
|
||||
turnover*cost/2`.
|
||||
- Warehouse generic feature store: `feature='close'` = PERP closes (crypto_close_panel); add
|
||||
`feature='spot_close'` for spot. `feature='funding'` already ingested (A4).
|
||||
- `sleeve_daily_carry(prev_positions, x_by_symbol) = Σ(qty·entry)·x / Σ|qty·entry|` — generic; A4 fed it raw
|
||||
funding. Feeding it the **delta-neutral return** (basis+funding) makes it faithful with NO change to the fn.
|
||||
|
||||
## Decisions
|
||||
|
||||
- **Ingest Binance SPOT daily closes** for the perp universe → `feature='spot_close'`; accessor
|
||||
`crypto_spot_panel()`. Mirror the funding ingest (CLI `crypto-spot-ingest` + Dagster asset, daily,
|
||||
single-writer). Reuse `BinanceSpotHistory` (per-symbol never-raise; coins with a perp but no spot are simply
|
||||
absent → funding-only fallback for those, exactly as the live strategy's documented fallback).
|
||||
- **`delta_neutral_return_panel(spot_panel, perp_panel, funding_panel) -> {sym:{day: dn_ret}}`** where
|
||||
`dn_ret(d) = (spot[d]/spot[d-1] − 1) − (perp[d]/perp[d-1] − 1) + funding[d]`; **funding-only fallback**
|
||||
(`dn_ret = funding[d]`) for a coin/day lacking a spot pair or a prior spot/perp close. (= the live formula.)
|
||||
- **Feed the carry sleeve the delta-neutral return, not raw funding:** the backfill (and snapshot) build
|
||||
`dn_by_date = {iso:{sym: dn_ret}}` from the three panels and pass it where `funding_by_date` was;
|
||||
`sleeve_daily_carry` is unchanged (`Σ w·dn_ret / gross`). The basis term injects real variance.
|
||||
- **Sizing = the existing ISV-adaptive overlay, unchanged** ("match the system"). No vol-floor, no carry-only
|
||||
cap, no separate allocation. The faithful return is what makes the system's risk estimate (and thus its
|
||||
inverse-vol weight + vol-target leverage) correct.
|
||||
- **Gross in the book (matches the system):** carry return is gross (basis+funding); costs are captured
|
||||
post-hoc on turnover (the existing cost analysis already costs xsfunding's trades) — consistent with the
|
||||
directional sleeves being gross. (The live strategy nets cost; the paper book's convention is gross + a
|
||||
post-hoc cost lens.)
|
||||
|
||||
## Components
|
||||
|
||||
1. **Spot ingest** — `application/spot_ingest.py` `ingest_spot(store, fetcher, symbols) -> int` (per symbol
|
||||
`daily_close` → `feature='spot_close'` rows `(day·86400, {'spot_close': c})` → `write_features_bulk`;
|
||||
per-symbol skip-on-empty, idempotent). `DuckDbFeatureStore.crypto_spot_panel(*, suffix='USDT')` (mirror
|
||||
`crypto_funding_panel`, `feature='spot_close'`). CLI `crypto-spot-ingest [--days]` + asset `crypto_spot`.
|
||||
2. **`application/funding_panel.py` `delta_neutral_return_panel(spot, perp, funding) -> {sym:{day: dn_ret}}`**
|
||||
(basis + funding; funding-only fallback). Pure.
|
||||
3. **`cli.py` `_paper_backfill_*` / `paper_backfill`**: build the spot panel + `dn_by_date` from
|
||||
`delta_neutral_return_panel`; pass `funding_by_date=dn_by_date` (the carry input) to `backfill_paper_book`.
|
||||
(xsfunding sleeve construction unchanged — still `XsFundingReplay(warehouse_funding_panel(...))` for its
|
||||
current_weights; only the RETURN input becomes delta-neutral.)
|
||||
4. **`assets.py` snapshot assembly** (live path): same — provide the delta-neutral return per symbol for as_of
|
||||
so live==replay (carry-mode booking consistent).
|
||||
5. **Verify**: deploy → ingest spot (+ funding already done) → re-backfill → check **xsfunding standalone
|
||||
Sharpe is now plausible (~1–3, not 30)** and the book is sane; per-sleeve + OOS H1/H2 + cost. Report
|
||||
honestly — **if the Sharpe is STILL absurd after the basis term, that proves the daily book can't represent
|
||||
carry → fall back to live-only** (the rigorous, pre-agreed exit).
|
||||
|
||||
## Risks / safety
|
||||
|
||||
- **Spot ingest external:** per-symbol never-raise (reuse `BinanceSpotHistory`'s retry/`_get`), idempotent
|
||||
writes; coins without spot → funding-only (documented, = live fallback). One-shot single-writer.
|
||||
- **Causal:** `dn_ret(d)` uses `d` and `d-1` closes + `d` funding; carry booked = prior weights × `dn_ret(d)`.
|
||||
No look-ahead (same as A4's carry causality).
|
||||
- **Read-only/additive:** new `spot_close` feature + a return-input change for the carry sleeve. A3f
|
||||
accounting, A3h overlay, directional sleeves, live xsfunding_nav untouched. Sizing system unchanged.
|
||||
- **Honest limit:** even the full delta-neutral daily return omits deep tails (squeeze/liquidation/funding-
|
||||
flip/exchange failure). The replay is therefore indicative; the **live `xsfunding_nav` forward-track remains
|
||||
the ground truth** for the carry's real risk. The verify's success bar is "plausible, non-dominating," not
|
||||
"exactly 1.27".
|
||||
|
||||
## Out of scope
|
||||
|
||||
Netting cost into the carry return (kept gross to match the system; post-hoc cost lens); modeling deep
|
||||
liquidation/counterparty tails; switching live xsfunding off its snapshot.
|
||||
|
||||
## Acceptance criteria
|
||||
|
||||
- `ingest_spot` + `crypto_spot_panel` (feature='spot_close'); idempotent; tested (fake fetcher + in-memory store).
|
||||
- `delta_neutral_return_panel`: `dn = basis + funding`, funding-only fallback on missing spot/prior; tested.
|
||||
- backfill/snapshot feed the carry sleeve the delta-neutral return (basis injects variance — a unit test
|
||||
shows a carry return that MOVES with basis, not the near-constant funding-only value).
|
||||
- CLI `crypto-spot-ingest` + asset wired to the warehouse path/universe; full suite green.
|
||||
- in-cluster: ingest spot → re-backfill → **xsfunding standalone Sharpe drops to a plausible band**; report
|
||||
the 3-edge curve + per-sleeve + OOS H1/H2 + cost HONESTLY (incl. the live-only fallback verdict if still
|
||||
absurd). `/paper` + `/paper/replay` render.
|
||||
- read-only/additive; sizing system unchanged; no `*.dbn` staged.
|
||||
@@ -0,0 +1,135 @@
|
||||
# Funding-history ingest + replayable xsfunding (3rd edge) — Design (Phase 2F-A4)
|
||||
|
||||
**Date:** 2026-06-22
|
||||
**Status:** Design (approved)
|
||||
**Repo:** fxhnt
|
||||
**Follows:** A3h. Adds the 3rd validated edge (cross-sectional funding carry) to the paper replay by ingesting
|
||||
historical funding rates into the warehouse.
|
||||
|
||||
## Motivation
|
||||
|
||||
The paper replay runs only 2 of the 3 validated edges; **xsfunding (funding-dispersion carry) is excluded —
|
||||
"live-only"** — because its `current_weights` reads a live funding snapshot and historical funding rates are
|
||||
**not in the warehouse** (unlock falls back to `funding=0`; the funding backtest reads the absent
|
||||
`crypto_pit_dir`). The A3h OOS split showed the 2-edge book is **regime-concentrated** (H1 Sharpe ~0.9, H2
|
||||
~5.6); a 3rd uncorrelated edge is the principled fix (diversification, not more tuning). This ingests funding
|
||||
history → warehouse → makes xsfunding replayable → 3-edge book.
|
||||
|
||||
## Key facts (grounded in code)
|
||||
|
||||
- Warehouse = generic feature store `features(symbol, ts, feature, value)`; `ts` = epoch-SECONDS, daily rows
|
||||
use `ts = epoch_day·86400`; folded via `ts//86400`. `write_features_bulk` is idempotent per (symbol,
|
||||
ts-range). `crypto_close_panel` reads `feature='close'`; `read_panel(symbols, feature)` reads any feature.
|
||||
→ **Funding needs NO new table — just `feature='funding'` rows + a mirror accessor.**
|
||||
- xsfunding signal (`domain/cross_sectional_funding.py`, pure): Panel = `{sym:{day:(close,qvol,funding)}}`;
|
||||
`eligible_asof` (qvol + history floor) → `funding_score` (robust_z of trailing-`lookback_days` mean funding)
|
||||
→ `construction_weights` (market-neutral long-low/short-high). Same panel shape unlock uses.
|
||||
- Binance perp funding history: public `GET /fapi/v1/fundingRate?symbol=&startTime=&endTime=&limit=1000`
|
||||
(8h cadence, `{symbol, fundingTime(ms), fundingRate}`). httpx is in the cockpit image (`web` extra).
|
||||
|
||||
## Decisions
|
||||
|
||||
- **Storage = the existing feature store, `feature='funding'`** (daily, `ts=epoch_day·86400`,
|
||||
value = that UTC day's summed funding = the day's total carry). No schema change.
|
||||
- **Ingest Binance perp `fundingRate` history**, 8h → **aggregated to daily (sum)**, for the warehouse's
|
||||
existing USDT-perp universe (`crypto_close_panel` symbols), via a never-raise-per-symbol httpx adapter +
|
||||
a CLI command + a Dagster asset (daily refresh keeps the live path consistent too).
|
||||
- **Accessor** `crypto_funding_panel(suffix='USDT') -> {sym:{day:funding}}` mirroring `crypto_close_panel`.
|
||||
- **`warehouse_funding_panel(close_panel, funding_panel, *, qvol_fallback)` -> `{sym:{day:(close,qvol,
|
||||
funding)}}`** (application): real funding now; qvol fallback as today. Serves BOTH xsfunding AND fixes
|
||||
unlock's `funding=0` fallback (bonus).
|
||||
- **Replayable xsfunding sleeve**: a `PositionSource` whose `current_weights(as_of)` runs the domain pipeline
|
||||
(`eligible_asof`→`funding_score`→`construction_weights`) over the warehouse panel, using the SAME params as
|
||||
the live xsfunding (mode/quantile/lookback/min_qvol/min_history from the live config) — faithful replay,
|
||||
just sourced from history instead of the live snapshot.
|
||||
- **Add xsfunding to the backfill sleeve set** → 3-edge replay. The overlay (A3h) + accounting (A3f) are
|
||||
unchanged; xsfunding is just another sleeve feeding the same `paper_risk_overlay`.
|
||||
- **DuckDB single-writer:** ingestion is a one-shot (CLI/asset) that must not run concurrently with other
|
||||
warehouse writers (documented; mirror the existing ingest ops' single-writer discipline).
|
||||
|
||||
## Components
|
||||
|
||||
1. **`adapters/exchange/binance_funding_history.py`** — `BinanceFundingHistory.fetch(symbol, start_ms,
|
||||
end_ms) -> list[tuple[int, float]]` (fundingTime ms, rate); paginated (limit 1000, walk startTime);
|
||||
per-symbol try/except → `[]` on error (never abort the batch); last-good. httpx, public base
|
||||
`https://fapi.binance.com`.
|
||||
2. **`DuckDbFeatureStore.crypto_funding_panel(*, suffix='USDT')`** — mirror of `crypto_close_panel` for
|
||||
`feature='funding'`.
|
||||
3. **`application/funding_ingest.py`** — `ingest_funding(store, fetcher, symbols, start, end) -> int`:
|
||||
per symbol fetch → aggregate 8h→daily (sum per UTC epoch_day) → `FeatureRow(ts=day·86400,
|
||||
{'funding': daily})` → `write_features_bulk`. Returns symbols written. Per-symbol skip-on-empty.
|
||||
4. **CLI `crypto-funding-ingest [--days N]`** + **Dagster asset `crypto_funding`** (daily) — wire (3) to the
|
||||
live warehouse store + the `crypto_close_panel` symbol universe + `BinanceFundingHistory`.
|
||||
5. **`application/funding_panel.py`** `warehouse_funding_panel(close_panel, funding_panel, *,
|
||||
qvol_fallback=1e7) -> Panel` — assemble `(close, qvol, funding)`; funding 0.0 only where a symbol/day
|
||||
genuinely lacks an ingested rate (logged). Reused by the xsfunding sleeve AND unlock (replaces its
|
||||
`funding=0` fallback).
|
||||
6. **Replayable xsfunding `PositionSource`** (`application/xsfunding_replay.py`): `current_weights(as_of)`
|
||||
= `construction_weights(funding_score(panel, eligible_asof(panel, day, ...), day, lookback), mode,
|
||||
quantile, ...)` with live params. `{}` when no eligible names that day.
|
||||
7. **`cli.py _paper_backfill_sleeves`**: add the xsfunding replay sleeve (built from
|
||||
`warehouse_funding_panel`) to the sleeve set; drop the "xsfunding excluded" note → now 3 edges.
|
||||
8. **Verify**: re-ingest + re-backfill in-cluster; run the OOS-half + cost analysis on the 3-edge book; check
|
||||
xsfunding contributes and the H1/H2 regime-concentration improves (more stable H1, higher blended Sharpe,
|
||||
lower DD) — the actual test of whether the 3rd edge helped.
|
||||
|
||||
## Risks / safety
|
||||
|
||||
- **External API:** rate-limited, paginated, per-symbol try/except → `[]` (a bad/missing symbol never aborts
|
||||
the run); idempotent writes (re-run replaces per (symbol, ts-range)). The `fundingRate` endpoint is light.
|
||||
- **Read-only/additive to the book:** new feature rows + a new replayable sleeve; A3f accounting, A3h overlay,
|
||||
shadow signal, the other 2 sleeves all unchanged. xsfunding's LIVE path is untouched (replay is additive).
|
||||
- **Faithful replay:** the replay sleeve uses the SAME pure domain pipeline + live params as live xsfunding;
|
||||
only the data source differs (warehouse history vs live snapshot). Causal (`eligible_asof`/`funding_score`
|
||||
use only `≤ asof_day`).
|
||||
- **DuckDB single-writer:** ingest one-shot, not concurrent with other warehouse writers.
|
||||
- **Funding aggregation:** 8h→daily SUM = the day's total carry (matches a daily-rebalanced book holding
|
||||
through the 3 funding stamps). Documented.
|
||||
- **Honest caveat:** still gross/frictionless for the *new* sleeve too; the cost model (already estimated)
|
||||
applies to the 3-edge turnover at verify time.
|
||||
|
||||
## ADDENDUM (post-T4 finding): xsfunding is a CARRY edge — needs a carry return-source
|
||||
|
||||
**Finding (verified in code):** xsfunding's P&L is **delta-neutral carry**, not directional price. The domain
|
||||
`book_return_xs = Σ wᵢ·fundingᵢ`; the live sleeve books `w·((spot_ret−perp_ret)+funding)`. The paper book's
|
||||
per-sleeve return is price-MTM (`Σ qty·Δclose`) on a single perp-close series — which would measure the
|
||||
*directional* price drift of funding-sorted perps (~3%/day noise) and completely miss the ~0.03%/day carry.
|
||||
So xsfunding **cannot** be added via the price-MTM path; it needs a **carry return-source**. (Approved by user:
|
||||
"extend the book.")
|
||||
|
||||
**Design:**
|
||||
- New pure **`sleeve_daily_carry(prev_positions, funding_by_symbol) -> float | None`** (`paper_book.py`):
|
||||
`Σ (qty·entry_price)·funding / Σ|qty·entry_price|`. `qty·entry_price` recovers the signed unit-weight from
|
||||
the shadow book; normalized by gross. Mirrors `sleeve_daily_return` (funding instead of Δclose). `None` on
|
||||
zero notional. (= the domain `book_return_xs` per unit gross.)
|
||||
- **Per-sleeve return mode:** `backfill_paper_book` + `PaperSnapshotService` compute each sleeve's daily
|
||||
return by `getattr(sleeve, "return_mode", "price")`: `"price"` → `sleeve_daily_return(prev_shadow, closes)`
|
||||
(unchanged); `"carry"` → `sleeve_daily_carry(prev_shadow, funding_today)`. `XsFundingReplay.return_mode =
|
||||
"carry"`. Both paths need the per-(day,symbol) funding (backfill: from `crypto_funding_panel`; snapshot:
|
||||
latest funding per symbol). Everything downstream consumes `paper_sleeve_ret` unchanged → live==replay.
|
||||
- **Consequence (intended):** xsfunding's low carry-vol → large inverse-vol weight (risk-parity), then
|
||||
vol-target levers the blend — the correct risk-parity 3-edge behavior; the verify step tests whether it
|
||||
de-concentrates the H1/H2 regime risk. The directional sleeves (unlock, ts-trend) stay price-MTM unchanged.
|
||||
- Revised tasks: **5a** `sleeve_daily_carry`; **5b** per-sleeve return-mode wiring (backfill+snapshot) + add
|
||||
xsfunding (carry) to the backfill sleeve set; **6** deploy/ingest/verify (does the carry edge help?).
|
||||
|
||||
## Out of scope
|
||||
|
||||
Switching live xsfunding off its live snapshot (replay is additive; live unchanged); spot-perp basis modeling
|
||||
(xsfunding replay uses funding-only, as the funding-only fallback the backtest documents); walk-forward
|
||||
re-selection of overlay params on the 3-edge book (separate); per-sleeve cost modeling.
|
||||
|
||||
## Acceptance criteria
|
||||
|
||||
- `BinanceFundingHistory.fetch`: paginates, parses `{fundingTime, fundingRate}`, per-symbol error → `[]`;
|
||||
unit-tested with a mocked httpx (multi-page + error).
|
||||
- `crypto_funding_panel` reads `feature='funding'` → `{sym:{day:funding}}`; tested.
|
||||
- `ingest_funding`: 8h→daily sum, idempotent write; tested with a fake fetcher + in-memory store (re-run =
|
||||
same rows).
|
||||
- `warehouse_funding_panel`: assembles `(close,qvol,funding)` with real funding; tested.
|
||||
- replayable xsfunding `current_weights(as_of)`: matches the domain pipeline on a fake panel (long-low /
|
||||
short-high, market-neutral); causal; `{}` on no-eligible; tested.
|
||||
- backfill includes xsfunding (3 sleeves); CLI ingest + asset wired; full suite green.
|
||||
- in-cluster: ingest funding → re-backfill 3-edge → xsfunding has positions/returns; OOS-half + cost rerun
|
||||
reported (does the 3rd edge de-concentrate H1/H2?); `/paper` + `/paper/replay` render; verified.
|
||||
- read-only/additive (accounting/overlay/other sleeves); no `*.dbn` staged.
|
||||
@@ -0,0 +1,85 @@
|
||||
# Paper book: ISV-adaptive vol budget — Design (Phase 2F-A3h)
|
||||
|
||||
**Date:** 2026-06-22
|
||||
**Status:** Design (approved)
|
||||
**Repo:** fxhnt
|
||||
**Follows:** A3g (fixed 30% full-Kelly overlay). Makes the vol BUDGET itself state-adaptive.
|
||||
|
||||
## Motivation
|
||||
|
||||
A3g set a fixed 30% full-Kelly vol budget. The adaptive-budget A/B sweep on the corrected curve showed an
|
||||
**ISV-adaptive budget bounded [25%, 35%] beats both fixed 30% and fixed 35% on Sharpe** (3.39 vs 3.24/3.28),
|
||||
at +221% / −14.5% DD. The win is NOT from pressing into strength — it's from **easing the budget toward 25%
|
||||
during weak patches** (when trailing book Sharpe softens) and running near the 35% ceiling when the edge is
|
||||
firing. User cap: 35% ("40% is massive"); this design never exceeds it.
|
||||
|
||||
## Decisions (from the sweep + user)
|
||||
|
||||
- **Adaptive target-vol scaled by trailing book Sharpe**, bounded **[25%, 35%]**, reference Sharpe **2.0**
|
||||
(the "eager" winner — reaches the 35% ceiling once the trailing book Sharpe ≥ 2, eases to 25% as it →0).
|
||||
Bootstrap to 30% until enough history.
|
||||
- **Same signal in BOTH sizing paths** — `paper_book_returns` (per historical date, from the running book
|
||||
series it already builds) AND `paper_risk_overlay` (latest date, from the full prior book series) — so live
|
||||
`/paper` == `/paper/replay` stays exact.
|
||||
- **Causal:** the budget for date d uses the trailing realized book returns STRICTLY BEFORE d.
|
||||
- Everything else from A3g unchanged: full-Kelly, dormant wide −25/−12 killswitch, marginal-gate OFF.
|
||||
|
||||
## Components
|
||||
|
||||
1. **`adaptive_target_vol(book_returns, *, vol_min=_VOL_MIN, vol_max=_VOL_MAX, sh_ref=_VOL_SH_REF,
|
||||
base=_VOL_BASE, win=_VOL_LOOKBACK, min_obs=_VOL_MIN_OBS) -> float`** (pure, `paper_book.py`):
|
||||
- `w = book_returns[-win:]`; if `len(w) < min_obs` → return `base` (bootstrap).
|
||||
- `sd = pstdev(w)`; `ts = mean(w)/sd·√periods_per_year` if `sd>0` else `0.0` (trailing ann. book Sharpe).
|
||||
- `frac = clip(ts / sh_ref, 0.0, 1.0)`; return `vol_min + (vol_max − vol_min)·frac`.
|
||||
- Constants: `_VOL_MIN=0.25`, `_VOL_MAX=0.35`, `_VOL_SH_REF=2.0`, `_VOL_BASE=0.30` (rename of the A3g
|
||||
`_TARGET_VOL=0.30` → kept as the bootstrap base), `_VOL_MIN_OBS=20`, reuse `_VOL_LOOKBACK=60`,
|
||||
`_PERIODS_PER_YEAR=365`.
|
||||
2. **`paper_book_returns`:** in the per-date loop, after `out[d]` is set, compute
|
||||
`tv = adaptive_target_vol(list(out.values()))` and pass `target_vol=tv` to `vol_target_leverage` (replacing
|
||||
the fixed `target_vol`). The book series `out` it already accumulates is the causal trailing signal.
|
||||
3. **`paper_risk_overlay`:** after `bret = paper_book_returns(returns_by_sleeve)`, compute
|
||||
`tv = adaptive_target_vol(list(bret.values()))` and use it for the latest-date
|
||||
`vol_target_leverage(weights, returns_by_sleeve, target_vol=tv, ...)`. (Killswitch replay over `bret`
|
||||
unchanged.) This matches what `paper_book_returns` would compute for the next date → consistent.
|
||||
4. `vol_target_leverage` itself is the unchanged primitive (still takes an explicit `target_vol`).
|
||||
|
||||
## Data flow
|
||||
|
||||
Per date (both paths): trailing realized book returns (strictly prior, from the series `paper_book_returns`
|
||||
builds / the `bret` the overlay gets) → `adaptive_target_vol` → bounded [25%,35%] budget → `vol_target_leverage`
|
||||
→ leverage. Bootstrap 30% until 20 obs. The adaptive budget rides near 35% in strong stretches, eases to 25%
|
||||
in weak ones; the dormant −25/−12 kill backstops a true crash.
|
||||
|
||||
## Risks / safety
|
||||
|
||||
- **Read-only/additive w.r.t. accounting + strategy mechanics:** only the vol BUDGET becomes adaptive (a
|
||||
function of the book's own trailing returns). A3f equity accounting, shadow signal, ISV weights, killswitch,
|
||||
and the `vol_target_leverage` primitive are unchanged. No book_allocator/sleeve/broker change.
|
||||
- **Bounded — never "massive":** hard `[vol_min=0.25, vol_max=0.35]` clip; `_MAX_LEVERAGE=3.0` still caps
|
||||
notional; the budget can only move within the user's ceiling.
|
||||
- **Procyclicality** (the honest risk of any performance-scaled budget): mitigated because (a) the win comes
|
||||
from EASING in weak patches, not pressing into tops; (b) it's bounded to a narrow [25,35] band; (c) the
|
||||
vol-target + dormant kill remain. Validated on the corrected curve (Sharpe 3.39 > fixed) — but it is ONE
|
||||
~1yr sample; documented, not a walk-forward claim.
|
||||
- **Causality:** `adaptive_target_vol` consumes only the strictly-prior book series (`out` through d−1 /
|
||||
`bret` over prior dates). No look-ahead.
|
||||
- **Consistency:** identical `adaptive_target_vol` call in both paths → live `/paper` == `/paper/replay`.
|
||||
- **Degenerate/cold:** `<min_obs` → base 0.30; zero-vol window → ts 0 → budget vol_min (25%); never NaN
|
||||
(guarded `sd>0`).
|
||||
|
||||
## Out of scope
|
||||
|
||||
Fee/slippage model; walk-forward / multi-window validation of the adaptive rule; per-sleeve adaptive budgets;
|
||||
re-enabling the marginal-gate; adding xsfunding to replay.
|
||||
|
||||
## Acceptance criteria
|
||||
|
||||
- `adaptive_target_vol`: bootstrap (`<min_obs` → base 0.30); strong trailing Sharpe (≥ sh_ref) → vol_max
|
||||
(0.35); weak/negative → vol_min (0.25); monotone in trailing Sharpe; bounded [0.25,0.35]; never NaN;
|
||||
unit-tested.
|
||||
- `paper_book_returns` + `paper_risk_overlay` both use it on the strictly-prior book series; causal; the two
|
||||
paths produce identical budgets for identical history (live==replay); unit-tested.
|
||||
- Full suite green (overlay tests updated for the adaptive budget where they implied a fixed one).
|
||||
- Re-run backfill in-cluster → `/paper/replay` curve ≈ the sweep's ADAPT[25→35] row (~+220% gross, Sharpe
|
||||
~3.4, DD ~−14.5%), capped at the 35% budget; `/paper` live agrees; verified post-deploy.
|
||||
- Read-only/additive (accounting/strategy mechanics); no `*.dbn` staged.
|
||||
@@ -0,0 +1,84 @@
|
||||
# Paper-book foundation audit (Phase 2F — pause & audit)
|
||||
|
||||
**Date:** 2026-06-22
|
||||
**Status:** Audit (no code) — decide the architecture before resuming
|
||||
**Trigger:** "going in circles" — A3c→A4→A5 each layered a fix that revealed the next gap. Stop, audit.
|
||||
|
||||
## Layers (current architecture, as built)
|
||||
|
||||
1. **Data (warehouse, DuckDB feature store)** — `feature='close'` (perp, 2020-09→2026, 217 sym),
|
||||
`'spot_close'` (2017→2026, 162 sym), `'funding'` (2020-06→2026, 217 sym). Daily granularity.
|
||||
**Survivorship-biased** (only currently-listed symbols have history; delisted coins absent).
|
||||
2. **Per-sleeve return signal (`paper_sleeve_ret`)** — per (date, sleeve) daily return, from a **unit shadow
|
||||
book** (`paper_shadow_positions`, never throttled). Directional sleeves: `sleeve_daily_return` = price MTM
|
||||
(`Σ qty·Δclose / gross`). Carry (xsfunding): `sleeve_daily_carry` = delta-neutral `Σ w·((spot_ret−perp_ret)
|
||||
+funding)/gross`. **Per-sleeve return MODE** (price vs carry) via `getattr(sleeve,'return_mode')`.
|
||||
3. **Sizing overlay (`paper_risk_overlay`)** — ISV-adaptive inverse-vol weights (`isv_adaptive_weights`) +
|
||||
vol-target leverage (`vol_target_leverage`, adaptive budget [25,35]% A3h, cap 3.0) + drawdown killswitch
|
||||
(`DrawdownKillSwitch`, −25/−12). Computed from the strictly-prior `paper_sleeve_ret` window.
|
||||
4. **Equity accounting (`paper_nav`, `persist_paper_book`)** — compounding MTM (A3f): `equity_d = equity_{d-1}
|
||||
+ Σ positions_before.qty·(close − entry)`; book sized off running equity.
|
||||
5. **Verification** — offline sweeps over `paper_sleeve_ret`; the real in-cluster backfill is ground truth.
|
||||
|
||||
## CONFIRMED issues (with evidence)
|
||||
|
||||
### F1 — ⛔ Equity ignores the per-sleeve return mode (carry P&L is dropped). [LOAD-BEARING]
|
||||
`persist_paper_book` computes `gain = Σ p.qty·(price − entry)` over ALL positions — **price MTM only, no carry
|
||||
branch.** The carry sleeve's positions are market-neutral perp longs/shorts → marked to perp close they net to
|
||||
~0 (basis only). **So xsfunding's +1.8-Sharpe funding P&L NEVER reaches `paper_nav`.** The signal layer (F2)
|
||||
uses carry; the equity layer uses price. **Basis mismatch between sizing and equity.** Evidence: all-xsfunding
|
||||
dates (2020-10) where equity barely moves; `persist_paper_book:30` is price-only.
|
||||
→ The "3-edge" equity is really the 2 directional edges; carry only crowds *weights*, contributes ~0 *return*.
|
||||
|
||||
### F2 — ⛔ Leverage caps the vol-target MULTIPLIER, not actual GROSS exposure.
|
||||
`vol_target_leverage` caps `lev ≤ 3.0`. But effective gross = `lev × Σ w_sleeve·gross_sleeve`; the
|
||||
market-neutral carry has `gross_sleeve=2` → **observed effective gross 6.0×** (cluster: 2020 mean 5.82, max
|
||||
6.00). The cap is on the wrong quantity.
|
||||
|
||||
### F3 — ⛔ No warmup: max leverage on cold-start thin history.
|
||||
2020 (series start, thin history) ran mean 5.8× leverage → the multi-year book's **−98.9% maxDD trough was
|
||||
2021-02-28** ($100k→$175k→$1,986). The vol-target levers maximally before it has a stable vol estimate.
|
||||
|
||||
### F4 — ⚠️ Multi-year OOS: the overlay was overfit to a 1-yr bull.
|
||||
1-yr (2025-26) showed +147%/Sharpe 2.74; the SAME overlay over 2020-2026 = **−98.9% maxDD**. The A3g/A3h
|
||||
re-tune (30% full-Kelly) was calibrated on a benign window.
|
||||
|
||||
### F5 — ⚠️ Offline sweep under-models risk → unreliable for tuning.
|
||||
The cycle sweep reported −3% to −23% maxDD; the real deployed backfill was −98.9% (it ignores the gross-2
|
||||
stacking + uses a has-history active-set proxy). **Tune/verify on the real backfill, never the sweep.**
|
||||
|
||||
### F6 — ⚠️ Carry deep-tail unmodellable from this data.
|
||||
Carry maxDD only −3.4% even through 2022, because the catastrophic carry losses (FTX counterparty, LUNA
|
||||
delist, ADL, intraday squeezes) are absent from **daily + survivorship-biased + Binance-surviving** data. The
|
||||
*normal* carry risk is realistic (Sharpe 1.80 multi-year); the *tail* is structurally invisible here.
|
||||
|
||||
## What is SOUND (keep)
|
||||
- The edges are real (multi-year standalone Sharpe: ts-trend 2.39, unlock 1.73, **xsfunding 1.80**).
|
||||
- A3f compounding accounting is correct **for price-MTM (directional) sleeves**; live==replay holds for them.
|
||||
- The shadow-signal design (kill-proof, causal) is sound for directional sleeves.
|
||||
- Data ingestion (funding/spot/perp, multi-year) is correct + reusable.
|
||||
|
||||
## Architectural decisions needed
|
||||
- **D1 (carry):** carry has TWO independent blockers (F1 equity-gap + F6 unmodellable tail). Decide:
|
||||
(a) **carry live-only** (revert from replay; keep `xsfunding_nav` forward-track + the data) — converges; OR
|
||||
(b) wire carry P&L into equity (fix F1) AND accept the tail stays optimistic (F6) + cap its weight.
|
||||
**Audit recommendation: (a)** — the daily-survivor replay cannot faithfully carry a delta-neutral funding
|
||||
edge; its truth is a live forward record.
|
||||
- **D2 (leverage):** cap **actual gross exposure** (e.g. ≤3× gross incl. market-neutral legs), not the
|
||||
vol-target multiplier (fixes F2). Apply post-sizing in `persist_paper_book`/`derive_and_persist`.
|
||||
- **D3 (warmup):** no leverage > 1.0 until ≥ `vol_lookback` (60) real book-return obs (fixes F3).
|
||||
- **D4 (validation):** re-derive/verify the overlay on the **full 6-yr cycle via the real backfill** (not the
|
||||
sweep); the target is a *survivable* maxDD across the bear, accepting lower bull-year return (fixes F4/F5).
|
||||
- **D5 (scope discipline):** one coherent change implementing the chosen D1+D2+D3, then verify — no more
|
||||
incremental layering.
|
||||
|
||||
## Recommended target architecture (if D1=a)
|
||||
A **2-edge directional paper book** (ts-trend + unlock), price-MTM compounding (sound today), with a
|
||||
**gross-capped, warmed-up** overlay re-derived on the full cycle. **xsfunding stays live-only** (`xsfunding_nav`
|
||||
+ the ingested funding/spot data retained). This is internally consistent (signal==equity basis), structurally
|
||||
safe (gross cap + warmup), and validated cross-cycle — and it stops the layering. The carry edge is real and
|
||||
tracked where its risk is real (live), not faked in a daily replay.
|
||||
|
||||
## Immediate state
|
||||
Cockpit currently shows the 6-yr multi-year backfill (+4320% / −98.9% — the directional book at 6× gross). It
|
||||
should be restored to a trustworthy view once D1–D4 are decided + implemented.
|
||||
@@ -0,0 +1,111 @@
|
||||
# Paper book: correct (compounding) equity accounting — Design (Phase 2F-A3f)
|
||||
|
||||
**Date:** 2026-06-22
|
||||
**Status:** Design (approved)
|
||||
**Repo:** fxhnt
|
||||
**Follows:** A3e. Fixes the load-bearing P&L-accounting bug that made the cockpit curve show a loss while the
|
||||
edges are strongly profitable.
|
||||
|
||||
## Motivation (root cause, confirmed)
|
||||
|
||||
The cockpit equity curve showed ≈−8% while the two replayable edges are intrinsically **+92.6%
|
||||
(crypto_tstrend, Sharpe 2.39)** and **+195.7% (unlock, Sharpe 1.73)** over 2025-06→2026-06 — a 50/50 blend of
|
||||
the clean per-sleeve signals compounds to **+153.5%**. Two sleeves at +92%/+195% cannot blend to a loss, so
|
||||
the equity curve is wrong. Root cause in `persist_paper_book`:
|
||||
- `target_positions` re-derives every position each day at that day's close → `entry_price` = today's close.
|
||||
- `unrealized = Σ qty·(price − entry_price)` with `price` = today's close = `entry_price` → **`unrealized` ≡ 0
|
||||
every day.**
|
||||
- `equity = capital + realized` then comes only from `realized_delta` (rebalance slivers, cost basis reset
|
||||
daily), which **discards the entire mark-to-market appreciation of held positions.**
|
||||
The risk overlays (A3c/A3d/A3e: −29%→−10%→−9% DD) were therefore tuned against a meaningless curve; they are
|
||||
valid machinery but must be re-judged on the corrected equity.
|
||||
|
||||
## Decisions (from brainstorming, approved)
|
||||
|
||||
- **Equity = mark-to-market of the held book, compounding.** The book's true daily P&L = the prior day's
|
||||
throttled book marked to today's close: `gain_d = Σ positions_before(d).qty·(close_d − entry_price)`
|
||||
(`entry_price` = prior close). `equity_d = equity_{d-1} + gain_d`.
|
||||
- **Compounding (user choice):** size each date's throttled book off the **running equity**, not fixed
|
||||
`capital`. Then `gain_d = equity_{d-1}·r_book_d` and `equity_d = equity_{d-1}·(1 + r_book_d)` — clean
|
||||
compounding falls out. The fixed `capital` ($100k) is only the initial seed + the nav `realized` baseline.
|
||||
- **Killed/flat days handled for free:** a kill makes today's book empty → `positions_before(next)` empty →
|
||||
`gain = 0` → equity flat while in cash. Correct.
|
||||
- **Both equity consumers fixed consistently:** the replay curve (`persist_paper_book` → `paper_nav`) and the
|
||||
live `/paper` (`PaperBookService.view`).
|
||||
- **Trades ledger unchanged** (`diff_trades`/`replace_trades` for display); the `realized_delta`-based equity
|
||||
contribution is dropped (superseded by MTM). The now-unused realized-pnl repo methods are left in place
|
||||
(out-of-scope cleanup) to bound blast radius.
|
||||
- **Frictionless gross** (idealized daily rebalance to `current_weights`, no fees/slippage) — a cost model is
|
||||
a separate later refinement; noted, not in scope.
|
||||
|
||||
## Components
|
||||
|
||||
1. **`PaperRepo.nav_equity_before(run_date) -> float | None`** — the most recent `paper_nav.equity` strictly
|
||||
before `run_date` (the prior equity to compound from; `None` → caller seeds with `capital`).
|
||||
2. **`PaperRepo.latest_nav_equity() -> float | None`** — the most recent `paper_nav.equity` (for the live view).
|
||||
3. **`PaperBookService.derive_and_persist(..., capital_base: float)`** — size the throttled book off
|
||||
`capital_base` (the running equity) instead of `self._capital`: `target_positions(...,
|
||||
capital=capital_base · leverage ...)`. Drop the `realized_delta`/`set_realized_for_run_date` equity calls;
|
||||
keep `diff_trades`/`replace_trades` (display) + `replace_positions`.
|
||||
4. **`persist_paper_book`** — the orchestrator computes the compounded equity then sizes off it:
|
||||
```
|
||||
prior = repo.positions_before(run_date) # prior throttled book (entry = prior close)
|
||||
gain = Σ prior.qty·(prices.get(sym, entry) − entry) # prior book marked to today's close
|
||||
e_prev = repo.nav_equity_before(run_date) or capital # equity_{d-1} (seed = capital on day 0)
|
||||
equity = e_prev + gain # = e_prev·(1 + r_book) under this sizing
|
||||
svc.derive_and_persist(... capital_base=equity, leverage=leverage) # size today's book off the new equity
|
||||
repo.upsert_nav(run_date, capital=capital, realized=e_prev−capital, unrealized=gain, equity=equity, at)
|
||||
```
|
||||
`realized = e_prev − capital` (cumulative through prior day), `unrealized = gain` (today's MTM) →
|
||||
`capital + realized + unrealized = equity` keeps the schema meaningful. Idempotent (ascending recompute /
|
||||
upsert per run_date; `positions_before` + `nav_equity_before` are both strictly-prior, unaffected by
|
||||
rewriting today's rows).
|
||||
5. **`PaperBookService.view`** — `equity = (latest_nav_equity or capital) + Σ current_positions.qty·(live_price
|
||||
− entry)`: the curve through last close plus the live intraday move. Drop the `realized_pnl()` term.
|
||||
|
||||
## Data flow
|
||||
|
||||
Per date (both backfill + snapshot, unchanged callers — the change is internal to `persist_paper_book`):
|
||||
prior throttled book marked to today's close → `gain_d` → `equity_d = equity_{d-1} + gain_d` → size today's
|
||||
book off `equity_d` → persist nav. The shadow signal (`paper_sleeve_ret`), `paper_risk_overlay`
|
||||
(weights/leverage/kill), and `unit_shadow_positions` are all UNCHANGED — only the throttled-book sizing base
|
||||
(capital → running equity) and the nav equity computation change.
|
||||
|
||||
## Risks / safety
|
||||
|
||||
- **Read-only/additive w.r.t. strategy:** no change to `book_allocator`, sleeve `run()`/`advance()`,
|
||||
`combine_book`, the gate record, `paper_risk_overlay`, the shadow signal, or any broker/live path. Only the
|
||||
paper-book equity accounting + sizing base.
|
||||
- **Causality:** `gain_d` uses the strictly-prior book marked to today's close; equity compounds from
|
||||
strictly-prior equity. No look-ahead.
|
||||
- **Idempotency:** ascending backfill recompute is deterministic; `nav_equity_before`/`positions_before` are
|
||||
strictly-prior; `upsert_nav`/`replace_positions` per run_date. Re-run reproduces identical equity.
|
||||
- **No runaway:** leverage stays capped at 3.0; notional = weight·equity·leverage scales with equity (a
|
||||
growing book takes proportionally larger positions — intended for compounding). A drawdown shrinks equity →
|
||||
smaller positions (de-risking), which is correct compounding behavior.
|
||||
- **Day 0 / empty prior:** `nav_equity_before` None → seed `capital`; `positions_before` empty → gain 0 →
|
||||
equity = capital. Clean start.
|
||||
- **Test churn:** existing tests asserting the old equity/realized semantics (e.g. realized-monotone backfill,
|
||||
equity==capital-with-no-pnl) will be updated to the compounding semantics, preserving intent.
|
||||
|
||||
## Out of scope
|
||||
|
||||
Fees/slippage cost model; removing the now-dead realized-pnl repo methods; re-tuning the A3d/A3e risk-overlay
|
||||
constants against the corrected curve (separate follow-up once the real curve is visible); the live-cache vs
|
||||
warehouse price-source coupling.
|
||||
|
||||
## Acceptance criteria
|
||||
|
||||
- `nav_equity_before`/`latest_nav_equity` return the right strictly-prior / latest equity; `None`-when-empty;
|
||||
tested.
|
||||
- `persist_paper_book` compounds: a rising sleeve grows equity multiplicatively off the prior equity (a
|
||||
two-day rising series gives `equity_2 ≈ equity_1·(1+r_2)`, not `capital + sliver`); a killed day → flat
|
||||
equity; day 0 → equity = capital; idempotent re-run; unit/integration-tested.
|
||||
- `derive_and_persist` sizes off `capital_base` (notional = weight·capital_base·leverage); `leverage=0` →
|
||||
flat; default behavior covered.
|
||||
- `PaperBookService.view` equity = latest nav equity + live intraday MTM of the current book; tested.
|
||||
- Full suite green (old equity/realized assertions updated to compounding semantics, intent preserved).
|
||||
- Re-run backfill in-cluster → `/paper/replay` curve is now strongly **profitable** (reflecting the +92%/+195%
|
||||
edges, compounded, after the risk overlay — not −8%); `/paper` live equity matches; verified post-deploy.
|
||||
Report the new final equity + max drawdown on the corrected curve.
|
||||
- Read-only/additive (strategy); no `*.dbn` staged.
|
||||
@@ -0,0 +1,96 @@
|
||||
# Paper book: marginal-gate edge pruning + A3c/A3d cleanup — Design (Phase 2F-A3e)
|
||||
|
||||
**Date:** 2026-06-22
|
||||
**Status:** Design (approved)
|
||||
**Repo:** fxhnt
|
||||
**Follows:** A3d vol-target + killswitch ([[project_fxhnt_a3d_vol_target_killswitch]]). Adds the leave-one-out
|
||||
edge-quality gate and clears the two logged A3c/A3d follow-ups.
|
||||
|
||||
## Motivation
|
||||
|
||||
The paper book sizes every active sleeve by inverse-vol regardless of whether it still contributes. The live
|
||||
`book_allocator` has a `marginal_gate` (leave-one-out marginal-Sharpe pruning) for exactly this — bench a
|
||||
sleeve only when **removing** it raises the trailing book Sharpe, which keeps a legitimate anticorrelated
|
||||
hedge while benching a dead, non-diversifying sleeve (a plain Sharpe floor can't tell them apart). The user
|
||||
asked to **set the marginal_gate** on the paper book. Two small follow-ups logged during A3c/A3d are cleared
|
||||
in the same change: dead `equal_weights`, and a zero-fill bias in `_regime_lookback`.
|
||||
|
||||
## Decisions (from brainstorming, approved)
|
||||
|
||||
- **marginal_gate is a faithful reuse**, not a reinvention: `book_allocator.align_edges` +
|
||||
`book_allocator._marginal_prune` (which itself uses `_inverse_vol_weights` + `per_period_sharpe`). Trusted
|
||||
live code.
|
||||
- **Cold sleeves pass through.** Sleeves with `<2` returns can't be Sharpe-scored — they bypass the gate
|
||||
(benefit of the doubt; the isv cold-start floor still sizes them), mirroring the mixed-history rule in
|
||||
`isv_adaptive_weights`. Only has-history sleeves are eligible for pruning.
|
||||
- **Applied in BOTH paths** (`paper_risk_overlay` sizing AND `paper_book_returns` shadow-replay) so the actual
|
||||
book and the killswitch regime signal agree → live `/paper` == `/paper/replay` is preserved.
|
||||
- **Causal + always-on.** Uses the strictly-prior `returns_by_sleeve` already passed in; no flag (the user
|
||||
asked to set it on for the paper book).
|
||||
- **Cleanup:** remove dead `equal_weights` (+ its dedicated test); make `_regime_lookback` NaN-aware so
|
||||
disjoint sleeve coverage no longer biases the regime ratio.
|
||||
|
||||
## Components
|
||||
|
||||
1. **`marginal_gate_active(returns_by_sleeve, active_sleeves, *, vol_lookback=_VOL_LOOKBACK,
|
||||
max_sleeve_weight=_PHI) -> list[str]`** (pure, `paper_book.py`):
|
||||
- `has = [s in active with len(returns)>=2]`; `cold = [s in active not in has]`.
|
||||
- If `len(has) < 2` → return `active` unchanged (nothing to score / shrink).
|
||||
- `window = align_edges({s: returns[s] for s in has})[1]`, truncated to the last `vol_lookback` rows
|
||||
per sleeve.
|
||||
- `kept_has = set(_marginal_prune(window, max_sleeve_weight))`.
|
||||
- Return `[s for s in active_sleeves if s in cold or s in kept_has]` (preserve input order).
|
||||
Imports `align_edges`, `_marginal_prune` from `fxhnt.application.book_allocator`.
|
||||
2. **`paper_risk_overlay`** (`paper_book.py`): before computing weights, `active =
|
||||
marginal_gate_active(returns_by_sleeve, active_sleeves)`; then `weights =
|
||||
isv_adaptive_weights(returns_by_sleeve, active)` (rest unchanged: leverage, killswitch replay).
|
||||
3. **`paper_book_returns`** (`paper_book.py`): inside the per-date loop, after computing `before_d`/`active`,
|
||||
`active = marginal_gate_active(before_d, active)` before `isv_adaptive_weights(before_d, active)` — so the
|
||||
shadow regime signal reflects the same pruned book.
|
||||
4. **`_regime_lookback`** (`paper_book.py`): build the matrix with `np.nan` for missing cells (`np.full(...,
|
||||
np.nan)`), and use `np.nanmean(axis=0)` for the representative series; guard a fully-NaN slice (→ hold
|
||||
`L_max`). Replaces the `np.zeros` zero-fill so disjoint coverage doesn't depress `ρ`.
|
||||
5. **Remove** `equal_weights` from `paper_book.py` and delete `tests/unit/test_equal_weights.py`. The
|
||||
`*_equal_weights_*` tests in backfill/snapshot stay (they assert cold-start sizing, not the function).
|
||||
|
||||
## Data flow
|
||||
|
||||
Per date (both paths): `returns_by_sleeve` (strictly-prior) + active set → `marginal_gate_active` benches any
|
||||
dead/non-diversifying has-history sleeve → `isv_adaptive_weights` over the kept set → leverage + killswitch as
|
||||
in A3d. Identical helper in both paths keeps live == replay.
|
||||
|
||||
## Risks / safety
|
||||
|
||||
- **Read-only/additive.** No change to `book_allocator` (only imports from it), sleeve `run()`/`advance()`,
|
||||
`combine_book`, the gate record, or any broker/live path. No new tables.
|
||||
- **Causality:** `marginal_gate_active` consumes only the strictly-prior `returns_by_sleeve`; the leave-one-out
|
||||
Sharpe is over the trailing window; no look-ahead.
|
||||
- **Never empties the book:** `_marginal_prune` shrinks to at least one sleeve; cold sleeves always pass
|
||||
through; `<2` has-history → unchanged.
|
||||
- **Consistency:** same `marginal_gate_active` in `paper_risk_overlay` + `paper_book_returns` → live == replay.
|
||||
- **3-sleeve reality:** usually keeps all three (each diversifies); benches one only when genuinely dead in a
|
||||
window. The gate cannot make the book worse than the un-gated inverse-vol on the trailing window by
|
||||
construction (it only removes a sleeve when removal raises trailing book Sharpe).
|
||||
- **Cleanup safety:** `equal_weights` has no production caller (verified — only its own test); removing it +
|
||||
its test leaves the suite green. `_regime_lookback` NaN-aware change only affects `L_t` (bounded
|
||||
`[L_min, L_max]`), never crashes.
|
||||
|
||||
## Out of scope
|
||||
|
||||
`min_sleeve_sharpe` raw Sharpe-floor gate (inferior to marginal — skip); making marginal_gate a tunable flag
|
||||
(YAGNI — always-on for the paper book); the live-cache vs warehouse price-source coupling and the
|
||||
`_live_sleeves`↔backfill dedup (separate tech debt).
|
||||
|
||||
## Acceptance criteria
|
||||
|
||||
- `marginal_gate_active`: benches a dead/non-diversifying has-history sleeve (removal raises trailing book
|
||||
Sharpe) while keeping a diversifying one; cold (`<2`) sleeves pass through; `<2` has-history → unchanged;
|
||||
never returns empty when active non-empty; unit-tested.
|
||||
- `paper_risk_overlay` + `paper_book_returns` both apply it (same pruned active set); unit-tested that a
|
||||
clearly-dead sleeve is dropped from the overlay weights.
|
||||
- `_regime_lookback` NaN-aware: disjoint-coverage sleeves no longer bias `ρ` toward `L_max` via zero-fill;
|
||||
unit-tested (a sleeve with a late start doesn't zero-depress the ratio); existing A3c regime tests stay green.
|
||||
- `equal_weights` removed; `tests/unit/test_equal_weights.py` deleted; full suite green (the backfill/snapshot
|
||||
cold-start `*_equal_weights_*` tests still pass).
|
||||
- Re-run backfill in-cluster → book still populated (gate didn't empty it); `/paper` + `/paper/replay` render;
|
||||
report whether any sleeve gets benched in any window. Read-only/additive; no `*.dbn` staged.
|
||||
@@ -0,0 +1,78 @@
|
||||
# Paper book: risk-overlay re-tune (vol budget + dormant kill + drop gate) — Design (Phase 2F-A3g)
|
||||
|
||||
**Date:** 2026-06-22
|
||||
**Status:** Design (approved)
|
||||
**Repo:** fxhnt
|
||||
**Follows:** A3f (compounding equity fix) + the overlay A/B sweep on the corrected curve.
|
||||
|
||||
## Motivation
|
||||
|
||||
With the equity accounting fixed (A3f), the overlay A/B sweep on the corrected curve showed the live overlay
|
||||
(12% vol-target, ½-Kelly, +killswitch, +marginal-gate) was strangling the edges to +10.5% / Sharpe 1.20.
|
||||
The data identified the right overlay for these Sharpe-2 crypto edges:
|
||||
- **vol-target is good** but the **12% / ½-Kelly level was far too timid** (~0.1 leverage). Full-Kelly at a
|
||||
25–40% vol budget gives **Sharpe 3.2–3.3, +138%–298%, DD −11%–18%** and *dominates* raw inverse-vol
|
||||
(higher Sharpe AND lower DD).
|
||||
- **marginal-gate HURTS** on this 2-sleeve book (Sharpe 1.78→1.33; benches contributors). Drop it.
|
||||
- **killswitch at −15/−7 whipsaws** (worse return AND worse DD). BUT a **wider kill (−25/−12) is dormant** at
|
||||
the chosen vol budget (vol-target caps DD below the threshold) → zero cost, retained as tail insurance.
|
||||
|
||||
## Decisions (from the sweep + user)
|
||||
|
||||
- **vol budget: 30% full-Kelly** (`_TARGET_VOL=0.30`, `_KELLY_FRACTION=1.0`) — the balanced middle
|
||||
(+183%, Sharpe 3.24, −13.6% DD). A single constant, trivially re-dialable to 25/35/40.
|
||||
- **killswitch: keep, widened to −25/−12** (`_KILL_DD=0.25`, `_REENTER_DD=0.12`) — dormant at this vol budget
|
||||
(the A/B showed identical results with/without it at 25–40% vol), so it's free tail-insurance for a
|
||||
worse-than-−25% crash. (User: "drop gate, keep a wider killswitch.")
|
||||
- **marginal-gate: OFF** — remove the `marginal_gate_active` call from both `paper_risk_overlay` and
|
||||
`paper_book_returns`. Keep the `marginal_gate_active` function + its tests in place (validated tool, may be
|
||||
re-enabled when the book grows past ~3 sleeves) but document it as intentionally disabled.
|
||||
|
||||
## Components
|
||||
|
||||
1. **Constants in `application/paper_book.py`:** `_TARGET_VOL = 0.30`, `_KELLY_FRACTION = 1.0`,
|
||||
`_KILL_DD = 0.25`, `_REENTER_DD = 0.12`. (Were `0.12 / 0.5 / 0.15 / 0.07`.)
|
||||
2. **`paper_risk_overlay`:** remove the `active = marginal_gate_active(returns_by_sleeve, active_sleeves)`
|
||||
line — weight directly over `active_sleeves`. Leverage + killswitch paths unchanged (they read the new
|
||||
constants).
|
||||
3. **`paper_book_returns`:** remove the `active = marginal_gate_active(before_d, active)` line in the per-date
|
||||
loop — weight over the has-history `active` directly. (Keeps the shadow replay consistent with the overlay
|
||||
so live == replay.)
|
||||
4. Docstring note on `marginal_gate_active` (+ the overlay): the gate is intentionally OFF for the current
|
||||
small book; the wider killswitch is a dormant backstop at the 30% vol budget.
|
||||
|
||||
## Data flow
|
||||
|
||||
Unchanged shape — only the overlay parameters + the (removed) gate step change. `paper_risk_overlay` and
|
||||
`paper_book_returns` both weight over the same `active` set (no gate) and apply the new vol-target/kill
|
||||
constants → live `/paper` == `/paper/replay` preserved.
|
||||
|
||||
## Risks / safety
|
||||
|
||||
- **Read-only/additive w.r.t. accounting + strategy mechanics:** only overlay *parameters* + the gate toggle
|
||||
change. No change to `book_allocator`, sleeve `run()`/`advance()`, the equity accounting (A3f), the shadow
|
||||
signal, or any broker path.
|
||||
- **Higher leverage (full-Kelly 30% vs ½-Kelly 12%):** deploys more exposure (the point). Still capped at
|
||||
`_MAX_LEVERAGE=3.0`; the vol-target de-risks as realized vol rises; the −25/−12 kill backstops a true crash.
|
||||
- **Consistency:** identical overlay in both paths (gate removed from both).
|
||||
- **Honest caveats (documented, not in scope):** gross/frictionless (no fee/slippage model); a single
|
||||
~1yr crypto window (2025-06→2026-06) — the 30% choice is a risk-appetite dial, not an optimization claim;
|
||||
xsfunding (3rd edge) still excluded from replay.
|
||||
|
||||
## Out of scope
|
||||
|
||||
Fee/slippage model; removing the now-uncalled `marginal_gate_active` + tests (kept as a re-enablable tool);
|
||||
adding xsfunding to replay; per-sleeve vol-target; walk-forward parameter validation.
|
||||
|
||||
## Acceptance criteria
|
||||
|
||||
- Constants updated to `0.30 / 1.0 / 0.25 / 0.12`; the existing overlay unit tests still pass (they assert
|
||||
shape/behaviour, not the specific constant values — update any that hard-coded 0.12/0.5/0.15/0.07, preserving
|
||||
intent).
|
||||
- `marginal_gate_active` no longer called by `paper_risk_overlay` or `paper_book_returns`; the
|
||||
`test_overlay_drops_dead_sleeve_from_weights` test (which asserted the gate dropped a dead sleeve via the
|
||||
overlay) is updated/removed to reflect the gate being off (the gate fn keeps its own direct unit tests).
|
||||
- Full suite green.
|
||||
- Re-run backfill in-cluster → `/paper/replay` curve reflects the re-tuned overlay (~+150–200% gross, Sharpe
|
||||
~3, DD ~−14% — directionally matching the C-grid 30% row); `/paper` live agrees; verified post-deploy.
|
||||
- Read-only/additive (accounting/strategy mechanics); no `*.dbn` staged.
|
||||
@@ -0,0 +1,123 @@
|
||||
# Paper book: vol-target leverage + drawdown killswitch — Design (Phase 2F-A3d)
|
||||
|
||||
**Date:** 2026-06-22
|
||||
**Status:** Design (approved)
|
||||
**Repo:** fxhnt
|
||||
**Follows:** A3c ISV-adaptive sleeve weighting ([[project_fxhnt_a3c_isv_adaptive_paper_weighting]]). Adds the
|
||||
portfolio-level risk layer ON TOP of the (sum-to-1) sleeve weights.
|
||||
|
||||
## Motivation
|
||||
|
||||
A3c sizes the paper book by ISV-adaptive inverse-vol *sleeve weights* (sum 1) but applies **no
|
||||
portfolio-level risk control** — no vol-target, no leverage, no drawdown control. The verified replay showed
|
||||
a ≈−29% drawdown over the year precisely because the book is sleeve-weighted only. This adds the same
|
||||
risk overlay the live `book_allocator.combine_book` already runs: **vol-target + fractional-Kelly leverage**
|
||||
(normalizes risk contribution) and a **drawdown killswitch** (the piece that actually caps the drawdown).
|
||||
Both must show in live `/paper` and `/paper/replay`.
|
||||
|
||||
## Decisions (from brainstorming)
|
||||
|
||||
- **Mirror `book_allocator` faithfully** (it is the trusted live implementation; do not refactor it):
|
||||
- vol-target: `lev = kelly_fraction · (target_vol_daily / trailing_port_vol)`, `target_vol_daily =
|
||||
target_vol/√periods_per_year`, capped `[0, MAX_LEVERAGE]`. Constants = live defaults: `target_vol=0.12`,
|
||||
`kelly_fraction=0.5`, `MAX_LEVERAGE=3.0`, `vol_lookback=60`, `periods_per_year=365`.
|
||||
- killswitch: flatten (stand in cash) once running drawdown of the **un-throttled shadow equity** exceeds
|
||||
`kill_dd=0.15`; re-enter when it recovers inside `reenter_dd=0.07` (hysteresis). Causal (decision uses
|
||||
strictly-prior days). Mirrors `apply_drawdown_killswitch`.
|
||||
- **Leverage scales notional only.** Threaded as a `leverage` arg through `persist_paper_book` →
|
||||
`target_positions(... capital=capital·leverage ...)`. The `$100k` equity *base* stays unscaled, so equity =
|
||||
`capital + leverage·(realized+unrealized)` — the levered book on a fixed base. `leverage=0` (killed) → zero
|
||||
notional → flat book (exit to cash; realized booked, unrealized 0).
|
||||
- **Decouple the return signal from the throttled book (the key correctness fix).** A killswitch flattens
|
||||
the actual book, but the per-sleeve return *signal* must NOT vanish during kills — else re-entry can never
|
||||
be detected and the ISV vol estimate corrupts. `sleeve_daily_return` is already leverage- and
|
||||
weight-invariant (the scalars cancel in pnl/notional), so we compute it from a **unit "shadow" book** (the
|
||||
sleeve's `current_weights` composition at unit notional, `sleeve_weight=1, capital=1`) that is persisted
|
||||
EVERY date and NEVER flattened. `paper_sleeve_ret` is fed from this clean signal. On non-killed days this is
|
||||
identical to A3c's value; on killed days it keeps the signal alive. (Strictly improves A3c too.)
|
||||
- **Both paths, consistent.** A single `paper_risk_overlay(returns_by_sleeve)` returns `(weights, leverage,
|
||||
killed)` for the latest date given its trailing window; backfill and snapshot both call it on their
|
||||
strictly-prior window → live `/paper` == `/paper/replay`.
|
||||
|
||||
## Components
|
||||
|
||||
1. **`vol_target_leverage(weights, returns_by_sleeve, *, target_vol=0.12, kelly_fraction=0.5,
|
||||
vol_lookback=60, periods_per_year=365, max_leverage=3.0) -> float`** (pure, `paper_book.py`): build the raw
|
||||
portfolio return over the last `vol_lookback` of the union date axis (`Σ wᵢ·rᵢ`, 0-fill missing days);
|
||||
`pv = std`; `lev = clip(kelly·target_vol/√ppy / pv, 0, max_leverage)`. **Cold-start (pv≤1e-12 or <2 obs) →
|
||||
`1.0`** (unlevered = A3c behavior; never flat from lack of data). Mirrors `combine_book` step 4.
|
||||
2. **`DrawdownKillSwitch(kill_dd=0.15, reenter_dd=0.07)`** (small stateful class, `paper_book.py`): `.observe(
|
||||
book_return)` updates shadow_eq/peak/killed (hysteresis, mirrors `apply_drawdown_killswitch`); `.killed`
|
||||
read BEFORE observing the next date = the state to apply. Pure (no I/O).
|
||||
3. **`paper_book_returns(returns_by_sleeve, *, <vol-target params>) -> dict[int, float]`** (pure,
|
||||
`paper_book.py`): the UN-throttled book return series. Ascending over the union date axis; at each date
|
||||
compute `w=isv_adaptive_weights(returns<d)`, `lev=vol_target_leverage(w, returns<d)` (held into the next
|
||||
date); `bret(d) = lev_held·Σ w_held·r_d`. Causal (weights/leverage use strictly-prior returns).
|
||||
4. **`paper_risk_overlay(returns_by_sleeve, active_sleeves, *, <params>) -> (dict weights, float leverage,
|
||||
bool killed)`** (pure, `paper_book.py`): the single entry point for the latest date. `weights =
|
||||
isv_adaptive_weights(returns, active)`; `leverage = vol_target_leverage(weights, returns)`; `killed =`
|
||||
replay a `DrawdownKillSwitch` over `paper_book_returns(returns)`; returns the effective `(weights,
|
||||
leverage, killed)`. Backfill + snapshot both call this on their trailing window. (O(N²) over a ≤365-date
|
||||
book — negligible; keeps ONE code path → live == replay.)
|
||||
5. **`paper_shadow_positions` table** (`cockpit_models.py`): `(run_date, sleeve, symbol)` PK, `qty`,
|
||||
`entry_price`, `src_updated_at` — the unit composition, never throttled. **PaperRepo**:
|
||||
`replace_shadow_positions(run_date, positions, *, at)` and `shadow_positions_before(run_date) ->
|
||||
list[Position]` (prior date's shadow book; mirrors `positions_before`).
|
||||
6. **`persist_paper_book(..., leverage: float = 1.0)`** + `PaperBookService.derive_and_persist(...,
|
||||
leverage=1.0)`: notional scaled by `leverage` (`target_positions(... capital=self._capital·leverage ...)`).
|
||||
Equity base unscaled. Default `1.0` keeps every existing caller/test unchanged.
|
||||
7. **`backfill_paper_book`** (`paper_backfill.py`): per date ascending — `returns_before_d =
|
||||
sleeve_returns(before=d)`; `w, lev_raw, killed = paper_risk_overlay(returns_before_d, active)`; `eff_lev =
|
||||
0.0 if killed else lev_raw`; `persist_paper_book(... sleeve_weights=w, leverage=eff_lev ...)`; persist the
|
||||
date's **unit shadow** book; compute the clean per-sleeve return from `shadow_positions_before(d)` and
|
||||
`upsert_sleeve_ret`. (Shadow + clean-signal replace A3c's persisted-position return source.)
|
||||
8. **`PaperSnapshotService`** (`paper_snapshot.py`): same — `paper_risk_overlay` off `sleeve_returns(
|
||||
before=as_of)`, persist throttled book with `eff_lev`, persist unit shadow, book the clean return from
|
||||
`shadow_positions_before(as_of)` + `self._prices`.
|
||||
|
||||
## Data flow
|
||||
|
||||
Per date (both paths): trailing clean per-sleeve returns → `paper_risk_overlay` → `(weights, leverage,
|
||||
killed)` → throttled actual book (`persist_paper_book` with `eff_lev`, flat if killed) + unit shadow book
|
||||
(always) → clean per-sleeve return appended to `paper_sleeve_ret` for the next date. The shadow book keeps
|
||||
the signal alive through kills so re-entry + vol estimates stay correct.
|
||||
|
||||
## Risks / safety
|
||||
|
||||
- **Read-only/additive.** No change to `book_allocator`, sleeve `run()`/`advance()`, `combine_book`, the gate
|
||||
record, or any broker/live-order path. New table + new pure fns + a `leverage` param (default 1.0).
|
||||
- **Causality:** all of weights/leverage/kill use strictly-prior returns; `bret(d)` uses weights/leverage
|
||||
held from before d. The kill decision for d uses shadow drawdown through d-1.
|
||||
- **No flat-from-starvation:** vol-target cold-start → leverage 1.0; killswitch starts un-killed; missing
|
||||
vol → unlevered. The book degrades to A3c, never to an unintended flat curve.
|
||||
- **Leverage cap 3.0** bounds exposure ($300k on $100k). `leverage=0` only via an explicit kill.
|
||||
- **Consistency:** one `paper_risk_overlay` + one clean signal → live `/paper` == `/paper/replay`.
|
||||
- **Idempotent:** shadow positions replaced per run_date (like `replace_positions`); `paper_sleeve_ret`
|
||||
upsert per (run_date, sleeve); re-running the backfill reproduces identical state.
|
||||
- **Known limitation:** vol-target can lever up in a low-vol downtrend; the killswitch is the backstop that
|
||||
caps the resulting drawdown (the reason both ship together).
|
||||
|
||||
## Out of scope
|
||||
|
||||
`marginal_gate`/`min_sleeve_sharpe` edge-quality pruning (live allocator has them OFF by default too);
|
||||
correlation-aware sizing; per-sleeve leverage; tuning the constants beyond the live defaults; the
|
||||
`_live_sleeves`↔backfill composition dedup (tech debt); cleaning up the now-dead `equal_weights`.
|
||||
|
||||
## Acceptance criteria
|
||||
|
||||
- `vol_target_leverage`: correct formula vs `combine_book`; cap at `max_leverage`; cold-start → 1.0; low
|
||||
trailing vol → higher leverage; unit-tested.
|
||||
- `DrawdownKillSwitch`: kills past `kill_dd`, re-enters inside `reenter_dd` (hysteresis), causal; unit-tested
|
||||
(incl. a kill→recover→re-enter sequence).
|
||||
- `paper_book_returns` / `paper_risk_overlay`: causal; `killed` reflects only strictly-prior returns; returns
|
||||
consistent `(weights, leverage, killed)`; unit-tested.
|
||||
- `paper_shadow_positions` table + `replace_shadow_positions`/`shadow_positions_before`; idempotent; tested.
|
||||
- `persist_paper_book(leverage=…)` scales notional, not the equity base; default 1.0 leaves existing tests
|
||||
green; tested (leverage=2 → double notional/positions, equity base still capital).
|
||||
- backfill + snapshot persist the unit shadow book, feed `paper_sleeve_ret` from it, and apply
|
||||
`paper_risk_overlay`; integration-tested (a kill period flattens the actual book while the shadow signal
|
||||
continues and re-entry later restores positions; both paths identical).
|
||||
- Re-run backfill in-cluster → `paper_nav` reflects leverage + a capped drawdown (shallower than A3c's −29%);
|
||||
`paper_shadow_positions` populated; `/paper` + `/paper/replay` render the levered, kill-aware curve;
|
||||
verified post-deploy.
|
||||
- Full suite green; read-only/additive; no `*.dbn` staged.
|
||||
63
docs/superpowers/specs/2026-06-23-bybit-ingestion-design.md
Normal file
63
docs/superpowers/specs/2026-06-23-bybit-ingestion-design.md
Normal file
@@ -0,0 +1,63 @@
|
||||
# Bybit ingestion + carry re-validation on the live venue — Design (A14)
|
||||
|
||||
**Date:** 2026-06-23
|
||||
**Status:** Design (for sign-off)
|
||||
**Repo:** fxhnt
|
||||
|
||||
## Why
|
||||
Live execution will be **Bybit EU (MiCA)**, but every edge is backtested on **Binance** data. Before any live
|
||||
capital — especially carry, which is venue-sensitive (funding + universe differ) — we must re-validate on the
|
||||
*actual* venue's data. This builds the Bybit data stack and re-runs the edges on it.
|
||||
|
||||
## Feasibility (probed from the cluster — strong)
|
||||
- **Universe**: `GET /v5/market/instruments-info?category=linear` → **590 active USDT perps** (vs Binance ~376).
|
||||
- **Funding**: `GET /v5/market/funding/history` is **SURVIVORSHIP-FREE** — delisted `LUNAUSDT` returns rows
|
||||
(retCode 0); paginates back to **2022-01+** via `endTime`; 8h funding (~66 days/200-row page). This is the
|
||||
carry signal and it's fully available incl. delisted (better than Binance REST, which purges delisted).
|
||||
- **Klines**: `GET /v5/market/kline?interval=D` → daily, ~2.7yr/1000-row page, paginates further via `start`.
|
||||
- **Ever-listed universe**: `public.bybit.com/trading/` enumerates per-symbol dumps (active + delisted) for the
|
||||
survivorship-free symbol list (the funding API then retains each one's history).
|
||||
- **Reachability**: api.bybit.com + public.bybit.com both reachable from the cluster (443 egress). Market data
|
||||
is public/global (no auth, no EU geo-restriction — the MiCA restriction is on *trading*, not data).
|
||||
|
||||
## Scope (phased — build A+B+C now; D later)
|
||||
- **A. Bybit funding adapter** (`BybitFundingHistory`): paginate funding/history (survivorship-free) for the
|
||||
universe (active 590 + delisted from public.bybit.com) back to ~2022 → a Bybit funding panel.
|
||||
- **B. Bybit klines adapter** (`BybitKlines`): perp + spot daily close (for basis + sleeve returns) → Bybit
|
||||
price panels. (Carry's return = funding + basis, so we need both legs on Bybit too.)
|
||||
- **C. Carry (+ directional) re-validation on Bybit**: run `XsFundingReplay` + the carry accounting
|
||||
(funding + basis, fees=5.5bp Bybit taker) on the Bybit panels → carry Sharpe/DD **on the venue we'll trade**.
|
||||
Compare to the Binance numbers (carry ~1.33 intraday-honest). This is the GATE before live carry.
|
||||
- **D. (LATER, not now) Live execution**: Bybit EU order placement, the **MiCA-restricted tradeable subset**
|
||||
(fewer coins / leverage caps), credentials. Separate workstream.
|
||||
|
||||
## Architecture
|
||||
- New adapters in `src/fxhnt/adapters/data/`: `bybit_funding.py`, `bybit_klines.py`, `bybit_universe.py`
|
||||
(mirror the Binance adapters' shapes so the existing carry pipeline / `XsFundingReplay` reuse them).
|
||||
- **Storage = Bybit-namespaced, Parquet side-store** (NOT the single-writer warehouse DuckDB — the lesson from
|
||||
the worst_basis OOM/contention): `bybit_funding.parquet`, `bybit_perp_close.parquet`, `bybit_spot_close.parquet`
|
||||
under the surfer data dir. A loader exposes `{symbol:{epoch_day:value}}` panels.
|
||||
- **Universe**: active (instruments-info) ∪ delisted (public.bybit.com/trading enumeration), with the same
|
||||
rename/redenomination care as Binance (1000X tokens, delisted intervals).
|
||||
- **Re-validation**: reuse `XsFundingReplay` + `carry_return_by_symbol` on the Bybit panels (same code, Bybit
|
||||
data) → a Bybit carry curve; surface it as a track in the sim (`cost_bps`=5.5).
|
||||
- **Memory/infra**: paginated fetch is light (funding <100MB; daily klines small) — run as a resumable CLI /
|
||||
k8s job with adequate memory (NOT the 2Gi dagster daemon; the 1m-ingest OOM lesson). Funding+daily klines are
|
||||
far lighter than 1m, so this is tractable.
|
||||
|
||||
## MiCA / Bybit EU (explicit)
|
||||
Ingestion uses the **global** market data (full 590 universe) — correct for *backtesting* the edge. Live
|
||||
execution (Phase D) must constrain to the **Bybit EU tradeable subset** (MiCA: fewer coins, leverage caps) —
|
||||
so the live carry universe will be SMALLER than the backtest universe. Flag: the Bybit-data re-validation is an
|
||||
upper bound; the live EU-tradeable subset is the real constraint, applied at execution.
|
||||
|
||||
## Tests / acceptance
|
||||
- Adapters: paginate funding + klines correctly (mock the HTTP; no network in unit tests); survivorship
|
||||
(delisted symbol returns history); rename handling.
|
||||
- Parquet side-store round-trip.
|
||||
- Carry re-validation runs on the Bybit panels and produces a carry curve (Sharpe/DD) comparable to Binance.
|
||||
- Full suite green; no `*.dbn` staged; Bybit cache/parquet gitignored.
|
||||
|
||||
## Out of scope (now)
|
||||
- Live order execution / credentials / the MiCA tradeable-subset enforcement (Phase D).
|
||||
- 1m intraday worst_basis on Bybit (daily basis first, mirror the Binance staging).
|
||||
@@ -0,0 +1,93 @@
|
||||
# Combined risk stack — Design (Phase 2F-A10)
|
||||
|
||||
**Date:** 2026-06-23
|
||||
**Status:** Design (for review)
|
||||
**Repo:** fxhnt
|
||||
|
||||
## Motivation
|
||||
We established (with data) that the drawdown problem has THREE orthogonal risk questions, each best served
|
||||
by a different mechanism — and crucially that one of them (edge-health) HURTS if mis-applied:
|
||||
|
||||
| Question | Mechanism | Cadence |
|
||||
|---|---|---|
|
||||
| "How much do I bet?" | `K(D*)` ex-ante Kelly sizing | continuous (the DD dial) |
|
||||
| "Is this *specific edge* still alive?" | `WindowedEdgeHealth` per-sleeve | rare / structural |
|
||||
| "Am I in a sudden catastrophic drawdown *now*?" | `DrawdownKillSwitch` | rare / book-level tail |
|
||||
|
||||
**The hard-won discipline (from the A/B):** reactive edge-health as a *continuous book-sizing throttle*
|
||||
WHIPSAWS a momentum book (cut after a losing window → small into the recovery): killswitch +1856%/Sh2.09
|
||||
vs edge_health +80%/Sh0.84. So `WindowedEdgeHealth` MUST be used as a **slow, binary, per-sleeve RETIREMENT
|
||||
gate** ("drop a sleeve whose edge is structurally dead"), NEVER as a daily sizing multiplier. That single
|
||||
distinction is what makes the three work together instead of fighting.
|
||||
|
||||
## Architecture
|
||||
A new `controller="combined"` mode (alongside `"killswitch"` default and the rejected/inert `"edge_health"`):
|
||||
```
|
||||
active = sleeves NOT structurally retired (WEH retirement gate)
|
||||
weights = isv_adaptive_weights(returns, active) # ISV over survivors (natural re-normalization)
|
||||
leverage = vol_target_leverage(weights, kelly_fraction=K) # K(D*) sizing; default K=1.0 (current)
|
||||
book = killswitch-gated(weights · leverage) # existing DrawdownKillSwitch as the tail backstop
|
||||
```
|
||||
The ONLY behavioural delta vs `"killswitch"` is the retirement gate filtering `active`. When all sleeves are
|
||||
healthy (the current 2-edge book), `combined == killswitch` exactly (gate inert) — it cannot hurt the current
|
||||
curve; its value is future-proofing (auto-drop a sleeve that later dies).
|
||||
|
||||
## The retirement gate — per-sleeve hysteresis state machine (the part to sanity-check)
|
||||
State per sleeve: `status ∈ {active, retired}`, counters `below_run`, `above_run`, and a `WindowedEdgeHealth`.
|
||||
Each day, feed every sleeve its **SHADOW** daily return (the kill-proof `unit_shadow_positions` series — alive
|
||||
even while retired, so recovery is detectable), then:
|
||||
```
|
||||
weh.update(shadow_ret); h = weh.health()
|
||||
if status == active:
|
||||
below_run = below_run + 1 if h < retire_floor else 0
|
||||
if below_run >= retire_window: status = retired; below_run = above_run = 0
|
||||
else: # retired
|
||||
above_run = above_run + 1 if h >= readmit_floor else 0
|
||||
if above_run >= readmit_window: status = active; below_run = above_run = 0
|
||||
active_sleeves = [s for s if status == active]
|
||||
```
|
||||
Hysteresis (`readmit_floor > retire_floor`) + long confirmation windows make it structural, not tactical —
|
||||
**it cannot whipsaw daily** (a sleeve must be dead `retire_window` days to drop, recovered `readmit_window`
|
||||
days to return). Bootstrap: a young sleeve has `health()==1.0` → never retired early. All-retired → empty
|
||||
book (0 exposure) is acceptable (don't trade dead edges).
|
||||
|
||||
### Parameters (fixed v1 defaults; ISV-adaptation is future work)
|
||||
- `retire_floor = 0.15`, `retire_window = 30` (≈ a month of structural deadness to drop)
|
||||
- `readmit_floor = 0.50`, `readmit_window = 30` (clear hysteresis gap; a month of recovery to return)
|
||||
- WEH: `window=63` (v1b defaults); `kelly_fraction = 1.0` (current sizing — `kelly_for_dd_budget(D*)` available)
|
||||
- `kill_dd / reenter_dd`: existing killswitch defaults (0.25 / 0.12)
|
||||
|
||||
## Integration (live==replay)
|
||||
- Add `controller="combined"` to `paper_risk_overlay` (stateless replay) + `IncrementalRiskOverlay` (stateful).
|
||||
Retirement state (per-sleeve WEH + counters + status) is maintained incrementally in `IncrementalRiskOverlay`
|
||||
and replayed from scratch in `paper_risk_overlay` — the existing pattern for the killswitch + per-sleeve WEH.
|
||||
- The gate reuses the active-set convention (cf. `marginal_gate_active`): it produces `active_sleeves`, which
|
||||
feeds `isv_adaptive_weights` exactly as today. Retired sleeves get weight 0.
|
||||
- `controller="killswitch"` stays **bit-identical** (default; existing tests unchanged).
|
||||
|
||||
## Validation (the honest test — don't measure on in-sample where nothing dies)
|
||||
1. **Synthetic dead-sleeve test (the decisive one):** two sleeves — A healthy throughout, B healthy then a
|
||||
sustained structural decay (then optionally a recovery). Assert:
|
||||
- `combined` retires B within ~`retire_window` of the decay onset; `killswitch`-alone keeps allocating to B.
|
||||
- final NAV(`combined`) > NAV(`killswitch`-alone) on the dying-sleeve scenario (the gate earns its keep).
|
||||
- if B recovers, `combined` re-admits it within ~`readmit_window` (no permanent exile).
|
||||
2. **Inert-on-healthy:** on the real 2-edge book (both healthy), `combined` ≡ `killswitch` (no retirements) —
|
||||
proves it doesn't touch the current curve.
|
||||
3. **live==replay golden:** `IncrementalRiskOverlay(controller="combined")` == `paper_risk_overlay(...,
|
||||
controller="combined")` per date, exact.
|
||||
4. **No-whipsaw guard:** a sleeve that merely *dips* below `retire_floor` for `< retire_window` days is NOT
|
||||
retired (distinguishes structural death from a tactical drawdown).
|
||||
|
||||
## Out of scope (future, only if the forward gate shows need)
|
||||
- ISV-adaptive gate params (windows/floors derived from data variance).
|
||||
- Re-normalization policy alternatives (currently: ISV over survivors).
|
||||
- Wiring the gate into the live snapshot default (stays behind the flag until the synthetic test + a forward
|
||||
period justify flipping the default).
|
||||
|
||||
## Acceptance criteria
|
||||
- Retirement gate is a pure, unit-tested hysteresis state machine (retire/re-admit windows + bootstrap + dip-not-death).
|
||||
- `controller="killswitch"` bit-identical (backward-compat; existing tests pass).
|
||||
- `controller="combined"` wired in BOTH overlays; live==replay golden passes.
|
||||
- Synthetic dead-sleeve test passes (combined > killswitch-alone on a dying sleeve; re-admits on recovery).
|
||||
- Inert-on-healthy: combined ≡ killswitch on the real 2-edge book.
|
||||
- Full suite green; no `*.dbn` staged.
|
||||
@@ -0,0 +1,82 @@
|
||||
# Interactive simulator cockpit + multi-track wiring — Design (A13)
|
||||
|
||||
**Date:** 2026-06-23
|
||||
**Status:** Design (for sign-off)
|
||||
**Repo:** fxhnt
|
||||
|
||||
## Goal
|
||||
A *usable*, configurable paper-simulation page in the cockpit: choose capital / date range / strategy
|
||||
composition / leverage / costs, see the equity curve, **scrub a timeline slider (fast-forward) to watch the
|
||||
book evolve**, and compare tracks (2-edge / 3-edge / per-strategy). Plus: wire the live nightly to keep
|
||||
**2-edge as the trusted live book** and run **3-edge** and **xsfunding-standalone** as separate forward
|
||||
tracks for the gate to judge.
|
||||
|
||||
## Why now (decided upstream)
|
||||
The numbers: 2-edge Sharpe 2.10 / −36% worst-year vs 3-edge Sharpe 1.54 / +1% worst-year / lower DD / higher
|
||||
CAGR. Carry diversifies the tail but lowers Sharpe — a genuine trade-off best resolved on *forward* data, not
|
||||
in-sample. So: run all three as parallel tracks + give a tool to explore the trade-offs interactively.
|
||||
|
||||
## Architecture — 3 pieces over one shared engine
|
||||
### 1. `simulate_book` — pure in-memory sim engine (`src/fxhnt/application/paper_sim.py`, NEW)
|
||||
`simulate_book(returns_by_sleeve, *, capital, sleeves, start, end, controller, kelly_fraction, cost_bps,
|
||||
worst_basis=None) -> SimResult` where `SimResult = {dates[], equity[], drawdown[], per_day:[{weights,
|
||||
leverage, killed}], metrics:{cagr, sharpe, max_dd, worst_year, best_year, end_value}}`.
|
||||
- Reuses `paper_risk_overlay`/`IncrementalRiskOverlay` + the `paper_book_returns` compounding loop (the
|
||||
architecture map confirms the exact loop already exists). PURE — consumes only the cached
|
||||
`{sleeve:{epoch_day:ret}}` dict; no warehouse, no network, no live prices → milliseconds → live-slider-fast.
|
||||
- `cost_bps` applies a per-turnover cost haircut (config knob — also how we model Bybit fees, see below).
|
||||
|
||||
### 2. Sim page + endpoints (`src/fxhnt/adapters/web/app.py` + `templates/sim.html`, NEW template)
|
||||
- `GET /paper/sim` — the page (config panel + chart + slider + state panel + metrics).
|
||||
- `GET /paper/sim/run?capital=&start=&end=&sleeves=&kelly=&controller=&cost_bps=` — recompute → HTML partial:
|
||||
the inline-SVG curve(s) + the metrics row + (optionally) the per-day state series for the slider.
|
||||
- Reads `PaperRepo.sleeve_returns()` ONCE, runs `simulate_book`, renders with the existing `charts.py`
|
||||
`nav_sparkline()` (extended to multi-line + a cursor marker).
|
||||
|
||||
### 3. Live multi-track wiring (the "wire it that way" part)
|
||||
- The nightly snapshot/backfill (and the read path) produce/serve THREE book tracks via `simulate_book` over
|
||||
the SAME cached sleeve returns: **`book_2edge`** (tstrend+unlock+stablecoin — the trusted live book),
|
||||
**`book_3edge`** (+xsfunding), **`xsfunding_solo`**. Stored as distinct ids (reuse `forward_nav`
|
||||
strategy_id, or a small `paper_book_tracks` table) so the gate sees three forward records.
|
||||
- The live `/paper` page stays **2-edge** (unchanged trust). 3-edge + xsfunding are comparison tracks on the
|
||||
cockpit + in the sim. Decision on promoting 3-edge → live is made later on forward data.
|
||||
|
||||
## The UI (usable — mirrors the existing `replay.html` slider/fetch pattern; no new JS libs)
|
||||
- **Config panel**: capital ($ input); start/end date pickers; composition = preset buttons
|
||||
(`2-edge` / `3-edge` / `tstrend` / `unlock` / `stablecoin` / `xsfunding`) + custom sleeve checkboxes;
|
||||
Kelly/leverage slider (0.1–1.5); controller dropdown (killswitch / combined); costs toggle + `cost_bps`
|
||||
field.
|
||||
- **Chart**: inline-SVG equity curve(s); overlay selected tracks for comparison; reveal up to the cursor.
|
||||
- **Timeline slider + ▶Play**: scrub a date cursor (vanilla `fetch` → partial, the `replay.html` model);
|
||||
the chart reveals to the cursor and a **state panel** shows equity / drawdown / active sleeves / weights /
|
||||
killed-flag AT the cursor. ▶Play animates the cursor = fast-forward.
|
||||
- **Metrics row** (live-updates on config change): CAGR, Sharpe, max DD, worst & best calendar year, final $.
|
||||
|
||||
## Bybit EU (forward note — informs, doesn't block)
|
||||
Live execution will likely be **Bybit EU (MiCA-regulated)**. Implications captured now:
|
||||
- **Costs**: Bybit perp taker ≈ 5.5bp — the sim's `cost_bps` knob models this directly (vs Binance 5bp).
|
||||
- **Universe**: Bybit-listed perps are a SUBSET of Binance; the binance.vision backtest is a **proxy** for the
|
||||
carry universe — live carry needs a Bybit-universe check.
|
||||
- **Funding signal**: live carry must read **Bybit** funding (not Binance) — the edge re-validated on Bybit
|
||||
funding before capital.
|
||||
- **MiCA**: possible leverage caps / coin restrictions.
|
||||
- **Action now**: make cost configurable (done via `cost_bps`); FLAG that live carry on Bybit needs a
|
||||
Bybit-specific re-validation (universe + funding + fees). NOT building Bybit ingestion in this feature.
|
||||
|
||||
## Phasing
|
||||
1. `simulate_book` engine + tests (the shared core).
|
||||
2. Sim page (config + multi-line chart + slider + state panel + metrics).
|
||||
3. Live multi-track wiring (3 nightly tracks; 2-edge stays live-trusted).
|
||||
|
||||
## Out of scope (v1)
|
||||
- Bybit data ingestion / live execution wiring.
|
||||
- Persisting user sim configs; auth. (Sim is read-only + stateless per request.)
|
||||
- Monte-Carlo / parameter sweeps in the UI (the engine supports it; UI is single-config + compare).
|
||||
|
||||
## Acceptance criteria
|
||||
- `simulate_book` pure + unit-tested; reproduces the backfill's causal book curve for the 2-edge config
|
||||
(golden vs `paper_nav`), and per-strat/3-edge configs compute correctly.
|
||||
- `/paper/sim` page usable: config controls recompute the curve+metrics in <1s; slider scrubs the cursor with
|
||||
the state panel; ▶Play animates.
|
||||
- 3 live tracks produced (2-edge/3-edge/xsfunding) without changing the trusted 2-edge `/paper`.
|
||||
- Full suite green; no `*.dbn` staged.
|
||||
@@ -0,0 +1,80 @@
|
||||
# Warehouse → Postgres consolidation — Design (A15)
|
||||
|
||||
**Date:** 2026-06-23
|
||||
**Status:** Design (for review)
|
||||
**Repo:** fxhnt
|
||||
|
||||
## Why
|
||||
The warehouse is a single **DuckDB file on an RWO block volume** → single-writer, pinned to ONE node (the
|
||||
8GB DEV1-L platform node `dagster` runs on). All warehouse compute is trapped there → batch jobs OOM (worst_basis
|
||||
ingest, the carry+4-sleeve backfill), pod churn, serialized reads. The data is **small** (~5–10M rows, hundreds
|
||||
of MB) — this is a *topology* problem, not a scale problem. Fix: move the feature store to the **networked
|
||||
Postgres we already run**, so compute decouples from the storage node and batch can run anywhere (incl. a
|
||||
bigger node). ClickHouse is overkill at this size; revisit only if we commit to tick-scale data.
|
||||
|
||||
## Target architecture
|
||||
- **Feature store = TimescaleDB** (we ALREADY run `timescale/timescaledb:latest-pg16`, TimescaleDB 2.25.1, same
|
||||
instance as the operational DB `FXHNT_OPERATIONAL_DSN`; `forward_nav` is already a hypertable). Networked →
|
||||
cockpit, sim, backfill, ingest, dagster all connect over the network from any node/pod. **Zero new infra.**
|
||||
- The `features` table is a **TimescaleDB hypertable with columnar compression** → near-DuckDB analytical speed
|
||||
(chunk exclusion on time-range + compressed columnar segments by feature/symbol) AND networked AND smaller on
|
||||
disk than the DuckDB file. This RESOLVES the row-store scan-speed caveat.
|
||||
- **No RWO warehouse PVC, no podAffinity** for batch → batch jobs run on a bigger / scale-to-zero node pool.
|
||||
- DuckDB kept behind a flag as a **fallback / cold backup** during/after cutover (reversible).
|
||||
|
||||
## The interface to preserve (drop-in)
|
||||
`DuckDbFeatureStore` (`src/fxhnt/adapters/warehouse/duckdb_feature_store.py`) — long format
|
||||
`features(symbol VARCHAR, ts BIGINT, feature VARCHAR, value DOUBLE)` + `ticker_membership`. Methods used across
|
||||
19 sites: `write_features`, `write_features_bulk`, `read_features`, `read_panel(symbols, feature)`,
|
||||
`crypto_close_panel`, `crypto_funding_panel`, `crypto_spot_panel`, `upsert_membership` (+ membership reads).
|
||||
A `PostgresFeatureStore` must implement the SAME methods identically (same return shapes, same `suffix`/keying,
|
||||
`ts//86400` → epoch_day).
|
||||
|
||||
## Phases
|
||||
### 1. `PostgresFeatureStore` + factory
|
||||
- New `src/fxhnt/adapters/warehouse/postgres_feature_store.py` — same interface, backed by Postgres tables
|
||||
`features(symbol text, ts bigint, feature text, value double precision)` (PK `(feature, symbol, ts)`) +
|
||||
`ticker_membership`. Index `(feature, symbol, ts)` for panel reads + `(symbol, ts)` for point reads.
|
||||
Writes = upsert (`INSERT ... ON CONFLICT (feature,symbol,ts) DO UPDATE`), preserving the scoped-idempotent
|
||||
semantics of `write_features` (re-running a day overwrites, doesn't duplicate).
|
||||
- `get_feature_store(settings)` factory in one place; `FXHNT_FEATURE_STORE=postgres|duckdb` setting (default
|
||||
duckdb until cutover). Tests: parity — same fixture written to both stores yields identical
|
||||
`crypto_*_panel`/`read_panel`/`read_features` output.
|
||||
|
||||
### 2. Data migration (idempotent, one-shot)
|
||||
- A CLI `fxhnt warehouse-migrate-pg`: read all `features` + `ticker_membership` from the DuckDB file, bulk-upsert
|
||||
into Postgres (chunked, memory-bounded — stream by symbol/feature, don't load all in RAM). Idempotent
|
||||
(ON CONFLICT). Verify: row counts + a few panel checksums match DuckDB.
|
||||
|
||||
### 3. Cutover
|
||||
- Replace the 19 direct `DuckDbFeatureStore(...)` instantiations with `get_feature_store(settings)`.
|
||||
- Flip `FXHNT_FEATURE_STORE=postgres`. Cockpit/sim/backfill/ingest now read/write Postgres.
|
||||
- Keep DuckDB as cold backup (the flag flips back if needed).
|
||||
|
||||
### 4. Unpin batch compute (the payoff)
|
||||
- Remove the warehouse RWO PVC mount + `podAffinity: dagster` from the backfill/ingest Jobs (they now reach the
|
||||
warehouse over Postgres). Give batch Jobs adequate memory.
|
||||
- Add a **scale-to-zero batch node pool** (POP2-4C-16G — same type as `ci-compile-cpu`, in stock) via the
|
||||
Kapsule terragrunt (`infra/live/production/kapsule`); batch Jobs target it (nodeSelector + toleration),
|
||||
scaled 0 when idle (≈free). Platform pool stays DEV1-L. Re-run the carry+4-sleeve backfill + the Bybit/worst_basis
|
||||
ingests there without OOM.
|
||||
|
||||
## Schema notes
|
||||
- One `features` table for all features (close/funding/high/low/spot_close + futures), as today — new features
|
||||
need no migration. Bybit data lands as additional symbols/features (or a `venue` column if we want to keep
|
||||
Binance/Bybit separate — decide at Bybit ingest time; default: prefix/suffix the symbol or add `venue`).
|
||||
- Postgres disk: hundreds of MB of features — trivial for the existing Postgres PVC.
|
||||
|
||||
## Risks / rollback
|
||||
- Parity bug → caught by Phase-1 parity tests + Phase-2 checksum verify.
|
||||
- Cutover → reversible via `FXHNT_FEATURE_STORE` flag (DuckDB retained).
|
||||
- Panel-read latency: Postgres indexed panel load ~1–3s vs DuckDB ~0.35s — acceptable (the sim's bottleneck is
|
||||
the overlay compute, not the load); if it ever matters, add a read cache or revisit Parquet/DuckDB-on-S3.
|
||||
|
||||
## Acceptance criteria
|
||||
- `PostgresFeatureStore` parity-tested against `DuckDbFeatureStore` (identical panel/read output on a fixture).
|
||||
- Migration copies all data; row-count + checksum verification passes.
|
||||
- Cutover via flag; full suite green with `FXHNT_FEATURE_STORE=postgres`.
|
||||
- A batch job (the carry backfill) runs to completion on the new batch pool **without the warehouse PVC** and
|
||||
without OOM.
|
||||
- No `*.dbn` staged; DuckDB retained as fallback.
|
||||
@@ -0,0 +1,142 @@
|
||||
# Bybit Testnet "Real-Paper" Execution Leg — Design (v2)
|
||||
|
||||
**Date:** 2026-06-28
|
||||
**Status:** Design (pending review) — v2 after critical self-review
|
||||
**Goal (REVISED):** Validate the **execution mechanics** of the `bybit_4edge` book on **Bybit testnet** — order placement, real fills, position reconciliation, and the risk-overlay applied to the *real* book state — with **measured, decomposed slippage** as first-class, tested output. Fake money.
|
||||
|
||||
> **Honest scope correction (from review #2):** Bybit testnet has thin liquidity in the small alts the unlock-shorts trade, so testnet **cannot hard-validate the slippage *magnitudes*** (esp. the 35bp unlock assumption). Testnet validates that the *loop is correct* (orders/fills/reconcile/overlay-on-real-state) and yields an **indicative / upper-bound** slippage. The slippage-magnitude truth — the 5/8/6/35bp tiers — is only confirmed by **mainnet micro-live** (a later, separate, real-money step gated on the recon gate). This leg de-risks the *mechanics* of that step and instruments the slippage measurement so micro-live inherits a tested calculation.
|
||||
|
||||
---
|
||||
|
||||
## 1. Key decisions (locked)
|
||||
|
||||
| Decision | Choice | Why |
|
||||
|---|---|---|
|
||||
| Execution adapter | **ccxt** (`ccxt.bybit` + `set_sandbox_mode(True)`) | Reuses the ccxt crypto-exec config; exchange-agnostic. |
|
||||
| Order type | **Market** | Honest slippage; deterministic reconcile. |
|
||||
| Orchestration | **Separate Dagster job + schedule** (no CronJob) | Decoupled → a broker hiccup never taints the forward gate's clean-execution record. |
|
||||
| Money | **Testnet only** (triple-gate) | Mechanics now; real money is post-gate micro-live. |
|
||||
| **Weights to execute** | **RAW pre-overlay sleeve weights**, overlay **re-applied on testnet state** (review #1) | The overlay (vol-target/gross-cap/killswitch) must act on the testnet book's OWN equity/drawdown, not the sim's. |
|
||||
| **Slippage baseline** | **Live orderbook mid at order time** (review #3) | Isolates execution slippage from price drift; the mark-close drift is a separate, measured term. |
|
||||
|
||||
---
|
||||
|
||||
## 2. Architecture
|
||||
|
||||
```
|
||||
combined_book_forward_job (23:30 UTC, EXISTING) — UNCHANGED
|
||||
└─ persists RAW per-sleeve target weights (pre-overlay) [see §6 — load-bearing dependency]
|
||||
|
||||
bybit_testnet_execution_job (NEW, SEPARATE, schedule bybit_testnet_daily)
|
||||
├─ bybit_testnet_reconcile
|
||||
│ 1. read RAW sleeve weights (latest) + testnet equity + testnet positions
|
||||
│ 2. re-apply the SAME risk overlay on the TESTNET equity/drawdown → effective target weights
|
||||
│ 3. plan_orders(effective_weights, testnet_positions, equity, instrument_limits)
|
||||
│ 4. for each order: capture live mid → place MARKET order → fill
|
||||
└─ bybit_testnet_record
|
||||
write fills (execution + timing slippage decomposed) + post-rebalance testnet NAV
|
||||
→ bybit_testnet_fills / bybit_testnet_nav
|
||||
```
|
||||
|
||||
Decoupled by schedule timing only (not a Dagster dep): a broker failure fails this job alone.
|
||||
|
||||
---
|
||||
|
||||
## 3. Slippage — the measurement model (pure, fully tested) ⭐ (answers "can we test slippage?")
|
||||
|
||||
All in `src/fxhnt/application/slippage.py` — **pure functions over floats**, no I/O, so they're unit-tested directly. The adapter supplies `fill_price`, `mid_at_order`, `book_mark`; the math is tested in isolation.
|
||||
|
||||
```
|
||||
sign(side) = +1 for BUY, -1 for SELL
|
||||
execution_slippage_bps(fill, mid_at_order, side)
|
||||
= sign(side) * (fill - mid_at_order) / mid_at_order * 1e4 # >0 = paid worse than mid (the cost we model as 5/8/6/35bp)
|
||||
timing_drift_bps(mid_at_order, book_mark, side)
|
||||
= sign(side) * (mid_at_order - book_mark) / book_mark * 1e4 # market moved between the book's mark and execution
|
||||
total_fill_vs_mark_bps = execution_slippage_bps + timing_drift_bps
|
||||
slippage_vs_assumed_bps(execution_slippage_bps, assumed_tier_bps)
|
||||
= execution_slippage_bps - assumed_tier_bps # the validation delta vs the cost-model tier
|
||||
```
|
||||
|
||||
- `assumed_tier_bps` per symbol from the A8 cost tiers (taker 5bp + tier 8/6/35bp; unlock symbols → 35bp).
|
||||
- **Decomposition is the whole point:** `execution_slippage` is what the cost model claims to capture; `timing_drift` is the close→execution price move (a separate, instrument-able effect). Conflating them (v1's bug) made the number meaningless.
|
||||
- **Guards (tested):** `mid<=0` or `book_mark<=0` → return `None` (undefined), never divide-by-zero or emit a garbage bp.
|
||||
- **Tests:** sign both sides; a known fill/mid → exact bps; decomposition sums to total; the assumed-delta; the None-guards; a thin-book sentinel (if `mid` from an empty/one-sided book → flagged `low_confidence`, recorded but excluded from the aggregate verdict — review #2).
|
||||
|
||||
---
|
||||
|
||||
## 4. Components
|
||||
|
||||
### 4.1 `BybitExecution` adapter — `src/fxhnt/adapters/broker/bybit.py`
|
||||
ccxt-backed; shape mirrors `adapters/broker/ibkr.py`.
|
||||
- `__init__(api_key, api_secret, *, testnet=True)` → `ccxt.bybit(...)`; `set_sandbox_mode(testnet)`.
|
||||
- `equity() -> float`; `positions() -> dict[str,float]` (signed qty); `instrument_limits() -> dict[str,InstrumentLimit]` (`minOrderQty`, `qtyStep`, `minNotional` from `load_markets`) — review #6.
|
||||
- `mid(symbol) -> float | None` — `fetch_order_book` best bid/ask → mid; `None` if empty/one-sided (thin testnet book).
|
||||
- `place_market(symbol, signed_qty, *, reduce_only) -> Fill` — `create_order('market', side, abs(qty), params={'reduceOnly': reduce_only})`.
|
||||
- Errors → typed `BrokerError`; **never logs secrets**.
|
||||
|
||||
### 4.2 Reconcile + overlay re-application — `src/fxhnt/application/bybit_testnet_execution.py` (pure where possible)
|
||||
- `effective_weights(raw_sleeve_weights, testnet_nav_history) -> dict[str,float]` — **re-runs the existing risk overlay** (vol-target, gross-cap ≤1.0×, drawdown killswitch) against the **testnet** equity/drawdown (review #1). Reuses the SAME overlay code the book uses (single source of truth — no reimplementation), fed testnet state. If the testnet drawdown trips the killswitch, the testnet book de-risks **on its own state** — that's the behavior we're validating.
|
||||
- `plan_orders(effective_weights, current_positions, equity, marks, limits) -> list[OrderIntent]`:
|
||||
- target_qty[s] = (weight·equity)/marks[s], snapped to `qtyStep`; drop if `|target_qty·mark| < max(minNotional, min_order_usd)` or `< minOrderQty` (review #6).
|
||||
- delta = target_qty − current; `reduce_only=True` on reductions/closes.
|
||||
- assert Σ|weight·equity| ≤ equity·`max_gross`(=1.0) → clip + log (no leverage).
|
||||
- Pure → the bulk of TDD.
|
||||
|
||||
### 4.3 Dagster job — `assets.py` + `definitions.py`
|
||||
- `@asset bybit_testnet_reconcile` (builds adapter from settings, runs §2 steps 1–4; per-order: `mid` → place → fill; best-effort `BrokerError` → log + structured result, never crash the daemon; **idempotency guard:** a per-day `(date)` marker in `bybit_testnet_nav` + an open-orders check before placing → re-run is a no-op, review #idempotency).
|
||||
- `@asset bybit_testnet_record` (writes fills with decomposed slippage + post-rebalance NAV).
|
||||
- Job `bybit_testnet_execution_job`; schedule `bybit_testnet_daily` (configurable; default 00:30 UTC). **Not** in `combined_book_forward_job`.
|
||||
|
||||
### 4.4 Persistence — migration
|
||||
- `bybit_testnet_nav(run_date PK, equity, realized_pnl, gross, n_orders, mean_exec_slippage_bps, mean_timing_drift_bps, ts)`.
|
||||
- `bybit_testnet_fills(id PK, run_date, symbol, side, qty, fill_price, mid_at_order, book_mark, exec_slippage_bps, timing_drift_bps, assumed_tier_bps, slippage_delta_bps, low_confidence bool, fee, order_id, ts)`.
|
||||
- Separate from sim `paper_nav` → cockpit shows real-paper vs sim; recon gate keeps reading the sim track unchanged.
|
||||
|
||||
### 4.5 Config — `config.py`
|
||||
`BybitExecutionConfig` (`FXHNT_BYBIT_EXEC_`): `api_key`, `api_secret`, `testnet=True`, `allow_live=False`, `kill_switch=False`, `min_order_usd=10`, `max_gross=1.0`, `schedule_cron`. **Triple-gate** for real money: `testnet=False` AND `allow_live=True` AND `kill_switch=False`.
|
||||
|
||||
### 4.6 Infra
|
||||
k8s secret `bybit-testnet-credentials`; NetworkPolicy egress `api-testnet.bybit.com:443` for the dagster pod. **No CronJob** — Dagster schedule.
|
||||
|
||||
---
|
||||
|
||||
## 5. Execution-gap verdict (review #5) — `src/fxhnt/application/execution_gap.py` (pure)
|
||||
Defines the analog of the recon gate, for execution:
|
||||
- **MECHANICS PASS:** over the window, post-rebalance testnet positions track the effective target within `pos_tolerance` (e.g. each symbol within 1 qtyStep + dust), no reconcile errors, no execution gaps (clean daily record).
|
||||
- **SLIPPAGE REPORT (indicative):** aggregate `exec_slippage_bps` per cost-tier vs `assumed_tier_bps`, **excluding `low_confidence` (thin-book) fills**; report the delta + the coverage (how many symbols had real liquidity). Explicitly labeled "indicative — magnitude pending mainnet micro-live."
|
||||
- Pure fn over the recorded rows → unit-tested (pass/fail on synthetic mechanics; slippage aggregation with low-confidence exclusion).
|
||||
|
||||
---
|
||||
|
||||
## 6. ⚠️ Load-bearing dependency (review #7) — verify FIRST in the plan
|
||||
The reconcile needs the **raw, pre-overlay per-sleeve target weights** — NOT sim-marked positions. **Task 1 of the plan = locate/expose this.** If the forward job only persists post-overlay sim positions, add a small read-model that exposes the raw sleeve weights (the strategies already compute `current_weights(as_of)`; expose that for the latest as_of). If this turns out unavailable cleanly, STOP and re-evaluate — the whole leg depends on a clean weight source.
|
||||
|
||||
---
|
||||
|
||||
## 7. Phasing (review — scope)
|
||||
- **Phase 1 — execution loop (dry-runnable):** slippage.py + execution_gap.py (pure + fully tested) → BybitExecution adapter (mocked-ccxt tests) → reconcile/overlay-re-apply → Dagster job + tables + config + secret + netpol. Acceptance: a testnet run reconciles the book, records fills with decomposed slippage, mechanics-verdict computes. Cockpit not required.
|
||||
- **Phase 2 — surfacing:** cockpit real-paper card (equity curve, mechanics verdict, indicative slippage vs assumed) reading `bybit_testnet_*`.
|
||||
|
||||
---
|
||||
|
||||
## 8. Safety
|
||||
Testnet-only default + triple-gate; `kill_switch` → zero orders (log intent only); `max_gross=1.0` hard assert (no leverage); idempotent diff + per-day marker + open-orders check; decoupled job.
|
||||
|
||||
## 9. Error handling
|
||||
`BrokerError` → asset logs + records `{"error":...}`, **no NAV row** (a visible gap, not a silent pass); partial fills self-correct next day; testnet demo-fund reset → discontinuity marker + NAV re-baseline.
|
||||
|
||||
## 10. Testing (TDD)
|
||||
- **slippage.py** (§3): sign/bps/decomposition/assumed-delta/None-guards/low-confidence — pure, exhaustive.
|
||||
- **execution_gap.py** (§5): mechanics pass/fail; slippage aggregation excluding low-confidence.
|
||||
- **plan_orders / effective_weights**: target>current→buy; <→reduce_only; qtyStep snap; minNotional/minOrderQty filter; gross>1.0 clip; overlay killswitch trips on testnet drawdown.
|
||||
- **BybitExecution**: mocked ccxt — sandbox set, market params (reduceOnly), fill/mid/limits parsing, secret never logged.
|
||||
- **Job wiring**: job exists, on `bybit_testnet_daily`, NOT in `combined_book_forward_job`, idempotent re-run = no-op, kill_switch = zero orders.
|
||||
- Full suite green.
|
||||
|
||||
## 11. Out of scope
|
||||
Real-money micro-live (separate, post-gate); limit/post-only; multi-venue. **Slippage-magnitude validation for the slippage-sensitive edges is explicitly deferred to mainnet micro-live** — testnet gives mechanics + indicative slippage only.
|
||||
|
||||
## 12. Open risks (honest)
|
||||
- Testnet alt liquidity → many `low_confidence` fills; the unlock-35bp magnitude likely NOT validated here (the core caveat — drives the goal restatement).
|
||||
- Testnet funding ≠ mainnet → funding economics indicative only.
|
||||
- Close→execution timing drift is measured (`timing_drift_bps`) but not eliminated; for cleaner magnitude, mainnet micro-live should execute nearer the book's mark time.
|
||||
@@ -0,0 +1,86 @@
|
||||
# Bybit 4-edge Leverage Overlay — Design
|
||||
|
||||
**Status:** draft for review
|
||||
**Date:** 2026-07-03
|
||||
**Author:** research + design dialogue (see session)
|
||||
|
||||
## Goal
|
||||
|
||||
Add **validated, differentiated leverage** to the `bybit_4edge` deploy book so the small ($35k) pilot compounds faster — WITHOUT breaching a hard drawdown/ruin tolerance — by **re-tuning and validating the existing `paper_book` risk overlay** on the funding-net Bybit baseline. We do NOT build new leverage machinery; the vol-target + killswitch overlay already exists and is the right tool. The deliverable is a re-tuned, capped, stress-tested overlay plus a shadow forward track, gated so real capital only ever rides the levered book after it validates out-of-sample.
|
||||
|
||||
## Why (research findings that justify this design)
|
||||
|
||||
All numbers below are from the funding-net baseline (`bybit_sleeve_ret`, 2010 days, 2021-01…2026-07) computed this session.
|
||||
|
||||
1. **The venue does not cap us.** At $35k spread over ~68 liquid USDT-perps, every position sits in Bybit Tier 1: max leverage (125x BTC / ~50x alts) and lowest MMR (0.5% / ~1–2.5%) on every symbol. We never approach a tier limit.
|
||||
2. **Liquidation does not cap us either.** A diversified cross-margin book never margin-called in bootstrap up to 3x gross (P(liq)=0%). Fully-hedged positions in cross margin are ADL-exempt, protecting the delta-neutral sleeves.
|
||||
3. **The binding constraint is the multi-year drawdown tail.** Book-level (uniform-leverage) ruin bootstrap (<5% chance of −35% in a year, <1% liq):
|
||||
|
||||
| gross L | P(maxDD≥35%)/yr | realized maxDD (5.5yr path) |
|
||||
|---|---|---|
|
||||
| 1.0x | 0% | −27.9% |
|
||||
| 1.5x | 0.3% | **−39.6%** |
|
||||
| 2.0x | 2.0% | **−49.9%** |
|
||||
| 2.5x | 6.7% | −56% |
|
||||
|
||||
The 1-year bootstrap alone suggests L_max≈2.2x, but the **realized 5.5-year path hit −50% at 2x / −40% at 1.5x** — a fund holds for years, so the multi-year tail governs. Honest book-gross ceiling: **~1.3–1.5x**.
|
||||
4. **Per-edge leverage capacity varies enormously** — leverage belongs on the low-vol neutral sleeves, not uniformly:
|
||||
|
||||
| sleeve | Sharpe | vol/d | worst day | 0.25-Kelly | role |
|
||||
|---|---|---|---|---|---|
|
||||
| xsfunding (carry) | 3.56 | 0.20% | −1.5% | 22.8x | **workhorse** |
|
||||
| positioning | 2.97 | 1.69% | −4.9% | 2.3x | moderate |
|
||||
| unlock | 1.04 | 1.01% | −5.9% | 1.3x | light |
|
||||
| crypto_tstrend | 0.70 | 2.25% | −13.7% | 0.4x | **~1x, liquidation vector** |
|
||||
5. **Carry is genuinely robust, not a steamroller.** It made money through the two worst crashes of the cycle: LUNA (2022-05) +1.01%, FTX (2022-11) +3.11%; worst single day in 5.5y = −1.5%. Leveraged: 5x → +96% CAGR / −19% realized DD / 0% ruin; but 10x → −35% DD, 20x → −60% DD, 50% ruin. **Sharpe never improves with leverage.** Safe carry ceiling under the moderate tolerance: **~5–8x**.
|
||||
|
||||
**Conclusion:** don't lever the naive eq-wt book uniformly (that levers the weak trend sleeve into its liquidation vector). Concentrate leverage on carry (~5–8x) and positioning, keep trend ~1x, cap book-gross ≤1.5x — exactly what an ISV-adaptive inverse-vol weight + vol-target overlay does. Re-tune it on these numbers, cap it, stress it, gate it.
|
||||
|
||||
## Non-goals (YAGNI)
|
||||
|
||||
- No new leverage engine — reuse `paper_book.IncrementalRiskOverlay` / `paper_risk_overlay`.
|
||||
- No reactive de-sizing controllers — A/B-rejected 3× (pro-cyclical); final = ex-ante K(D*) + rare killswitch. (See `pearl_fxhnt_reactive_risk_controllers_rejected`.)
|
||||
- No change to the honest naive forward track — it stays as the unlevered floor.
|
||||
- No real-capital deployment as part of this work — capital gating is a later, separate decision.
|
||||
|
||||
## Phasing (approved: shadow track first, learn, then re-tune)
|
||||
|
||||
- **Phase 1 — Static-leverage shadow track (observe-only, ship first).** Apply a SIMPLE, explicit differentiated leverage vector taken straight from the research (starting point: carry 5x, positioning 2x, unlock 1.5x, trend 1x, then scaled so book-gross ≤ 1.5x) to the same daily 4-edge book, and record it as the shadow track `bybit_4edge_levered` alongside the honest naive track. NO capital, NO adaptive re-tune, NO killswitch yet — just observe the levered book live vs its backtest and vs the naive track. We learn: does the levered book reconcile OOS? does the realized DD behave? is carry's live tail as benign as the backtest?
|
||||
- **Phase 2 — Adaptive re-tune + hard caps + killswitch + stress (later, informed by Phase 1).** Only if Phase 1's shadow track validates: replace the static vector with the re-tuned `isv_adaptive_weights` overlay, add the safety rails and the DD-calibrated killswitch, and the stress tests. This is where capital-readiness is decided.
|
||||
|
||||
The components below are the full design; **Phase 1 implements only Component 3 with a static leverage vector** (Components 1–2 and the killswitch are Phase 2).
|
||||
|
||||
## Design
|
||||
|
||||
### Component 1 — Re-tune the overlay on the funding-net Bybit baseline
|
||||
The overlay (`isv_adaptive_weights` + `adaptive_target_vol` + `vol_target_leverage` + `DrawdownKillSwitch`) is currently Binance-mirage-tuned. Recompute its parameters (target vol, gross cap, kill/re-enter drawdown) against `bybit_sleeve_ret`. The ISV-adaptive inverse-vol weighting already overweights the low-vol carry sleeve and de-levers the high-vol trend sleeve — verify it reproduces the target envelope (carry-heavy, trend-light) on these numbers.
|
||||
|
||||
### Component 2 — Hard safety rails (on top of the adaptive weights)
|
||||
Adaptive is not enough for real money; add explicit caps as constants (ISV-derived where a bound is natural, fixed where it is a risk limit):
|
||||
- **Book-gross cap:** `MAX_BOOK_GROSS = 1.5x`.
|
||||
- **Per-sleeve leverage caps:** carry ≤ 8x, positioning ≤ 2.5x, unlock ≤ 1.5x, trend ≤ 1.0x. (Rails, not targets; the adaptive weights choose within them.)
|
||||
- **Killswitch** calibrated to the ruin line: flatten booked exposure when the shadow drawdown breaches the level that keeps realized maxDD inside −35%; re-enter per the existing `reenter_dd`.
|
||||
|
||||
### Component 3 — Shadow levered forward track
|
||||
The nightly already records the naive unlevered `bybit_4edge` track. Add a SHADOW track `bybit_4edge_levered` that applies the re-tuned overlay to the same daily book, recorded alongside — so we can compare the levered book live before any capital rides it. Same `ForwardTracker`, its own state file; reconciliation gate wired the same way (needs its own `backtest_summary` row — see `reference_fxhnt_forward_gate_two_store_sync`).
|
||||
|
||||
### Objective & success criterion (honest)
|
||||
Leverage does not raise Sharpe; its value is **capital efficiency / growth** for the sub-viable $35k pilot. Success = **maximize CAGR subject to** (multi-year realized maxDD ≤ 35% with the killswitch active) AND (bootstrap P(maxDD≥35%/yr) ≤ 5%, P(liq) ≤ 1%) AND (survives the LUNA/FTX windows) AND (survives a stress test with a deliberately fattened carry tail). If the levered book cannot beat the naive book's CAGR at equal-or-lower drawdown-tolerance breach, it is not worth the risk — ship nothing.
|
||||
|
||||
## Data flow
|
||||
`bybit_sleeve_ret` (funding-net baseline) → overlay re-tune (offline, own-memory Job) → overlay params + `backtest_summary(bybit_4edge_levered)` persisted → nightly `bybit_4edge_levered_nav` asset steps the shadow ForwardTracker → `cockpit_forward` ingests → cockpit shows naive vs levered side by side.
|
||||
|
||||
## Testing (TDD)
|
||||
- Overlay re-tune reproduces the target envelope (carry-weighted, trend ≤1x) on a seeded funding-net store.
|
||||
- `MAX_BOOK_GROSS` and per-sleeve caps are enforced (levered daily gross never exceeds the cap).
|
||||
- Killswitch fires at the calibrated DD line and re-enters correctly.
|
||||
- Crisis-window survival (LUNA/FTX slices) — levered book DD within tolerance.
|
||||
- Bootstrap ruin of the levered book ≤ tolerance (P(maxDD≥35%/yr) ≤ 5%).
|
||||
- Stress test: inject a synthetic −10% carry day (unseen in history) and assert the killswitch + gross cap bound the loss short of wipeout.
|
||||
- Shadow track: `bybit_4edge_levered` reconciliation gate gets its `backtest_summary` reference (regression against the silent-degrade bug).
|
||||
|
||||
## Risks & mitigations
|
||||
- **Backtest tail understates reality** (winsorized ±25%/day; no dislocation worse than LUNA/FTX in-sample) → the stress test + killswitch + the 1.5x book cap are the backstops; err toward WAIT.
|
||||
- **Concentration** — carry at 8x makes the book a levered carry bet, losing 4-edge diversification → the book-gross cap and per-sleeve caps bound how far concentration goes; keep trend/unlock live for uncorrelated ballast.
|
||||
- **Regime dependence** — carry Sharpe 3.56 rides the current funding/basis structure; erosion + leverage compounds losses → the shadow track must validate OOS before capital; monitor funding-regime decay.
|
||||
- **False-PASS on the levered gate** — the levered shadow track MUST reconcile against its own funding-net backtest reference (not silently degrade). Real capital only after BOTH the naive gate PASSes AND the levered shadow validates.
|
||||
@@ -0,0 +1,46 @@
|
||||
# Cockpit: surface the levered shadow track in Backtest + Replay — Design
|
||||
|
||||
**Status:** draft for review
|
||||
**Date:** 2026-07-05
|
||||
|
||||
## Goal
|
||||
|
||||
Let the user visually compare the naive `bybit_4edge` book against its levered shadow (`bybit_4edge_levered`) in the two cockpit views that currently show only the naive book: the **Backtest** (`/paper/sim`) and the **Replay** (`/paper/replay`). The overview/gate list already shows both gate rows; these two curve/track views do not. Observe-only — no capital, no new positions.
|
||||
|
||||
## Why it is not a trivial toggle (the constraint that shapes the design)
|
||||
|
||||
The levered shadow track exists ONLY as a `forward_nav` return series + `forward_summary` gate row + `backtest_summary` reference. It has **no positions** (observe-only) and **no precomputed measured curve**. The two views consume different data:
|
||||
|
||||
- **Backtest** reads a precomputed **MEASURED per-coin cost curve** (`compare_measured_cost._bybit_measured_net` → `bybit_liquidity_sweep.per_coin_book_returns`), cached in `compare_measured_cache`, read via `read_cached_single_measured`. The Backtest deliberately shows ONLY measured cost, never flat (a flat/cost-blind curve is the +7097% mirage foot-gun). So the levered Backtest curve must ALSO be measured — it cannot be a cheap flat combine.
|
||||
- **Replay** reads the venue paper-book `nav_history()` + scrubbable per-day positions (`_venue_repo(venue).nav_history()`). The shadow has no positions, so a levered replay must read its `forward_nav` and render position-less (exactly as carry/xsfunding already does — "return-defined, no scrubbable positions").
|
||||
|
||||
## Design
|
||||
|
||||
### Component 1 — Backtest: a measured levered curve
|
||||
- Thread an optional `leverage_shape: dict[str,float] | None` (+ `max_gross`) through `per_coin_book_returns` and `_bybit_measured_net`. When set, the per-sleeve per-coin weights are scaled by `shape[sleeve]/n` and the whole book down-scaled so summed gross ≤ `max_gross` (the SAME transform as `levered_eqwt_daily_returns`, applied at the per-coin weight layer, so the GROSS book and per-coin measured turnover cost stay internally consistent). `leverage_shape=None` → the current naive eq-wt path, byte-identical.
|
||||
- `precompute_single_book_measured` also computes + caches the levered book under key `bybit_4edge_levered` (reusing `_LEVERED_SLEEVE_SHAPE` / `MAX_BOOK_GROSS` from `bybit_forward_track`).
|
||||
- Add `bybit_4edge_levered` to `_SIM_BOOKS` in `web/app.py` as a 4th book-switch pill ("Bybit 4-edge levered · shadow"). `_sim_returns` / `read_cached_single_measured` resolve it like the other single books.
|
||||
- Re-run the `compare-measured-precompute` Job so the levered measured cache is populated (in-cluster, git-synced — like the sleeve-ret Job).
|
||||
|
||||
### Component 2 — Replay: naive↔levered track toggle
|
||||
- `paper_replay` / `paper_replay_at` gain an optional `track` param (default `naive`). `track=naive` → the current venue paper-book `nav_history()` path (unchanged). `track=levered` → read the `bybit_4edge_levered` `forward_nav` series (via `ForwardNavRepo` / the DashboardService `detail`) and render the NAV curve position-less, with the same "return-defined — no scrubbable positions" note carry uses.
|
||||
- `replay.html` gains a small naive↔levered toggle (like the existing venue toggle), relative-URL, htmx-consistent.
|
||||
|
||||
### Objective & honesty guardrails
|
||||
- The Backtest levered curve MUST be measured-cost (never flat) — the same anti-mirage principle as the naive book. Label it clearly as the levered SHADOW.
|
||||
- The Replay levered track is the recorded forward_nav (0 days at inception; grows as it accrues) — clearly labelled observe-only, no capital.
|
||||
- Naive paths stay byte-identical (`leverage_shape=None`, `track=naive`).
|
||||
|
||||
## Testing
|
||||
- Unit: `per_coin_book_returns(leverage_shape=uniform)` == naive; carry-weighted shape scales + gross-caps; `_bybit_measured_net` levered curve differs from naive with the same measured cost model.
|
||||
- Web: `/paper/sim?book=bybit_4edge_levered` renders (measured curve or graceful pending note); the book-switch shows the 4th pill; `/paper/replay?track=levered` renders the forward_nav curve position-less; `track=naive` unchanged; both leak-clean.
|
||||
- **LOCAL BROWSER (mandatory, per feedback_verify_cockpit_locally_not_prod):** run the cockpit locally, open both pages in a real browser (playwright), read the actual rendered numbers/curves, screenshot naive vs levered side by side. Never claim "works" off prod/synthetic.
|
||||
|
||||
## Non-goals
|
||||
- No levered Paper view (`/paper`) — the shadow has no positions; Paper stays naive-only.
|
||||
- No adaptive overlay / killswitch (that is Phase 2 of the leverage spec).
|
||||
- No capital.
|
||||
|
||||
## Risks
|
||||
- **Measured-cost core is sensitive** — the leverage_shape must not change the GROSS series or the per-coin cost model, only the sleeve weighting; the uniform-shape==naive test is the guard.
|
||||
- **Precompute Job must re-run** or the levered Backtest shows the graceful pending note until it does — acceptable (never a 500, never a flat curve).
|
||||
129
docs/superpowers/specs/2026-07-07-alpaca-execution-leg-design.md
Normal file
129
docs/superpowers/specs/2026-07-07-alpaca-execution-leg-design.md
Normal file
@@ -0,0 +1,129 @@
|
||||
# Alpaca execution leg (fractional US-equity venue) — Design
|
||||
|
||||
**Status:** draft for review
|
||||
**Date:** 2026-07-07
|
||||
|
||||
## Goal
|
||||
|
||||
Build the **Alpaca execution FOUNDATION** — a broker adapter on the clean `Broker` port + fractional
|
||||
order sizing + a paper-first deploy — so any book that runs through the fxhnt `ExecutionService` can
|
||||
trade on a **commission-free, fractional-share** US-equity venue. It runs immediately on the book
|
||||
`ExecutionService` already rebalances (the survivor/research book via `fxhnt execute`), proving the
|
||||
leg end-to-end. Fractional shares let a book implement its target weights **precisely** at small
|
||||
capital, where IBKR's whole-share rounding distorts them. Paper-first (Alpaca paper account); live
|
||||
is gated behind the existing `allow_live`.
|
||||
|
||||
**Scope note (from investigation):** the validated multi-strat ETF book's POSITION logic is NOT on
|
||||
the clean port today — `domain/strategies/multistrat.py` exposes only a return series (`book_series`),
|
||||
and its live IBKR execution is an archived, out-of-repo foxhunt script (`multistrat_bot.py` in a
|
||||
configmap). Putting the multi-strat book itself on Alpaca additionally requires exposing its
|
||||
per-instrument target weights, bridging them into `ExecutionService`, and handling its BTC-USD leg
|
||||
(Alpaca equities can't trade BTC). That is a deliberate FOLLOW-UP (see Non-goals); this spec delivers
|
||||
the shared foundation the follow-up will reuse.
|
||||
|
||||
## Why (fit + what changes)
|
||||
|
||||
The Broker port (`ports/broker.py`) is exactly two methods — `account_state()` + `place_order()` —
|
||||
and `Order.quantity` is already a `float`, so the port ALREADY supports fractional; only the
|
||||
adapters and the execution sizing round to whole shares. Two whole-share assumptions exist today:
|
||||
- `adapters/broker/ibkr.py:54` — `int(round(order.quantity))` (IBKR rejects fractional, error 10243);
|
||||
- `application/execution.py:90` — `qty = int(round(abs(delta)))` (the app-layer sizing).
|
||||
|
||||
Alpaca removes both: fractional orders + precise weights. This is an EXECUTION-venue change, not a
|
||||
new alpha signal — the multi-strat book's edge is unchanged; it just gets a cheaper, fractional home.
|
||||
|
||||
## Design
|
||||
|
||||
### 1. `AlpacaSettings` (config.py, mirrors `IbkrSettings`)
|
||||
|
||||
```python
|
||||
class AlpacaSettings(BaseSettings):
|
||||
model_config = SettingsConfigDict(env_prefix="FXHNT_ALPACA_")
|
||||
api_key: str = ""
|
||||
api_secret: str = ""
|
||||
base_url: str = "https://paper-api.alpaca.markets" # paper default; live = https://api.alpaca.markets
|
||||
timeout: int = 15
|
||||
min_order_notional: float = 1.0 # Alpaca min $1/order; skip dust below this
|
||||
```
|
||||
Added to `Settings` as `alpaca: AlpacaSettings`. Credentials come from env (k8s secret), never code.
|
||||
|
||||
### 2. `adapters/broker/alpaca.py` — `AlpacaBroker` (httpx REST)
|
||||
|
||||
Implements the `Broker` port against the Alpaca REST API (httpx is already a dep; no heavy SDK).
|
||||
Mirrors the compact IBKR adapter.
|
||||
|
||||
- `supports_fractional = True` (a capability flag the execution layer reads; `IbkrBroker` gets
|
||||
`supports_fractional = False`).
|
||||
- Auth via headers `APCA-API-KEY-ID` / `APCA-API-SECRET-KEY`; one reused `httpx.Client`.
|
||||
- `account_state()` → `GET /v2/account` (`portfolio_value`→nlv, `cash`) + `GET /v2/positions`
|
||||
(`{symbol: float(qty)}`, and `Σ|market_value|` → `gross_position_value` for the existing
|
||||
data-consistency sanity check). `is_paper = "paper-api" in base_url`.
|
||||
- `place_order(order)` → `POST /v2/orders` `{symbol, qty: str(quantity), side: lower, type: "market",
|
||||
time_in_force: "day"}` (fractional requires market + day). Returns the response `status`.
|
||||
- Raises on non-2xx with a trimmed message; `close()` / context-manager like `IbkrBroker`.
|
||||
|
||||
### 3. `execution.py` — fractional-aware sizing
|
||||
|
||||
Replace the hard whole-share round with a capability branch (whole-share path UNCHANGED for IBKR):
|
||||
|
||||
```python
|
||||
frac = getattr(self._broker, "supports_fractional", False)
|
||||
...
|
||||
if frac:
|
||||
qty = round(abs(delta), 6) # fractional shares
|
||||
if qty * px < self._s.alpaca.min_order_notional: # skip sub-$1 dust
|
||||
continue
|
||||
else:
|
||||
qty = int(round(abs(delta))) # whole shares (IBKR)
|
||||
if qty == 0:
|
||||
continue
|
||||
```
|
||||
The entry-floor / hysteresis band logic above it is unchanged (it already works in notional terms).
|
||||
`PlannedOrder.quantity` stays a float; the whole-share path just happens to emit integers.
|
||||
|
||||
### 4. Wiring — CLI + deploy (paper-first, gated)
|
||||
|
||||
- **CLI:** add `--broker {ibkr,alpaca}` to `execute` (default `ibkr`). `alpaca` builds `AlpacaBroker`
|
||||
from settings and runs the SAME `ExecutionService(...).rebalance(book, execute=...)`. The existing
|
||||
`--live`/`allow_live` gate and dry-run default carry over unchanged (paper is the default).
|
||||
- **Deploy:** a k8s CronJob `fxhnt-alpaca-rebalancer` mirroring the IBKR multistrat rebalancer —
|
||||
scheduled daily, `--broker alpaca` (paper base_url), reading `alpaca-credentials` (a k8s secret
|
||||
with `api_key`/`api_secret`). **Ship SUSPENDED** until the secret is populated (mirrors the Bybit
|
||||
testnet leg awaiting its key). A network policy allowing egress to `*.alpaca.markets:443`.
|
||||
|
||||
## Validation gates
|
||||
|
||||
1. **Adapter unit tests (mocked httpx, NO network):** `account_state()` parses
|
||||
portfolio_value/cash/positions/market_value; `place_order()` POSTs a FRACTIONAL qty payload
|
||||
(e.g. `qty="1.2345"`, `type=market`, `tif=day`) and returns the status; `is_paper` true on the
|
||||
paper base_url, false on live; a non-2xx raises. `supports_fractional is True`.
|
||||
2. **Execution fractional path:** with a fake `supports_fractional=True` broker, a target implying
|
||||
1.2345 shares emits a fractional order (NOT rounded to 1); a sub-$1 delta is skipped; the
|
||||
whole-share path (fractional=False) is byte-unchanged (regression on the existing IBKR tests).
|
||||
3. **Live-account guard still bites:** a non-paper Alpaca account with `allow_live=False` is BLOCKED
|
||||
(reuses the ExecutionService gate — no code path bypasses it).
|
||||
4. Full suite green.
|
||||
|
||||
## Non-goals (explicit follow-ups)
|
||||
|
||||
- **The multi-strat ETF book on Alpaca — BUILT (follow-up completed same session).** `multistrat.target_weights`
|
||||
exposes the final-day `L·tw·scale` weights (byte-identical `book_series` via a shared `_volnorm_scale`);
|
||||
`ExecutionService.rebalance_weights` bridges pre-computed weights + caller prices through the same gates +
|
||||
fractional sizing; BTC-USD routes to **Alpaca crypto** ("BTC/USD", time_in_force gtc) via the crypto-aware
|
||||
`AlpacaBroker.place_order` (equities stay day). CLI `fxhnt execute-multistrat`; the CronJob runs it. The
|
||||
Alpaca-crypto position-symbol normalisation is validated on first real (paper) run.
|
||||
- **No new alpha / signal change.** Purely a cheaper, fractional execution venue.
|
||||
- **No live trading now.** Paper-first; `allow_live` stays the gate. The CronJob ships suspended.
|
||||
- **No Alpaca crypto / options / streaming.** Only the two REST endpoints the Broker port needs.
|
||||
- No migration OFF IBKR — the IBKR leg stays; Alpaca is added alongside (the fund can run both).
|
||||
|
||||
## Risks
|
||||
|
||||
- **Credentials required to run.** The leg is code-complete + tested but idle until Alpaca paper API
|
||||
keys are provided (like the Bybit testnet leg). The CronJob is suspended until then.
|
||||
- **Alpaca fractional constraints:** fractional orders must be market + day and are US-market-hours
|
||||
only; the adapter fixes `type=market, tif=day` so a fractional order is never rejected for tif.
|
||||
- **Weekend/holiday runs:** a rebalance outside market hours queues/rejects; the CronJob schedule
|
||||
should target a weekday market-open slot (documented in the manifest), and a rejected order is
|
||||
logged per-order (the existing `place_order` try/except in the rebalance loop) — never aborts the
|
||||
batch.
|
||||
@@ -0,0 +1,189 @@
|
||||
# Momentum-sleeve volatility-target overlay (H2) — Design
|
||||
|
||||
**Status:** ❌ FALSIFIED — do NOT implement (2026-07-07)
|
||||
**Date:** 2026-07-07
|
||||
|
||||
## FALSIFICATION VERDICT (2026-07-07)
|
||||
|
||||
Running this spec's own gate 2 (anti-pro-cyclical) EARLY — before any build — killed it. The
|
||||
headline +0.24 OOS book Sharpe is **not a return edge; it is pure volatility-shrinkage from
|
||||
de-risking, and the de-risking is pro-cyclical** (the exact rejected-controller failure mode,
|
||||
[[pearl_fxhnt_reactive_risk_controllers_rejected]]):
|
||||
|
||||
- **Return contribution is negative and insignificant:** daily book-return difference
|
||||
(vol-targeted − raw) = **−0.31 bp/day, t = −0.65**. The overlay earns *less*, not more; Sharpe
|
||||
rises only because vol falls faster than return (de-leveraging, not alpha).
|
||||
- **It misses more upside than downside it avoids:** exposure removed on down days avoided 99.4%
|
||||
of loss, but removed on up days missed 110.3% of gain → **net −10.8% return drag** from the
|
||||
de-risking itself.
|
||||
- **It cuts hardest right before the biggest up days:** mean scale the day before a top-decile up
|
||||
day = **0.854 vs 0.904 unconditional** — the "cut before the bounce" signature.
|
||||
|
||||
Root cause the design missed: in crypto-bull, high realized vol and drawdown-then-bounce are
|
||||
strongly correlated, so conditioning on **vol** ≈ conditioning on **drawdown** *in effect* — the
|
||||
overlay is the 3×-rejected reactive de-sizing in a new guise, not a distinct mechanism. The
|
||||
modest maxDD improvement (−6.7→−5.3) is bought with a regime-fragile (bull-only sample, no 2022
|
||||
bear), return-insignificant, pro-cyclical mechanism → for real capital this is a false-PASS risk.
|
||||
**Verdict: not built. Raw momentum stands. See [[pearl_fxhnt_voltarget_overlay_is_reactive_desizing_rejected]].**
|
||||
|
||||
The design below is retained as the falsified record (what was proposed and why it was rejected).
|
||||
|
||||
---
|
||||
|
||||
## Goal
|
||||
|
||||
Add an **ex-ante, de-risk-only sleeve-level volatility-target overlay** to the crypto momentum
|
||||
sleeve (`crypto_tstrend`, driven by `TrendRunner`), scaling the whole position vector down in
|
||||
high-realized-volatility regimes and never levering up. This raises the risk-adjusted return of
|
||||
the sleeve — and thereby the naive `bybit_4edge` book — without adding leverage, funding cost, or
|
||||
meaningful turnover cost. Deploy only if the improvement survives on the fully net-of-cost book
|
||||
and clears an explicit anti-pro-cyclical falsification gate.
|
||||
|
||||
## Why (the investigation)
|
||||
|
||||
Investigation-first, out-of-fold (expanding window, min 250d train, 21d step, 881 OOS test days,
|
||||
2023-01 → 2026-07), measured on the exact 4-sleeve deploy book (tstrend/positioning/unlock/xsfunding)
|
||||
using persisted per-sleeve daily returns (`bybit_sleeve_ret`):
|
||||
|
||||
| Momentum variant | CAGR | Sharpe | maxDD |
|
||||
|---|---|---|---|
|
||||
| raw (current) | 77.0% | 3.68 | −6.7% |
|
||||
| **de-risk-only vol-target (cap 1.0)** | 75.4% | **3.92 (+0.24)** | **−5.3%** |
|
||||
| leveraged Pareto (cap 1.5, ref) | 79.4% | 3.80 (+0.12) | −5.7% |
|
||||
|
||||
Key findings:
|
||||
|
||||
1. **The free (defensive) half wins.** Scaling exposure *down* in high-vol regimes (`cap=1.0`,
|
||||
never lever) delivered a **larger** Sharpe gain (+0.24) than the leveraged version (+0.12).
|
||||
Levering up in calm regimes adds vol-adjusted return but also risk, diluting the Sharpe.
|
||||
2. **Net-of-cost robust.** The overlay adds only ~0.62% |Δscale|/day (the 40-day trailing vol
|
||||
moves slowly). Charging the incremental rebalance turnover at 10/15/25/40 bp leaves the gain at
|
||||
+0.24 / +0.23 / +0.23 / +0.22 Sharpe — above the 0.2 kill-gate even at an absurd 40 bp/turn.
|
||||
3. **Parameter-robust, not fit.** L ∈ {20, 40} and floor ∈ {0.25, 0.5} all give ~+0.24 Sharpe and
|
||||
the same DD improvement — the effect is structural (momentum vol clustering), not a lucky pick.
|
||||
4. **Not redundant with per-symbol vol targeting.** `TrendRunner` already vol-targets each symbol
|
||||
cross-sectionally; the sleeve overlay responds to the sleeve's *realized* return vol, which
|
||||
carries the correlation/regime vol that per-symbol targeting cannot see.
|
||||
|
||||
Falsified alternatives from the same investigation (do NOT revisit): **H1** — ex-ante sleeve
|
||||
re-weighting (Sharpe/inverse-vol) fails its gate (equal-weight is at the efficient point; tilting
|
||||
costs CAGR). **H3** — `xvenue_carry` is vestigial (~0 return; dilutes CAGR) → separate cleanup spec.
|
||||
|
||||
## The load-bearing distinction: this is NOT the rejected reactive de-sizing
|
||||
|
||||
[[pearl_fxhnt_reactive_risk_controllers_rejected]]: reactive de-sizing was A/B-rejected 3× as
|
||||
pro-cyclical (it cut after drawdowns and got whipsawed — cut at the bottom, missed the bounce).
|
||||
This overlay is different in kind, and the difference is load-bearing:
|
||||
|
||||
- **Rejected:** conditioned on **drawdown / recent PnL** — path-dependent, sticky, directional.
|
||||
- **This:** conditions on **realized volatility** — direction-agnostic, mean-reverting, symmetric.
|
||||
High vol ≠ recent loss.
|
||||
- **Empirical proof it is not the trap:** it *improves* maxDD (−6.7 → −5.3) and Sharpe across all
|
||||
folds and params. The rejected controllers did the opposite (net worse). Same failure mode →
|
||||
same signature; this shows the inverse signature.
|
||||
|
||||
Because this is real capital and the pattern superficially resembles the rejected one, the spec
|
||||
mandates an **explicit anti-pro-cyclical falsification gate** (below) before any deploy.
|
||||
|
||||
## Design
|
||||
|
||||
### The overlay (inside `TrendRunner`)
|
||||
|
||||
Add two optional kwargs to `TrendRunner.__init__`: `sleeve_vol_target: float | None = None`
|
||||
(annualized; `None` = current behaviour, overlay off) and `sleeve_vol_window: int = 40`, plus a
|
||||
`sleeve_vol_floor: float = 0.25`. When `sleeve_vol_target` is set, `run()`:
|
||||
|
||||
1. Forms the gross-capped `pos` for day `d` exactly as today (`_positions_for_day`).
|
||||
2. Books the **raw** realized return `raw_r` for `[d, nxt]` (price + funding, cost-free) — the
|
||||
unscaled sleeve return.
|
||||
3. Computes `scale[nxt] = clip(sleeve_vol_target_daily / trailing_vol, sleeve_vol_floor, 1.0)`
|
||||
where `trailing_vol` = realized daily vol of the **raw** booked returns over the last
|
||||
`sleeve_vol_window` days strictly BEFORE `nxt` (no lookahead; empty/short buffer → scale 1.0).
|
||||
4. Books the scaled return: `scale[nxt] · raw_r` and computes turnover on the **scaled** position
|
||||
vector `scale[nxt] · pos` versus the previous day's scaled positions — so the vol-target's
|
||||
own rebalance turnover flows through the existing `turnover` haircut automatically (faithful
|
||||
net-of-cost; no separate cost path).
|
||||
|
||||
Using the **raw** (unscaled) returns for `trailing_vol` breaks the feedback loop (scale depends
|
||||
only on the underlying strategy's vol, not on the already-de-risked vol) and matches the
|
||||
investigation's measurement exactly.
|
||||
|
||||
### `sleeve_vol_target` is adaptive, not a tuned constant
|
||||
|
||||
Per [[feedback_adaptive_not_tuned]] and [[feedback_isv_for_adaptive_bounds]], `sleeve_vol_target`
|
||||
is **not** a hand-picked number. The deploy wiring sets it to the **expanding-window realized daily
|
||||
vol of the raw sleeve returns** (the sleeve's own long-run vol, growing with data), so the ratio
|
||||
`target/trailing` is ≈1 on average and deviates only in vol regimes. `sleeve_vol_window` (40) and
|
||||
`sleeve_vol_floor` (0.25) are held at values shown robust across the tested range — the plan pins a
|
||||
robustness check (the metric must not degrade materially at L=20 or floor=0.5), so they are chosen
|
||||
for stability, not to maximise the backtest number.
|
||||
|
||||
### SSOT: `run()` and `weights_and_contributions()`
|
||||
|
||||
`weights_and_contributions()` (which feeds `sleeve_returns_from_store` → the persisted sleeve
|
||||
returns → the book combine) must apply the **same** `scale` series so persisted returns match
|
||||
`run()`. Both iterate the same `all_days` deterministically; factor the scale computation so both
|
||||
share one source of truth (a private helper that, given the raw-return buffer, returns the per-day
|
||||
scale) — no drift between the booked return and the surfaced weights, matching the existing
|
||||
invariant that `Σ weight·return == run()`'s booked gross return.
|
||||
|
||||
### Deploy: back-compute + swap (deterministic → no separate shadow wait)
|
||||
|
||||
The momentum sleeve is **warehouse-backed and deterministic** — its returns are a pure function of
|
||||
the price/funding data already stored. Unlike the `xsfunding` live-snapshot track (which could not
|
||||
backfill a missed past day), the vol-targeted momentum sleeve can be **back-computed over both the
|
||||
historical backtest and the already-elapsed forward window** ([[reference_fxhnt_forward_gate_two_store_sync]]:
|
||||
warehouse-backed tracks recompute from persistent data; live-snapshot tracks cannot). So a separate
|
||||
2–3-week shadow-observation wait is unnecessary and is NOT used. Instead:
|
||||
|
||||
1. Back-compute the vol-targeted momentum sleeve + the `bybit_4edge` book over the backtest **and**
|
||||
the elapsed forward window (the ~16 nights `crypto_tstrend` already has). Run gates 1–3 on that.
|
||||
2. If all gates pass, swap the vol-targeted sleeve into the deploy `bybit_4edge` book. This changes
|
||||
the book's return definition → it **re-inceptions** (`15/21` → `0/21`), and the book's own
|
||||
21-day reconciliation gate becomes the frozen-forward validation of the new definition. No
|
||||
separate `_voltgt` shadow track is created — the book's existing gate polices it.
|
||||
|
||||
This is equally gated (the book gate validates the swapped-in definition over unseen nights either
|
||||
way) but ~2–3 weeks faster than a separate shadow, because determinism lets the elapsed-forward
|
||||
reconciliation stand in for a fresh shadow observation. Resetting `15/21`→`0/21` is the honest cost
|
||||
of a definition change (the old 15 nights observed a different book); shadow-first would incur the
|
||||
same reset, only later.
|
||||
|
||||
## Validation gates (all must pass before the swap into the deploy book)
|
||||
|
||||
1. **Net-book re-measure — multi-condition (not a single Sharpe point).** Build the overlay in
|
||||
`TrendRunner`; re-run the real `bybit_4edge` backtest with the full 5.5 bp net cost model (not
|
||||
the pre-cost sleeve stream), over the historical backtest **and** the elapsed forward window.
|
||||
Deploy only if ALL hold: net book Sharpe improves by **≥ +0.15** (the pre-cost +0.24 will
|
||||
compress net — a brittle +0.2 point-gate risks false-killing a real edge), **maxDD does not
|
||||
worsen**, gate 3 (per-year consistency) passes, and gate 2 (anti-pro-cyclical) is clean. The
|
||||
rigor lives in the robustness conditions — hard to fake — not in a single number a fluke could
|
||||
clear. If the net improvement is below +0.15 or DD worsens, H2 is killed; raw momentum stands.
|
||||
2. **Anti-pro-cyclical falsification gate.** Test the specific rejected failure mode: the overlay
|
||||
must NOT systematically cut exposure just before the momentum sleeve's best recovery days.
|
||||
Concretely — condition scale on next-day sleeve return; the mean scale ahead of the top-decile
|
||||
positive return days must not be materially below the unconditional mean scale (the rejected
|
||||
controllers failed exactly here). Fail → do not deploy.
|
||||
3. **Per-year / per-fold consistency.** The Sharpe/DD improvement must hold in a majority of
|
||||
calendar years (not driven by one regime). A single-year artifact fails.
|
||||
4. **Full suite green** + a new invariant test: with `sleeve_vol_target=None`, `run()` and
|
||||
`weights_and_contributions()` are byte-identical to today (backward compat); with it set,
|
||||
`Σ weight·return == run()`'s booked gross return still holds day-by-day.
|
||||
|
||||
## Non-goals
|
||||
|
||||
- No leverage / offensive scaling (`cap` is fixed at 1.0 — the whole point is the free defensive
|
||||
half). The leveraged Pareto variant is explicitly out of scope.
|
||||
- No sleeve re-weighting (H1, falsified). No `xvenue_carry` changes (H3, separate spec).
|
||||
- No change to the per-symbol vol targeting or `gross_cap` already in `TrendRunner`.
|
||||
- No adaptive/CVaR book overlay (that is the dormant Phase-2, not this).
|
||||
|
||||
## Risks
|
||||
|
||||
- **Pre-cost → net gap.** The +0.24 was on pre-cost sleeve returns; the net-book re-measure
|
||||
(gate 1) is the real test. If costs on the actual book erode it below +0.2, we stop.
|
||||
- **Regime dependence.** 3.5 years OOS spans a bull-heavy crypto period; gate 3 (per-year) guards
|
||||
against a single-regime artifact, but the re-inceptioned `bybit_4edge` book's own 21-day forward
|
||||
gate (post-swap) is the ultimate arbiter before capital.
|
||||
- **Resemblance to the rejected pattern.** Mitigated by gate 2, but reviewers must treat any sign
|
||||
of pro-cyclical cutting as a hard stop, not a tuning problem.
|
||||
@@ -0,0 +1,166 @@
|
||||
# Positioning sleeve — capacity-aware realized returns — Design
|
||||
|
||||
**Status:** draft for review
|
||||
**Date:** 2026-07-07
|
||||
|
||||
## Goal
|
||||
|
||||
Make the positioning sleeve's booked return **honest about capacity AND robust to tail-concentration**,
|
||||
via two adaptive, gains-only haircuts in `_book_breakdown` (the SSOT engine for the positioning
|
||||
forward track, the reconciliation-gate reference, and the 4-edge book's positioning sleeve):
|
||||
|
||||
1. **Capacity haircut** — cap each coin's realized GAIN by a market-impact fraction driven by the
|
||||
coin's liquidity (turnover) and the fund's capital, so the number is honest as capital scales.
|
||||
2. **Tail-concentration cap** — cap each coin's per-day CONTRIBUTION to the book at an ISV-driven
|
||||
multiple of the book's own daily-return scale, so no single coin/day can dominate the book (the
|
||||
+15% "anomaly" days are ~one coin), reducing the fat-right-tail dependence.
|
||||
|
||||
**Framing correction (from the OHLC/spot investigation):** the +150% microcap days are NOT data
|
||||
glitches and NOT uncapturable at $100k — HUSDT +154% is spot-corroborated (spot +153.8%) on $214M
|
||||
turnover; a ~$9k position is <0.01% of volume → capturable. So the issue is not bad prints; it is
|
||||
(a) capacity at LARGE capital and (b) tail-concentration (28% of return from 10 days). Both haircuts
|
||||
can only LOWER measured P&L (gains-only), the safe direction for a real-capital gate reference. A
|
||||
data-glitch guard is deliberately NOT built — it would suppress real, spot-confirmed gains.
|
||||
|
||||
## Why (the investigation, 2026-07-07)
|
||||
|
||||
Decomposing positioning's returns (in-cluster, 1283 days, liquid-64 universe):
|
||||
- **28% of total return (+103% of +371%) comes from 10 days (0.8%)**, each driven by ONE microcap
|
||||
doubling: HUSDT +153.7%, HUSDT +156%, LAB +162%, 1000BONK +57%.
|
||||
- These are sub-dollar tokens; +150% single-day closes are uncapturable at large size.
|
||||
- The +15% forward day (2026-06-27) does NOT reproduce on recompute (net −0.09%) — its driver coin
|
||||
dropped out of the current universe: a transient, uncapturable name.
|
||||
- **The edge survives winsorization** (flat ±20% cap → Sharpe 2.46 / CAGR +75% vs raw 3.19 / 172%),
|
||||
so positioning is a REAL contrarian edge whose HEADLINE is inflated by uncapturable tails — not a
|
||||
fake edge.
|
||||
|
||||
**Key capital insight:** capturability is size-dependent. At the configured `paper_capital`
|
||||
($100k), a ~9% weight in a ≥$10M-turnover coin is a ~$9k position = <0.1% of daily volume →
|
||||
capturable. The overstatement bites only at large capital ($10M+). So the honest fix is
|
||||
**capacity-aware** (attenuates by position/turnover), not a flat cap: at $100k it barely binds
|
||||
(confirming the edge is real at fund scale); as capital scales it caps thin names automatically.
|
||||
|
||||
## Design
|
||||
|
||||
### The capacity haircut (in `_book_breakdown`)
|
||||
|
||||
For each coin `s` on day `d`, replace the booked per-coin return with a capacity-attenuated one:
|
||||
|
||||
```
|
||||
position_notionalₛ = capital · |wₛ| # unit-gross weight → dollars in coin s
|
||||
executable_notionalₛ = π · med_turnoverₛ # what you can trade w/o prohibitive impact
|
||||
capture_fractionₛ = min(1, executable_notionalₛ / position_notionalₛ) # ∈ (0, 1]
|
||||
realized_retₛ = retₛ if retₛ ≤ 0 # losses kept IN FULL (conservative — you're stuck)
|
||||
= retₛ · capture_fractionₛ if retₛ > 0 # GAINS capped: can't bank more than you can exit
|
||||
```
|
||||
|
||||
- **`med_turnoverₛ`** = the coin's MEDIAN turnover over a trailing window (the SAME robust measure
|
||||
`liquid_universe` uses — median, not the squeeze-day spike, so a normally-thin coin cannot rely
|
||||
on squeeze-day volume to bail out its position). Adaptive per-coin (the ISV-style bound), read
|
||||
from the existing `turnover` feature.
|
||||
- **`π`** = max participation = **0.10** (you can trade up to ~10% of a coin's daily $ volume in a
|
||||
day without prohibitive impact). A documented, conservative CAPACITY assumption — like a cost
|
||||
assumption, NOT a return-maximising fit. Held constant; the adaptivity is in `med_turnoverₛ`.
|
||||
- **Gains-only, conservative:** capping only favourable moves (never reducing losses) guarantees
|
||||
the haircut can only LOWER measured P&L — the safe direction for a real-capital gate reference
|
||||
([[feedback_adaptive_not_tuned]]: caps are adaptive-liquidity-driven, not tuned magic numbers).
|
||||
|
||||
`capacity_capital=None` ⇒ no haircut (current behaviour) so the change is backward-compatible and
|
||||
unit-testable with explicit capital.
|
||||
|
||||
### The tail-concentration cap (in `_book_breakdown`)
|
||||
|
||||
After the capacity haircut, cap each coin's per-day CONTRIBUTION so no single coin can dominate the
|
||||
book beyond a multiple of the book's own normal daily move:
|
||||
|
||||
```
|
||||
cₛ = wₛ · realized_retₛ / gross # the coin's contribution to the book return that day
|
||||
σ_book = trailing std of the RAW (pre-cap) book daily returns over a window (bootstrapped:
|
||||
first-observation → expanding until the window fills, then rolling) # the ISV scale
|
||||
capped_cₛ = min(cₛ, K · σ_book) if cₛ > 0 # GAINS-only: clip a dominating positive contribution
|
||||
= cₛ if cₛ ≤ 0 # losses kept in full (conservative)
|
||||
book_return_d = Σ capped_cₛ
|
||||
```
|
||||
|
||||
- **`σ_book`** is the ADAPTIVE, ISV-style scale — the book's OWN trailing daily-return volatility,
|
||||
computed from the RAW (pre-cap) contributions so the cap does not feed back on its own reference
|
||||
([[feedback_isv_for_adaptive_bounds]], [[pearl_wiener_optimal_adaptive_alpha]]: bootstrap the
|
||||
estimator on first observation, no hardcoded seed). A single coin contributing +14% when σ_book ≈
|
||||
1.5% is a ~9σ dominator → clipped to `K·σ_book`.
|
||||
- **`K`** = z-multiple = **4.0** (a 4σ single-coin contribution ceiling). A documented robustness
|
||||
constant — the adaptivity is in `σ_book`, NOT in K ([[feedback_adaptive_not_tuned]]). Not fitted
|
||||
to a return target; gate 6 (below) checks it behaves sensibly.
|
||||
- **Gains-only** (never clip a negative contribution): the cap can only reduce the fat RIGHT tail,
|
||||
never flatter a loss — same conservative direction as the capacity haircut.
|
||||
|
||||
The two haircuts compose: capacity first (models "can I exit this size"), then tail-cap (models "no
|
||||
single name should carry the book"). Both are off when `capacity_capital=None` AND the tail-cap is
|
||||
disabled — controlled by one `capacity_capital` gate plus a `tail_cap_sigma=None` opt-out for
|
||||
backward-compatible byte-reproduction in tests.
|
||||
|
||||
### SSOT threading (one engine, forward + ref + book all fixed)
|
||||
|
||||
The forward track, the reconciliation-gate reference, AND the 4-edge book's positioning sleeve all
|
||||
resolve to `positioning_returns_from_store → _book_breakdown`. Thread one optional param down that
|
||||
single chain so all three are fixed identically (basis-matched):
|
||||
|
||||
```
|
||||
_book_breakdown(store, *, universe, cost_bps, direction, capacity_capital=None, participation=0.10)
|
||||
positioning_returns_from_store(..., capacity_capital=None)
|
||||
bybit_book_eval._positioning_series(store, *, universe, cost_bps, capacity_capital=None)
|
||||
bybit_book_eval.sleeve_returns_from_store(..., capacity_capital=None) # positioning branch only
|
||||
bybit_forward_track.BybitFourEdgeStrategy(..., capacity_capital=None) # passes through
|
||||
```
|
||||
|
||||
Wiring passes `s.paper_capital`:
|
||||
- `build_positioning_forward_nav` / `positioning_nav` (the standalone forward track),
|
||||
- `bybit_4edge_backtest_summary(sleeves=["positioning"], ...)` (the gate ref, persist Job),
|
||||
- `build_bybit_4edge_forward_nav` + `bybit_4edge_backtest_summary` (the FULL book — its positioning
|
||||
sleeve is capacity-honest too, so the deploy book's number matches).
|
||||
|
||||
Turnover panel read is universe-bounded (memory-safe), mirroring the existing `long_ratio`/`close`
|
||||
reads in `_book_breakdown`.
|
||||
|
||||
## Validation gates
|
||||
|
||||
1. **Capacity curve (the core check).** Sweep `capacity_capital` ∈ {$100k, $1M, $10M, $100M} and
|
||||
report positioning Sharpe/CAGR. Expect: ≈unchanged at $100k (edge is capturable at fund scale),
|
||||
monotonically degrading as capital grows, the top-10 spike days attenuating at high capital. A
|
||||
flat/no degradation at high capital means the haircut isn't binding (bug); a big drop at $100k
|
||||
means the participation π is too tight (re-examine).
|
||||
2. **Conservative direction.** For every (capital, day), the capacity-booked return ≤ the raw
|
||||
booked return (gains-only haircut never raises P&L). Assert on a seeded microcap-squeeze day.
|
||||
3. **SSOT basis-match.** With the same `capacity_capital`, the standalone positioning forward
|
||||
series == the book's positioning-sleeve series == the gate ref's series (one engine).
|
||||
4. **Backward-compat.** `capacity_capital=None` reproduces today's `_book_breakdown` byte-for-byte.
|
||||
5. **Tail-concentration reduction.** Report the share of cumulative return from the top-10 |days|
|
||||
before vs after the tail-cap. Expect it to fall materially from the current 28%; the per-day
|
||||
book std should shrink and the daily-return kurtosis drop (fewer extreme days). Sharpe should
|
||||
stay in the winsorization-test ballpark (~2.5–2.9), NOT collapse — if it collapses, K is too
|
||||
tight (the cap is eating the edge, not just the tail).
|
||||
6. **K/σ_book sanity.** On a seeded day with one dominating coin, the capped book return equals
|
||||
`K·σ_book + Σ(other contributions)`, and with `tail_cap_sigma=None` it equals the raw book. σ_book
|
||||
bootstraps on the first observations (no NaN, no hardcoded seed).
|
||||
7. Full suite green.
|
||||
|
||||
## Non-goals
|
||||
|
||||
- **Data-glitch / wick filtering — deliberately NOT built.** The investigation showed the +150%
|
||||
days are REAL (HUSDT +154% spot-corroborated at +153.8%, $214M turnover), so a glitch guard would
|
||||
suppress genuine, spot-confirmed gains. The tail-concentration cap handles the RISK (one coin
|
||||
dominating) without falsely branding a real move as an error.
|
||||
- **Other sleeves.** tstrend/unlock/carry may have their own capacity/tail limits; this spec is
|
||||
positioning-only (the sleeve the investigation implicated). Extending to other sleeves is later.
|
||||
- No change to the contrarian SIGNAL (weights) — both haircuts cap realized RETURN / CONTRIBUTION,
|
||||
not sizing (the chosen approach; sizing-based capacity was the rejected alternative).
|
||||
|
||||
## Risks
|
||||
|
||||
- **At $100k the numbers barely change** — because positioning IS largely capturable at fund scale.
|
||||
This is the honest outcome, not a weak fix: the value is a capacity-honest reference + automatic
|
||||
protection as capital scales, and it removes the overstatement that would mislead at growth.
|
||||
- **π and the median-turnover window are modelling assumptions.** Documented + conservative; the
|
||||
capacity curve (gate 1) is the sanity check that they behave sensibly, not fitted to a target.
|
||||
- The gate ref becomes slightly lower (gains capped) → the band centre drops → forward reconciles
|
||||
more easily. Still false-WAIT-safe (a lower expected return only makes shortfall less likely, and
|
||||
the min-band floor keeps the band from collapsing).
|
||||
@@ -0,0 +1,51 @@
|
||||
# Tail-budgeted levered shape — Design
|
||||
|
||||
**Status:** draft for review
|
||||
**Date:** 2026-07-07
|
||||
|
||||
## Goal
|
||||
|
||||
Re-tune the levered shadow book's per-sleeve leverage shape from the Sharpe-maximal fixed shape (carry ≈53% of gross) to a **tail-budgeted** shape (carry ≈40% of gross) that bounds a plausible carry dislocation to an acceptable book loss, at a small Sharpe cost. The shadow track then observes the more robust deployable candidate OOS instead of the fat-tailed Sharpe-max one.
|
||||
|
||||
## Why (the tail-budget analysis)
|
||||
|
||||
The current `_LEVERED_SLEEVE_SHAPE = {crypto_tstrend 1.0, unlock 1.5, xsfunding 5.0, positioning 2.0}` gives effective weights (gross 1.5×) carry 0.79 (53%). Carry is 53% of the WEIGHT but only ~4% of the daily variance — it is low-vol ballast whose real risk is a rare DISLOCATION (funding spike / stablecoin depeg / exchange halt), a fat tail the 5.5-yr backtest (worst carry day −1.5%) cannot rule out. The frontier (carry weight vs risk, gross 1.5×, non-carry split inverse-vol):
|
||||
|
||||
| −20% carry stress → book | carry% | Sharpe | CAGR | normal maxDD | P(−35%/yr, $35k) |
|
||||
|---|---|---|---|---|---|
|
||||
| −5.0% | 25% | 2.42 | +50% | −20.6% | 0% |
|
||||
| **−8.0%** | **40%** | **2.70** | **+44%** | **−11.3%** | 0% |
|
||||
| −10.5% (current) | 52% | 3.05 | +40% | −6.8% | 0% |
|
||||
|
||||
Key findings: (1) P(−35%/yr) is 0% for ALL shapes — no ruin risk in normal times. (2) Capping carry is NOT a free lunch — it shrinks the rare dislocation tail but RAISES the common normal drawdown (freed weight → volatile sleeves). It is a risk SWAP (rare-severe ↔ common-moderate), not a strict improvement. (3) The **carry ≈40% / −8% budget** is the balance point: Sharpe 2.70 (still well above naive 2.26), CAGR +44%, normal maxDD −11.3% (survivable), and the dislocation black-swan **halved** to −8% vs the Sharpe-max shape's −10.5%.
|
||||
|
||||
For a real-money pilot facing an UNMODELED carry tail, bounding the black-swan to −8% while keeping most of the return/Sharpe advantage is the prudent default.
|
||||
|
||||
## Design
|
||||
|
||||
Change ONE constant. Because the whole levered chain (`levered_eqwt_daily_returns`, `BybitFourEdgeStrategy`, `bybit_4edge_backtest_summary`, `_bybit_measured_net`/precompute, the sim pill, the replay toggle) reads `_LEVERED_SLEEVE_SHAPE`, re-tuning it re-tunes everything.
|
||||
|
||||
New shape (chosen so the gross-cap transform `effective[s] = shape[s]·1.5/Σshape` yields carry 0.60 / unlock 0.44 / positioning 0.26 / trend 0.20, i.e. carry 40% of gross, non-carry inverse-vol):
|
||||
|
||||
```
|
||||
_LEVERED_SLEEVE_SHAPE = {crypto_tstrend: 1.0, unlock: 2.2, xsfunding: 3.0, positioning: 1.3}
|
||||
```
|
||||
(Σ = 7.5; /4 = {0.25, 0.55, 0.75, 0.325}; gross 1.875 > 1.5 → scale 0.8 → effective {0.20, 0.44, 0.60, 0.26}.)
|
||||
|
||||
A new invariant test PINS the tail budget: the levered book's effective carry weight ≈ 40% of gross (±2pp), so a −20% carry day is bounded to ≈ −8% of the book. This makes the tail budget a permanent, checked property — not a number that silently drifts on a future edit.
|
||||
|
||||
## Data flow / deploy
|
||||
Constant change → push → build. Then re-run the two own-memory Jobs so prod reflects the new shape: `bybit-persist-sleeve-ret` (levered backtest reference) and `compare-measured-precompute` (levered measured curve). The levered shadow track has only ~1 old-shape day (observe-only, just started); RESTART it with a fresh T0 (delete its state + .bak so the next nightly re-inceptions) so it observes the new shape cleanly from day 0.
|
||||
|
||||
## Testing
|
||||
- Invariant: effective carry weight of `_LEVERED_SLEEVE_SHAPE` (via `levered_eqwt` weight math or a small helper) is 40% ± 2pp of gross; the shape's non-carry ratios match inverse-vol.
|
||||
- The existing levered-vs-naive tests still hold (they assert relative difference, not absolute values).
|
||||
- Full suite green.
|
||||
- Local browser (per feedback_verify_cockpit_locally_not_prod): the Backtest levered curve still renders (new shape → new metrics ~Sharpe 2.7 / CAGR 44% measured); the Replay toggle unchanged.
|
||||
|
||||
## Non-goals
|
||||
- No adaptive/CVaR-optimizing overlay yet (that is the fuller Phase-2 — this is a static tail-budgeted shape, the robust default the shadow observes now). No capital.
|
||||
|
||||
## Risks
|
||||
- The −20% carry stress is a modeling assumption (worst historical was −1.5%); the tail budget is a prudent guard against an unmodeled event, not a measured one. The shadow track must still observe a real carry stress OOS before capital.
|
||||
- Restarting the shadow track resets its (1-day) OOS record — acceptable because the book definition changed; the old day observed a different book.
|
||||
@@ -0,0 +1,238 @@
|
||||
# Deterministic Forward State (Sub-project A) — Design
|
||||
|
||||
**Date:** 2026-07-08
|
||||
**Status:** Approved (brainstorming) — ready for implementation plan
|
||||
**Sub-project:** A of the "Unified data-driven pipeline" vision (foundation layer)
|
||||
|
||||
## North-star context (the target the sub-project serves)
|
||||
|
||||
The whole system converges on ONE data-driven loop:
|
||||
|
||||
```
|
||||
market data → warehouse (SSOT) → deterministic strategy compute
|
||||
→ gate/health (policy evaluation) → allocation (capital policy, human-ceilinged)
|
||||
→ execution engine (orders/fills) → fills back as data → loop
|
||||
```
|
||||
|
||||
Five planes: (1) Data, (2) Compute, (3) **Deterministic state ← THIS sub-project**,
|
||||
(4) Policy (gate/health/promotion/allocation, declarative), (5) Execution (the coming
|
||||
trading engine, capital human-gated). The guiding principle: **data-driven mechanics,
|
||||
human-gated capital** — automate the pipeline up to the point of committing real money;
|
||||
gate the capital with tested policy + a human ceiling + killswitch. Sub-project A does
|
||||
NOT touch capital; it makes the mechanics deterministic so B and C can build on solid
|
||||
ground.
|
||||
|
||||
**The problem A solves.** Today each paper-forward tracker accumulates a path-dependent
|
||||
ledger in a mutable JSON on the PVC (`*_state.json`: `inception`/T0, `last_mark_day`,
|
||||
`positions`, `equity`, `log`). Operations are hand-driven: re-inception = manually delete
|
||||
the JSON + delete DB rows + re-persist the backtest ref; a two-store drift exists (JSON =
|
||||
truth, DB = display). Every operational error observed recently (hollow gates, empty fleet
|
||||
table, stale display name, the manual BTC-drop re-inception dance) traces to this mutable,
|
||||
hand-editable state that drifts from what a clean deterministic recompute would produce.
|
||||
|
||||
**North-star for A:** system state = `f(committed policy, observed data)`. With no
|
||||
hand-editable accumulated state, there is nothing to hand-drive or corrupt.
|
||||
|
||||
## Goal
|
||||
|
||||
Replace the mutable, accumulated tracker state with a **recompute-from-anchor** model:
|
||||
persist only an immutable anchor `(T0, definition_hash)` per track; recompute the full
|
||||
forward record over `[T0, today]` from point-in-time warehouse data on every run;
|
||||
auto-re-inception when the declared definition changes. Idempotent, self-healing, single
|
||||
source of truth.
|
||||
|
||||
## Core mechanism: recompute-from-anchor (chosen over event-sourced ledger / JSON+guards)
|
||||
|
||||
The paper strategies are already (or can be refactored to be) **pure functions of the
|
||||
warehouse data**. So nothing needs to be accumulated: given a frozen `T0` and the current
|
||||
data, `forward_series(data, T0)` IS the forward record. The anchor is the only persistent
|
||||
state; everything else is derived and cacheable.
|
||||
|
||||
Rejected alternatives:
|
||||
- **Event-sourced ledger** — append-only, no hand-edits, but still *accumulated* (a
|
||||
missed/wrong day stays wrong unless history is rewritten) and doesn't leverage the
|
||||
determinism we already have. More machinery for less guarantee.
|
||||
- **JSON + validation guards** — keeps the mutable state (the root cause); a band-aid that
|
||||
conflicts with the single-SSOT / wire-or-delete discipline.
|
||||
|
||||
## Components
|
||||
|
||||
### 1. The anchor (new SSOT)
|
||||
|
||||
A new Postgres table `forward_anchor`:
|
||||
|
||||
| column | type | notes |
|
||||
|---|---|---|
|
||||
| `strategy_id` | text | part of PK |
|
||||
| `t0` | date | frozen inception; the OOS clock start |
|
||||
| `definition_hash` | text | sha256 of the declared definition (see §2) |
|
||||
| `definition_version` | int | the dev-controlled escape-hatch component |
|
||||
| `params_json` | jsonb | the return-affecting params that were hashed (audit) |
|
||||
| `created_at` | timestamptz | when this anchor was set |
|
||||
| `archived_at` | timestamptz null | set when superseded by a re-inception |
|
||||
|
||||
PK is `(strategy_id, created_at)` so re-inception archives the old row and inserts a new
|
||||
one (full lineage kept, nothing deleted). The **active** anchor for a strategy is the row
|
||||
with `archived_at IS NULL`. This is the ONLY persistent per-track state. It **retires the
|
||||
PVC `*_state.json` files** and collapses the two-store drift: `forward_nav` /
|
||||
`forward_summary` become pure derived caches of the recompute, not an independent store.
|
||||
|
||||
### 2. The declared definition + hash
|
||||
|
||||
Each tracked strategy declares, in the registry (the existing SSOT), a `definition`:
|
||||
|
||||
- the **return-affecting params**: instrument list, `cost_bps`, universe key, sleeve set,
|
||||
and any key numeric constants that change the return series;
|
||||
- a `definition_version: int` — an explicit escape-hatch the dev bumps for pure-code
|
||||
changes (e.g., a formula change inside `book_series`) that the params don't capture;
|
||||
- a `record_mode: "recompute" | "observed"` (see §3b) — a static property of the track.
|
||||
|
||||
`definition_hash = sha256(canonical_json(sorted(params)) + str(definition_version))`.
|
||||
|
||||
This catches param changes automatically (the BTC-drop — `FUND_INSTRUMENTS` 6→5 — would
|
||||
have auto-triggered re-inception), never resets on comments/refactors, and keeps the human
|
||||
in control of "did the return-generating logic change." Erring slightly toward extra
|
||||
re-inception is the safe direction for real money (a changed strategy must re-prove itself
|
||||
OOS = false-WAIT-safe).
|
||||
|
||||
### 3. The recompute engine (replaces accumulation)
|
||||
|
||||
`_run_paper_tracker` changes from *accumulate-and-write* to *recompute-from-anchor*:
|
||||
|
||||
1. Read the active anchor for `strategy_id`; compute the current `definition_hash`.
|
||||
2. **No active anchor** → inception: insert anchor `(T0=today, current hash)`.
|
||||
3. **Hash changed** → auto-re-inception: set `archived_at` on the old anchor, insert a new
|
||||
one `(T0=today, new hash)`.
|
||||
4. **Hash unchanged** → produce the forward record over `[T0, today]` per the track's
|
||||
**record mode** (see §4): `recompute` tracks recompute the full series from PIT data;
|
||||
`observed` tracks append today's booked row to their immutable log. Write the derived
|
||||
`forward_nav`/`forward_summary` (a cache/projection of the record).
|
||||
|
||||
For `recompute` tracks, no accumulated equity/positions/log is written anywhere.
|
||||
|
||||
### 3b. Two record modes (recompute + observed)
|
||||
|
||||
Not every tracked strategy is recomputable. Each track declares a `record_mode`:
|
||||
|
||||
| mode | tracks | how the record is produced |
|
||||
|---|---|---|
|
||||
| `recompute` | multistrat, sixtyforty, crypto_tstrend, bybit_4edge, bybit_4edge_levered, positioning, stablecoin, unlock | anchor + PIT data → recompute `[T0, today]` each run (stateless except anchor; self-healing) |
|
||||
| `observed` | xsfunding, combined | anchor + append-only immutable booked log + minimal carried book-state, all in DB |
|
||||
|
||||
**`observed` tracks** book off a live snapshot that has no reproducible full-history series
|
||||
(e.g. xsfunding books today's live funding snapshot; `persist_backtest_ref=False`). They
|
||||
cannot be recomputed. So their forward record is a genuinely *observed* series: each run
|
||||
appends today's booked `(date, ret)` to an append-only, immutable log in the DB, plus the
|
||||
minimal carried book-state the strategy needs to book the next day (e.g. the current
|
||||
delta-neutral positions). This removes the mutable, hand-editable PVC JSON (append-only in
|
||||
DB ≠ hand-editable) but does NOT provide the recompute/self-heal guarantee — which these
|
||||
tracks fundamentally cannot have. `observed` is the *same* machinery sub-project C reuses
|
||||
for execution fills (fills are observed events too).
|
||||
|
||||
`combined` currently carries a latching kill/re-enter drawdown state; it starts as
|
||||
`observed` and MAY later move to `recompute` if the kill overlay is refactored to a pure
|
||||
fold over the recomputed equity path (out of scope for A — a `definition_version` bump when
|
||||
it happens).
|
||||
|
||||
### 4. Determinism contract + the point-in-time (PIT) requirement
|
||||
|
||||
Each `recompute`-mode strategy exposes a pure full-history series with no hidden state.
|
||||
The primitive already exists: `build_strategy().advance(None, {})` returns the full
|
||||
inception series at the exact basis the forward books (this is what
|
||||
`_persist_track_backtest_ref` uses). Recompute-from-anchor is therefore
|
||||
`advance(None, {})` filtered to `date >= T0`. Path-dependent pieces (e.g., a latching
|
||||
killswitch) stay pure because they are a deterministic function of the — itself
|
||||
deterministic — return path. `observed`-mode tracks (§3b) are exempt from this contract.
|
||||
|
||||
**Critical correctness requirement:** recompute is only stable if *all* return-affecting
|
||||
inputs are **point-in-time (PIT)** in the warehouse. Crypto (`bybit_features`) already is
|
||||
(bars are immutable). **Yahoo (multistrat) is not**: adjusted closes are revised
|
||||
retroactively (dividends/splits), so a live re-fetch would silently shift the record.
|
||||
Therefore **A includes persisting the Yahoo daily closes as PIT snapshots into the
|
||||
warehouse**, and the multistrat recompute reads those snapshots — never a live re-fetch.
|
||||
Without this, the anchor model would only *appear* deterministic while drifting at the data
|
||||
layer (the very drift we are killing, displaced downward).
|
||||
|
||||
### 5. Boundary to sub-project C (fills are out of scope)
|
||||
|
||||
Actual execution fills (Alpaca / Bybit paper) are **append-only observed data**, not
|
||||
recomputable — they belong to the trading-engine sub-project (C). A covers only the
|
||||
*modeled* forward tracks (the gate's subject). The fills log is untouched by A. This is a
|
||||
clean seam: modeled tracks are `f(anchor, data)`; fills are observed events.
|
||||
|
||||
### 6. Migration (preserve T0 + reconcile) — one-time
|
||||
|
||||
For each existing live track:
|
||||
|
||||
1. Read the current `*_state.json` inception → seed the anchor `T0`. Compute + store the
|
||||
current `definition_hash` and the `record_mode`.
|
||||
2. **`recompute` tracks:** recompute `[T0, today]` and compare to the accumulated
|
||||
`forward_nav` within tolerance.
|
||||
- **Match** → seamless: the honestly-accrued clock is preserved (e.g. `bybit_4edge`
|
||||
keeps its ~15 days).
|
||||
- **Diverge beyond tolerance** → flag it (log + a migration-report row). Divergence *is*
|
||||
evidence the old accumulated record was drifting; the deterministic recompute is the
|
||||
honest truth and the gate rules on it henceforth. **Never silently overwrite** without
|
||||
surfacing the divergence.
|
||||
3. **`observed` tracks:** import the accumulated booked `(date, ret)` series from the JSON
|
||||
into the append-only log verbatim (it is the only record of those observations — it
|
||||
cannot be recomputed), and carry the current book-state (positions) into the DB. No
|
||||
reconciliation is possible or attempted; the accrued clock is preserved as-is.
|
||||
4. After migration is verified, retire the `state_reader` JSON path — the DB anchor +
|
||||
record are SSOT.
|
||||
|
||||
### 7. Self-healing + idempotency
|
||||
|
||||
A missed nightly run self-heals (the next run recomputes `[T0, today]` in full). A
|
||||
corrupted or hand-edited record cannot persist (nothing is accumulated beyond the
|
||||
immutable anchor). Re-running an asset is byte-identical (deterministic). The manual
|
||||
BTC-drop dance collapses to: change the declared params → the nightly detects the hash
|
||||
change → it re-inceptions itself.
|
||||
|
||||
## Error handling
|
||||
|
||||
- **Insufficient data** for `[T0, today]` → the recompute returns an empty/partial series →
|
||||
the gate stays **WAIT** (never fabricates a PASS). Consistent with the real-money rule:
|
||||
err toward false-WAIT, never false-PASS.
|
||||
- **Migration divergence** → flagged in a report row + log, not silently applied.
|
||||
- **Non-deterministic track** (fails the `forward_series` contract, or a required input is
|
||||
not PIT in the warehouse) → migration/build fails loudly. Never silently fall back to
|
||||
accumulation.
|
||||
|
||||
## Testing
|
||||
|
||||
- **Determinism:** recompute twice → byte-identical (property test).
|
||||
- **Auto-re-inception:** change a declared param → hash changes → new active anchor, old
|
||||
archived, T0 resets. Change a comment/refactor → hash unchanged → clock preserved.
|
||||
- **Migration reconcile:** seed an anchor from a known accumulated record; a matching
|
||||
recompute preserves the clock; an injected drift is flagged (not silently overwritten).
|
||||
- **Self-heal:** skip a run → the next run's record equals the uninterrupted record.
|
||||
- **Gate honesty:** recompute with insufficient data → WAIT.
|
||||
- **PIT Yahoo:** a retroactive adjusted-close revision does not change an already-anchored
|
||||
record (it reads the PIT snapshot, not a live re-fetch).
|
||||
- **Observed mode:** an `observed` track appends today's booked row to the immutable log
|
||||
and carries its book-state; re-running the same day does not double-append (idempotent
|
||||
on `(strategy_id, date)`); a param change re-inceptions and starts a fresh log.
|
||||
- **Fills boundary:** the fills log is untouched by A's code paths.
|
||||
|
||||
## Out of scope (YAGNI)
|
||||
|
||||
No event log; no fills / execution (sub-project C); no policy-layer changes (sub-project
|
||||
B); no new UI beyond surfacing the anchor. Capital is untouched.
|
||||
|
||||
## Affected code (initial map — the plan refines this)
|
||||
|
||||
- New: `forward_anchor` table + repo methods (`adapters/persistence/forward_nav.py` or a
|
||||
new `forward_anchor.py`); `cockpit_models.py` row.
|
||||
- New: `ForwardDefinition` + hash helper; `definition` entries in `registry.py`.
|
||||
- New: `forward_record` append-only log table (for `observed`-mode tracks + reused by C)
|
||||
keyed `(strategy_id, date)`; repo append/read methods.
|
||||
- Modify: `_run_paper_tracker` (`adapters/orchestration/assets.py`) → recompute-from-anchor
|
||||
for `recompute` tracks; append-to-log for `observed` tracks (branch on `record_mode`).
|
||||
- Modify: each tracked strategy to expose the pure `forward_series(data, t0)` contract
|
||||
(`application/paper_strategies.py`, `domain/strategies/*`).
|
||||
- New: Yahoo PIT close-snapshot persistence into the warehouse + a PIT-reading client for
|
||||
the multistrat recompute.
|
||||
- New: one-time migration (seed anchors from existing `*_state.json`, reconcile, report).
|
||||
- Retire: `adapters/persistence/state_reader.py` JSON path once migration is verified.
|
||||
```
|
||||
@@ -0,0 +1,171 @@
|
||||
# Declarative Strategy-Level Allocation Policy + Capital Ceiling (Sub-project B2a) — Design
|
||||
|
||||
**Date:** 2026-07-09
|
||||
**Status:** Approved (brainstorming) — ready for implementation plan
|
||||
**Sub-project:** B2a — the core of the allocation plane (layer 4, plane 2) of the "unified data-driven
|
||||
pipeline" north-star.
|
||||
**Builds on:** sub-project A (deterministic forward state — the DB record SSOT, merged) + B1 (declarative
|
||||
gate + promotion policy, merged). B2a reads A's `forward_nav` records (per-strategy return series) and B1's
|
||||
deploy-tier/promotion status.
|
||||
|
||||
## North-star context (layer 4, plane 2 of the allocation layer)
|
||||
|
||||
The allocation layer turns the set of {live + healthy} deployed strategies into target capital, driven by ONE
|
||||
declarative, versioned, git-committed allocation policy, bounded by a HUMAN CAPITAL CEILING:
|
||||
|
||||
```
|
||||
{live+healthy strategies} (B1 deploy status; B3 health later — "healthy" defaults to all deploy-tier now)
|
||||
→ STRATEGY-LEVEL allocation policy → per-strategy target WEIGHT (ex-ante sized) ← THIS sub-project (B2a)
|
||||
→ each strategy's WITHIN-book weights → per-instrument weight (already built; B2b later, maybe never)
|
||||
→ × CAPITAL (≤ human ceiling) → per-instrument target POSITIONS
|
||||
→ rare aggregate KILLSWITCH (ex-ante) → (C) execution engine
|
||||
```
|
||||
|
||||
Guiding principle (mirror of A/B1): allocation = `f(committed policy, A's records, B1's status)`. Policy
|
||||
declarative + versioned + committed in git (a change is a reviewed commit + version bump). The human sets the
|
||||
policy + a HARD CAPITAL CEILING (the one human gate) + arms venue keys. B2a produces the target weights +
|
||||
target capital; it does NOT move capital (that is sub-project C — the execution engine). "Data-driven
|
||||
mechanics, human-gated capital."
|
||||
|
||||
**The problem B2a solves.** Allocation knobs are scattered as function defaults (`book_allocator.combine_book`:
|
||||
`target_vol=0.12`, `kelly_fraction=0.5`, `max_sleeve_weight=0.4`, `marginal_gate=False`; `_MAX_LEVERAGE=3.0`;
|
||||
`apply_drawdown_killswitch`: `kill_dd=0.15`, `reenter_dd=0.07`) + config settings (`kelly_cap`, `max_gross`,
|
||||
`combined_book_kill_dd`, `paper_capital`). There is no versioned allocation policy, no explicit capital
|
||||
ceiling as a human gate, and `combine_book`'s leverage (`kelly · target_vol / trailing_port_vol`) is
|
||||
**reactive de-sizing** — the approach rejected 3× (pearl_fxhnt_reactive_risk_controllers_rejected;
|
||||
pearl_fxhnt_voltarget_overlay_is_reactive_desizing_rejected: pro-cyclical, cuts before the bounce). The
|
||||
rejected-controllers conclusion is **ex-ante K + a rare killswitch, no reactive de-sizing**.
|
||||
|
||||
## Goal
|
||||
|
||||
A versioned in-code `AllocationPolicy` + a pure `compute_allocation` that turns the deploy-tier strategies
|
||||
into ex-ante-sized per-strategy target weights (equal-weight the marginal-Sharpe-gate survivors; ex-ante
|
||||
vol-target leverage from the **full-history** vol, never trailing), a rare latching aggregate killswitch, and
|
||||
a `target_capital` that caps total deployed gross at a hard human `capital_ceiling`. The allocation is a
|
||||
deterministic function of (policy + records + status), recomputable and audit-stamped with the policy version.
|
||||
|
||||
## Components
|
||||
|
||||
### 1. The allocation policy object (`allocation_policy.py`, in-code typed, versioned)
|
||||
|
||||
A new module `src/fxhnt/application/allocation_policy.py` (mirrors B1's `gate_policy.py`):
|
||||
|
||||
- `AllocationPolicy` — a frozen dataclass:
|
||||
- `target_vol: float = 0.12` — the ex-ante annualised vol target for the book.
|
||||
- `kelly_fraction: float = 0.5`, `max_leverage: float = 1.5` — fractional-Kelly + a hard leverage cap K.
|
||||
- `max_strategy_weight: float = 0.5` — per-strategy weight cap (before leverage).
|
||||
- `marginal_gate: bool = True` — the leave-one-out quality filter.
|
||||
- `min_history_days: int = 60` — a strategy needs at least this many forward-return days for a stable
|
||||
ex-ante vol estimate; below it, it is EXCLUDED (never sized on a short/noisy vol).
|
||||
- `kill_dd: float = 0.25`, `reenter_dd: float = 0.10` — the RARE aggregate killswitch thresholds (deep/high,
|
||||
latching; deliberately far above the day-to-day noise — a circuit-breaker, not a de-sizer).
|
||||
- `capital_ceiling: float = 35_000.0` — the HARD fund-level dollar cap on total deployed gross (the human
|
||||
gate). Initial value = the $35k pilot capital floor; the human raises it deliberately (a reviewed policy
|
||||
change + version bump) as the fund grows.
|
||||
- `ALLOCATION_POLICY: AllocationPolicy` — the single current instance.
|
||||
- `ALLOCATION_POLICY_VERSION: int = 1` — bumped on any policy change (auditable).
|
||||
- `allocation_policy_hash(policy, version) -> str` — sha256 of the canonical fields + version (content-derived;
|
||||
a forgotten version bump still changes the hash → a self-check test fails, forcing the bump — same
|
||||
discipline as A's definition_hash and B1's policy_hash).
|
||||
|
||||
A capital-ceiling change is a policy change → a version bump → audited. The ceiling lives IN the versioned
|
||||
policy.
|
||||
|
||||
### 2. The allocation engine (pure, ex-ante — `allocation_engine.py`)
|
||||
|
||||
`compute_allocation(returns_by_strategy: dict[str, dict[str, float]], policy: AllocationPolicy) -> dict[str, float]`
|
||||
where `returns_by_strategy` is each live+healthy strategy's `{date: ret}` from A's record. Steps:
|
||||
|
||||
1. **Eligibility:** drop any strategy with fewer than `policy.min_history_days` return observations (err safe
|
||||
— excluded until it has a stable ex-ante vol). If none remain → return `{}` (deploy nothing).
|
||||
2. **Marginal-Sharpe gate** (if `policy.marginal_gate`): leave-one-out over the equal-weight book — bench a
|
||||
strategy only when REMOVING it raises the equal-weight book's full-history Sharpe. Reuse
|
||||
`book_allocator._marginal_prune`'s logic adapted to equal weights. Survivors kept.
|
||||
3. **Equal-weight** the survivors: `w_i = 1/N`, each capped at `policy.max_strategy_weight`, renormalised so
|
||||
the base weights sum to 1.
|
||||
4. **Ex-ante leverage** `K = min(policy.max_leverage, policy.kelly_fraction · policy.target_vol /
|
||||
ex_ante_book_vol)`, where `ex_ante_book_vol` = the annualised realised vol of the equal-weight book's daily
|
||||
returns over the **FULL history** (all of A's record — stable, ex-ante), NOT a trailing window. Target
|
||||
weights = `K · w_i`. **A recent vol spike does not change K** (the anti-reactive invariant).
|
||||
5. **Rare aggregate killswitch:** compute the equal-weight book's running drawdown over the full record; if it
|
||||
breaches `policy.kill_dd` and has not recovered inside `policy.reenter_dd` (latching), return all-zero
|
||||
weights (flatten). This is the ONLY drawdown-reactive element and is deliberately rare/latching.
|
||||
|
||||
Output: `{strategy_id: target_weight}` (ex-ante-sized, gated, killswitch-aware). Deterministic → recomputable.
|
||||
|
||||
### 3. Capital ceiling → target capital (`target_capital`)
|
||||
|
||||
`target_capital(target_weights: dict[str, float], equity: float, policy: AllocationPolicy) -> dict[str, float]`:
|
||||
|
||||
- `gross = sum(abs(w) for w in target_weights.values()) · equity` (the notional the weights imply at full
|
||||
equity), then `deployed = min(gross, policy.capital_ceiling)` — the **hard human gate: total deployed gross
|
||||
never exceeds `capital_ceiling`**, whatever the policy computes.
|
||||
- Per-strategy dollars = `(abs(w_i) · equity / gross) · deployed` for each strategy (i.e. scale the notional
|
||||
down by `deployed/gross` when the ceiling binds; unchanged when it does not), carrying the sign of `w_i`.
|
||||
- Edge cases: `gross == 0` (killswitch/empty) → all zero. `capital_ceiling >= gross` → no scaling.
|
||||
|
||||
### 4. Auditability + output
|
||||
|
||||
`compute_allocation`/`target_capital` are pure and deterministic from (policy + A's records + B1's status), so
|
||||
the allocation is recomputable (no accumulated state — same principle as A). B2a records the latest
|
||||
target-weight snapshot to a `strategy_allocation` table (`strategy_id, target_weight, target_dollars,
|
||||
policy_version, policy_hash, as_of, computed_at`) — for the cockpit/audit and for sub-project C to consume.
|
||||
A nightly asset (or a step in `cockpit_forward`) computes + records it after B1's ingest/promotion runs (so
|
||||
the deploy-tier set is fresh). "Allocated under allocation-policy v2" is queryable.
|
||||
|
||||
### 5. Where the human stays in the loop
|
||||
|
||||
The human sets the `AllocationPolicy` (weighting/sizing/gate/killswitch thresholds + the `capital_ceiling`,
|
||||
versioned in git — reviewed) and arms venue keys. B2a computes the target weights + target capital
|
||||
automatically. NO capital is moved by B2a — it produces the TARGETS; moving capital is sub-project C
|
||||
(execution) + manual arming. The capital ceiling is the hard human gate on how much can ever be deployed.
|
||||
|
||||
## Error handling
|
||||
|
||||
- **No eligible strategies** (none deploy-tier, or all below `min_history_days`) → empty allocation (deploy
|
||||
nothing). Safe.
|
||||
- **Short/insufficient history** for a strategy → excluded (never sized on a noisy short-window vol, which
|
||||
would edge toward the reactive trap).
|
||||
- **Killswitch latched** → all-zero weights (flatten) until recovery inside `reenter_dd`.
|
||||
- **Capital ceiling binds** → scale the notional down so gross ≤ ceiling; never deploy more than the ceiling
|
||||
or more than equity.
|
||||
- **NO reactive de-sizing:** leverage/sizing derive from the FULL-history vol (stable), never a trailing
|
||||
window. The only drawdown-reactive element is the rare, latching aggregate killswitch.
|
||||
|
||||
## Testing
|
||||
|
||||
- **Anti-reactive invariant (load-bearing):** appending a recent vol spike to a strategy's record does NOT
|
||||
change `K` (the ex-ante vol uses the full history) — the core guarantee that this is not the rejected
|
||||
reactive de-sizing.
|
||||
- **Equal-weight + marginal gate:** a strategy whose removal raises the book Sharpe is benched; survivors are
|
||||
equal-weighted (capped + renormalised).
|
||||
- **Ex-ante vol-target:** `K = min(max_leverage, kelly_fraction · target_vol / full_history_vol)`; the cap
|
||||
binds when the implied K exceeds `max_leverage`.
|
||||
- **Capital ceiling:** total deployed gross ≤ `capital_ceiling` (binds when `equity·K > ceiling`), signs
|
||||
preserved; `ceiling >= gross` → unscaled.
|
||||
- **Killswitch:** aggregate DD > `kill_dd` → all-zero (latching), recovery < `reenter_dd` → re-enter.
|
||||
- **Eligibility:** a strategy with < `min_history_days` obs is excluded; no eligible strategies → `{}`.
|
||||
- **Policy hash self-check** pins the current hash (a field change without a version bump fails it); the
|
||||
recorded snapshot carries `ALLOCATION_POLICY_VERSION` + hash.
|
||||
|
||||
## Out of scope (YAGNI)
|
||||
|
||||
No within-book allocation (B2b — each strategy keeps its own instrument weights). No execution / capital
|
||||
movement (C). No health/decay (B3 — "healthy" is all deploy-tier for now; the killswitch is catastrophe
|
||||
protection, NOT a decay detector — a quietly-dying single edge is B3's job, and must be ex-ante/structural,
|
||||
not reactive). No inverse-vol reweighting (evidence: equal-weight already efficient). The rare killswitch is
|
||||
the only drawdown-reactive element.
|
||||
|
||||
## Affected code (initial map — the plan refines this)
|
||||
|
||||
- New: `src/fxhnt/application/allocation_policy.py` — `AllocationPolicy`, `ALLOCATION_POLICY`,
|
||||
`ALLOCATION_POLICY_VERSION`, `allocation_policy_hash`.
|
||||
- New: `src/fxhnt/application/allocation_engine.py` — `compute_allocation`, `target_capital`, the full-history
|
||||
ex-ante vol + equal-weight + marginal-gate + killswitch composition (reusing `book_allocator._marginal_prune`
|
||||
/ `apply_drawdown_killswitch` where they fit an EX-ANTE, full-history use — NOT `combine_book`'s reactive
|
||||
trailing-vol leverage).
|
||||
- New: `strategy_allocation` table (`cockpit_models.py`) + repo write/read (`forward_nav.py`).
|
||||
- Modify: a nightly asset / `cockpit_forward` step computes + records the allocation after B1's promotion runs.
|
||||
- Reuse: `ForwardNavRepo.nav_history` (per-strategy returns), the deploy-tier set (registry `tier=="deploy"` ∪
|
||||
`promoted_strategy_ids()`), `book_allocator` marginal-gate + killswitch helpers.
|
||||
```
|
||||
@@ -0,0 +1,159 @@
|
||||
# Declarative Gate + Promotion Policy (Sub-project B1) — Design
|
||||
|
||||
**Date:** 2026-07-09
|
||||
**Status:** Approved (brainstorming) — ready for implementation plan
|
||||
**Sub-project:** B1 of the policy plane (layer 4) of the "unified data-driven pipeline" north-star.
|
||||
**Builds on:** sub-project A (deterministic forward state — the DB anchor + record SSOT, branch
|
||||
`feat/deterministic-forward-state`). Requires A merged/available: B1 reads `forward_summary` +
|
||||
`forward_nav` (A's record) and extends `forward_ingest`.
|
||||
|
||||
## North-star context (layer 4, plane 1 of 3)
|
||||
|
||||
The policy plane turns A's record into decisions, driven by ONE declarative, versioned, git-committed
|
||||
policy — not scattered constants. The three planes:
|
||||
|
||||
```
|
||||
A's record (per-track forward series + summary)
|
||||
→ B1 GATE + PROMOTION policy → per-track STATUS {WAIT | PASS | deploy-tier} ← THIS sub-project
|
||||
→ B3 HEALTH / DECAY policy → per-edge {healthy | decaying | killed} (ex-ante, rare)
|
||||
→ B2 ALLOCATION policy → per-strategy target weights (bounded by a human capital ceiling)
|
||||
→ (C) execution engine
|
||||
```
|
||||
|
||||
Guiding principle (mirror of A): a decision = `f(committed policy, A's record)`. Policy is declarative +
|
||||
versioned + committed in **git** (a change is a reviewed commit + a version bump), NOT runtime-editable in
|
||||
the DB (that would reintroduce the hand-editable real-money state A eliminated). Evaluation is automatic.
|
||||
The human sets policy + (later, in B2) a capital ceiling. B1 touches NO capital — "deploy-tier" is a
|
||||
tracking tier; real capital is B2 + manual venue-key arming.
|
||||
|
||||
**The problem B1 solves.** Gate/promotion policy is scattered: default thresholds live as module constants
|
||||
(`reconciliation_gate.MIN_FORWARD_DAYS`, `RECON_TOLERANCE_FRAC`, `RECON_MIN_BAND`, `RECON_MIN_CORR`) and
|
||||
inline `.get(..., default)` literals in `gate.py`, per-track overrides live in registry `gate_spec` dicts,
|
||||
and promotion is a `promote_on_pass` flag + `evaluate_promotions`. Changing a threshold means editing a
|
||||
constant or a function default in code, with no version/audit trail on the capital-affecting verdict.
|
||||
|
||||
## Goal
|
||||
|
||||
Consolidate the scattered gate + promotion defaults into ONE declarative, versioned, in-code `GatePolicy`;
|
||||
resolve the effective per-track policy at one point; stamp every gate verdict with the `policy_version` +
|
||||
content-derived `policy_hash` that produced it (auditability); and formalize promotion as
|
||||
`gate==PASS AND human flag AND automatic correlation backstop`. Behavior is byte-identical to today under the
|
||||
current thresholds (a pure consolidation + audit + one new promotion guard).
|
||||
|
||||
## Components
|
||||
|
||||
### 1. The policy object (`gate_policy.py`, in-code typed, versioned)
|
||||
|
||||
A new module `src/fxhnt/application/gate_policy.py`:
|
||||
|
||||
- `GatePolicy` — a frozen dataclass holding the DEFAULTS currently scattered as module constants / inline
|
||||
literals:
|
||||
- reconciliation: `min_forward_days: int`, `recon_tolerance_frac: float`, `recon_min_band: float`,
|
||||
`recon_min_corr: float`, `max_gap_days: int`
|
||||
- absolute: `min_days: int`, `min_total_return: float`, `min_sharpe: float`
|
||||
- promotion: `promotion_max_corr: float` (the auto-corr backstop threshold) + `promotion_min_overlap_days:
|
||||
int` (minimum overlapping days for a valid correlation — below this the corr is indeterminate and
|
||||
promotion is withheld). Both are covered by `policy_hash`.
|
||||
- `POLICY_VERSION: int` — bumped on ANY change to the policy defaults (auditable; dev-controlled).
|
||||
- `policy_hash(policy: GatePolicy, version: int) -> str` — sha256 of the canonical policy fields + version
|
||||
(content-derived; a forgotten version bump still changes the hash → the self-check test fails).
|
||||
- `POLICY: GatePolicy` — the single current policy instance with the values that are byte-identical to
|
||||
today's constants (so the consolidation changes no verdict).
|
||||
|
||||
The module constants in `reconciliation_gate.py` and the inline defaults in `gate.py` are REMOVED — those
|
||||
files read from the resolved policy (single source of truth for the defaults). Per-track overrides stay in
|
||||
the registry `gate_spec` dict (unchanged shape).
|
||||
|
||||
### 2. Effective-policy resolution (one point)
|
||||
|
||||
`resolve_policy(policy: GatePolicy, gate_spec: dict) -> ResolvedPolicy` layers the per-track `gate_spec`
|
||||
overrides on top of the `GatePolicy` defaults, producing a complete `ResolvedPolicy` (every threshold
|
||||
explicit). `evaluate_reconciliation_gate` and `evaluate_gate` take the `ResolvedPolicy` (not raw `gate_spec`
|
||||
+ module-constant fallbacks). This is the single place defaults and overrides combine; there are no hidden
|
||||
fallbacks elsewhere.
|
||||
|
||||
### 3. Auditability — stamp the verdict
|
||||
|
||||
`forward_summary` (ORM `ForwardSummaryRow`) gains two columns: `policy_version: int` and
|
||||
`policy_hash: str` (both nullable for backfill/migration tolerance). When `forward_ingest.ingest` writes a
|
||||
verdict via `upsert_summary`, it stamps the `POLICY_VERSION` + `policy_hash(POLICY, POLICY_VERSION)` that
|
||||
produced it. "PASSed under policy v3" is queryable; a policy version bump means future verdicts carry the
|
||||
new version while history keeps the old — you can see exactly which policy each capital-affecting PASS fell
|
||||
under. `upsert_summary`'s signature gains the two fields (defaulted so no other caller breaks).
|
||||
|
||||
### 4. Promotion policy (human flag + automatic correlation backstop)
|
||||
|
||||
`evaluate_promotions` (in `forward_ingest`) promotes research→deploy iff ALL hold:
|
||||
|
||||
1. `gate_status == "PASS"` (unchanged), AND
|
||||
2. the registry `promote_on_pass` flag is set (human intent — the diversification candidate), AND
|
||||
3. **NEW — the automatic correlation backstop:** the track's daily-return correlation to the current
|
||||
deploy book is `< POLICY.promotion_max_corr`.
|
||||
|
||||
The **deploy book** is the equal-weight aggregate of the currently-deploy-tier tracks' daily returns
|
||||
(from A's record: `nav_history`/`record_series` per deploy track, aligned by date, averaged per day). The
|
||||
candidate's daily returns come from A's record too. Correlation is computed over the overlapping window.
|
||||
|
||||
- **Empty deploy book** (no deploy-tier track yet — the first edge): the backstop passes (nothing to
|
||||
correlate against — the first edge trivially diversifies). PROMOTE.
|
||||
- **A deploy book exists but no/short overlap** with the candidate: err safe — WITHHOLD promotion (log it);
|
||||
promotion is capital-adjacent, so an indeterminate diversification is not auto-promoted.
|
||||
- A flagged candidate at PASS whose corr `>= threshold` is WITHHELD with a clear log line
|
||||
(`"<sid> gate=PASS but corr 0.55 >= 0.40 → promotion withheld"`), catching a human mis-flag with data.
|
||||
|
||||
Promotion stays ONE-WAY + persistent (unchanged): once promoted, it does not revert if corr later drifts up
|
||||
(the fund-membership decision is made deliberately once, not flickering daily).
|
||||
|
||||
### 5. Where the human stays in the loop
|
||||
|
||||
The human: sets the POLICY (thresholds + `POLICY_VERSION`, in git — reviewed) and flags promotion candidates
|
||||
(`promote_on_pass`). The policy AUTOMATES the gate verdict + promotion (given flag + corr backstop). NO
|
||||
capital is committed by B1 — deploy-tier is a tracking/display tier; real capital is B2 (allocation + a hard
|
||||
human capital ceiling) + manual venue-key arming.
|
||||
|
||||
## Error handling
|
||||
|
||||
- **Forgotten version bump:** the content-derived `policy_hash` changes whenever the policy fields change,
|
||||
even if `POLICY_VERSION` is not bumped. A self-check test pins `policy_hash(POLICY, POLICY_VERSION)` to an
|
||||
expected value; changing the policy without bumping the version fails the test, forcing the bump (mirrors
|
||||
A's `definition_version` discipline).
|
||||
- **Corr backstop indeterminate:** empty deploy book → pass (first edge); non-empty book but insufficient
|
||||
overlap → withhold (err safe, capital-adjacent).
|
||||
- **Gate verdict logic unchanged:** the consolidation must produce byte-identical verdicts under the current
|
||||
thresholds — a regression anchor test asserts this.
|
||||
- **WAIT-safe preserved:** insufficient data → WAIT, never a fabricated PASS (unchanged from A).
|
||||
|
||||
## Testing
|
||||
|
||||
- **Policy resolution:** `GatePolicy` defaults ⊕ a track's `gate_spec` overrides → the correct
|
||||
`ResolvedPolicy` (override wins; unset falls back to the default).
|
||||
- **Byte-identical verdicts (regression anchor):** the same forward records produce the same gate verdicts
|
||||
as before the consolidation, under the current `POLICY` values.
|
||||
- **Verdict stamping:** an ingested verdict carries `POLICY_VERSION` + `policy_hash(POLICY, POLICY_VERSION)`.
|
||||
- **Policy-hash self-check:** `policy_hash(POLICY, POLICY_VERSION)` equals a pinned expected value (changing
|
||||
a policy field without bumping the version fails this).
|
||||
- **Promotion:** flag + PASS + corr `<` threshold → promoted; corr `>=` threshold → withheld; empty deploy
|
||||
book → promoted (first edge); non-empty book + no overlap → withheld; no flag → not promoted; already
|
||||
promoted → stays promoted (one-way).
|
||||
- **Deploy-book aggregate:** the equal-weight daily aggregate over ≥2 deploy tracks aligns by date and
|
||||
averages correctly.
|
||||
|
||||
## Out of scope (YAGNI)
|
||||
|
||||
No allocation (B2), no health/decay (B3), no runtime-editable/DB policy, no capital. No UI beyond surfacing
|
||||
the policy version (a later cockpit touch, optional). Registry `gate_spec` per-track overrides stay as-is.
|
||||
|
||||
## Affected code (initial map — the plan refines this)
|
||||
|
||||
- New: `src/fxhnt/application/gate_policy.py` — `GatePolicy`, `POLICY`, `POLICY_VERSION`, `policy_hash`,
|
||||
`resolve_policy`, `ResolvedPolicy`.
|
||||
- Modify: `src/fxhnt/application/reconciliation_gate.py` — read from the resolved policy; remove the module
|
||||
constants (or re-export from `gate_policy` for back-compat if other code imports them — the plan checks).
|
||||
- Modify: `src/fxhnt/application/gate.py` — take the resolved policy instead of inline `.get` defaults.
|
||||
- Modify: `src/fxhnt/application/forward_ingest.py` — resolve the policy per track, stamp the verdict, and
|
||||
add the correlation backstop to `evaluate_promotions` (build the equal-weight deploy book from A's records).
|
||||
- Modify: `src/fxhnt/adapters/persistence/cockpit_models.py` — `ForwardSummaryRow` gains `policy_version` +
|
||||
`policy_hash`; `src/fxhnt/adapters/persistence/forward_nav.py` — `upsert_summary` stamps them.
|
||||
- Modify: `src/fxhnt/application/forward_models.py` — `ForwardSummary` DTO if it must carry the stamp
|
||||
(the plan decides whether the stamp is a DTO field or a separate upsert arg).
|
||||
```
|
||||
@@ -0,0 +1,156 @@
|
||||
# Execution Engine — Alpaca/multistrat Loop (Sub-project C1) — Design
|
||||
|
||||
**Date:** 2026-07-09
|
||||
**Status:** Approved (brainstorming) — ready for implementation plan
|
||||
**Sub-project:** C1 — the first slice of the execution plane (layer 5) of the "unified data-driven pipeline"
|
||||
north-star. C2 (Bybit venue) + C3 (real-money arming / multi-venue aggregate) come after.
|
||||
**Builds on:** A (deterministic forward state + `forward_record` observed machinery, merged) + B1 (gate/
|
||||
promotion policy, merged) + B2a (allocation policy → `strategy_allocation` target dollars, merged).
|
||||
|
||||
## North-star context (layer 5, first slice)
|
||||
|
||||
The execution engine is a data-driven RECONCILIATION LOOP: given the desired state (B2a's target positions)
|
||||
and the actual state (the venue account), compute the minimal orders to close the gap, place them, and record
|
||||
the fills as observed data. C1 closes that loop for ONE venue/strategy — Alpaca / multistrat, on paper:
|
||||
|
||||
```
|
||||
B2a strategy_allocation[multistrat] (target $)
|
||||
→ BRIDGE: envelope = min(target_$, account_NLV); per-instrument target $ = within_weight_i · envelope
|
||||
→ RECONCILE: target vs Alpaca account → minimal orders [reuse ExecutionService._plan / rebalance_weights]
|
||||
→ PLACE: fractional orders via AlpacaBroker (PAPER; --execute; safety gates)
|
||||
→ FILLS: executed-book equity delta → observed record for "multistrat_exec" (A's forward_record)
|
||||
```
|
||||
|
||||
Guiding principle (mirror A/B1/B2a): the engine reconciles; it does NOT authorize capital. Capital stays
|
||||
human-armed (Alpaca keys; live is a separate deliberate step) + ceiling-bounded (B2a). The reconciliation is
|
||||
deterministic given (targets + account state).
|
||||
|
||||
**The problem C1 solves.** The execution legs exist (`execute-multistrat` CLI, `ExecutionService`,
|
||||
`AlpacaBroker`, the safety gates hard-won from live paper trading) but do NOT consume B2a's per-strategy
|
||||
dollar allocation — `execute-multistrat` sizes `multistrat.target_weights` against the full Alpaca account
|
||||
NLV, ignoring how much capital B2a allocated to multistrat and never de-risking when B2a's killswitch/prune
|
||||
zeroes the allocation. And the executed reality is never recorded back as an observed track, so the forward
|
||||
gate can only ever rule on the MODELED book, not the real executed one.
|
||||
|
||||
## Goal
|
||||
|
||||
Make `execute-multistrat` B2a-driven: deploy exactly `strategy_allocation[multistrat]` dollars into the
|
||||
multistrat book on Alpaca (flatten to cash when B2a allocates $0 or has no row), reconcile + place via the
|
||||
existing `ExecutionService` (dry-run default), and record the executed book's daily return as an observed
|
||||
`forward_record` for a new `multistrat_exec` track — so the real executed track is trackable and comparable
|
||||
to the modeled one. Paper only; real capital stays human-armed.
|
||||
|
||||
## Components
|
||||
|
||||
### 1. The bridge — B2a dollars → target positions
|
||||
|
||||
Read `ForwardNavRepo.current_allocation()["multistrat"]` (B2a's target dollars; from `strategy_allocation`).
|
||||
The **envelope** = `min(target_dollars, account_NLV)` (never over-deploy the account, never exceed B2a's
|
||||
allocation). The within-book instrument weights come from `multistrat.target_weights(closes,
|
||||
FUND_INSTRUMENTS)` (the book's own weights, unchanged). Per-instrument target dollars = `within_weight_i ·
|
||||
envelope`.
|
||||
|
||||
- **B2a row absent, or `target_dollars == 0`** (killswitch flattened / marginal-gate pruned / not deploy-tier)
|
||||
→ envelope = 0 → all target positions 0 → the book FLATTENS to cash. The engine de-allocates exactly when
|
||||
B2a de-allocates — the killswitch/prune propagates to real positions.
|
||||
|
||||
### 2. Reconcile + place — reuse `ExecutionService`
|
||||
|
||||
Feed the bridge output to the EXISTING `ExecutionService.rebalance_weights(name, weights, prices, execute)` by
|
||||
passing `weights_i = target_dollars_i / account_NLV` (so the engine's `weight · NLV` sizing lands exactly on
|
||||
`target_dollars_i`). `ExecutionService._plan` reconciles vs the Alpaca account, sizes FRACTIONAL (Alpaca
|
||||
`supports_fractional = True`), and applies the hard-won safety gates (NLV-vs-cash consistency / `data_gap`,
|
||||
`allow_live`, `max_gross`) before placing. **Dry-run is the default; `--execute` places.** C1 changes only
|
||||
the DRIVER (the B2a envelope instead of full NLV) + the flatten-on-zero + the recording — the reconciliation
|
||||
engine, adapter, and gates are unchanged.
|
||||
|
||||
### 3. Fills → the executed observed track (the loop closure)
|
||||
|
||||
After a real (`--execute`) run, compute the executed multistrat book equity **on B2a's envelope, NOT the full
|
||||
account NLV**: `exec_equity = Σ(multistrat_position_qty · price) + reserve_cash`, where `reserve_cash =
|
||||
envelope − deployed_notional` (deployed = Σ of the position notionals just targeted). Store the prior
|
||||
`exec_equity` in A's observed book-state (`ForwardNavRepo.set_book_state("multistrat_exec", ...)`). The daily
|
||||
executed return = `exec_equity_today / exec_equity_prior − 1`, appended via
|
||||
`ForwardNavRepo.append_record("multistrat_exec", today, exec_return, at)` (A's append-only observed
|
||||
`forward_record`). On the FIRST run there is no prior equity → record no return (inception, exactly like A).
|
||||
|
||||
**Load-bearing correctness invariant:** the executed return is measured on the B2a-allocated **envelope**,
|
||||
not the full account NLV. Otherwise idle account cash (e.g. a $90k idle balance behind a $10k envelope)
|
||||
dilutes the return ~10× and the modeled-vs-executed comparison is meaningless. A test pins this: a large
|
||||
account with a small envelope yields an executed return reflecting the envelope's P&L, undiluted.
|
||||
|
||||
**Capital flows are not return (time-weighted):** the daily return is the P&L of the HELD positions since the
|
||||
last rebalance (`marked_value_of_prior_positions_at_today's_prices / value_at_prior_rebalance − 1`), NOT the
|
||||
raw equity delta. A B2a re-allocation (e.g. multistrat $10k → $15k) is a capital FLOW, not a gain — it enters
|
||||
at the rebalance, not in the return. The plan specifies the exact computation; a test pins that a pure
|
||||
re-allocation with flat prices records a ~0% return, not a jump.
|
||||
|
||||
### 4. Registry — the executed track
|
||||
|
||||
Add `multistrat_exec` to `STRATEGY_REGISTRY` as an OBSERVED track (`record_mode: "observed"`, venue
|
||||
`alpaca-paper`, a `definition` per B1's shape, tier research). Its reconciliation backtest reference is the
|
||||
MODELED multistrat's series (so B1's gate reconciles executed-vs-modeled — the executed track PASSes only if
|
||||
it tracks the modeled book, catching execution decay). It flows through A's engine (observed mode) + B1's
|
||||
gate + the cockpit like any other track, appearing beside the modeled `multistrat`.
|
||||
|
||||
### 5. Shape + safety
|
||||
|
||||
Keep the existing `execute-multistrat` CLI (run by the SUSPENDED `fxhnt-alpaca-rebalancer` CronJob when
|
||||
armed) — deliberately SEPARATE from the nightly `cockpit_forward` (a broker hiccup must not taint the forward
|
||||
track, mirroring why the Bybit testnet leg is its own job). C1 extends the CLI: read B2a's allocation as the
|
||||
envelope, flatten-on-zero, and on `--execute` record the `multistrat_exec` observed track. Safety = the
|
||||
existing `ExecutionService` gates + dry-run default. Real capital = `--live` + `allow_live` + live Alpaca keys
|
||||
— a separate, deliberate human step (unchanged).
|
||||
|
||||
### Where the human stays in the loop
|
||||
|
||||
The human arms Alpaca keys (paper first; live separately) and sets the B2a capital ceiling. The engine
|
||||
deploys exactly B2a's multistrat envelope, reconciles, and records — it never authorizes capital. Moving to
|
||||
live is `--live` + `allow_live` + live keys, deliberately human.
|
||||
|
||||
## Error handling
|
||||
|
||||
- **No B2a row / `$0` allocation** → envelope 0 → flatten to cash (deploy nothing). Safe.
|
||||
- **Account data inconsistent** (`data_gap`) → `ExecutionService` blocks (existing gate).
|
||||
- **Live account without `allow_live`** → blocked (existing gate).
|
||||
- **First `--execute` run** → no prior exec equity → record no return (inception), like A.
|
||||
- **Envelope > account NLV** → envelope capped at NLV (never over-deploy the account).
|
||||
- The executed return is on the ENVELOPE, never the full NLV (the correctness invariant).
|
||||
|
||||
## Testing
|
||||
|
||||
- **Bridge:** B2a $10k × within-weights → per-instrument target dollars sum to ≤ $10k; B2a $0 / absent row →
|
||||
all-zero targets (flatten).
|
||||
- **Envelope basis (load-bearing):** a $100k account with a $10k B2a envelope → the executed return reflects
|
||||
the $10k book's P&L (undiluted by the idle $90k), NOT the full-account NLV return.
|
||||
- **Capital flows excluded (load-bearing):** a pure B2a re-allocation ($10k → $15k) with FLAT prices records a
|
||||
~0% executed return (the $5k is a capital flow, not a gain), not a jump.
|
||||
- **Reconcile driver:** the bridge produces `weights_i = target_dollars_i / NLV` such that
|
||||
`ExecutionService` sizes to the intended target dollars (reuses the tested engine + gates; C1's diff is the
|
||||
driver).
|
||||
- **Observed record:** after `--execute`, `multistrat_exec` gets an observed `forward_record`; the first run
|
||||
is inception (no return); a flatten run sells the book to cash and the exec equity reflects it.
|
||||
- **Dry-run default:** without `--execute`, no orders are placed and no record is written.
|
||||
- **Registry:** `multistrat_exec` is an observed track with the modeled multistrat as its backtest ref.
|
||||
|
||||
## Out of scope (YAGNI)
|
||||
|
||||
No Bybit venue (C2 — where per-fill slippage decomposition belongs, since crypto slippage matters and the
|
||||
`bybit_testnet` leg already has it). No per-fill slippage decomposition for Alpaca (commission-free fractional
|
||||
ETFs → net equity-delta suffices). No real-money live path beyond the existing `allow_live` gate. No
|
||||
multi-venue aggregate reconciliation (C3). Reuses `ExecutionService`, `AlpacaBroker`, and A's observed
|
||||
`forward_record`/book-state machinery.
|
||||
|
||||
## Affected code (initial map — the plan refines this)
|
||||
|
||||
- Modify: `src/fxhnt/cli.py` `execute_multistrat` — read `current_allocation()["multistrat"]` as the envelope
|
||||
(min with NLV), size the within-book weights to it, flatten-on-zero, and on `--execute` record the
|
||||
`multistrat_exec` observed return.
|
||||
- New: an `application/` helper (e.g. `multistrat_exec_record.py`) — compute the envelope-basis exec equity +
|
||||
daily return from the account positions + prior book-state, and `append_record` it. Pure where possible
|
||||
(the equity math), with the repo write at the edge.
|
||||
- Modify: `src/fxhnt/registry.py` — add the `multistrat_exec` observed track (definition + backtest-ref
|
||||
mapping to the modeled multistrat).
|
||||
- Reuse: `ExecutionService.rebalance_weights`, `AlpacaBroker`, `ForwardNavRepo.current_allocation` (B2a) +
|
||||
`append_record` / `get_book_state` / `set_book_state` (A observed machinery).
|
||||
```
|
||||
@@ -0,0 +1,200 @@
|
||||
# Bybit Execution Leg (Sub-project C2) — Design
|
||||
|
||||
**Date:** 2026-07-10
|
||||
**Status:** Approved (brainstorming) — ready for implementation plan
|
||||
**Sub-project:** C2 — the SECOND venue of the execution plane (layer 5) of the "unified data-driven pipeline"
|
||||
north-star, after C1 (Alpaca/multistrat, merged). C3 (real-money arming / multi-venue aggregate) comes after.
|
||||
**Builds on:** A (deterministic forward state + `run_track` observed machinery) + B1 (gate/promotion policy) +
|
||||
B2a (allocation → `strategy_allocation` target dollars) + B3 (funding revocation → the allocation is
|
||||
funding-filtered) + C1 (the execution pattern: envelope bridge + `record_exec_track` observed track) — all
|
||||
merged. Reuses the EXISTING Bybit-testnet execution leg (`bybit_testnet_reconcile`/`bybit_testnet_record`
|
||||
Dagster assets, `plan_orders`, `slippage.py`, `BybitTestnetRepo`).
|
||||
|
||||
## North-star context (execution plane, second venue)
|
||||
|
||||
The execution engine is a data-driven RECONCILIATION LOOP: given the desired state (B2a's funding-filtered
|
||||
target dollars) and the actual state (the venue account), compute the minimal orders to close the gap, place
|
||||
them, and record the fills as observed data. C1 closed that loop for Alpaca/multistrat. C2 closes it for the
|
||||
CRYPTO book on Bybit — the venue the fund's real edges actually live on (`bybit_4edge` is the `tier==deploy`
|
||||
edge):
|
||||
|
||||
```
|
||||
B2a strategy_allocation[bybit_4edge] (target $, ALREADY B3-funding-filtered)
|
||||
→ BRIDGE: envelope = min(target_$, testnet_equity); size the raw 4-sleeve weights to the envelope
|
||||
→ RECONCILE: target vs Bybit-testnet positions → minimal orders [reuse the existing plan_orders + leg]
|
||||
→ PLACE: market orders on Bybit TESTNET (kill-switch; per-fill decomposed slippage) [existing]
|
||||
→ FILLS: envelope-basis, funding/fee/slippage-NET executed return → observed `bybit_4edge_exec` track
|
||||
(A's run_track), reconciling executed-vs-modeled against `bybit_4edge`
|
||||
```
|
||||
|
||||
Guiding principle (mirror A/B1/B2a/B3/C1): the engine reconciles; it does NOT authorize capital. Capital stays
|
||||
human-armed (Bybit keys; real-money is a separate deliberate step = C3) + ceiling-bounded (B2a) +
|
||||
funding-gated (B3). The reconciliation is deterministic given (funding-filtered targets + account state).
|
||||
|
||||
**The problem C2 solves.** The Bybit-testnet execution leg exists and is sophisticated (per-fill decomposed
|
||||
slippage, kill-switch, idempotent NAV marker), but it (1) sizes the book against the FULL testnet equity via
|
||||
`plan_orders(weights, current, equity=adapter.equity(), ...)`, IGNORING how much capital B2a allocated to
|
||||
`bybit_4edge` and never de-risking when B3 de-funds it or B2a's killswitch zeroes it; and (2) records the
|
||||
executed book only to its own `bybit_testnet_nav` (full-testnet-equity basis), NOT as an A-style observed
|
||||
forward track — so the real executed crypto book is not comparable to the modeled `bybit_4edge` through the
|
||||
gate/cockpit.
|
||||
|
||||
## Goal
|
||||
|
||||
Make the Bybit-testnet leg B2a/B3-driven — deploy exactly `strategy_allocation[bybit_4edge]` dollars (min with
|
||||
the testnet equity; flatten to cash when B3 de-funds it, B2a killswitches, or there is no allocation) — and
|
||||
record the executed book's daily return as an envelope-basis, capital-flow-excluded, **funding/fee/slippage-net**
|
||||
observed `bybit_4edge_exec` track via A's `run_track`, reconciling executed-vs-modeled against `bybit_4edge`.
|
||||
Reuse the existing slippage decomposition unchanged. Testnet only; real capital stays human-armed (C3).
|
||||
|
||||
## Components
|
||||
|
||||
### 1. The envelope bridge — B2a/B3 dollars → sized crypto book
|
||||
|
||||
Read `ForwardNavRepo.current_allocation()["bybit_4edge"]` (B2a's target dollars; ALREADY B3-funding-filtered —
|
||||
a de-funded/killswitched `bybit_4edge` is absent → `.get(..., 0.0)` = 0). The **envelope** =
|
||||
`min(target_dollars, testnet_equity)` (never over-deploy the account, never exceed B2a's allocation). The
|
||||
within-book weights are the existing raw 4-sleeve weights (`latest_raw_sleeve_weights` — unchanged; the 4-edge
|
||||
book has no overlay, so the raw weights are the effective weights). Size them to the envelope by passing the
|
||||
**envelope as the sizing equity** to `plan_orders` (targets = `weight · envelope`), instead of
|
||||
`adapter.equity()`.
|
||||
|
||||
- **`bybit_4edge` de-funded / killswitched / no allocation → envelope = 0** → all target positions 0 → the
|
||||
book FLATTENS to cash (reduce-only orders close the positions). The engine de-allocates exactly when
|
||||
B3/B2a de-allocate — funding revocation and the killswitch propagate to real (testnet) positions.
|
||||
- `max_gross` now binds against the envelope (deploy ≤ `envelope · max_gross`), tighter and correct.
|
||||
|
||||
Everything else in `bybit_testnet_reconcile` is unchanged: the kill-switch (log intent, place nothing), the
|
||||
idempotent per-day NAV marker (no-op re-run), the resting-open-orders abort, the per-order `BrokerError` →
|
||||
structured error row, and the per-fill decomposed-slippage capture.
|
||||
|
||||
### 2. The executed observed track — `bybit_4edge_exec` (loop closure)
|
||||
|
||||
Register `bybit_4edge_exec` in `STRATEGY_REGISTRY` as an OBSERVED track (`record_mode: "observed"`, venue
|
||||
`bybit-testnet`, tier `research`, `gate_type reconciliation`), mirroring C1's `multistrat_exec`. Its
|
||||
reconciliation reference is the MODELED `bybit_4edge` via a ref-provider alias `bybit_4edge_exec →
|
||||
bybit_4edge` (extending C1's `CockpitBacktestRefProvider._REF_ALIAS`), so B1's gate reconciles
|
||||
executed-vs-modeled — the executed crypto track PASSes only if it tracks the modeled book (execution decay =
|
||||
divergence), and it flows through A's engine + B1's gate + the cockpit + B3's funding beside `bybit_4edge`.
|
||||
|
||||
After a real (non-killed, non-noop) reconcile, compute the executed daily return (§3) and book it via A's
|
||||
`run_track(repo, "bybit_4edge_exec", ...)` — the same observed-booking path C1 uses (`record_exec_track`).
|
||||
The next run's position/cash basis is carried in A's book-state under a key kept SEPARATE from run_track's own
|
||||
(e.g. `bybit_4edge_exec_pos`), exactly as C1 does with `multistrat_exec_pos`.
|
||||
|
||||
### 3. The executed return — envelope-basis, flow-excluded, funding/fee/slippage-net (the C2 crux)
|
||||
|
||||
C1 (Alpaca, commission-free equities) computed the executed return by marking the held positions at prices +
|
||||
idle reserve. Crypto perps add two real components C1 did not have: **funding** (a periodic cash flow that IS
|
||||
edge return — several `bybit_4edge` sleeves harvest it) and **fees/slippage** (non-zero, unlike commission-free
|
||||
ETFs). The executed return MUST be measured on the SAME basis as the modeled `bybit_4edge` it reconciles
|
||||
against (which is funding-net and cost-modeled), or the gate diverges spuriously.
|
||||
|
||||
Computation (extends C1's `compute_exec_return`):
|
||||
|
||||
```
|
||||
exec_equity_today = Σ(held_qty_s · mark_today_s) + reserve_cash
|
||||
```
|
||||
where `reserve_cash` carries forward `envelope − deployed_notional` from the last rebalance AND accrues the
|
||||
realized **funding − fees** since the last mark. The daily executed return is:
|
||||
```
|
||||
exec_return = exec_equity_today / envelope_prior − 1
|
||||
```
|
||||
On the FIRST run there is no prior book → record no return (inception, exactly like A/C1).
|
||||
|
||||
Three points make the attribution exact and unambiguous:
|
||||
- **Dedicated account:** the Bybit-testnet account holds ONLY the `bybit_4edge` book (all positions are the
|
||||
book's), so ALL realized funding/fees are the book's — there is no cross-strategy attribution to untangle.
|
||||
The idle balance (`testnet_equity − envelope`) holds no positions, hence accrues no funding.
|
||||
- **Per-period delta:** the funding/fees folded into `reserve_cash` must be the realized amount SINCE THE LAST
|
||||
RECORD (a per-run delta), not a cumulative account figure. If the existing leg exposes `realized_pnl` as a
|
||||
cumulative, the plan derives the per-period delta (this-run cumulative − last-run cumulative, carried in the
|
||||
book-state).
|
||||
- **Re-allocation resets the basis:** a B2a re-allocation (e.g. `bybit_4edge` $5k → $8k) is a capital FLOW,
|
||||
not a gain. The return for the period is computed against `envelope_prior` FIRST; then the stored basis
|
||||
(`envelope`, post-rebalance `positions`, `reserve_cash = envelope − deployed`) resets to the NEW envelope —
|
||||
so the injected/withdrawn capital enters at the rebalance, never in the return (mirrors C1's "next basis is
|
||||
the target book" rule).
|
||||
|
||||
**Load-bearing correctness invariant:** the executed return is measured on the B2a-allocated **envelope**
|
||||
(not the full testnet equity), is **capital-flow-excluded** (a pure re-allocation with flat marks + zero
|
||||
funding records ~0%), and is **funding/fee/slippage-net** (funding and fees flow through `reserve_cash`; fills
|
||||
embed slippage). A test pins each clause: a large testnet balance behind a small envelope yields the
|
||||
envelope's undiluted P&L; a flat-mark re-allocation records ~0%; and a funding credit with flat marks records
|
||||
a POSITIVE return (funding is edge return, not a flow).
|
||||
|
||||
### 4. Slippage — reuse the decomposition unchanged
|
||||
|
||||
The existing per-fill decomposition (`execution_slippage_bps` = fill vs live mid; `timing_drift_bps` =
|
||||
mid-at-order vs book mark; `slippage_vs_assumed_bps` = validation delta vs the A8 assumed cost tier) is
|
||||
first-class and already recorded to `bybit_testnet_fills` by `bybit_testnet_record`. C2 does NOT change it —
|
||||
the executed-track return in §3 is net of the REAL fills (which embed the actual slippage), and the per-fill
|
||||
rows remain the A8 cost-validation output. This is the concrete reason C2 differs from C1: crypto slippage is
|
||||
material, and it is measured, not assumed.
|
||||
|
||||
### 5. Where the human stays in the loop
|
||||
|
||||
The human arms Bybit TESTNET keys (real-money keys are a separate C3 step) and sets the B2a capital ceiling +
|
||||
the `GatePolicy` (whose B3 windows gate funding). C2 deploys exactly B2a's `bybit_4edge` envelope (funding-
|
||||
filtered), reconciles, and records — it never authorizes real capital. Moving to real money is deliberate
|
||||
Bybit-mainnet keys + an explicit arming step (C3). The kill-switch remains an always-available circuit-breaker.
|
||||
|
||||
## Error handling
|
||||
|
||||
- **No `bybit_4edge` allocation / de-funded / `$0`** → envelope 0 → flatten to cash (reduce-only). Safe.
|
||||
- **Kill-switch on** → log intended orders, place NOTHING (existing), record NO return (a killed run is a
|
||||
visible gap, not a silent pass — existing `bybit_testnet_record` writes nothing).
|
||||
- **Idempotent re-run** (NAV marker for today) → no-op, no double-execution (existing).
|
||||
- **Resting open orders** → abort (existing); **per-order `BrokerError`** → structured error row, no crash
|
||||
(existing).
|
||||
- **First run** → no prior exec equity → record no return (inception), like A/C1.
|
||||
- **Envelope > testnet equity** → capped at equity (never over-deploy).
|
||||
- **Low-confidence mid** (missing book) → excluded from the slippage verdict (existing); the return marks use
|
||||
the best available mark.
|
||||
- The executed return is on the ENVELOPE, funding/fee/slippage-net (the correctness invariant).
|
||||
|
||||
## Testing
|
||||
|
||||
- **Envelope bridge:** B2a `bybit_4edge` $X (funded) × raw sleeve weights → per-symbol targets sum to ≤ $X;
|
||||
B2a `$0` / de-funded / absent row → all-zero targets (flatten, reduce-only).
|
||||
- **Envelope basis (load-bearing):** a large testnet equity with a small `bybit_4edge` envelope → the executed
|
||||
return reflects the envelope's P&L (undiluted by the idle balance), NOT the full-testnet-equity return.
|
||||
- **Funding is return, not flow (load-bearing):** a funding CREDIT with flat marks records a POSITIVE executed
|
||||
return (funding accrues into `reserve_cash`); a pure re-allocation with flat marks + zero funding records
|
||||
~0% (capital flow excluded).
|
||||
- **Fee/slippage net (load-bearing):** fees reduce `reserve_cash` → a flat-mark period with a fee records a
|
||||
slightly NEGATIVE return; the per-fill slippage rows are unchanged (decomposition reused).
|
||||
- **Observed record:** after a real reconcile, `bybit_4edge_exec` gets an observed `forward_record`; the first
|
||||
run is inception (no return); a flatten run sells the book to cash and the exec equity reflects it.
|
||||
- **Kill-switch / no-op:** a killed or idempotent-no-op reconcile records NO order and NO return.
|
||||
- **Registry + gate:** `bybit_4edge_exec` is an observed track whose reconciliation reference resolves (via
|
||||
the alias) to the modeled `bybit_4edge`.
|
||||
|
||||
## Out of scope (YAGNI)
|
||||
|
||||
- **No real-money Bybit** (C3 — mainnet keys + an explicit arming step). Testnet only.
|
||||
- **No multi-venue aggregate reconciliation** (C3).
|
||||
- **No change to the edges / the 4-sleeve book / the slippage model.** C2 reuses `latest_raw_sleeve_weights`,
|
||||
`plan_orders`, `slippage.py`, the `bybit_testnet` leg, and A's observed machinery.
|
||||
- **`bybit_testnet_nav` is kept** (per the brainstorm decision) as the per-run full-equity + mean-slippage
|
||||
detail; the new observed track is the envelope-basis forward record. Cockpit VISUAL surfacing of the exec
|
||||
track follows the same deferral as C1/B3 (surfaced later alongside allocation).
|
||||
|
||||
## Affected code (initial map — the plan refines this)
|
||||
|
||||
- Modify: `src/fxhnt/adapters/orchestration/assets.py` `bybit_testnet_reconcile` — read
|
||||
`current_allocation()["bybit_4edge"]` as the envelope (min with testnet equity), size the sleeve weights to
|
||||
it via `plan_orders(..., equity=envelope, ...)`, flatten-on-zero; carry the realized-funding/fee delta +
|
||||
post-rebalance positions into the return computation. `bybit_testnet_record` — after recording the fills +
|
||||
`bybit_testnet_nav` (unchanged), compute the envelope-basis exec return (§3) and book the
|
||||
`bybit_4edge_exec` observed track via A's `run_track` + book-state.
|
||||
- New: a small `application/` helper (e.g. extend/parallel `multistrat_exec_record.py`) — the envelope-basis,
|
||||
funding/fee/slippage-net exec-equity + return + `record_exec_track` for the crypto book. Pure where possible
|
||||
(the equity math), repo/`run_track` at the edge. Reuse C1's `compute_exec_return`/`record_exec_track` shape.
|
||||
- Modify: `src/fxhnt/registry.py` — add the `bybit_4edge_exec` observed track (venue `bybit-testnet`, tier
|
||||
research, gate_type reconciliation, record_mode observed, definition per B1's shape).
|
||||
- Modify: `src/fxhnt/application/forward_ingest.py` `CockpitBacktestRefProvider._REF_ALIAS` — add
|
||||
`bybit_4edge_exec → bybit_4edge` (so the gate reconciles executed-vs-modeled).
|
||||
- Reuse: `ForwardNavRepo.current_allocation` (B2a+B3), `plan_orders`, `slippage.py`, `BybitTestnetRepo`,
|
||||
`run_track` / `append_record` / `get_book_state` / `set_book_state` (A), C1's `record_exec_track` pattern.
|
||||
```
|
||||
@@ -0,0 +1,159 @@
|
||||
# Cockpit Surfacing + Mobile Redesign — Design
|
||||
|
||||
**Date:** 2026-07-10
|
||||
**Status:** Approved (brainstorming; visual mockup approved) — ready for implementation plan
|
||||
**Sub-project:** the OBSERVABILITY layer of the "unified data-driven pipeline" north-star — make the pipeline's
|
||||
decisions visible in the cockpit, and redesign the cockpit for navigation + mobile. The "verify" enabler
|
||||
before the accumulated A/B1/B2a/B3/C1/C2 pipeline is trusted/deployed.
|
||||
**Builds on:** A/B1/B2a/B3/C1/C2 (all merged). Reads their already-written tables; adds NO pipeline logic and
|
||||
moves NO capital.
|
||||
**Approved mockup:** the redesign was validated as a visual artifact (mobile overview + detail + desktop) with
|
||||
real data before this spec.
|
||||
|
||||
## Goal
|
||||
|
||||
Two things at once, in the EXISTING server-rendered cockpit (`adapters/web/`, FastAPI + Jinja2 + HTMX):
|
||||
|
||||
1. **Surface the pipeline's decisions** (read-only) — fully integrated into the existing Overview + Detail
|
||||
pages (NO new page, NO JSON dumps): B2a **allocation** (per-edge target $, weight, gross-vs-$35k-ceiling),
|
||||
B3 **funding** health (funded / decaying / de-funded + why), and the C1/C2 **executed-vs-modeled**
|
||||
reconciliation (the executed twin nested under its modeled edge).
|
||||
2. **Redesign for usability** — a mobile-first, thumb-reachable **bottom-tab navigation** on small screens
|
||||
(top bar on desktop), edge **cards** instead of wrapping tables on mobile, clearer hierarchy — a refinement
|
||||
of the existing GitHub-dark terminal aesthetic, not a re-skin.
|
||||
|
||||
Read-only, human-gated capital unchanged. Verified LOCALLY in a real browser before deploy.
|
||||
|
||||
## Locked design decisions (from the approved mockup)
|
||||
|
||||
- **Navigation:** sticky bottom tab bar (Overview · Paper · Replay · Backtest) below ~640px; the existing top
|
||||
nav on desktop. htmx `hx-boost` preserved.
|
||||
- **Capital meter:** in the fund headline — `deployed $X / $35,000 ceiling` with a fill bar + "N funded · M
|
||||
de-funded".
|
||||
- **Funding badge:** a semantic pill on every edge — `funded` (green `#3fb950`), `decaying N/10` (amber
|
||||
`#d29922`, a NEW token distinct from the neutral-grey WAIT), `de-funded` (red `#f85149`).
|
||||
- **Executed twin:** nested UNDER its modeled edge (a sub-line / sub-row `executed +X% vs modeled +Y% · Δ Z% ·
|
||||
slip N bp`), never a separate top-level track.
|
||||
- **Detail page** = the full per-edge lifecycle: status → forward → **funding** → **allocation** →
|
||||
**executed reality** → backtest → sleeves → daily rows.
|
||||
- Aesthetic honored: dark-committed terminal, monospace, `tabular-nums`, existing palette + badge classes.
|
||||
|
||||
## Components
|
||||
|
||||
### 1. Read model (`DashboardService` + repo reads)
|
||||
|
||||
The dashboard reads three already-written tables. Add repo read methods (mirroring `current_allocation` /
|
||||
`current_funding`, which return only partial data today):
|
||||
|
||||
- `ForwardNavRepo.all_allocations() -> list[StrategyAllocationRow]` (target_weight, target_dollars,
|
||||
policy_version, as_of) — `current_allocation()` returns only `{sid: dollars}`, insufficient for the UI.
|
||||
- `ForwardNavRepo.all_funding() -> list[StrategyFundingRow]` (state, label, decay_run, recovery_run,
|
||||
first_pass_date, policy_version, as_of) — `current_funding()` returns only `{sid: state}`.
|
||||
|
||||
`DashboardService` gains pure read-model methods that JOIN these onto the existing `FleetRow` / `StrategyDetail`:
|
||||
|
||||
- **Allocation:** per-sid `{target_dollars, target_weight, policy_version}`; fund-level `gross_deployed =
|
||||
Σ target_dollars` and `ceiling` (from `ALLOCATION_POLICY.capital_ceiling`).
|
||||
- **Funding:** per-sid `{state, label, decay_run, recovery_run, first_pass_date, policy_version}`.
|
||||
- **Executed pairing:** for each MODELED sid that has an executed twin, resolve the twin via the REVERSE of
|
||||
C1/C2's `CockpitBacktestRefProvider._REF_ALIAS` (`{multistrat_exec: multistrat, bybit_4edge_exec:
|
||||
bybit_4edge}` → reverse `{multistrat: multistrat_exec, bybit_4edge: bybit_4edge_exec}`); read the twin's
|
||||
`forward_summary` (executed total_return, days) and compute `divergence = executed_total_return −
|
||||
modeled_total_return`. **Mean slippage** comes from `bybit_testnet_nav.mean_exec_slippage_bps` /
|
||||
`mean_timing_drift_bps` for the crypto twin only (Alpaca is commission-free → no slippage row; show "—").
|
||||
|
||||
The executed twins are `tier=research` observed tracks: on the Overview they are NOT rendered as their own
|
||||
top-level rows — they are folded into their modeled parent's row/card. (They keep their own detail page.)
|
||||
|
||||
### 2. `base.html` — navigation + CSS refinement
|
||||
|
||||
- **Bottom tab nav:** add a `<nav class="tabs">` (Overview/Paper/Replay/Backtest) rendered as a sticky bottom
|
||||
bar under `@media(max-width:640px)`; hidden on desktop. The existing top `nav.bar` shows on desktop, hidden
|
||||
on mobile. Both `hx-boost`. The active tab is derived from `request.url.path` exactly as today.
|
||||
- **CSS tokens:** add the `decaying` amber token + the funding/exec/meter/card classes from the mockup, into
|
||||
the existing inline `<style>`. Keep the existing badge classes (WAIT/GO/PASS/NO_GO/paper/live).
|
||||
- **Responsive:** the existing `table.fleet → cards` mobile pattern is extended to carry the new allocation +
|
||||
funding fields (`data-label` cells). `overflow-x:auto` wrappers on any wide table.
|
||||
- Accessibility: visible keyboard focus on tabs/links; `prefers-reduced-motion` respected (no essential motion).
|
||||
|
||||
### 3. `cockpit.html` (Overview)
|
||||
|
||||
- **Fund headline:** add the capital meter (`gross_deployed / ceiling` + fill bar + funded/de-funded count).
|
||||
- **"All forward tracks" table → "every gate + capital + health at a glance":** add `allocated` (target $ ·
|
||||
weight, or `$0`/`—`) and `funding` (badge) columns. Under a modeled deploy edge with an executed twin,
|
||||
render a nested `exrow` sub-line (executed return, Δ, slippage). Mobile: each row → card (the new fields as
|
||||
labeled rows; the exec twin as a nested block).
|
||||
- Grouping by tier (deploy/research) is unchanged.
|
||||
|
||||
### 4. `strategy.html` (Detail)
|
||||
|
||||
Add three lifecycle blocks (only when data exists), between the status header and the backtest verdict:
|
||||
|
||||
- **Funding health:** state badge + `first_pass_date`, `decay_run`/`recovery_run`, and a plain-language "why"
|
||||
(e.g. "would de-fund after 10 consecutive divergence days") + policy version.
|
||||
- **Allocation:** target capital, book weight, ceiling, sizing note + policy version. A de-funded / $0 edge
|
||||
shows "$0 — de-funded/killswitched, held as cash".
|
||||
- **Executed reality** (only for a modeled track WITH an executed twin): executed forward vs modeled forward,
|
||||
divergence, mean slippage (crypto) / "—" (Alpaca), a link to the exec track's own detail page.
|
||||
|
||||
### 5. Graceful degradation
|
||||
|
||||
Every new read is wrapped so a MISSING table/row/column (e.g. a not-yet-migrated DB, or an edge with no
|
||||
allocation/funding/exec row) renders `—` or hides the block — NEVER a 500. This matches the overview's existing
|
||||
discipline (`except Exception: return []`). Load-bearing because prod's schema/data lags until deploy + the
|
||||
first nightly.
|
||||
|
||||
## Data flow
|
||||
|
||||
```
|
||||
strategy_allocation ─┐
|
||||
strategy_funding ─┼─> ForwardNavRepo reads ─> DashboardService (pure joins) ─> Jinja2 (cockpit/strategy.html)
|
||||
forward_summary(exec)┤ + base.html nav/CSS
|
||||
bybit_testnet_nav ─┘ (server-rendered, read-only)
|
||||
```
|
||||
|
||||
## Error handling
|
||||
|
||||
- Missing table/column/row → `—` / hidden block, never 500 (wrap each new read).
|
||||
- No executed twin for a track → no "executed" section (most tracks).
|
||||
- No `bybit_testnet_nav` slippage row → slippage shows "—".
|
||||
- Read-only: the dashboard NEVER writes (dev loop builds repos without `.migrate()`; prod migrates at boot).
|
||||
|
||||
## Testing
|
||||
|
||||
- **Unit (read model):** `DashboardService` allocation/funding/exec-pairing methods over a seeded in-memory
|
||||
sqlite repo — correct join, gross-vs-ceiling, divergence math, reverse-alias pairing, and graceful `—` when a
|
||||
table/row is absent.
|
||||
- **Repo:** `all_allocations()` / `all_funding()` return the full rows; empty when absent.
|
||||
- **Local browser verification (the load-bearing acceptance):** the current `dev-cockpit-local` loop is
|
||||
CONFOUNDED — it runs master's code against prod's 24-day-old schema (missing `forward_summary.policy_version`
|
||||
etc.), so it crashes. Verification instead runs the cockpit against a **local DB migrated to the current
|
||||
schema + seeded** with representative allocation/funding/exec/forward rows, driven by Playwright:
|
||||
screenshot the Overview (capital meter, funding badges, nested exec twin) + a Detail page (the three
|
||||
lifecycle blocks) at BOTH desktop and mobile widths, and READ THE NUMBERS (per
|
||||
feedback_verify_cockpit_locally_not_prod). A tiny `scripts/seed_cockpit_demo.py` (or a pytest fixture the
|
||||
verification reuses) seeds the local DB. This same loop re-runs post-deploy against real prod data.
|
||||
|
||||
## Out of scope (YAGNI)
|
||||
|
||||
- No new pipeline logic, no capital movement, no new nightly assets — pure display of written tables.
|
||||
- No new top-level routes/pages — integrate into the existing Overview + Detail (and the existing
|
||||
Paper/Replay/Backtest tabs are untouched beyond the nav restyle).
|
||||
- No cockpit surfacing of the Paper/Replay/Sim internals beyond the nav restyle.
|
||||
- The DEPLOY itself (push + argo + migrate-forward-anchors) is a SEPARATE follow-up, gated on this being
|
||||
browser-verified locally.
|
||||
|
||||
## Affected code (initial map — the plan refines this)
|
||||
|
||||
- Modify: `src/fxhnt/adapters/persistence/forward_nav.py` — `all_allocations()`, `all_funding()` reads.
|
||||
- Modify: `src/fxhnt/application/dashboard_service.py` — allocation/funding/exec-pairing read-model methods +
|
||||
join onto `FleetRow`/`StrategyDetail` (extend those DTOs).
|
||||
- Modify: `src/fxhnt/adapters/web/templates/base.html` — bottom-tab nav + CSS tokens (decaying amber, meter,
|
||||
funding/exec/card classes) + responsive rules.
|
||||
- Modify: `src/fxhnt/adapters/web/templates/cockpit.html` — capital meter, allocation/funding columns, nested
|
||||
executed twin.
|
||||
- Modify: `src/fxhnt/adapters/web/templates/strategy.html` — funding / allocation / executed-reality blocks.
|
||||
- New: `scripts/seed_cockpit_demo.py` (or a shared pytest fixture) — seed a local DB for browser verification.
|
||||
- Reuse: `ALLOCATION_POLICY.capital_ceiling` (B2a), `CockpitBacktestRefProvider._REF_ALIAS` (C1/C2),
|
||||
`scripts/dev-cockpit-local.sh` pattern (repointed at the seeded local DB), `charts.py` sparklines.
|
||||
```
|
||||
@@ -0,0 +1,215 @@
|
||||
# Funding Revocation on Sustained Decay (Sub-project B3) — Design
|
||||
|
||||
**Date:** 2026-07-10
|
||||
**Status:** Approved (brainstorming) — ready for implementation plan
|
||||
**Sub-project:** B3 — the edge-health / decay plane of the allocation layer of the "unified data-driven
|
||||
pipeline" north-star. Scoped down during brainstorming from a general per-edge health layer to the specific,
|
||||
real hole it needed to close.
|
||||
**Builds on:** A (deterministic forward state — the immutable `forward_record` SSOT, merged) + B1 (gate /
|
||||
promotion policy — the reconciliation gate verdict is B3's signal, merged) + B2a (allocation policy — B3
|
||||
filters its input set, merged).
|
||||
|
||||
## North-star context (the allocation layer's health plane)
|
||||
|
||||
The allocation layer turns `{live + healthy}` deployed strategies into target capital (B2a). Today "healthy"
|
||||
defaults to *all* of the effective deploy set, so a strategy that was validated once and later decays keeps
|
||||
its capital forever. B3 makes "healthy" real:
|
||||
|
||||
```
|
||||
{effective deploy set} = registry tier=="deploy" ∪ promoted_strategy_ids()
|
||||
→ FUNDING FILTER (B3): drop any edge in SUSTAINED reconciliation-decay ← THIS sub-project
|
||||
→ {live + healthy} → B2a compute_allocation (unchanged) → target capital (≤ ceiling)
|
||||
```
|
||||
|
||||
Guiding principle (mirror A/B1/B2a): funding status = `f(committed policy, A's immutable record, B1's gate)`.
|
||||
Pure, recomputable, versioned, human-gated by the committed policy + the capital ceiling. B3 moves no capital
|
||||
(that is C); it only narrows B2a's input set.
|
||||
|
||||
## The problem B3 solves
|
||||
|
||||
Promotion is deliberately **one-way and persistent** (`forward_ingest.evaluate_promotions`: *"never reverts
|
||||
if the gate later dips … not flickering with the daily gate"*). That anti-flicker choice is correct, but it
|
||||
leaves a hole: a **promoted (or config-deployed) edge that later decays keeps its capital** even after its
|
||||
reconciliation gate has reverted to not-PASS. Closing that hole naively — de-fund the instant the gate isn't
|
||||
PASS — would reintroduce exactly the daily flicker the one-way design avoids, and daily flicker on a
|
||||
drawdown *is* the reactive-de-sizing pattern rejected 3× ([[pearl_fxhnt_reactive_risk_controllers_rejected]],
|
||||
[[pearl_fxhnt_voltarget_overlay_is_reactive_desizing_rejected]]). So the fix must be **sustained + hysteretic
|
||||
+ structural**, keyed off the right signal.
|
||||
|
||||
**Why the reconciliation gate is the right signal (not an absolute-drift detector).** An earlier design used
|
||||
`WindowedEdgeHealth` (a trailing t-stat). It only fires on sustained *strongly-negative* returns (t < ~−1.6):
|
||||
a quietly-dead edge whose forward mean drifts to ~0 scores health ≈ 0.8 and is **never** retired — it catches
|
||||
*bleeding*, not *stopping*. B1's reconciliation gate already asks the exact right question — *"is the forward
|
||||
edge still tracking what its backtest predicted?"* — and flips to a divergence WAIT when the forward return
|
||||
falls materially below the backtest's expectation. That is precisely "the edge stopped paying." B3 reuses it.
|
||||
|
||||
## Goal
|
||||
|
||||
De-fund an effective-deploy edge whose reconciliation gate has shown **sustained divergence** from its backtest
|
||||
(hysteresis, no flicker), and re-fund it when it sustainedly reconciles again — as a pure recompute over A's
|
||||
immutable record, with **no new mutable state**, scoped to edges the data once validated (that ever PASSed).
|
||||
|
||||
## Components
|
||||
|
||||
### 1. Machine-readable gate-verdict category (small additive B1 change)
|
||||
|
||||
`evaluate_reconciliation_gate` returns a `GateVerdict(status, reason)`. Add a machine-readable **category** so
|
||||
funding classification never string-matches the human reason. Category ∈:
|
||||
|
||||
- `PASS` — reconciles (status PASS).
|
||||
- `DIVERGENCE` — the decay signal: forward cumulative materially below the backtest band, or forward daily
|
||||
correlation negative (the two mirage/decay sub-checks).
|
||||
- `IMMATURE` — building (`< min_forward_days`) or no backtest reference (nothing to reconcile *yet*).
|
||||
- `EXECUTION_GAP` — a hole/duplicate in the daily record (a pipeline/ops issue, **not** edge decay).
|
||||
|
||||
Only `DIVERGENCE` counts as a decay day; only `PASS` counts as a healthy day; `IMMATURE`/`EXECUTION_GAP` are
|
||||
**neutral** (they neither de-fund nor re-fund — immaturity and ops gaps are not edge mortality). This keeps
|
||||
the funding decision honest: an edge is de-funded for *decaying*, never for being young or for a pipeline
|
||||
hiccup.
|
||||
|
||||
**Non-reconciliation gates.** An effective-deploy edge whose `gate_spec.gate_type` is not `reconciliation`
|
||||
(evaluated by `evaluate_gate`, not `evaluate_reconciliation_gate`) has no divergence signal to decay against.
|
||||
Its `GateVerdict.category` maps `PASS → PASS` and any non-PASS → `IMMATURE` (neutral), so B3 **never**
|
||||
de-funds a non-reconciliation edge — funding revocation applies only where a backtest reconciliation exists.
|
||||
The `category` field is set on **every** `GateVerdict` return path in both gate functions (default-safe).
|
||||
|
||||
### 2. Funding policy (folded into B1's versioned `GatePolicy`)
|
||||
|
||||
The funding decision is a direct function of gate verdicts, so its knobs live in the existing versioned
|
||||
`GatePolicy` (no 4th policy module):
|
||||
|
||||
- `defund_window: int = 10` — consecutive `DIVERGENCE` days that flip `funded → de-funded`.
|
||||
- `refund_window: int = 15` — consecutive `PASS` days that flip `de-funded → re-funded`.
|
||||
|
||||
Asymmetric by design (quicker to de-risk, slower to re-arm real capital — "err toward not deploying"). These
|
||||
are **debounce** windows over an already-*cumulative* gate signal (each day's verdict reflects the whole
|
||||
forward record to date), so they are modest, not a primary sustained-detector. Adding these fields bumps
|
||||
`GATE_POLICY_VERSION` and changes `gate_policy_hash` (the self-check test forces the bump — same discipline as
|
||||
A's definition_hash).
|
||||
|
||||
### 3. The funding engine (pure, recompute-from-record — `funding_status.py`)
|
||||
|
||||
`compute_funding(rows, backtest_ref, policy) -> FundingStatus` per edge, where `rows` is A's ascending
|
||||
`forward_record` for the edge and `backtest_ref` is the per-edge reconciliation reference (the same
|
||||
`BacktestRefProvider` B1 uses; the executed-track alias from C1 applies). Steps, **replayed from inception
|
||||
each night** (the proven `paper_book._retirement_active_stateless` pattern — deterministic, no persisted
|
||||
latch):
|
||||
|
||||
1. For each record-day `d` (ascending), reconstruct the as-of `ForwardSummary` (`days`, `total_return`) from
|
||||
`rows[:d]` and evaluate `evaluate_reconciliation_gate(summary_d, rows[:d], ref_d, policy)`, where `ref_d`
|
||||
is the backtest reference pro-rated to `max(days_d, min_forward_days)`. Take its **category** (§1).
|
||||
2. Replay a funding state machine over the category sequence, starting `funded`, tracking whether the edge has
|
||||
**ever** had a `PASS` (`has_passed`):
|
||||
- `funded`: a `DIVERGENCE` day increments the decay run **only if `has_passed`** (a DIVERGENCE before the
|
||||
edge's first PASS is pre-validation, not decay → neutral); **any** non-`DIVERGENCE` day resets the decay
|
||||
run to 0. `decay_run >= defund_window` → `de-funded` (reset runs).
|
||||
- `de-funded`: a `PASS` day increments the recovery run; any non-`PASS` day resets it. `recovery_run >=
|
||||
refund_window` → `funded` (reset runs).
|
||||
3. `FundingStatus` = the terminal state + a display label (`funded` / `decaying` (funded but a decay run is
|
||||
building) / `de-funded`) + the current run lengths (for the cockpit).
|
||||
|
||||
**Scope guard — only edges that ever PASSed are watchable.** The state machine begins de-fund-watching only
|
||||
*after* the edge's first `PASS` day; before that the edge is `IMMATURE`/pre-validation and its decay run
|
||||
cannot advance. So an edge that has never PASSed is **never** de-funded — a promoted edge always qualifies
|
||||
(it PASSed to get promoted); a hardcoded `tier=="deploy"` edge qualifies only once it earns a PASS, so a
|
||||
config-deploy-by-fiat edge that is merely still building is never auto-defunded, while one the data validated
|
||||
and then invalidated **is** (capital safety > a stale config), loudly logged.
|
||||
|
||||
### 4. Wiring — the funding filter in front of B2a (Approach A)
|
||||
|
||||
`allocation_ingest.record_allocation` filters the effective deploy set through `compute_funding` **before**
|
||||
`compute_allocation`:
|
||||
|
||||
```
|
||||
deploy_sids = {tier=="deploy"} ∪ promoted_strategy_ids()
|
||||
funded_sids = {sid for sid in deploy_sids if compute_funding(...).state != "de-funded"}
|
||||
returns = {sid: {r.date: r.ret} for sid in funded_sids}
|
||||
weights = compute_allocation(returns, ALLOCATION_POLICY) # B2a UNCHANGED
|
||||
```
|
||||
|
||||
A de-funded edge is simply *absent* from the input → gets $0. B2a's `max_strategy_weight` cap redistributes
|
||||
to survivors up to the cap and holds the remainder as cash; the aggregate killswitch is unchanged. Promotion
|
||||
is **not** touched — `promoted_strategy_ids()` still returns the historical promotion (audit: "was
|
||||
validated"); funding is now the conditional layer on top.
|
||||
|
||||
### 5. Auditability (+ cockpit surfacing deferred)
|
||||
|
||||
Record the per-edge funding snapshot each nightly to a `strategy_funding` table (`strategy_id`, `state`,
|
||||
`decay_run`, `recovery_run`, `first_pass_date`, `policy_version`, `policy_hash`, `as_of`, `computed_at`) —
|
||||
the auditable SSOT. "De-funded under gate-policy vN" is queryable, and it's the read-model a later cockpit
|
||||
pass will surface from.
|
||||
|
||||
**Cockpit VISUAL surfacing is deferred** (not in this sub-project): B2a's `strategy_allocation` is itself not
|
||||
yet surfaced in the cockpit (its only consumer today is C1's envelope read), so a funding-state view would be
|
||||
premature and inconsistent — the confusing "`$0` with no explanation" case cannot arise until allocation is
|
||||
surfaced. Funding + allocation should be surfaced together in a future cockpit pass (with the mandated local
|
||||
browser verification). B3 delivers the durable, queryable funding state; the visual is a clean follow-up.
|
||||
|
||||
### 6. Where the human stays in the loop
|
||||
|
||||
The human sets the `GatePolicy` (now including `defund_window`/`refund_window`, versioned in git — reviewed)
|
||||
and the B2a capital ceiling, and arms venue keys. B3 de-funds/re-funds automatically per the committed policy
|
||||
and the immutable record. No capital is moved (that is C); B3 only narrows B2a's input set. Re-arming a
|
||||
recovered edge is automatic (hysteresis), but slower than de-risking, and always bounded by the ceiling.
|
||||
|
||||
## Error handling
|
||||
|
||||
- **Edge never PASSed** → never de-funded (pre-validation; not decay). Safe.
|
||||
- **All effective-deploy edges de-funded** → empty allocation (deploy nothing). Safe.
|
||||
- **Record has a hole** → `EXECUTION_GAP` category = neutral (resets the decay run); an ops gap never de-funds
|
||||
an edge. The gap still surfaces via B1's gate.
|
||||
- **No backtest reference** → `IMMATURE` = neutral (cannot reconcile → cannot claim decay). Never de-funds.
|
||||
- **Transient dip** shorter than `defund_window` → decay run resets on the first non-divergence day; funding
|
||||
does **not** flicker (the anti-whipsaw guarantee).
|
||||
- **NO reactive de-sizing:** de-funding is binary + structural (sustained divergence over the window), never a
|
||||
continuous per-day throttle. Re-funding is symmetric + slower.
|
||||
|
||||
## Testing
|
||||
|
||||
- **Load-bearing (anti-whipsaw):** a transient `DIVERGENCE` run shorter than `defund_window`, then a `PASS`,
|
||||
leaves the edge `funded` — funding does not flicker on a normal dip.
|
||||
- **Sustained decay de-funds:** `defund_window` consecutive `DIVERGENCE` days flip `funded → de-funded`; the
|
||||
edge is then absent from `record_allocation`'s input and gets `$0`.
|
||||
- **Recovery re-funds:** after de-funding, `refund_window` consecutive `PASS` days flip back to `funded`;
|
||||
asymmetry holds (`refund_window > defund_window`).
|
||||
- **Never-PASSed is never de-funded:** an edge whose whole record is `IMMATURE`/`DIVERGENCE` but that never
|
||||
PASSed stays fundable (state never leaves the pre-validation guard).
|
||||
- **Neutral days don't de-fund:** an `EXECUTION_GAP` or `IMMATURE` run of any length never flips funding.
|
||||
- **Verdict categorisation:** each reconciliation-gate reason maps to the correct category
|
||||
(PASS/DIVERGENCE/IMMATURE/EXECUTION_GAP).
|
||||
- **Recompute determinism:** the same record + policy → the same `FundingStatus` (no persisted state).
|
||||
- **Policy-hash self-check:** adding `defund_window`/`refund_window` without a `GATE_POLICY_VERSION` bump fails
|
||||
the hash self-check; the recorded snapshot carries the version + hash.
|
||||
- **B2a redistribution:** with one of two funded edges de-funded, the survivor is equal-weighted up to the
|
||||
`max_strategy_weight` cap and the remainder is held as cash (B2a unchanged).
|
||||
|
||||
## Out of scope (YAGNI)
|
||||
|
||||
- **No `WindowedEdgeHealth` / absolute-drift layer** — the reconciliation gate is the correct "stopped paying"
|
||||
signal; WEH misses quiet death and is redundant with B2a's marginal gate for the bleeding case. `RetirementGate`
|
||||
/ `WindowedEdgeHealth` stay in `domain/edge_decay.py` (used by the `combined` paper controller), untouched.
|
||||
- **No change to promotion** — it stays one-way + persistent (the historical "was validated" fact). Funding is
|
||||
the new conditional layer; the two are deliberately separate.
|
||||
- **No continuous health throttle / per-day de-sizing** (the rejected reactive path).
|
||||
- **No capital movement** (C). B3 only narrows B2a's input set.
|
||||
- **No per-fill / execution-decay signal** — that is the executed-vs-modeled reconciliation the C1
|
||||
`multistrat_exec` track already carries via its own gate.
|
||||
|
||||
## Affected code (initial map — the plan refines this)
|
||||
|
||||
- Modify: `src/fxhnt/application/reconciliation_gate.py` — add the machine-readable `category` to `GateVerdict`
|
||||
(PASS/DIVERGENCE/IMMATURE/EXECUTION_GAP); each existing return path sets its category. Verdicts unchanged.
|
||||
- Modify: `src/fxhnt/application/gate_policy.py` — add `defund_window=10`, `refund_window=15` to `GatePolicy`;
|
||||
bump `GATE_POLICY_VERSION`; the hash self-check pins the new value.
|
||||
- New: `src/fxhnt/application/funding_status.py` — `compute_funding(rows, backtest_ref, policy) ->
|
||||
FundingStatus`: the per-day gate recompute + the inception-replayed funding state machine (pure).
|
||||
- Modify: `src/fxhnt/application/allocation_ingest.py` `record_allocation` — filter the effective deploy set
|
||||
through `compute_funding` before `compute_allocation`; record the `strategy_funding` snapshot.
|
||||
- New: `strategy_funding` table (`cockpit_models.py`) + repo write/read (`forward_nav.py`).
|
||||
- Deferred (NOT this sub-project): cockpit visual surfacing of funding `state` — deferred to a future cockpit
|
||||
pass alongside allocation surfacing (which is itself not yet surfaced). The `strategy_funding` table is the
|
||||
read-model that pass will consume.
|
||||
- Reuse: `evaluate_reconciliation_gate` + `resolve_policy` (B1), the `BacktestRefProvider` incl. C1's
|
||||
`multistrat_exec → multistrat` alias, `ForwardNavRepo.nav_history` / `all_backtest_summaries` (A),
|
||||
`deploy_book`/`compute_allocation` (B2a, unchanged).
|
||||
```
|
||||
@@ -0,0 +1,214 @@
|
||||
# C3 — Execution Unification: bybit→CLI port + paper-validation envelope + real exec-venue recording
|
||||
|
||||
**Date:** 2026-07-10 (rev 2026-07-11 — unified scope)
|
||||
**Status:** Approved (design)
|
||||
**Depends on:** A (deterministic forward state), B2a (allocation), B3 (funding), C1 (Alpaca exec leg), C2 (Bybit exec leg) — all merged.
|
||||
|
||||
## Problem
|
||||
|
||||
Three things converge:
|
||||
|
||||
1. **Two execution legs, two architectures.** Multistrat execution is a CLI command
|
||||
(`execute-multistrat`) run by a k8s CronJob; the Bybit leg is a pair of **Dagster assets**
|
||||
(`bybit_testnet_reconcile` + `bybit_testnet_record`) run by the `bybit_testnet_job` schedule.
|
||||
Dagster should orchestrate *pipelining* (research, forward-recording); *execution* legs — which
|
||||
need a scheduled pod-to-pod / external broker path independent of the research nightly — belong
|
||||
on the CLI + CronJob pattern. The Bybit leg contradicts that boundary.
|
||||
|
||||
2. **The exec track cannot run continuously.** Both executed paper tracks (`multistrat_exec`,
|
||||
`bybit_4edge_exec`) are funded by B2a's real-capital allocation
|
||||
(`envelope = min(current_allocation[sid], nlv)`). B2a correctly sets `multistrat = $0` (edge
|
||||
<60 forward days), so the next `--execute` books one return day then **flattens**. Paper-execution
|
||||
*validation* is a different concern from real-capital *allocation*, yet they share one envelope.
|
||||
|
||||
3. **The executed-reality badge lies about the venue.** The cockpit renders `d.venue` — the sleeve's
|
||||
*configured* venue (`alpaca-paper`) — but the first run executed on **IBKR paper**. Since IBKR
|
||||
whole-share and Alpaca fractional have different friction (the whole point of the divergence
|
||||
readout), the badge must reflect where fills actually happened.
|
||||
|
||||
## Goals
|
||||
|
||||
- **Port the Bybit execution leg off Dagster into a CLI command** (`execute-bybit`), run by a new
|
||||
`fxhnt-bybit-rebalancer` CronJob — mirroring `execute-multistrat`. Fully remove the two Dagster
|
||||
exec assets, the `bybit_testnet_job`, its schedule, and the `_testnet_*` asset seams (clean cut,
|
||||
no dead code).
|
||||
- A **separate paper-validation funding envelope**, uniform across both legs via the identical
|
||||
`--paper-envelope` CLI flag, NOT gated by B2a — so paper books run continuously while B2a still
|
||||
governs real capital unchanged.
|
||||
- Record the **actual execution venue** on each exec track and surface it in the cockpit badge.
|
||||
- Arm both CronJobs to run the continuous paper experiment.
|
||||
|
||||
## Non-Goals / Hard Constraints
|
||||
|
||||
- **Real money errs false-WAIT.** The paper envelope MUST hard-refuse (place zero orders) on any
|
||||
non-paper/non-testnet account. It never reads or writes `strategy_allocation` → real-capital
|
||||
policy untouched by construction.
|
||||
- `--paper-envelope 0` (default) ⇒ behavior identical to today (B2a-gated).
|
||||
- No schema migration — `venue` rides in existing JSONB book-state.
|
||||
- **Faithful port, no behavior change** to the Bybit exec logic itself: the CLI path must preserve
|
||||
every safety guard the assets have (kill-switch, per-day NAV idempotency marker, open-orders
|
||||
resting guard, per-order `BrokerError` → structured gap row, low-confidence slippage exclusion,
|
||||
funding-cursor advance). This is a relocation, not a rewrite.
|
||||
- Out of scope: the deferred crypto re-entry-cost accounting noted in `bybit_exec_record.py`.
|
||||
|
||||
## Design
|
||||
|
||||
### Component 1 — Port the Bybit execution leg to a CLI command
|
||||
|
||||
**New application function** `src/fxhnt/application/bybit_testnet_run.py`:
|
||||
|
||||
```python
|
||||
def run_bybit_testnet(settings, *, paper_envelope: float, execute: bool,
|
||||
log: Callable[[str], None]) -> dict:
|
||||
"""Reconcile + record the bybit_4edge book on Bybit TESTNET in one process (the Dagster
|
||||
reconcile→record split collapses into a single CLI run, like plan_and_record for multistrat).
|
||||
Preserves every guard: kill-switch, NAV-marker idempotency, open-orders resting guard, per-order
|
||||
BrokerError→gap row, funding-cursor advance. Returns a summary dict (placed, equity, envelope,
|
||||
slippage means, exec_return, venue)."""
|
||||
```
|
||||
|
||||
Its body is the current `bybit_testnet_reconcile` + `bybit_testnet_record` logic concatenated,
|
||||
with three mechanical changes: `context.log.info/warning/error` → `log(...)`; the reconcile→record
|
||||
dict handoff becomes local variables; envelope is paper-aware (Component 2). The five
|
||||
`_testnet_*` factory seams (`_testnet_store/_universe/_unlock_events/_marks/_adapter`) **move** from
|
||||
`assets.py` into this module (still monkeypatchable seams for tests). When `execute` is False it runs
|
||||
the plan + slippage estimate but places no orders and writes no NAV (dry-run), mirroring
|
||||
`execute-multistrat`'s dry-run.
|
||||
|
||||
**New CLI command** `execute-bybit` in `cli.py`:
|
||||
|
||||
```python
|
||||
@app.command("execute-bybit")
|
||||
def execute_bybit(
|
||||
do_execute: bool = typer.Option(False, "--execute", help="place orders (default: dry-run)"),
|
||||
paper_envelope: float = typer.Option(0.0, "--paper-envelope",
|
||||
help="PAPER-only validation envelope in $ (bypasses B2a; refused on live). 0 = B2a-gated."),
|
||||
) -> None:
|
||||
... # get_settings(); run_bybit_testnet(...); echo the summary
|
||||
```
|
||||
|
||||
**Cleanup (clean cut):**
|
||||
- `assets.py`: delete `bybit_testnet_reconcile`, `bybit_testnet_record`, and the five `_testnet_*`
|
||||
helpers (moved to `bybit_testnet_run.py`).
|
||||
- `definitions.py`: remove the `bybit_testnet_reconcile`/`bybit_testnet_record` imports,
|
||||
`bybit_testnet_job` (`define_asset_job`), and `bybit_testnet_schedule` (`ScheduleDefinition`).
|
||||
Confirm `Definitions(...)` still loads with the remaining pipelining assets.
|
||||
- Tests referencing the assets (`tests/integration/test_record_bybit_exec_track.py` and any Dagster
|
||||
materialization tests of these assets) are re-pointed at `run_bybit_testnet` / `execute-bybit`.
|
||||
|
||||
### Component 2 — Paper-validation envelope (uniform CLI flag)
|
||||
|
||||
**Config (shared default):** `PaperValidationSettings` in `config.py` (env prefix
|
||||
`FXHNT_PAPER_VALIDATION_`), one field `envelope: float = 0.0`; exposed as `settings.paper_validation`.
|
||||
|
||||
**Both legs** take the identical `--paper-envelope <float>` CLI flag (default `0.0`). Resolution in
|
||||
each command: `paper_env = flag if flag > 0 else settings.paper_validation.envelope`.
|
||||
|
||||
**Envelope selection** — `plan_and_record` (multistrat) and `run_bybit_testnet` (bybit), via the
|
||||
shared-shape guard:
|
||||
|
||||
```
|
||||
if paper_envelope > 0: # paper-validation mode — bypasses B2a entirely
|
||||
if not account_is_paper: # SAFETY GATE — refuse, place NO orders
|
||||
raise ValueError("paper-envelope refused: account is not paper/testnet")
|
||||
envelope = min(paper_envelope, nlv) # nlv = alpaca NLV / bybit testnet equity
|
||||
else: # real-capital mode — UNCHANGED
|
||||
envelope = min(current_allocation[sid], nlv)
|
||||
```
|
||||
|
||||
`account_is_paper` comes from `AccountState.is_paper` (multistrat: `broker.account_state().is_paper`,
|
||||
already `"paper-api" in base_url` / `DU`-prefix) and `settings.bybit_exec.testnet` (bybit). In paper
|
||||
mode `current_allocation` is never read.
|
||||
|
||||
For bybit the helper lives in `bybit_exec_record.py`:
|
||||
`paper_or_b2a_crypto_envelope(repo, equity, paper_envelope, is_paper) -> float`.
|
||||
|
||||
### Component 3 — Real exec-venue recording
|
||||
|
||||
Derive `venue = f"{broker.name}-{'paper' if is_paper else 'live'}"` (→ `alpaca-paper`,
|
||||
`ibkr-paper`); bybit records the literal `"bybit-testnet"`. Persist inside the exec-position
|
||||
book-state (JSONB, no migration): `build_pos_state(..., venue="")` and
|
||||
`build_crypto_pos_state(..., venue="")` add `"venue"`. `record_exec_track` / `record_bybit_exec_track`
|
||||
gain a `venue: str = ""` param, written on every run (incl. inception).
|
||||
|
||||
### Component 4 — Cockpit surfacing
|
||||
|
||||
- `DashboardService.detail()` reads `self._repo.get_book_state(f"{twin}_pos").get("venue")` via a
|
||||
safe helper → new `StrategyDetail.exec_venue: str | None`.
|
||||
- `strategy.html` exec-tag renders `{{ d.exec_venue or d.venue }}` (recorded venue wins; configured
|
||||
venue is the fallback for pre-C3 anchors).
|
||||
|
||||
### Component 5 — Wiring & deploy
|
||||
|
||||
- `infra/k8s/jobs/fxhnt-alpaca-rebalancer.yaml`: append `--paper-envelope 1000000`; `suspend: false`.
|
||||
- **New** `infra/k8s/jobs/fxhnt-bybit-rebalancer.yaml`: copy the alpaca-rebalancer template
|
||||
(git-sync init + `fxhnt-cockpit` image + `FXHNT_OPERATIONAL_DSN`/`PGPASSWORD`), running
|
||||
`fxhnt execute-bybit --execute --paper-envelope 1000000`; env `FXHNT_BYBIT_EXEC_*` from the bybit
|
||||
secret; NetworkPolicy allows egress to the Bybit API (external 443) + Postgres 5432 + gitea. Ship
|
||||
armed only after the bybit exec secret is confirmed (else `suspend: true` until keys present).
|
||||
- Revert the `$1M` `strategy_allocation[multistrat]` override → cockpit shows the true B2a `$0`.
|
||||
- Cancel the queued IBKR paper orders on DU9600528 (one-off `ib_async` job).
|
||||
|
||||
## Data Flow (multistrat; bybit is the mirror)
|
||||
|
||||
```
|
||||
CronJob(--paper-envelope 1e6) → execute-multistrat
|
||||
→ AlpacaBroker.account_state() → nlv, is_paper=True
|
||||
→ plan_and_record(paper_envelope=1e6, account_is_paper=True, venue="alpaca-paper")
|
||||
→ is_paper ✓ → envelope = min(1e6, nlv) [B2a NOT read]
|
||||
→ rebalance_weights(...) → compute_exec_return(prior)
|
||||
→ record_exec_track(..., venue="alpaca-paper")
|
||||
→ run_track("multistrat_exec", return) [A's forward store]
|
||||
→ set_book_state("multistrat_exec_pos", {positions, prices, envelope, venue})
|
||||
Cockpit → detail("multistrat") → exec_venue="alpaca-paper" → badge renders it
|
||||
```
|
||||
|
||||
## Error Handling
|
||||
|
||||
- Paper envelope on a live/non-paper account → hard refuse, no orders, non-zero exit.
|
||||
- `paper_envelope=0` → today's B2a behavior (regression-tested).
|
||||
- Missing `venue` in an older book-state → `.get("venue")` → `None` → badge falls back to `d.venue`.
|
||||
- Bybit CLI preserves the assets' guards: kill-switch (zero orders), NAV-marker no-op, open-orders
|
||||
resting → abort with structured error, per-order `BrokerError` → gap row (never a crash).
|
||||
|
||||
## Testing
|
||||
|
||||
Unit:
|
||||
- `settings.paper_validation.envelope` default/env.
|
||||
- `plan_and_record`: paper>0+paper→min(paper,nlv), B2a not read; paper>0+live→refuse, no orders;
|
||||
paper=0→B2a. Same three for `paper_or_b2a_crypto_envelope`.
|
||||
- venue string derivation; `build_pos_state`/`build_crypto_pos_state` persist `venue`.
|
||||
- `run_bybit_testnet`: kill-switch → zero orders; NAV-marker → no-op; dry-run (`execute=False`) →
|
||||
no orders/no NAV; paper-envelope refuse on non-testnet. (Uses the monkeypatchable `_testnet_*`
|
||||
seams to avoid Timescale/ccxt.)
|
||||
- `_exec_pair`/`detail`: `exec_venue` from book-state; missing → `None`.
|
||||
|
||||
Local cockpit verify (per `feedback_verify_cockpit_locally_not_prod`): seed exec book-state venues,
|
||||
`scripts/dev-cockpit-local.sh`, Playwright-screenshot `/strategy/multistrat` + `/strategy/bybit_4edge`,
|
||||
**read** the badge shows the real venue.
|
||||
|
||||
## File Touchpoints
|
||||
|
||||
| File | Change |
|
||||
|---|---|
|
||||
| `src/fxhnt/config.py` | `PaperValidationSettings` + `settings.paper_validation` |
|
||||
| `src/fxhnt/application/bybit_testnet_run.py` | **new** — `run_bybit_testnet` + moved `_testnet_*` seams (ported asset logic) |
|
||||
| `src/fxhnt/application/multistrat_exec_record.py` | paper-envelope branch + safety gate; `venue` in `build_pos_state`/`record_exec_track` |
|
||||
| `src/fxhnt/application/bybit_exec_record.py` | `paper_or_b2a_crypto_envelope`; `venue` in `build_crypto_pos_state`/`record_bybit_exec_track` |
|
||||
| `src/fxhnt/adapters/orchestration/assets.py` | **delete** `bybit_testnet_reconcile`, `bybit_testnet_record`, `_testnet_*` helpers |
|
||||
| `src/fxhnt/adapters/orchestration/definitions.py` | **remove** bybit exec imports, `bybit_testnet_job`, `bybit_testnet_schedule` |
|
||||
| `src/fxhnt/cli.py` | `execute-multistrat --paper-envelope` + venue; **new** `execute-bybit` command |
|
||||
| `src/fxhnt/application/dashboard_service.py` | `detail` reads book-state `venue` → `exec_venue` |
|
||||
| `src/fxhnt/ports/dashboard.py` | `StrategyDetail.exec_venue` |
|
||||
| `src/fxhnt/adapters/web/templates/strategy.html` | exec-tag → `{{ d.exec_venue or d.venue }}` |
|
||||
| `infra/k8s/jobs/fxhnt-alpaca-rebalancer.yaml` | `--paper-envelope 1000000`, `suspend: false` |
|
||||
| `infra/k8s/jobs/fxhnt-bybit-rebalancer.yaml` | **new** CronJob for `execute-bybit` |
|
||||
|
||||
## Deploy Sequence (post-merge)
|
||||
|
||||
1. Push master → `argo-deploy-cockpit.sh`. No schema migration.
|
||||
2. Revert `strategy_allocation[multistrat]` → `$0` (or let the nightly do it).
|
||||
3. Cancel queued IBKR paper orders on DU9600528.
|
||||
4. Arm `fxhnt-alpaca-rebalancer` (`suspend: false`); confirm bybit exec secret, then arm
|
||||
`fxhnt-bybit-rebalancer`.
|
||||
5. Verify cockpit LOCALLY, then confirm on `dashboard.fxhnt.ai`.
|
||||
@@ -0,0 +1,191 @@
|
||||
# Configurable Capital Ceiling + CVaR-Aware Ex-Ante Sizing (B2a)
|
||||
|
||||
**Date:** 2026-07-11
|
||||
**Status:** Approved (design)
|
||||
**Depends on:** B2a allocation engine (merged). Real-money-adjacent — sizes real capital.
|
||||
|
||||
## Problem
|
||||
|
||||
Two gaps in the B2a allocation engine (`allocation_policy.py` + `allocation_engine.py`):
|
||||
|
||||
1. **The hard `$35k` capital ceiling is a hardcoded frozen-dataclass default.** It cannot be set for
|
||||
different modes (e.g. run paper at a high ceiling while live stays hard-gated at `$35k`) without a code
|
||||
edit. The earlier `$1M`-paper experiment had to override `strategy_allocation` directly in Postgres — a
|
||||
hack the ceiling design should make unnecessary. But the ceiling is *deliberately* part of the versioned,
|
||||
content-hashed policy (`allocation_policy_hash`) so "a ceiling change is audited"; any fix must stay
|
||||
inside that audit discipline (a raw un-hashed env override would break the deterministic-forward-state
|
||||
invariant).
|
||||
|
||||
2. **The ex-ante vol-target sizes by standard deviation only.** `full_history_vol` = annualised std. For
|
||||
most strategies std ≈ risk, but a fat-left-tail short-vol sleeve (the planned Alpaca defined-risk VRP
|
||||
sleeve) has risk std systematically *understates* — it looks low-vol right up until it spikes. Std-based
|
||||
vol-targeting would over-allocate such a sleeve. Sizing needs a tail-aware component.
|
||||
|
||||
## Goals
|
||||
|
||||
- Make the capital ceiling **configurable per mode (paper vs live)** while keeping it **inside the audited
|
||||
policy hash** (a change still versions + audits deterministically).
|
||||
- Add a **CVaR-aware leverage floor** so fat-left-tail sleeves are de-levered by tail risk, while normal
|
||||
(Gaussian-ish) strategies are sized exactly as today.
|
||||
- Preserve the load-bearing **ex-ante / full-history / anti-reactive** property (CVaR over full history,
|
||||
never a trailing window; no reactive de-sizing).
|
||||
|
||||
## Non-Goals / Hard Constraints
|
||||
|
||||
- **Real money errs false-WAIT.** The CVaR floor only ever *reduces* leverage; a noisy short-history CVaR
|
||||
therefore errs toward under-leverage (safe). The ceiling caps deployed gross; never widens it.
|
||||
- **Determinism/audit invariant holds.** The resolved ceiling + `cvar_alpha` fold into
|
||||
`allocation_policy_hash`; no un-hashed runtime state affects sizing.
|
||||
- No reactive de-sizing. Sizing derives from full-history statistics; the only drawdown-reactive element
|
||||
remains the existing rare aggregate killswitch (unchanged).
|
||||
|
||||
## Design
|
||||
|
||||
### Component 1 — Paper/live capital ceiling (audit-preserving)
|
||||
|
||||
**Settings (new)** in `config.py` — a small `AllocationSettings` (env prefix `FXHNT_ALLOCATION_`):
|
||||
```python
|
||||
class AllocationSettings(BaseSettings):
|
||||
model_config = SettingsConfigDict(env_prefix="FXHNT_ALLOCATION_")
|
||||
capital_ceiling_paper: float = 1_000_000.0 # FXHNT_ALLOCATION_CAPITAL_CEILING_PAPER
|
||||
capital_ceiling_live: float = 35_000.0 # FXHNT_ALLOCATION_CAPITAL_CEILING_LIVE
|
||||
```
|
||||
Exposed as `settings.allocation`.
|
||||
|
||||
**Selector = the existing `execution.allow_live` gate** (no new mode concept):
|
||||
```
|
||||
ceiling = settings.allocation.capital_ceiling_live if settings.execution.allow_live
|
||||
else settings.allocation.capital_ceiling_paper
|
||||
```
|
||||
Paper by default (`allow_live=False`) → high ceiling; flipping the fund live → the `$35k` hard gate. Reuses
|
||||
the gate the system already has for real-money execution, so paper/live ceiling and paper/live execution
|
||||
are governed by one switch.
|
||||
|
||||
**Resolution + audit.** A new builder resolves the policy from settings and injects the ceiling:
|
||||
```python
|
||||
def resolve_allocation_policy(settings) -> AllocationPolicy:
|
||||
ceiling = (settings.allocation.capital_ceiling_live if settings.execution.allow_live
|
||||
else settings.allocation.capital_ceiling_paper)
|
||||
return AllocationPolicy(capital_ceiling=ceiling) # other fields = fixed policy defaults
|
||||
```
|
||||
`allocation_policy_hash(resolved_policy, VERSION)` is computed on the **resolved** policy → the effective
|
||||
ceiling is in the hash → a ceiling change (or paper→live flip) versions + audits deterministically.
|
||||
`record_allocation` calls `resolve_allocation_policy(get_settings())` instead of using the module constant
|
||||
`ALLOCATION_POLICY` for the ceiling-bearing path. (`ALLOCATION_POLICY = AllocationPolicy()` may remain for
|
||||
tests/back-compat, but the live caller uses the resolved policy.)
|
||||
|
||||
### Component 2 — CVaR-aware ex-ante sizing (tail-aware leverage floor)
|
||||
|
||||
The K formula becomes a **min over three caps** — size by whichever binds tightest:
|
||||
```
|
||||
K_vol = kelly_fraction * target_vol / full_history_vol(book) # unchanged (annualised std, √365)
|
||||
K_cvar = kelly_fraction * target_cvar_daily / cvar_daily(book, alpha) # NEW tail-aware floor
|
||||
K = min(max_leverage, K_vol, K_cvar)
|
||||
```
|
||||
|
||||
**Gaussian-calibrated `target_cvar` (the no-op-for-normal property):**
|
||||
- CVaR-α of a standard normal = `φ(Φ⁻¹(α)) / α`. For α=0.10 this is `0.1755/0.10 = 1.7550` (call it `M`).
|
||||
- `target_cvar_daily = M * (target_vol / sqrt(365.0))` — the daily expected-shortfall a Gaussian book at
|
||||
`target_vol` would have (√365 to match `full_history_vol`'s `_ANN` convention).
|
||||
- For a Gaussian-ish book, `cvar_daily(book) ≈ M * σ_daily = M * full_history_vol(book)/√365`, so
|
||||
`K_cvar ≈ K_vol` — **the floor is a no-op**. `K_cvar < K_vol` only when the book's left tail is fatter
|
||||
than Gaussian (a VRP short-vol blowup). `target_cvar` is **derived, not a new tunable** — matches
|
||||
adaptive-not-tuned + max-with-floor.
|
||||
|
||||
**`cvar_daily(book, alpha)`** — a new pure helper in `allocation_engine.py`:
|
||||
- `book` = the **aggregate deploy book** daily returns (`deploy_book_returns`, the same equal-weight series
|
||||
the std vol-target + killswitch already use). Aggregate, not per-strategy: VRP's fat tail shows up in the
|
||||
aggregate (diversified with the rest), so the whole book de-levers when the tail fattens.
|
||||
- CVaR = the mean of the worst `ceil(alpha * n)` returns over the **FULL history** (ex-ante, never
|
||||
trailing), returned as a **positive loss magnitude** (`-mean(worst tail)`, floored at 0). Needs
|
||||
`n_tail = max(1, ceil(alpha*n))` observations; with `min_history_days=60` and α=0.10 that is ~6 obs.
|
||||
- Degenerate cases: `< 2` obs → return 0.0 (so `K_cvar` is unconstrained, mirroring `full_history_vol`'s
|
||||
`< 2 → 0.0` → the guard `if cvar > 1e-12 else skip that cap`). A book with a non-negative worst-tail (all
|
||||
gains in-sample) → CVaR ≤ 0 → skip the CVaR cap (don't divide by ~0; K_vol governs).
|
||||
|
||||
**Integration in `compute_allocation`** (after the killswitch flatten check, alongside the existing vol
|
||||
block):
|
||||
```python
|
||||
vol = full_history_vol(book)
|
||||
k_vol = min(policy.max_leverage, policy.kelly_fraction * policy.target_vol / vol) if vol > 1e-12 else 0.0 # PRESERVED: vol-degenerate → flatten
|
||||
cv = cvar_daily(book, policy.cvar_alpha)
|
||||
if cv > 1e-12:
|
||||
target_cvar_daily = _GAUSSIAN_ES[policy.cvar_alpha] * policy.target_vol / math.sqrt(365.0)
|
||||
k_cvar = min(policy.max_leverage, policy.kelly_fraction * target_cvar_daily / cv)
|
||||
else:
|
||||
k_cvar = policy.max_leverage # CVaR INACTIVE (no in-sample tail loss / <2 obs) → don't constrain below k_vol
|
||||
k = min(k_vol, k_cvar)
|
||||
return {s: k * w for s, w in base.items()}
|
||||
```
|
||||
`_GAUSSIAN_ES = {0.10: 1.7550}` (a small lookup keyed by the supported alpha; assert the policy's alpha is a
|
||||
key). **The `k_vol` degenerate fallback stays `0.0`** (unchanged from today — vol-degenerate flattens). The
|
||||
CVaR cap is a *pure floor*: when `cv` is ~0 the cap is inactive (`k_cvar = max_leverage`, so `k = k_vol`),
|
||||
never forcing `k` to 0 on its own. When `vol==0`, `k_vol=0 → k=0` regardless of CVaR (today's behavior).
|
||||
|
||||
### Component 3 — Policy versioning
|
||||
|
||||
- `AllocationPolicy` gains `cvar_alpha: float = 0.10`. `capital_ceiling` keeps its `35_000.0` default (for
|
||||
the bare `ALLOCATION_POLICY` constant / tests) but is injected by `resolve_allocation_policy` for the live
|
||||
path.
|
||||
- New field ⇒ **`ALLOCATION_POLICY_VERSION` 1 → 2**. The content-derived `allocation_policy_hash` self-check
|
||||
forces the bump (any allocation recorded under v1 vs v2 is distinguishable + audited).
|
||||
- `target_cvar` is derived at compute time (not stored) → not a separate hashed tunable; `cvar_alpha` and
|
||||
the resolved `capital_ceiling` ARE in the hash.
|
||||
|
||||
## Data Flow
|
||||
|
||||
```
|
||||
cockpit_forward (nightly) → record_allocation(dsn, equity)
|
||||
→ policy = resolve_allocation_policy(get_settings()) # ceiling: paper(1M) unless allow_live → live(35k)
|
||||
→ returns = {funded deploy sid: daily returns} # unchanged
|
||||
→ weights = compute_allocation(returns, policy) # eligibility→marginal→equal→killswitch→K
|
||||
K = min(max_leverage, K_vol, K_cvar) # CVaR floor de-levers only fat-tail books
|
||||
→ dollars = target_capital(weights, equity, policy) # gross capped at policy.capital_ceiling
|
||||
→ replace_allocations(...) ; hash = allocation_policy_hash(policy, 2) # resolved ceiling in the hash
|
||||
```
|
||||
|
||||
## Error Handling
|
||||
|
||||
- CVaR with `< 2` obs or non-negative worst-tail → CVaR skipped (K_vol governs); never a divide-by-zero.
|
||||
- Unsupported `cvar_alpha` (not in `_GAUSSIAN_ES`) → explicit error at compute (fail-closed, not a silent
|
||||
wrong multiplier).
|
||||
- Ceiling: `target_capital` already clamps `deployed = min(gross, ceiling)`; a `0`/non-finite ceiling → the
|
||||
existing `gross <= 0 → all-zero` guard applies (no capital deployed = safe).
|
||||
- Determinism: the resolved ceiling is in the hash; two runs with the same settings + returns → identical
|
||||
weights + hash.
|
||||
|
||||
## Testing
|
||||
|
||||
Unit (`tests/unit/`):
|
||||
- `resolve_allocation_policy`: `allow_live=False` → paper ceiling; `allow_live=True` → live ceiling; the two
|
||||
produce **different `allocation_policy_hash`** (audit).
|
||||
- `cvar_daily`: worst-10% mean over full history; positive-loss sign; `<2` obs → 0.0; all-gains tail → ≤0
|
||||
(skipped); full-history not trailing (adding an old bad day changes it).
|
||||
- `compute_allocation` CVaR no-op: a Gaussian synthetic book → `K_cvar ≈ K_vol` (weights within tolerance of
|
||||
the std-only result) — normal strategies unchanged.
|
||||
- `compute_allocation` CVaR bites: a fat-left-tail book (steady small gains + rare large losses, same std as
|
||||
a Gaussian control) → `K_cvar < K_vol` → strictly lower weights than the std-only result.
|
||||
- `min(k_vol, k_cvar)` picks the tightest; `vol==0 → k_vol=0 → k=0` (unchanged); `cv~0 → k_cvar` inactive → `k=k_vol`.
|
||||
- Version: `ALLOCATION_POLICY_VERSION == 2`; hash changes vs a v1 policy.
|
||||
|
||||
Existing B2a tests: must still pass (normal strategies' K unchanged because CVaR is a no-op for them);
|
||||
update any test asserting `ALLOCATION_POLICY_VERSION == 1` or the exact hash.
|
||||
|
||||
## File Touchpoints
|
||||
|
||||
| File | Change |
|
||||
|---|---|
|
||||
| `src/fxhnt/config.py` | `AllocationSettings` (paper/live ceilings) + `settings.allocation` |
|
||||
| `src/fxhnt/application/allocation_policy.py` | `AllocationPolicy.cvar_alpha`; `resolve_allocation_policy(settings)`; `ALLOCATION_POLICY_VERSION` 1→2; `_GAUSSIAN_ES` |
|
||||
| `src/fxhnt/application/allocation_engine.py` | `cvar_daily(book, alpha)`; `compute_allocation` K = min(max_lev, K_vol, K_cvar) |
|
||||
| `src/fxhnt/application/allocation_ingest.py` | `record_allocation` uses `resolve_allocation_policy(get_settings())` for the ceiling-bearing policy + hash |
|
||||
| `tests/unit/test_allocation_engine.py` (+ policy/config tests) | CVaR + ceiling + version tests; update v1/hash asserts |
|
||||
|
||||
## Deploy Notes
|
||||
|
||||
- No schema change (allocation rows already carry `policy_version` + `policy_hash`; they'll record v2 + the
|
||||
new hash on the next nightly).
|
||||
- Behavior change is intentional and bounded: normal deploy strategies size identically (CVaR no-op); a
|
||||
future VRP sleeve gets de-levered by the CVaR floor. Paper ceiling default `1M` means the paper book is no
|
||||
longer pinned at `$35k` — verify that's intended for the paper track before deploy (live stays `35k` via
|
||||
`allow_live`).
|
||||
146
docs/superpowers/specs/2026-07-11-vrp-modeled-sleeve-design.md
Normal file
146
docs/superpowers/specs/2026-07-11-vrp-modeled-sleeve-design.md
Normal file
@@ -0,0 +1,146 @@
|
||||
# Defined-Risk Equity-VRP Modeled Sleeve (SPY put-credit-spreads)
|
||||
|
||||
**Date:** 2026-07-11
|
||||
**Status:** SUPERSEDED (2026-07-12) by `2026-07-12-ibkr-consolidation-xsp-vrp-ucits-design.md` — the
|
||||
flat-VIX Black–Scholes synthetic below was a biased, un-validatable placeholder; VRP is rebuilt on **real
|
||||
OPRA option prices** (XSP put-credit-spreads on IBKR, not synthetic SPY). Kept for history only.
|
||||
**Depends on:** the forward engine (A), the reconciliation gate (B1), the CVaR-aware allocation engine (merged `edeb40b`), the multistrat forward-track pattern. Real-money-adjacent (it becomes a sized, gated sleeve).
|
||||
|
||||
## Problem / Goal
|
||||
|
||||
The defined-risk equity-VRP sleeve is the validated cross-asset diversifier (existence-test PASSED this session: `^PUT` proxy, defined-risk Sharpe ~0.8–1.0, diversifies *idiosyncratic* crypto crashes but co-crashes systemic risk-off). Get it into the book as a **recomputable, forward-tracked, CVaR-sized, gated strategy** — the same first-class treatment as the crypto edges and the multistrat book — **without** the Alpaca-options execution leg, which is blocked on the 401 paper keys and is a large untestable API surface.
|
||||
|
||||
**Goal:** a `vrp` strategy that produces a recomputable daily return series (a synthetic *defined-risk* SPY put-credit-spread ladder priced from public SPY + VIX data), wired into the nightly forward engine + reconciliation gate + CVaR-aware allocation, and surfaced in the cockpit. The Alpaca-options execution twin is explicitly deferred.
|
||||
|
||||
## Non-Goals / Hard Constraints
|
||||
|
||||
- **No Alpaca-options execution leg now** (blocked on 401 keys; a later `vrp_exec` twin reconciling executed-vs-modeled, mirroring C1/C2).
|
||||
- **Defined-risk ONLY** — the modeled series is a *put-credit-spread* with a bounded max loss (width − credit), never a naked put-write. This is the load-bearing safety property: std under-sizes short-vol's fat tail, so the sleeve is safe only because (a) the CVaR floor de-levers it and (b) defined-risk bounds its max loss.
|
||||
- **Real money errs false-WAIT**: recompute-from-frozen-PIT (stable record); reconciliation gate before any capital; sized small (10–20% risk) via the existing engine; **it is NOT a systemic hedge** (co-crashes risk-off).
|
||||
- Verify the cockpit **locally** (read the NAV/gate) per `feedback_verify_cockpit_locally_not_prod`.
|
||||
- Faithful-but-approximate: the BS+VIX synthetic ignores skew and real slippage — the *deferred* exec-twin will measure that divergence; the modeled track is the honest baseline, not a claim of live fills.
|
||||
|
||||
## Design
|
||||
|
||||
### Component 1 — the synthetic VRP strategy (`src/fxhnt/application/vrp_book.py`)
|
||||
|
||||
Pure, testable Black–Scholes helpers + a `VrpStrategy` implementing the `ForwardStrategy` protocol
|
||||
(`advance(last_date, extra) -> (list[(iso_date, fractional_return)], extra)`), recomputing the full daily
|
||||
NAV each call (recompute mode).
|
||||
|
||||
**Rules (approved params):** weekly entry, **~30 DTE**, short-leg **~20-delta** put, fixed **$5 SPY width**,
|
||||
**manage at 50% profit or close 1 trading day before expiry**, held as a **ladder** (~4–5 concurrent spreads
|
||||
so the book is always-on → a smooth daily series).
|
||||
|
||||
**Pricing (Black–Scholes, VIX as IV, r=0):** for spot `S`, IV `σ = VIX/100`, `τ = DTE/365`:
|
||||
- Short strike from the ~20-delta target: put `|Δ| = N(-d1) = 0.20` ⇒ `d1 = Φ⁻¹(0.80) = 0.8416` ⇒
|
||||
`K_short = S·exp(σ²τ/2 − 0.8416·σ√τ)` (rounded to the nearest $1 SPY strike).
|
||||
- `K_long = K_short − 5`.
|
||||
- Entry `credit = BS_put(K_short, σ, τ) − BS_put(K_long, σ, τ)` where
|
||||
`BS_put(K) = K·N(−d2) − S·N(−d1)` (r=0), `d1 = [ln(S/K)+σ²τ/2]/(σ√τ)`, `d2 = d1 − σ√τ`.
|
||||
- **Daily mark:** each day re-price the open spread with updated `S`, decremented `τ`, current `σ` ⇒
|
||||
`mtm_value`; unrealized P&L = `credit − mtm_value`.
|
||||
- **Management:** close (realize P&L) when `mtm_value ≤ 0.5·credit` (50% profit) OR at DTE=1; then open a
|
||||
fresh 30-DTE spread the next weekly slot.
|
||||
- **Return basis (defined-risk):** per spread `margin = (5·100) = $500` max loss. The ladder's **daily
|
||||
fractional return = Σ(daily MTM change over open spreads) / (N_open · margin)** — return per dollar of
|
||||
defined-risk margin. (A fixed nominal capital = N·margin; no leverage inside the sleeve — the allocation
|
||||
engine sizes it.)
|
||||
|
||||
Same `VrpStrategy` recomputes the **full 2016–2026 history** (for the backtest ref) and the **recent forward
|
||||
window** — identical to how `MultiStratStrategy` produces both.
|
||||
|
||||
### Component 2 — data (SPY + VIX, PIT-frozen)
|
||||
|
||||
Fetch **SPY** and **^VIX** daily closes via the existing Yahoo path (`YahooDataProvider.fetch(Market(symbol=…))`
|
||||
takes any symbol in the URL — confirmed). Mirror `multistrat_nav`: **snapshot today's closes PIT into the
|
||||
warehouse** and recompute off the FROZEN PIT series (stable against Yahoo's retroactive adjustments). VIX is
|
||||
an index (not adjusted), SPY uses adjusted close.
|
||||
|
||||
### Component 3 — forward-track + gate wiring
|
||||
|
||||
- **`vrp_nav` Dagster asset** (`assets.py`, mirrors `multistrat_nav`): snapshot SPY+VIX PIT →
|
||||
`run_track(repo, "vrp", VrpStrategy(...))` → `persist_backtest_ref=True` (publishes the basis-matched ref
|
||||
via `_persist_track_backtest_ref(context, "vrp", build_strategy)` = `VrpStrategy().advance(None, {})`).
|
||||
- **`definitions.py`:** add `vrp_nav` to `combined_book_forward_job.selection` + `Definitions.assets`; add it
|
||||
to `cockpit_forward`'s `deps`.
|
||||
- **`registry.py`:** `"vrp"` entry — `display_name` "Equity VRP (SPY put-credit-spreads, defined-risk)",
|
||||
`sleeve` "equity-vol", `venue` "alpaca-options" (reference-only until the exec twin lands), `tier`
|
||||
"research", `execution` "paper", `record_mode` "recompute", `gate_spec = {gate_type: "reconciliation",
|
||||
min_forward_days: 21}`, `promote_on_pass` False initially (turn on after we've watched the forward
|
||||
reconcile — a deliberate human step, given it's real-capital-adjacent).
|
||||
- **`migration_builders.py`:** a `builders["vrp"] = ("vrp_state", VrpStrategy-builder)` entry (recompute
|
||||
track needs a builder so `migrate-forward-anchors` can seed its anchor — the CRITICAL-1 pattern).
|
||||
|
||||
### Component 4 — sizing (the end-to-end payoff)
|
||||
|
||||
No new sizing code: once `vrp` is a funded deploy/promoted strategy, `record_allocation` →
|
||||
`compute_allocation` sizes it automatically, and the **CVaR floor we just merged de-levers its fat left tail**
|
||||
(the synthetic VRP series is fat-left-tailed by construction). Add a unit test asserting a VRP-shaped return
|
||||
series produces `K_cvar < K_vol` (the two sub-projects connect end-to-end). Defined-risk bounds its
|
||||
contribution to the aggregate killswitch drawdown.
|
||||
|
||||
### Component 5 — cockpit + local verify
|
||||
|
||||
`vrp` surfaces as a research forward-tracked sleeve (equity-vol venue). Verify LOCALLY: seed/recompute the
|
||||
sleeve into the local dev DB, `scripts/dev-cockpit-local.sh`, screenshot `/strategy/vrp` — read the NAV,
|
||||
gate status (WAIT/building N/21), and the backtest ref.
|
||||
|
||||
### Component 6 — DEFERRED (explicitly out of scope now)
|
||||
|
||||
The Alpaca-options **execution leg** (`vrp_exec` twin: options-chain fetch, ~20-delta strike selection, atomic
|
||||
`mleg` put-credit-spread orders, the `execute-vrp` CLI + a `fxhnt-vrp-rebalancer` CronJob, reconcile
|
||||
executed-vs-modeled slippage). Blocked on valid Alpaca paper keys + a large new options-API surface — a later
|
||||
sub-project, built the same way as the C1/C2 exec twins once keys land.
|
||||
|
||||
## Data Flow
|
||||
|
||||
```
|
||||
combined_book_forward_job (nightly)
|
||||
→ vrp_nav: snapshot SPY+VIX PIT closes → recompute VrpStrategy (defined-risk SPY put-spread ladder, BS+VIX)
|
||||
→ run_track(repo, "vrp", VrpStrategy) → forward_summary/nav + persist_backtest_ref
|
||||
→ cockpit_forward: evaluate the reconciliation gate (forward vs ref), surface in the cockpit
|
||||
Nightly record_allocation (when vrp promotes to deploy):
|
||||
→ compute_allocation includes vrp; K = min(k_vol, k_cvar) → CVaR floor de-levers the fat-tail sleeve
|
||||
```
|
||||
|
||||
## Error Handling
|
||||
|
||||
- Missing/short SPY or VIX history → `advance` returns `[]` (inception, no rows) — the track warms, never crashes.
|
||||
- BS numerical guards: `σ>0`, `τ>0`, `S>0`; a degenerate day (σ→0) → skip/flat that day.
|
||||
- Recompute stability: frozen PIT series → the record doesn't drift on Yahoo re-adjustment (multistrat lesson).
|
||||
- Gate: no backtest ref → the reconciliation gate degrades to a bare timer (known failure mode) — the
|
||||
`persist_backtest_ref=True` in `vrp_nav` prevents it; a test asserts the ref row is written.
|
||||
|
||||
## Testing
|
||||
|
||||
Unit:
|
||||
- BS helpers: put price vs a known reference; `d1/d2`; 20-delta strike inversion (`|Δ(K_short)| ≈ 0.20`).
|
||||
- `VrpStrategy.advance`: on a synthetic SPY+VIX path, produces a plausible defined-risk return series (bounded
|
||||
per-spread loss = width; positive credit-decay on calm days; capped loss on a crash day); `advance(None,{})`
|
||||
returns the full inception series; recompute is deterministic (same input → same series).
|
||||
- The fat-left-tail property: the produced series has `cvar_daily/std > 1.755` (fatter than Gaussian) →
|
||||
feeding it to `compute_allocation` yields `K_cvar < K_vol` (connects to the CVaR engine).
|
||||
Integration:
|
||||
- `vrp_nav` asset recomputes + writes a `forward_summary` + a `backtest_summary` (ref) row; `definitions.py`
|
||||
loads with `vrp_nav` in the job + cockpit_forward deps; `migrate-forward-anchors` seeds the `vrp` anchor.
|
||||
Local cockpit verify (Playwright): `/strategy/vrp` renders NAV + gate + ref.
|
||||
|
||||
## File Touchpoints
|
||||
|
||||
| File | Change |
|
||||
|---|---|
|
||||
| `src/fxhnt/application/vrp_book.py` | **new** — BS helpers + `VrpStrategy` (defined-risk SPY put-spread ladder) |
|
||||
| `src/fxhnt/adapters/orchestration/assets.py` | **new** `vrp_nav` asset (mirror `multistrat_nav`) + `cockpit_forward` dep |
|
||||
| `src/fxhnt/adapters/orchestration/definitions.py` | add `vrp_nav` to the job + `Definitions.assets` |
|
||||
| `src/fxhnt/registry.py` | `"vrp"` entry (recompute, reconciliation gate, promote_on_pass False) |
|
||||
| `src/fxhnt/adapters/orchestration/migration_builders.py` | `builders["vrp"]` (anchor seeding) |
|
||||
| tests: `test_vrp_book.py` (unit), `test_vrp_nav_asset.py` / definitions + migration (integration) | BS + strategy + fat-tail→CVaR + wiring |
|
||||
|
||||
## Deploy Notes
|
||||
|
||||
- No schema change. Next nightly runs `vrp_nav` (recompute + ref); the gate shows building N/21.
|
||||
- `promote_on_pass` is **off** — `vrp` will NOT auto-fund even on gate PASS; promotion to deploy (and thus
|
||||
CVaR-sized capital) is a deliberate human step after watching the forward reconcile. Real money errs
|
||||
false-WAIT.
|
||||
- The Alpaca-options exec twin is a separate future sub-project (needs valid paper keys first).
|
||||
@@ -0,0 +1,295 @@
|
||||
# IBKR Consolidation — OPRA-backed XSP-VRP sleeve + UCITS-mapped multistrat routing
|
||||
|
||||
**Date:** 2026-07-12
|
||||
**Status:** Approved (design)
|
||||
**Supersedes:** `docs/superpowers/specs/2026-07-11-vrp-modeled-sleeve-design.md` (the synthetic
|
||||
flat-VIX Black–Scholes VRP approach — parked because the synthetic was a biased, un-validatable
|
||||
placeholder; this spec rebuilds VRP on **real OPRA option prices**).
|
||||
**Depends on:** the forward engine (`run_track`, recompute/observed), the reconciliation gate (B1),
|
||||
the CVaR-aware allocation engine (merged `edeb40b`), the multistrat forward-track + exec-twin pattern
|
||||
(C1/C2/C3), the Databento adapter (`src/fxhnt/adapters/data/databento.py`).
|
||||
|
||||
## Problem / Goal
|
||||
|
||||
Consolidate the fund's *tradfi* execution onto **IBKR** (EUR-native IBKR Ireland, the venue the operator
|
||||
already trusts), dropping Alpaca entirely. Crypto stays structurally on **Bybit** — our crypto edges are
|
||||
perpetual-futures-based (funding harvest, broad-alt cross-sectional momentum, unlock shorts, the Bybit
|
||||
4-edge book) and IBKR offers only spot crypto (~10 coins, no funding, no shorting), so *zero* crypto
|
||||
strategies can run there. The fund is inherently two venues: IBKR (tradfi) + Bybit (crypto).
|
||||
|
||||
Two forces shape the tradfi side:
|
||||
|
||||
1. **PRIIPs** blocks a Dutch *retail* IBKR account from buying US-domiciled ETFs (SPY/IEF/GLD/PDBC/DBMF —
|
||||
no EU KID). The multistrat book must *execute* on **UCITS-equivalent** ETFs. But EU-venue price history
|
||||
is short (Databento `EPRL.DOM` Euronext starts 2023-03-28) — so we **model on the US underlyings**
|
||||
(long clean history) and **execute on UCITS**, measuring the divergence with an exec twin.
|
||||
2. **OPRA** (verified empirically: `OPRA.PILLAR` carries real XSP *and* SPX option bars back to
|
||||
**2013-04-01**, `ohlcv-1d`, ~$0.88/mo for XSP) **un-parks the equity-VRP sleeve**. VRP was shelved
|
||||
because we had no real option data; now we can build a *faithful* defined-risk XSP put-credit-spread
|
||||
backtest on real prices — true credits, real skew, through COVID + 2022. **XSP** (cash-settled 1/10-SPX,
|
||||
European exercise, Section-1256, EU-retail-available on IBKR) is cleaner than the parked Alpaca-SPY plan
|
||||
(no early assignment, no pin risk).
|
||||
|
||||
**Goal:** one fused spec, one venue-consolidation goal, **two bounded components**, execution **coded but
|
||||
shipped SUSPENDED** throughout (neither XSP options nor UCITS-Euronext can be paper-executed on the current
|
||||
US-style `DU9600528` account):
|
||||
|
||||
- **Component A (centerpiece):** a `vrp` strategy producing a recomputable daily defined-risk XSP
|
||||
put-credit-spread return series from **real OPRA data**, wired into the forward engine + reconciliation
|
||||
gate + CVaR-aware allocation + cockpit. Execution twin (`execute-vrp`, IBKR XSP `mleg` orders) coded,
|
||||
suspended.
|
||||
- **Component B (thin):** UCITS-legal *execution routing* for the existing multistrat book — the modeled
|
||||
track is unchanged (still models US underlyings); B adds a US→UCITS instrument map + an `--venue ucits`
|
||||
execute path, coded and suspended.
|
||||
|
||||
## Non-Goals / Hard Constraints
|
||||
|
||||
- **No live execution this round.** Both exec legs ship SUSPENDED (the C1/C2/C3 pattern). Arming is a later
|
||||
operational step once an IBKR account with the right permissions (options; Euronext) exists.
|
||||
- **No change to the multistrat modeled track.** `multistrat_nav` keeps modeling US underlyings. B is
|
||||
execution routing only. The currently-armed US-ETF IBKR-paper track (`multistrat_exec`, DU9600528, $1M)
|
||||
is left running untouched — `--venue ucits` is an additive *second* mode, default stays `us`.
|
||||
- **No change to the crypto book / Bybit.** Out of scope by nature.
|
||||
- **Defined-risk ONLY** for VRP — the modeled series is a *put-credit-spread* with bounded max loss
|
||||
(width − credit), never a naked put-write. Load-bearing safety property: std under-sizes short-vol's fat
|
||||
tail, so the sleeve is safe only because (a) the CVaR floor de-levers it and (b) defined-risk bounds max
|
||||
loss.
|
||||
- **Determinism spine:** state is a deterministic function of committed policy + observed data. OPRA bars
|
||||
are **frozen into a PIT warehouse table** and recomputed off the frozen rows — never re-hitting OPRA
|
||||
live (determinism + respects the `$0.50` cost-guard). Mirrors the equity PIT-snapshot discipline.
|
||||
- **Real money errs false-WAIT:** recompute-from-frozen-PIT; reconciliation gate before any capital;
|
||||
`promote_on_pass=False` for `vrp` (deliberate human promote); sized small via the existing engine; VRP is
|
||||
**NOT a systemic hedge** (co-crashes systemic risk-off — it diversifies *idiosyncratic* crypto crashes).
|
||||
- **Verify the cockpit LOCALLY** (read the NAV/gate in a real browser) per
|
||||
`feedback_verify_cockpit_locally_not_prod`.
|
||||
- Every review Minor is a latent bug — fix in-loop.
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
Component A — VRP sleeve (NEW alpha, the centerpiece)
|
||||
OPRA.PILLAR (real XSP option chains, 2013→)
|
||||
→ xsp_option_bars PIT warehouse table (one-time backfill + nightly append; deterministic)
|
||||
→ VrpStrategy.advance(): put-call-parity forward + IV-inversion 20Δ strike + real-mark defined-risk P&L
|
||||
→ run_track("vrp") → forward + reconciliation gate + CVaR sizing → cockpit
|
||||
→ [SUSPENDED] execute-vrp: IBKR XSP mleg put-credit-spread orders + vrp_exec twin
|
||||
|
||||
Component B — UCITS multistrat routing (PRIIPs-legal live)
|
||||
Modeled track UNCHANGED (multistrat_nav models US underlyings — long clean history)
|
||||
→ UCITS instrument map (US ticker → Euronext UCITS ticker), config
|
||||
→ [SUSPENDED] execute-multistrat --venue ucits: IBKR whole-share UCITS routing + ibkr-ucits exec twin
|
||||
```
|
||||
|
||||
The two components are asymmetric on purpose: A is a genuinely new options-data pipeline feeding a
|
||||
newly-validatable strategy; B is thin because the multistrat modeled track already models US underlyings
|
||||
correctly, so B is only the execution *routing*. The plan builds A's substance first, then B's routing.
|
||||
|
||||
## Component A — VRP on real OPRA data
|
||||
|
||||
### A.1 — Data path (`databento.py` options extension + PIT warehouse)
|
||||
|
||||
Extend `DatabentoDataProvider` with an options path on `OPRA.PILLAR`, used two ways:
|
||||
|
||||
- **Chain discovery (entry days):** `definition` schema via **parent** symbology (`XSP.OPT`) → all listed
|
||||
contracts (expiries, strikes, put/call) active on a date. Filter to the target ~30-DTE expiry and puts.
|
||||
- **Per-contract marks (daily):** `ohlcv-1d` by **raw OSI symbol** (e.g. `XSP YYMMDDP00470000`) for the
|
||||
specific held contracts.
|
||||
|
||||
The current `_resolve()` raises `NotImplementedError` for options; add an `AssetClass.OPTION` branch and two
|
||||
new methods: `fetch_option_chain(underlying_parent, as_of) -> list[OptionContract]` and
|
||||
`fetch_option_bars(osi_symbols, start, end) -> dict[osi, PriceSeries]`. Both honor the existing
|
||||
`get_cost` guard; the one-time 2013→ backfill runs in a Job under a raised cap, chunked by month.
|
||||
|
||||
**PIT freeze:** a new warehouse table `xsp_option_bars(date, osi_symbol, expiry, strike, right, open, high,
|
||||
low, close)` (+ a small `xsp_option_bars` repo with `upsert_bars`, `bars_for(osi_symbols, start, end)`,
|
||||
`chain_asof(date, dte_window)`). One-time backfill for the ref; nightly append for the forward. The strategy
|
||||
recomputes off **frozen rows**, never re-hitting OPRA live.
|
||||
|
||||
### A.2 — Underlying forward + IV inversion (self-contained, OPRA-internal)
|
||||
|
||||
No SPX/XSP index level exists in any Databento dataset (OPRA is options-only), and SPY as a proxy injects
|
||||
dividend/tracking drift into the inversion. Instead derive the **implied forward from put-call parity** on
|
||||
the ATM pair: `F = K_atm + e^{rT}(C_atm − P_atm)` (also yields the implied rate `r`). Then per put, invert
|
||||
Black–Scholes on the real OPRA close (Newton, seeded from a Brenner–Subrahmanyam ATM guess, bracketed
|
||||
fallback) → implied vol → BS put delta `Δ = −N(−d1)` under forward `F`. A degenerate day (no arb-free
|
||||
solution, `τ≤0`, or a crossed/zero mark) → that contract is skipped; if the ladder can't be marked that day,
|
||||
the day books flat (track warms, never crashes).
|
||||
|
||||
### A.3 — Strike selection & the strategy (`src/fxhnt/application/vrp_book.py`)
|
||||
|
||||
Pure, testable BS/IV helpers + `VrpStrategy` implementing `advance(last_date, extra) -> (list[(iso_date,
|
||||
fractional_return)], extra)` (recompute mode). **Rules (approved):**
|
||||
|
||||
- **Weekly** entry (fixed weekday).
|
||||
- **~30 DTE** XSP expiry (nearest listed).
|
||||
- Short leg = strike nearest **|Δ| = 0.20** (from A.2 inversion); long leg a fixed **5-XSP-point width**
|
||||
below. Per-spread `margin = 5 × 100 = $500`; max loss = `(5 − credit) × 100`.
|
||||
- Held as a **ladder of ~4–5** staggered concurrent spreads (weekly entries) → always-on smooth daily
|
||||
series.
|
||||
- **Daily mark:** re-price each open spread on real OPRA closes (A.1); unrealized P&L = `credit − current
|
||||
spread value`.
|
||||
- **Management:** close (realize) at **50% profit** OR **1 trading day before expiry**. XSP is cash-settled
|
||||
European → no early assignment / no pin risk; close-early only avoids settlement-day noise.
|
||||
- **Return basis (defined-risk):** daily fractional return = `Σ(daily MTM change over open spreads) /
|
||||
(N_open × margin)` — return per dollar of defined-risk margin. Fixed nominal capital = `N × margin`; no
|
||||
leverage inside the sleeve (the allocation engine sizes it).
|
||||
|
||||
The same `VrpStrategy` recomputes the **full 2013→ backtest ref** (one-time, off frozen OPRA) and the recent
|
||||
**forward window** (nightly) — mirroring how `MultiStratStrategy` produces both.
|
||||
|
||||
### A.4 — Forward-track, gate, sizing (rides merged infra)
|
||||
|
||||
- **`vrp_nav` Dagster asset** (`assets.py`, mirrors `multistrat_nav`): append today's OPRA bars to the PIT
|
||||
table → `run_track(repo, "vrp", VrpStrategy(...), today, at)` → `persist_backtest_ref=True` (publishes the
|
||||
basis-matched real ref via the one-time full-history `VrpStrategy().advance(None, {})`).
|
||||
- **`definitions.py`:** add `vrp_nav` to `combined_book_forward_job.selection` + `Definitions.assets`; add it
|
||||
to `cockpit_forward`'s deps.
|
||||
- **`registry.py`:** `"vrp"` entry — `display_name` "Equity VRP (XSP put-credit-spreads, defined-risk)",
|
||||
`sleeve` "equity-vol", `venue` **"ibkr-xsp"**, `tier` "research", `execution` "paper", `state_file`
|
||||
"vrp_state", `record_mode` "recompute", `gate_spec = {gate_type: "reconciliation", min_forward_days: 21}`,
|
||||
**`promote_on_pass` False** (real-capital-adjacent → deliberate human promote after watching it
|
||||
reconcile).
|
||||
- **`migration_builders.py`:** `builders["vrp"] = ("vrp_state", _build_vrp)` so `migrate-forward-anchors`
|
||||
seeds the recompute anchor (the CRITICAL-1 recompute-track pattern).
|
||||
- **Sizing:** no new sizing code. Once promoted, `record_allocation → compute_allocation` sizes `vrp`; the
|
||||
**CVaR floor (merged `edeb40b`) de-levers the fat-left tail** — now genuinely fat because the ref carries
|
||||
real COVID/2022 crash marks, not a Gaussian synthetic. Defined-risk bounds its contribution to the
|
||||
aggregate killswitch drawdown.
|
||||
|
||||
### A.5 — Execution twin (coded, SUSPENDED)
|
||||
|
||||
- **`IbkrBroker` options path** (new): construct an atomic `mleg` combo (BAG) — sell the 20Δ short put, buy
|
||||
the wing — with a negative limit price = the target net credit. Requires IBKR options-trading permission.
|
||||
- **`execute-vrp` CLI:** fetch the live XSP chain, select the 20Δ/5-pt spread, place the `mleg` order,
|
||||
record executed fills; venue recorded `ibkr-xsp`. Guards mirror `execute-multistrat` (paper-envelope
|
||||
refuse on non-paper, per-order gap rows, kill-switch, idempotency marker).
|
||||
- **`fxhnt-vrp-rebalancer` CronJob** (weekly), `suspend: true` — arm only once the account has options
|
||||
permission.
|
||||
- Records a **`vrp_exec` twin** reconciling executed-vs-modeled slippage — surfaced in the cockpit
|
||||
"Executed reality" block, exactly like the multistrat/bybit twins.
|
||||
|
||||
## Component B — UCITS multistrat routing (thin)
|
||||
|
||||
### B.1 — UCITS instrument map (`config.py`)
|
||||
|
||||
A typed `US ticker → Euronext-UCITS ticker` map (env-overridable `FXHNT_UCITS_MAP`), defaults refinable when
|
||||
the account exists:
|
||||
|
||||
| Sleeve | Modeled (US, unchanged) | Executed (UCITS, suspended) |
|
||||
|---|---|---|
|
||||
| S&P 500 | SPY | CSPX (fallback VUAA) |
|
||||
| 7–10y UST | IEF | IBTM (resolved on Euronext `EPRL.DOM`) |
|
||||
| Gold | GLD | SGLN (fallback IGLN) |
|
||||
| Broad commodities | PDBC | ICOM-type commodity UCITS |
|
||||
| Managed futures | DBMF | DBMF.L (iMGP DBi Managed Futures UCITS) |
|
||||
|
||||
The map is **execution-only**. The modeled series always uses US underlyings; the map only rewrites which
|
||||
symbols the (suspended) execute path sends. This keeps determinism clean — modeled = US, executed = UCITS,
|
||||
the exec twin measures the tracking + FX + fee divergence.
|
||||
|
||||
### B.2 — Execute path (coded, SUSPENDED)
|
||||
|
||||
- Add `--venue us|ucits` to `execute-multistrat` (`cli.py`, default **`us`** — today's armed US-ETF
|
||||
IBKR-paper track is untouched). `--venue ucits` applies the B.1 map to `FUND_INSTRUMENTS` before building
|
||||
orders and records venue **`ibkr-ucits`**.
|
||||
- An **unmapped sleeve → the execute path refuses** (never routes an unmapped order).
|
||||
- No new sizing code: the `IbkrBroker` whole-share path already exists (`supports_fractional = False`,
|
||||
int-rounded `MarketOrder`). UCITS prices ($10–100/share) make whole-share rounding ~1–2%, acceptable for
|
||||
risk-parity at $35k.
|
||||
- A **suspended** `fxhnt-ucits-rebalancer` CronJob on a **monthly** schedule (per-order-min friction control
|
||||
at $35k). The modeled `multistrat_nav` stays daily; only execution cadence is monthly.
|
||||
- Records an `ibkr-ucits` exec twin (via the existing `multistrat_exec` twin path + `_safe_book_venue`
|
||||
badge) reconciling executed-UCITS-vs-modeled-US.
|
||||
|
||||
**Cutover (later, operational):** arm `fxhnt-ucits-rebalancer` + `suspend: true` the US-ETF
|
||||
`fxhnt-ibkr-rebalancer`, once a Euronext-enabled account exists. The modeled series never moves.
|
||||
|
||||
## Data Flow
|
||||
|
||||
```
|
||||
combined_book_forward_job (nightly)
|
||||
→ vrp_nav: append XSP OPRA bars to xsp_option_bars PIT → recompute VrpStrategy (real-mark defined-risk
|
||||
XSP put-spread ladder) → run_track(repo, "vrp", VrpStrategy) → forward_summary/nav + persist_backtest_ref
|
||||
→ multistrat_nav: UNCHANGED (models US underlyings)
|
||||
→ cockpit_forward: evaluate the reconciliation gates (forward vs real ref), surface in the cockpit
|
||||
Nightly record_allocation (when vrp promotes to deploy):
|
||||
→ compute_allocation includes vrp; K = min(k_vol, k_cvar) → CVaR floor de-levers the fat-left tail
|
||||
Execution (both SUSPENDED):
|
||||
→ execute-vrp (IBKR XSP mleg) → vrp_exec twin
|
||||
→ execute-multistrat --venue ucits (IBKR UCITS whole-share) → ibkr-ucits exec twin
|
||||
```
|
||||
|
||||
## Error Handling
|
||||
|
||||
- **VRP data:** missing/short OPRA history or a degenerate IV inversion (no arb-free solution, `τ≤0`,
|
||||
crossed/zero mark) → skip that contract; if the ladder can't be marked, book that day flat (track warms,
|
||||
never crashes). OPRA `get_cost` breach → refuse (backfill Job runs chunked under a raised cap).
|
||||
- **VRP gate:** no `backtest_summary` ref → the reconciliation gate silently degrades to a bare timer (known
|
||||
failure mode) — `persist_backtest_ref=True` in `vrp_nav` prevents it; a test asserts the ref row is
|
||||
written.
|
||||
- **VRP recompute stability:** frozen PIT bars → the record doesn't drift on any OPRA revision (the
|
||||
multistrat lesson, applied to options).
|
||||
- **UCITS:** an unmapped sleeve → `execute-multistrat --venue ucits` refuses (never routes an unmapped
|
||||
order); `--venue ucits` on a non-Euronext account → IBKR rejects → recorded as a gap row (suspended
|
||||
anyway).
|
||||
|
||||
## Testing
|
||||
|
||||
**VRP unit (`tests/unit/test_vrp_book.py`):**
|
||||
- BS/IV helpers: put price vs a known reference; `d1/d2`; IV-inversion round-trip (price → IV → price within
|
||||
tol); put-call-parity forward on a synthetic ATM pair.
|
||||
- 20Δ strike selection: on a synthetic chain, the chosen short strike has `|Δ| ≈ 0.20`.
|
||||
- `VrpStrategy.advance`: on a synthetic OPRA path, a plausible defined-risk series (bounded per-spread loss =
|
||||
width; positive credit-decay on calm days; capped loss on a crash day); `advance(None,{})` returns the full
|
||||
inception series; recompute is deterministic (same frozen input → same series).
|
||||
- Fat-left-tail property: the produced series has `cvar_daily/std > 1.755` → feeding it to
|
||||
`compute_allocation` yields `K_cvar < K_vol` (connects to the merged CVaR engine).
|
||||
|
||||
**VRP integration:**
|
||||
- `databento.py` options path: `fetch_option_chain` / `fetch_option_bars` parse OPRA `definition` +
|
||||
`ohlcv-1d` into the expected shapes (against a recorded fixture, not a live pull); cost-guard refuses over
|
||||
cap.
|
||||
- `xsp_option_bars` repo: `upsert_bars` / `bars_for` / `chain_asof` round-trip.
|
||||
- `vrp_nav` asset recomputes + writes `forward_summary` + a real `backtest_summary` (ref) row;
|
||||
`definitions.py` loads with `vrp_nav` in the job + `cockpit_forward` deps; `migrate-forward-anchors` seeds
|
||||
the `vrp` anchor.
|
||||
|
||||
**UCITS unit (`tests/unit/test_ucits_routing.py`):**
|
||||
- Map application: `--venue ucits` rewrites `FUND_INSTRUMENTS` → UCITS symbols; `--venue us` (default) is
|
||||
unchanged.
|
||||
- Unmapped-sleeve refusal.
|
||||
- Whole-share rounding on UCITS prices.
|
||||
|
||||
**Local cockpit verify (Playwright):** `/strategy/vrp` renders NAV + gate (building N/21) + the real
|
||||
backtest ref; multistrat detail shows the `ibkr-ucits` badge path via `_safe_book_venue`. Read the numbers —
|
||||
never claim "works" off synthetic events or a prod deploy.
|
||||
|
||||
## File Touchpoints
|
||||
|
||||
| File | Change |
|
||||
|---|---|
|
||||
| `src/fxhnt/adapters/data/databento.py` | **options path** — `AssetClass.OPTION` branch, `fetch_option_chain`, `fetch_option_bars` (OPRA.PILLAR, definition + ohlcv-1d) |
|
||||
| `src/fxhnt/adapters/persistence/` (+ migration) | **new** `xsp_option_bars` PIT table + repo (`upsert_bars`/`bars_for`/`chain_asof`) |
|
||||
| `src/fxhnt/application/vrp_book.py` | **new** — BS/IV helpers + put-call-parity forward + `VrpStrategy` (real-mark defined-risk XSP put-spread ladder) |
|
||||
| `src/fxhnt/adapters/orchestration/assets.py` | **new** `vrp_nav` asset (mirror `multistrat_nav`) + `cockpit_forward` dep |
|
||||
| `src/fxhnt/adapters/orchestration/definitions.py` | add `vrp_nav` to the job + `Definitions.assets` |
|
||||
| `src/fxhnt/registry.py` | `"vrp"` entry (venue `ibkr-xsp`, recompute, reconciliation gate, `promote_on_pass` False) |
|
||||
| `src/fxhnt/adapters/orchestration/migration_builders.py` | `builders["vrp"]` anchor seeding |
|
||||
| `src/fxhnt/adapters/broker/ibkr.py` | **options `mleg`** combo path (suspended exec) |
|
||||
| `src/fxhnt/cli.py` | `execute-vrp` command; `--venue us\|ucits` on `execute-multistrat` |
|
||||
| `src/fxhnt/config.py` | `FXHNT_UCITS_MAP` US→UCITS instrument map settings |
|
||||
| `infra/k8s/jobs/fxhnt-vrp-rebalancer.yaml` | **new** suspended weekly CronJob + NetworkPolicy (OPRA/IBKR egress) |
|
||||
| `infra/k8s/jobs/fxhnt-ucits-rebalancer.yaml` | **new** suspended monthly CronJob |
|
||||
| tests | `test_vrp_book.py`, options-adapter + `xsp_option_bars` + `vrp_nav` integration, `test_ucits_routing.py`, Playwright cockpit |
|
||||
|
||||
## Deploy Notes
|
||||
|
||||
- **Schema change:** the `xsp_option_bars` PIT table (a migration). The one-time OPRA backfill (2013→) runs
|
||||
as a separate Job under a raised cost cap, chunked by month — not on cockpit startup (no heavy DDL/backfill
|
||||
in app startup per `feedback_no_heavy_ddl_in_app_startup`).
|
||||
- Next nightly runs `vrp_nav` (append + recompute + ref); the gate shows building N/21.
|
||||
- `promote_on_pass` is **off** — `vrp` will NOT auto-fund even on gate PASS; promotion to deploy (and thus
|
||||
CVaR-sized capital) is a deliberate human step after watching the forward reconcile. Real money errs
|
||||
false-WAIT.
|
||||
- Both exec CronJobs ship `suspend: true`. Arming = later operational step once an IBKR account with options
|
||||
/ Euronext permission exists.
|
||||
@@ -0,0 +1,147 @@
|
||||
# Phase 0b — Venue Consolidation to Bybit + IBKR (Design)
|
||||
|
||||
> Sub-project of Phase 0 (uniform two-venue foundation). Follows 0a (unified DB-backed backtest→ref
|
||||
> architecture, branch `feat/unified-db-backtest-architecture`, pushed/unmerged). **0b depends on 0a** —
|
||||
> it modifies 0a's xsfunding ref wiring (see §7). Start 0b from the 0a branch or after 0a merges.
|
||||
|
||||
## Goal
|
||||
|
||||
Make **Bybit** (crypto) and **IBKR** (tradfi) the *only* venues in the system — for both **execution and
|
||||
data**. Remove the dead Binance/Alpaca **execution** surface, migrate the crypto edges that *actually work
|
||||
on Bybit* onto Bybit data, retire the ones that don't, and remove Binance **entirely** (execution + data).
|
||||
|
||||
## Governing principle (operator)
|
||||
|
||||
> "We can't *trade* Binance — but that doesn't mean we can't run strats on Bybit. Keep the strats that
|
||||
> **actually work on Bybit**; understand, per strat, whether the edge survives on Bybit."
|
||||
|
||||
A crypto strat survives 0b only if **(a)** Bybit lists the instruments it needs **and** **(b)** the edge
|
||||
holds on Bybit data. This was checked empirically per edge (§3).
|
||||
|
||||
## Current state (verified this session)
|
||||
|
||||
- **Two data worlds coexist today:** the Binance-fed `features` warehouse (default `warehouse_table`) feeds
|
||||
the standalone crypto *forward tracks* (`xsfunding_nav`, `crypto_tstrend_nav`, `unlock_nav`,
|
||||
`stablecoin_rotation_nav`); the Bybit-fed `bybit_features` feeds the **deploy book** `bybit_4edge`
|
||||
(sleeves tstrend/unlock/xsfunding/positioning) + the `positioning` standalone track.
|
||||
- **The crypto edges are duplicated:** xsfunding/tstrend/unlock exist as *both* a Binance-data standalone
|
||||
track *and* a Bybit-data sleeve in `bybit_4edge`. The Binance standalone tracks are research-era artifacts;
|
||||
the memory finding is that the **Binance broad book is a mirage; the Bybit liquid book is the real edge**.
|
||||
- **Only one live executor:** `fxhnt-ibkr-rebalancer` CronJob (active). `fxhnt-alpaca-rebalancer`,
|
||||
`fxhnt-bybit-rebalancer`, and the old `multistrat-rebalancer` are suspended.
|
||||
|
||||
## §3 — Per-edge Bybit-survival verdicts (measured, not assumed)
|
||||
|
||||
Leave-one-out marginal contribution to the equal-weight `bybit_4edge` book (2023–2026, cost-netted;
|
||||
book Sharpe +3.47, maxDD −8.9%) + crash behavior:
|
||||
|
||||
| Edge | Standalone Sharpe / maxDD | Marginal to book (Sharpe / maxDD) | Worst-20-day behavior | **Verdict** |
|
||||
|---|---|---|---|---|
|
||||
| **xsfunding** | +3.55 / −3.4% | +0.28 / −3.3pp | **+0.20%/day (hedges)** | **Migrate → Bybit track** |
|
||||
| **unlock** | +1.04 / −23.8% | **+0.34 / −3.7pp** | −0.71%/day | **Migrate → Bybit track** (uncorrelated, protective) |
|
||||
| positioning | +2.79 / −14.3% | +1.22 / −3.8pp | −0.95%/day | Keep (already Bybit) |
|
||||
| **crypto_tstrend** | +0.74 / −54.3% | **−0.21 / +1.2pp** | **−6.46%/day (amplifies)** | **Retire standalone track** (drag; not a hedge here) |
|
||||
| **stablecoin_rotation** | n/a | not in book | n/a | **Retire** (does not exist/work on Bybit — §3a) |
|
||||
|
||||
### §3a — Stablecoin does not port to Bybit
|
||||
The strat trades peg-reversion on **FDUSD/USDT, TUSD/USDT, USDP/USDT** (Binance spot). Bybit listing check:
|
||||
FDUSD **NOT LISTED**, USDP **NOT LISTED**, TUSD listed. Bybit's *entire* stablecoin/USDT spot set is
|
||||
{TUSD, PYUSD, USDC, USDE} — a poor peg-reversion basis (USDC/USDT deviations are tiny; USDE is a structural
|
||||
synthetic dollar), and the primary edge driver (FDUSD's Binance zero-fee microstructure) is Binance-specific.
|
||||
→ **Retire.** A Bybit-native stablecoin edge would be a *fresh research probe with its own gate*, explicitly
|
||||
out of 0b scope.
|
||||
|
||||
## Architecture — what changes (grouped)
|
||||
|
||||
### A. Migrate xsfunding + unlock → standalone Bybit forward tracks
|
||||
**Template = `positioning`.** `build_positioning_forward_nav(repo, store, sleeve='positioning', …)`
|
||||
(assets.py:177) builds a single-sleeve Bybit forward track from `sleeve_returns_from_store` /
|
||||
`naive_eqwt_daily_returns` on `bybit_features`; its reconciliation ref comes from the Bybit book persist
|
||||
(`bybit_4edge_backtest_summary(sleeves=['positioning'])`, one of `_BYBIT_BOOK_SIDS`).
|
||||
|
||||
- Generalize to `build_bybit_sleeve_forward_nav(repo, store, sleeve, …)` (or add xsfunding/unlock the same
|
||||
way positioning is built). Repoint `xsfunding_nav` and `unlock_nav` assets to construct the **Bybit**
|
||||
single-sleeve track over `TimescaleFeatureStore(dsn, table='bybit_features')` — dropping
|
||||
`XsFundingForward(BinanceXsFundingLive())` and the Binance `features` unlock construction.
|
||||
- Registry `xsfunding` / `unlock`: `venue` → `bybit-perp`; data basis → Bybit; `backtest.kind` moves to the
|
||||
Bybit-book report basis (like `positioning`) — xsfunding **from `sleeve`** (0a), unlock **from
|
||||
`recompute-replay`** (0a) — and both sids join `_BYBIT_BOOK_SIDS` in `_persist_bybit_book` so their refs
|
||||
are written from the Bybit sleeve series (§7).
|
||||
- Forward re-inception: the Bybit track is a NEW basis, so xsfunding/unlock re-inception on cutover
|
||||
(deterministic-forward-state; a re-inception, not silent). Document the `migrate-forward-anchors` step.
|
||||
|
||||
### B. Retire crypto_tstrend (standalone) + stablecoin_rotation
|
||||
- Delete `crypto_tstrend_nav` (assets.py:669) and `stablecoin_rotation_nav` (assets.py:691) assets +
|
||||
their registry entries. Delete `stablecoin_strategy.py` / `stablecoin_runner.py` (no other consumer).
|
||||
- **crypto_tstrend stays a sleeve in the `bybit_4edge` deploy book** — 0b retires only the redundant
|
||||
*standalone Binance track*, not the book sleeve. (The −0.21 marginal-drag finding is a **separate
|
||||
portfolio-composition question**, flagged in §Out-of-scope, NOT decided by 0b.)
|
||||
|
||||
### C. Remove the dead Alpaca execution surface (keep the Broker port, IBKR sole impl)
|
||||
- Delete `src/fxhnt/adapters/broker/alpaca.py` (`AlpacaBroker`), `AlpacaSettings` (config.py:45) + the
|
||||
`alpaca` field (config.py:185).
|
||||
- Remove the `--broker` option and its `alpaca` branches from `execute-multistrat` (cli.py:257–325) and any
|
||||
peer command (cli.py:206–223) — **IBKR becomes implicit** (the sole impl behind the retained Broker port).
|
||||
- Delete manifests `infra/k8s/jobs/fxhnt-alpaca-rebalancer.yaml`,
|
||||
`infra/k8s/secrets/alpaca-credentials.yaml`; delete the in-cluster `multistrat-rebalancer` CronJob
|
||||
(superseded by `fxhnt-ibkr-rebalancer`; locate its manifest) and the `alpaca-credentials` cluster secret.
|
||||
- **Keep the Broker port/interface** with `IBKRBroker` as the only implementation (clean seam; a future
|
||||
venue plugs in without reintroducing an abstraction).
|
||||
|
||||
### D. Remove the Binance execution surface
|
||||
- Delete `src/fxhnt/adapters/exchange/binance_ccxt.py`, the `crypto-rebalance` / `crypto-positions` /
|
||||
`crypto-flatten` CLI commands (cli.py:4746/4768/4776), and the Binance coupling in
|
||||
`crypto_execution.py`. Remove the execution-only parts of `BinanceSettings`.
|
||||
|
||||
### E. Remove Binance DATA entirely (after A + B land)
|
||||
- Delete the Binance data adapters: `binance_funding_history.py`, `binance_spot_history.py`,
|
||||
`binance_xsfunding_live.py`.
|
||||
- Delete/repoint the Binance ingest assets `crypto_bars` / `crypto_funding` / `crypto_spot` (assets.py:263/
|
||||
322/345) — with A/B done, nothing live reads the `features` table, so the Binance ingest chain and the
|
||||
`features` table are retired. Verify no residual reader (e.g. `bybit_edges_eval.py`'s Binance imports —
|
||||
remove or repoint) before dropping the table.
|
||||
|
||||
### F. Remove the vestigial registry `state_file` field
|
||||
The `state_file` field on every registry entry (the old JSON-tracker-on-PVC path) is vestigial since the
|
||||
single-DB-SSOT forward engine — `_run_paper_tracker` documents it as "retained ONLY for the one-time
|
||||
migration seed, unused at steady state." Remove the `state_file` field from all registry entries and drop
|
||||
the parameter from `_run_paper_tracker` / any `entry["state_file"]` reads. **Guard:** the JSON-based
|
||||
`ForwardTracker` (+ its `.bak` crash-recovery) is still used by the standalone `fxhnt forward-track` CLI —
|
||||
that command supplies its OWN path and must keep working; only the *registry field* + the Dagster-asset
|
||||
plumbing of it are removed (verify the CLI path is independent before deleting).
|
||||
|
||||
## §7 — 0a interaction (must-carry)
|
||||
0a Task 6 wired the `xsfunding` reconciliation ref via `get_feature_store(s)` (= `features` = **Binance**),
|
||||
deliberately, to match the *then*-live `XsFundingForward(BinanceXsFundingLive())`. 0b **moves xsfunding onto
|
||||
Bybit**, so that wiring must flip: xsfunding's (and unlock's) ref now comes from the **Bybit** sleeve series
|
||||
(`bybit_features`), via `_persist_bybit_book` / the `backtest-refs` CronJob, exactly like `positioning`.
|
||||
Net: after 0b, `_persist_track_backtest_ref`'s sleeve-kind branch no longer serves any Binance sid (the
|
||||
only inline sleeve sid, xsfunding, becomes a Bybit report-book sid). The 0a invariant test still must pass.
|
||||
|
||||
## Out of scope (explicit)
|
||||
- **Re-composing the `bybit_4edge` deploy book** (e.g. dropping tstrend from the book on its −0.21 marginal /
|
||||
crash-amplifier finding) — a real-capital portfolio decision, separate from venue cleanup. Flagged for a
|
||||
later, deliberate decision with the crash-protection question (does trend protect out-of-sample?) resolved.
|
||||
- **A Bybit-native stablecoin edge** — fresh research + its own gate, not a migration.
|
||||
- Alpaca/Bybit rebalancer *code* that is venue-generic and reused by IBKR stays; only Alpaca/Binance-specific
|
||||
execution is removed.
|
||||
|
||||
## Risks & verification
|
||||
- **Real-capital gate safety:** xsfunding/unlock re-inception on cutover must be a *visible* re-inception
|
||||
(deterministic-forward-state), never a silent reset; their new Bybit refs must exist before the gate can
|
||||
PASS (0a invariant + the `migrate-forward-anchors` post-deploy step). Errs false-WAIT, never false-FUND.
|
||||
- **No orphaned reads:** removing the `features` table must follow a repo-wide check that no live path reads
|
||||
it (the §E verify). A guard test (like 0a's `test_no_live_duckdb_files`) asserting no live Binance-data
|
||||
consumer remains.
|
||||
- **Cockpit:** verify locally (real browser) that the migrated xsfunding/unlock Bybit tracks render and the
|
||||
retired tracks (tstrend-standalone/stablecoin) disappear cleanly — no 500s, no dangling nav rows.
|
||||
- **Full suite green** + whole-branch review before merge.
|
||||
|
||||
## Success criteria
|
||||
1. Crypto is 100% Bybit (data + execution): xsfunding/unlock on Bybit tracks; tstrend-standalone + stablecoin
|
||||
retired; `bybit_4edge` book unchanged.
|
||||
2. No Binance code remains (execution *or* data adapters/ingests); `features` table retired; a guard test
|
||||
proves it.
|
||||
3. No Alpaca code/manifests/secret remain; IBKR is the sole equity executor behind the retained Broker port.
|
||||
4. The 0a reconciliation invariant still holds; xsfunding/unlock reconcile against **Bybit** refs.
|
||||
5. Cockpit verified locally; full suite green; whole-branch review clean.
|
||||
@@ -0,0 +1,201 @@
|
||||
# Unified DB-backed backtest architecture (Phase 0a)
|
||||
|
||||
**Date:** 2026-07-13
|
||||
**Status:** Approved (design)
|
||||
**Scope:** sub-project 0a of the "uniform Bybit+IBKR, DB-backed foundation" — the unified professional
|
||||
backtest system. Sibling 0b (Binance/Alpaca-execution deprecation) is a separate, simpler spec.
|
||||
**Depends on:** the forward engine (deterministic-forward-state, single-Postgres SSOT), the reconciliation
|
||||
gate (`reconciliation_gate.py`), the `AnalyticalStore` port (`ports/repository.py`) + its two implementations
|
||||
(`TimescaleFeatureStore` live / `DuckDbFeatureStore`), the nightly Dagster asset graph.
|
||||
|
||||
## Problem / Goal
|
||||
|
||||
The platform produces its `backtest_summary` reconciliation-gate reference through **three fragmented
|
||||
mechanisms**, and that fragmentation causes real, currently-live bugs:
|
||||
|
||||
1. **Inline** `_persist_track_backtest_ref(context, name, build_strategy)` (`assets.py:363`) — replays the
|
||||
strategy over full history in the nightly `*_nav` asset when `persist_backtest_ref=True`. Used by
|
||||
`multistrat`, `vrp`, `crypto_tstrend`, `stablecoin_rotation`, `unlock`.
|
||||
2. **The `bybit-persist-sleeve-ret` Job** (`cli.py:740`, `backtest_summaries_from_report`) — the ONLY writer
|
||||
of the `backtest_summary` ref for the deploy-tier `bybit_4edge` / `bybit_4edge_levered` / `positioning`.
|
||||
It is a plain `batch/v1` Job, **not scheduled** (run by hand/weekly; it OOM-killed the 4Gi Dagster daemon,
|
||||
so it was deliberately pulled out of the nightly graph).
|
||||
3. **The B3 `equity_backtest_runner`** (`application/equity_backtest_runner.py`) — a survivorship-free
|
||||
equity-factor backtest against a **separate DuckDB warehouse on its own PVC**, whose verdicts are keyed to
|
||||
equity-factor constructions that have since been **retired from the registry**. Orphaned: no live gated
|
||||
consumer.
|
||||
|
||||
**Confirmed live bug (A):** `xsfunding` — a flagship crypto edge — has `gate_type: reconciliation` but its
|
||||
`xsfunding_nav` asset does NOT set `persist_backtest_ref`, and nothing else writes a `backtest_summary` row
|
||||
for it. `reconciliation_gate.py` therefore permanently returns WAIT ("no backtest reference to reconcile
|
||||
against"). The edge can never gate-PASS. *(`sixtyforty` uses an **absolute** gate — `{"min_days":20,...}` —
|
||||
so it needs no ref; it is NOT affected.)*
|
||||
|
||||
**Confirmed live bug (B):** the deploy-tier Bybit book's gate reference comes only from the unscheduled Job
|
||||
(#2). If it lapses, `bybit_4edge` / `positioning` silently cannot gate-PASS.
|
||||
|
||||
**Persistence is split** between Timescale (the live operational + feature SSOT — the live nav assets already
|
||||
read via `get_feature_store()` → `TimescaleFeatureStore`) and file-based stores (DuckDB warehouse files,
|
||||
`worst_basis.parquet`) used by orphaned/retired research.
|
||||
|
||||
**Goal:** a uniform, professional backtest system where
|
||||
1. **every reconciliation-gated strategy produces its `backtest_summary` via ONE consistent path** — so a
|
||||
gated strategy without a ref cannot exist (a structural *invariant*, not per-strategy patches). Fixes A,
|
||||
B, and the F-orphaning by construction.
|
||||
2. **Timescale is the single durable store**; DuckDB is repositioned as an **ephemeral columnar engine** that
|
||||
reads from Timescale and writes verdicts back — no persistent DuckDB files / PVC in the live path.
|
||||
3. heavy backtests run in a **dedicated, resourced, scheduled** step (own memory), never the OOM-prone shared
|
||||
Dagster daemon.
|
||||
|
||||
## Non-Goals / Hard Constraints
|
||||
|
||||
- **Real money errs false-WAIT:** a backtest that fails or produces no ref → the gate WAITs, NEVER
|
||||
false-PASSes. Absence of a ref is safe (WAIT); the invariant just makes absence impossible for a properly
|
||||
onboarded strategy.
|
||||
- **Single-SSOT / deterministic-forward-state** preserved — Timescale remains the one truth; the DuckDB
|
||||
engine is stateless (ephemeral per run).
|
||||
- **Out of scope:** reviving equity-factor research now (the architecture *supports* it — an equity strat
|
||||
would land its data in Timescale and plug into the unified path — but we do NOT migrate the Tiingo equity
|
||||
DuckDB warehouse in 0a); the Binance/Alpaca-execution cleanup (0b); the cockpit Ops/Fleet UI (Phase 2).
|
||||
- Verify the cockpit **locally** (the gate un-sticks for `xsfunding`) per `feedback_verify_cockpit_locally_not_prod`.
|
||||
- Every review Minor is a latent bug — fix in-loop.
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
Dagster ingest ─────────────► TIMESCALE ◄──────────── the ONE durable store
|
||||
(bars/features/funding/ (SSOT: raw data + backtest_summary verdicts
|
||||
xsp_option_bars, unchanged) + forward_nav/summary + allocation/funding/book_state)
|
||||
▲ │
|
||||
unified backtest step ─────────┤ │ reconciliation gate reads the ref
|
||||
(registry-driven dispatch) │ ▼
|
||||
│ recompute-replay ──────┘ forward_ingest: gate → promotion → allocation
|
||||
│ (cheap: replay ForwardStrategy over the Timescale panel)
|
||||
│ report-based ──── ephemeral in-mem DuckDB: ATTACH Timescale (postgres ext,
|
||||
│ (heavy) verified OK on duckdb 1.5.3) → columnar compute → verdict
|
||||
└──────────────────► upsert_backtest_summary → TIMESCALE
|
||||
│
|
||||
▼
|
||||
COCKPIT (read-only) — gate PASS-able for every edge
|
||||
```
|
||||
|
||||
The **`AnalyticalStore` port is the seam**: strategies and backtests ask the port for panels; the port is
|
||||
backed by Timescale live, or by an ephemeral DuckDB-over-Timescale for heavy compute. Swapping the backend is
|
||||
an adapter choice, not a rewrite.
|
||||
|
||||
## Component 1 — The unified backtest→ref contract (the heart)
|
||||
|
||||
A single registry-driven dispatch replaces the three mechanisms:
|
||||
|
||||
- Each `STRATEGY_REGISTRY` entry declares a **backtest kind** in its `definition` (new field
|
||||
`backtest: {"kind": "recompute-replay" | "report" | "none"}`; `none` = absolute-gated benchmarks like
|
||||
`sixtyforty` that need no ref, made EXPLICIT rather than silently ref-less).
|
||||
- A `backtest_ref(strategy_id, store: AnalyticalStore) -> BacktestSummary | None` function dispatches on the
|
||||
kind:
|
||||
- **`recompute-replay`** (multistrat, vrp, crypto_tstrend, stablecoin_rotation, unlock, **xsfunding**):
|
||||
build the strategy, `advance(None, {})` over full history off the Timescale panel → `derive_nav` →
|
||||
`BacktestSummary`. This generalizes today's `_persist_track_backtest_ref`. **Fixes A**: `xsfunding` gets a
|
||||
ref via this path (add `backtest.kind = "recompute-replay"` + a `build_strategy` entry).
|
||||
- **`report`** (bybit_4edge, bybit_4edge_levered, positioning): compute the sleeve-return report and
|
||||
`backtest_summaries_from_report` → `BacktestSummary`. Generalizes the `bybit-persist-sleeve-ret` logic,
|
||||
moved INTO the unified path and scheduled. **Fixes B**.
|
||||
- **`none`** (sixtyforty): no ref; the gate is absolute — asserted explicitly.
|
||||
- **The invariant:** a test iterates `STRATEGY_REGISTRY` and asserts every `gate_type: reconciliation`
|
||||
strategy has a non-`none` backtest kind + a working provider. A new reconciliation-gated strategy without a
|
||||
backtest declaration FAILS the test — so onboarding a strategy that can never gate-PASS is impossible.
|
||||
This is the structural fix that turns "POC with silent gaps" into a professional invariant.
|
||||
|
||||
## Component 2 — DuckDB as an ephemeral engine (no persistent files)
|
||||
|
||||
- `report`-kind (heavy) backtests use an **ephemeral in-memory DuckDB**: `duckdb.connect()` →
|
||||
`INSTALL postgres; LOAD postgres; ATTACH '<timescale dsn>'` (feasibility verified: duckdb 1.5.3 postgres
|
||||
extension loads) → columnar scan/aggregate the needed slice from Timescale → verdict → written back to
|
||||
Timescale via `upsert_backtest_summary`. The DuckDB connection is per-run and discarded — no file, no PVC.
|
||||
- The `AnalyticalStore` port already abstracts panel reads; a thin `TimescaleDuckDbEngine` adapter provides
|
||||
the ATTACH-and-scan path for heavy report backtests while the cheap recompute-replay stays on the plain
|
||||
Timescale reads it already uses.
|
||||
- **Retire** the persistent DuckDB files (`warehouse.duckdb`, `analytical.duckdb`) and `worst_basis.parquet`
|
||||
from the live path (see Component 3 for the classification).
|
||||
|
||||
## Component 3 — File-store inventory + migrate/park classification
|
||||
|
||||
| store / module | role | classification | action in 0a |
|
||||
|---|---|---|---|
|
||||
| `TimescaleFeatureStore` (crypto panels, `bybit_features`, Yahoo-PIT, `xsp_option_bars`) | LIVE feature/data reads | already Timescale ✓ | none |
|
||||
| `equity_backtest_runner` + B3 DuckDB warehouse (`/backtest-data/*.duckdb` PVC) | equity-factor backtest | ORPHANED (no live strat; retired constructions) | **park** — keep code; equity strats plug into the unified path (data→Timescale) when actually pursued. Retire the PVC/Job. |
|
||||
| `worst_basis*` / `bybit_carry_revalidate` / `bybit_worst_basis_ingest` + `worst_basis.parquet` | cross-venue carry research | tied to the RETIRED `xvenue_carry` | **park** — research-only, revivable; migrate to Timescale only if carry revives |
|
||||
| `warehouse.duckdb` / `analytical.duckdb` (dev/opt-in `DuckDbFeatureStore`) | dev/test embedded backend | opt-in (prod uses postgres) | keep as the dev/test backend behind the port; not in the live path |
|
||||
| `paper_book` / `paper_sleeves` / `paper_backfill` DuckDB imports | paper-book/replay services | **VERIFY** (prod uses `get_feature_store()`=postgres; the DuckDB import may be a dev/fallback) | the plan MUST confirm whether these read DuckDB in the LIVE path; if live-adjacent → migrate to the Timescale port; if dev-only → leave |
|
||||
|
||||
Net: **no persistent DuckDB file / PVC in the live path**; Timescale is the one store. Parked research keeps
|
||||
its code (revivable) but its file-stores are decommissioned from the running system.
|
||||
|
||||
## Component 4 — Orchestration + the OOM lesson
|
||||
|
||||
- The unified backtest step runs in **dedicated, own-memory, scheduled** compute — NOT the 4Gi Dagster daemon
|
||||
(which OOM'd on the bybit sleeve-ret). Two acceptable homes: (a) a resourced Dagster op with a k8s/
|
||||
per-op-executor, or (b) a scheduled `CronJob` running `fxhnt backtest-refs --all` writing to Timescale
|
||||
(mirrors the batched-OPRA-backfill / compare-precompute pattern). **Fixes B**: the ref becomes a scheduled
|
||||
guarantee, not a manual Job.
|
||||
- **Cheap `recompute-replay` refs stay inline** in the nightly `*_nav` assets (they already run there and are
|
||||
light) — only the **heavy `report` refs** move to the resourced scheduled step. So the nightly daemon load
|
||||
is unchanged; the OOM-prone work is where it belongs (own memory).
|
||||
- Dagster ingest → Timescale is unchanged. The gate/promotion/allocation chain in `cockpit_forward` reads the
|
||||
refs exactly as today — it just now finds one for every gated strategy.
|
||||
|
||||
## Data Flow
|
||||
|
||||
```
|
||||
nightly Dagster job (unchanged ingest → *_nav → cockpit_forward → gate/promote/allocation)
|
||||
+ recompute-replay refs inline (xsfunding NOW included → gate un-sticks)
|
||||
scheduled unified-backtest step (own memory):
|
||||
for each registry strategy with backtest.kind == "report":
|
||||
ephemeral DuckDB ATTACH Timescale → compute report → upsert_backtest_summary (Timescale)
|
||||
gate (cockpit_forward): reconciliation vs the ref → PASS-able for every edge → cockpit
|
||||
```
|
||||
|
||||
## Error Handling
|
||||
|
||||
- A backtest provider that raises / returns None → NO ref written → the gate WAITs (safe, false-WAIT). The
|
||||
step logs the failure per strategy; one bad strategy never aborts the others.
|
||||
- The invariant test guarantees a *declared* provider exists for every reconciliation-gated strategy; a
|
||||
runtime failure still degrades to WAIT, never false-PASS.
|
||||
- The ephemeral DuckDB ATTACH failing (extension/DSN) → the report backtest raises → WAIT for those
|
||||
strategies (never a silent partial ref).
|
||||
- `sixtyforty` (`backtest.kind == "none"`, absolute gate) is explicitly exempt — asserted, not silently
|
||||
ref-less.
|
||||
|
||||
## Testing
|
||||
|
||||
- **Unit:** `backtest_ref` dispatch produces a `BacktestSummary` for a `recompute-replay` strategy (replay a
|
||||
synthetic ForwardStrategy) and for a `report` strategy (synthetic report); returns None/ skips for `none`.
|
||||
A regression test: `xsfunding`'s provider yields a ref (bug A). The `TimescaleDuckDbEngine` ATTACH-and-scan
|
||||
reads a panel from a seeded (sqlite/duck) store and computes.
|
||||
- **Invariant test (integration):** iterate `STRATEGY_REGISTRY`; assert every `gate_type: reconciliation`
|
||||
entry declares a non-`none` `backtest.kind` AND its provider is constructible. This test makes a
|
||||
gate-un-passable strategy a CI failure.
|
||||
- **Integration:** the scheduled backtest step writes `backtest_summary` rows for the `report` strategies;
|
||||
the gate evaluates PASS-able (not stuck WAIT) for `xsfunding` and the bybit book.
|
||||
- **Local cockpit (Playwright):** `/strategy/xsfunding` no longer shows a permanent "no backtest reference"
|
||||
WAIT; read the gate state. Never claim off synthetic/prod.
|
||||
|
||||
## File Touchpoints
|
||||
|
||||
| File | Change |
|
||||
|---|---|
|
||||
| `src/fxhnt/registry.py` | add `definition.backtest.kind` per entry (`recompute-replay`/`report`/`none`); `xsfunding` → `recompute-replay` |
|
||||
| `src/fxhnt/application/backtest_refs.py` (new) | `backtest_ref(strategy_id, store) -> BacktestSummary\|None` dispatch (recompute-replay / report / none) |
|
||||
| `src/fxhnt/adapters/warehouse/timescale_duckdb_engine.py` (new) | `TimescaleDuckDbEngine` — ephemeral DuckDB ATTACH Timescale for heavy report backtests |
|
||||
| `src/fxhnt/adapters/orchestration/assets.py` | route inline recompute refs through `backtest_ref`; add `xsfunding` ref; keep heavy report out of the daemon |
|
||||
| `src/fxhnt/adapters/orchestration/definitions.py` | schedule the resourced unified-backtest step (op or CronJob), replacing the manual bybit-sleeve-ret Job |
|
||||
| `src/fxhnt/cli.py` | `backtest-refs --all` command (for the scheduled step); retire/redirect `bybit-persist-sleeve-ret` into it |
|
||||
| `infra/k8s/jobs/` | new scheduled `fxhnt-backtest-refs` CronJob (own memory); retire the B3 backtest DuckDB PVC + the manual sleeve-ret Job |
|
||||
| tests | `test_backtest_refs.py` (dispatch + xsfunding regression), the registry invariant test, `test_timescale_duckdb_engine.py`, cockpit-local |
|
||||
|
||||
## Deploy Notes
|
||||
|
||||
- The ephemeral DuckDB `INSTALL postgres` runs at runtime (already verified on duckdb 1.5.3); the resourced
|
||||
backtest step needs its own k8s memory (like the OPRA backfill / compare-precompute).
|
||||
- Retiring the B3 DuckDB PVC + the manual sleeve-ret Job is a decommission step; the parked research code
|
||||
stays in the tree (revivable) but its Jobs/PVCs are removed from the running system.
|
||||
- No forward-state migration; `xsfunding` gets its first ref on the next run and its gate begins reconciling.
|
||||
@@ -0,0 +1,267 @@
|
||||
# IBKR Paper Cockpit Parity — Design
|
||||
|
||||
**Date:** 2026-07-14
|
||||
**Status:** Approved (brainstorming) → ready for implementation plan
|
||||
**Branch:** `feat/ibkr-paper-cockpit-parity`
|
||||
|
||||
## Goal
|
||||
|
||||
Bring the IBKR paper strategies to cockpit parity with the Bybit book: show their **backtest** in the sim
|
||||
and their **real paper-account trading** (actual DU-account fills / positions / NAV) across the cockpit —
|
||||
overview (at-a-glance), strategy detail (the slice), and an account-level view (the whole DU pot) — so the
|
||||
paper trading that is already working is *visible and verifiable* at every level, as the honest pre-live
|
||||
confidence path. UX is a first-class requirement, and the design scales to multiple executing strategies by
|
||||
adding a row, not a redesign.
|
||||
|
||||
## Scope
|
||||
|
||||
**In:** (information architecture VALIDATED via UX mockup — consolidated, plain-language, click-through-to-holdings)
|
||||
- D1 — the IBKR strategies' honest-cost backtest curve on the **Backtests** page (both venues, one place).
|
||||
- D2 — capture the **real** IBKR DU paper-account state (fills, positions, NAV), render a **holdings detail
|
||||
page** (what it's holding: assets + shares + value + %; recent trades; value-over-time), and reconcile it
|
||||
against the recomputed (idealized) sim ("how closely the real trades follow the plan").
|
||||
- D3 — surface the executed reality on the **overview**: a "real trades" sub-line under an executing strategy
|
||||
(reusing the console's existing exec-twin sub-row) with a "behind/ahead of plan" pill.
|
||||
- D4 — a **consolidated "Paper books" page** (`/paper`) showing BOTH books side by side — the crypto 4-edge
|
||||
book (simulated) and the IBKR paper account (real) — each a clickable card leading to its holdings detail.
|
||||
The IBKR book's detail carries the per-strategy breakdown that scales by row as more strategies trade.
|
||||
|
||||
**UX rules (VALIDATED, binding):**
|
||||
- **Plain language, no dev jargon** in ANY user-facing copy. Human names only: "Crypto 4-edge book",
|
||||
"Multi-asset ETF portfolio", "Options income (put spreads)", "real trades", "0.3% behind plan", "cost to
|
||||
trade 0.04%", "building", "not trading — options access pending". NEVER surface internal IDs/terms
|
||||
(`bybit_4edge`, `multistrat`, `vrp`, the DU account number, `bps`, "sim", "recompute-replay", "exec-twin").
|
||||
Keep only the fund's own vocabulary (Sharpe, backtest, worst drop). A single `display_name` mapping is the
|
||||
source of every rendered name.
|
||||
- **`REAL PAPER`** badge (blue) marks the one real-execution surface (the IBKR paper account); **`PAPER`**
|
||||
(grey) marks the simulated crypto book. Reuse the existing `.badge-exec` style.
|
||||
- The crypto book and the IBKR account are the SAME kind of thing (paper books you watch) — consolidate onto
|
||||
one Paper-books page; click either for its holdings.
|
||||
|
||||
Designed for scale (UX is a first-class requirement): today only `multistrat` executes on DU (vrp exec is
|
||||
suspended on IBKR options permission; sixtyforty is a benchmark), so the account currently *is* the multistrat
|
||||
book. But ALL captured data is tagged by strategy and the views are built at both the per-strategy AND the
|
||||
account level, so a second executor arming is a new ROW, never a redesign.
|
||||
|
||||
**Out (explicit):**
|
||||
- No live / real-capital execution, no arm step, no promotion→allocation lifecycle changes ("still on paper,
|
||||
no live yet"). Gate/promotion/allocation semantics are UNCHANGED.
|
||||
- Bybit stays exactly as-is (a pure simulated book — no EU Bybit-testnet API keys are available, so there is
|
||||
no real Bybit account to ingest). This design does not touch the Bybit `/paper` book or its sim.
|
||||
|
||||
## Current state (grounded)
|
||||
|
||||
- **Backtesting IS integrated:** `multistrat` has a real 0a `backtest_ref` (Sharpe ~1.67), `vrp` has one; both
|
||||
reconcile against the gate. The `/paper/sim` honest-cost curve, however, reads ONLY `bybit_sleeve_ret`
|
||||
(`app.py` `_SIM_BOOKS = ("bybit_4edge", "bybit_4edge_levered")`, `_sim_returns` →
|
||||
`_paper_repo().bybit_sleeve_returns()`) — the IBKR backtests exist but are not shown there.
|
||||
- **Real paper execution happens but is not captured:** `IbkrBroker` (`adapters/broker/ibkr.py:39-40`) already
|
||||
reads real `accountSummary()` (NLV/cash/gross) + `ib.positions()`. `execute-multistrat` (`cli.py:248`) →
|
||||
`ExecutionService.rebalance` → `multistrat_exec_record.plan_and_record` books an *observed* return for the
|
||||
`multistrat_exec` track. BUT it records the **intended** target book (`weights·nlv/price`), not the actual
|
||||
fills; the `exec_fills` table (`ExecFillRow`: client_id, rebalance_id, symbol, side, qty, price, fee, status,
|
||||
shortfall_bps, at) is **empty**; and there is **no cockpit view** of the DU account's real positions/NAV.
|
||||
|
||||
So this is a faithful EXTENSION of existing machinery, not greenfield: populate `exec_fills`, snapshot the real
|
||||
account NAV, and add the cockpit view + reconciliation.
|
||||
|
||||
## D1 — IBKR backtest curve in `/paper/sim`
|
||||
|
||||
Generalize the sim's data source so a book draws its honest-cost backtest curve from its registry
|
||||
`backtest.kind` engine (the 0a `backtest_ref` dispatch: `recompute-replay` for `multistrat`/`vrp`) instead of
|
||||
only `bybit_sleeve_ret`.
|
||||
|
||||
- **Book list:** replace the hardcoded Bybit-only `_SIM_BOOKS` with a source that includes the IBKR recompute
|
||||
books (`multistrat`, `vrp`) alongside the Bybit books. Each book declares (via the registry) which engine
|
||||
produces its curve.
|
||||
- **Curve source:** `_sim_returns(book)` dispatches on the book's `backtest.kind`: Bybit books keep reading
|
||||
`bybit_sleeve_ret` (unchanged); `recompute-replay` books source their series from the 0a
|
||||
`backtest_ref(sid, ...)` / the recompute engine that already produces their backtest. NO new backtest
|
||||
compute on the request path — read precomputed/cached series as the Bybit path does (persist the IBKR
|
||||
recompute-replay curve where the sim can read it cheaply, mirroring `bybit_sleeve_ret`; the
|
||||
`fxhnt-backtest-refs` Job / the nightly asset is the natural writer).
|
||||
- **UI:** the existing `/paper/sim` book selector renders the added IBKR books; the honest-cost curve + summary
|
||||
stats render identically. No template redesign — a data-source generalization.
|
||||
|
||||
**Boundary:** D1 changes only the sim's book list + curve-source dispatch. It does not touch the gate,
|
||||
`forward_summary`, or the Bybit sim behavior.
|
||||
|
||||
## D2 — Real DU paper-account book + reconciliation
|
||||
|
||||
### D2.1 Capture (extend the exec path)
|
||||
At each `execute-multistrat` run (post-execution, when not blocked by a safety gate), capture the REAL account:
|
||||
- **Fills:** collect the actual fills from the placed trades (ib-async trade `.fills` / `ib.reqExecutions`) and
|
||||
persist to the existing `exec_fills` table — one row per fill (symbol, side, qty, filled price, fee/commission,
|
||||
status, `shortfall_bps` = filled-vs-intended slippage, `rebalance_id`, `at`). This populates the currently-empty
|
||||
table.
|
||||
- **Account snapshot:** persist a per-run snapshot of the real account — `NetLiquidation`, cash, gross position
|
||||
value, and the real held positions (`ib.positions()`) — into a NEW idempotent table `ibkr_account_nav`
|
||||
(run_date PK; nlv, cash, gross, positions_json, at). One row per run_date (idempotent replace), Timescale/
|
||||
Postgres-safe migrate following the `forward_nav`/`strategy_health` create-if-not-exists pattern.
|
||||
- **Reuse:** the broker already reads NLV+positions; add fill collection + the two persistence calls. Do NOT
|
||||
change the `multistrat_exec` return basis (it is gate-relevant, C1-designed) — the capture is additive.
|
||||
|
||||
### D2.2 Holdings detail page
|
||||
Clicking a book (from Paper books or the overview) opens its detail page. For the IBKR paper account book:
|
||||
- **What it's holding right now** — a holdings table: asset (SPY/IEF/…), a plain "what it is" description
|
||||
(US stocks / US Treasuries / Gold / Commodities / Managed futures), shares, value, % of book (from the
|
||||
latest `ibkr_account_nav` positions snapshot).
|
||||
- **Recent trades** — from `exec_fills`: time, asset, action (Buy/Sell), shares, price, "cost to trade"
|
||||
(slippage as a %, NOT bps), fee.
|
||||
- **Value over time** — the **real** account NAV curve (from `ibkr_account_nav`) with the plan it's trying to
|
||||
follow overlaid as a dashed line.
|
||||
Empty/absent data → "no trades recorded yet" (never a 500, never a fabricated number). The crypto book's detail
|
||||
renders its (simulated) holdings the same way.
|
||||
|
||||
### D2.3 Reconciliation — "how closely the real trades follow the plan"
|
||||
The one shared signal, in plain language. Compute the divergence between the **real DU account NAV**
|
||||
(`ibkr_account_nav`, new) and the recomputed idealized sim (`forward_summary`) → a "X% behind/ahead of plan"
|
||||
pill (reuse the existing amber "decaying"/green pill vocabulary). Surface it: a pill on the overview real-trades
|
||||
line, the "Following the plan" card + headline on the book detail, and the per-strategy column on the account
|
||||
detail. This gap — real fill prices, whole-share rounding, cost-to-trade, fees — is the pre-live verification.
|
||||
`multistrat_exec` (the observed envelope-return track) remains the gated exec twin, unchanged.
|
||||
|
||||
## D3 — Overview page: the executed reality at a glance
|
||||
|
||||
The overview (`/`, `dashboard_service.fleet()`/`overview()`) already renders each strategy with gate/health/
|
||||
backtest badges and — for a modeled edge with an executed twin — a nested sub-row. For a strategy executing on
|
||||
the real paper account, render that sub-row as a **"real trades"** line (reuse the existing `.exec-tag` +
|
||||
`tr.exrow` style): what actually filled (real return, trade count, holdings), a **"X% behind/ahead of plan"**
|
||||
pill (the shared D2.3 signal), and a plain "cost to trade / fees" note. Mark the strategy's row with the
|
||||
**`REAL PAPER`** badge. Clicking the strategy (or "N trades · M holdings →") opens its holdings detail (D2.2).
|
||||
No separate rollup card — the consolidated Paper-books page (D4) is the account home.
|
||||
|
||||
## D4 — Consolidated "Paper books" page (`/paper`)
|
||||
|
||||
Replace the current Bybit-only `/paper` landing with a page that shows BOTH books side by side, each a
|
||||
clickable card:
|
||||
- **Crypto 4-edge book** (`PAPER`, simulated) — value / since / Sharpe / worst drop / status, a mini curve.
|
||||
- **IBKR paper account** (`REAL PAPER`, real) — account value / strategies trading / holdings count / the
|
||||
"X% behind plan" pill / a mini curve with the plan overlaid.
|
||||
Both read identically (value, track record, status). Clicking a card opens that book's holdings detail (D2.2).
|
||||
|
||||
The IBKR book's detail page carries the **per-strategy breakdown** (one row per executing strategy: its real
|
||||
return, "behind/ahead of plan", holdings + trade counts) — one row today, N as strategies arm ("scales by row,
|
||||
not redesign"). The account is one pot; the per-strategy rows are its slices, driven by the strategy-tagged
|
||||
capture. A dimmed "not started yet" row illustrates the scale path.
|
||||
|
||||
## UX & information architecture
|
||||
|
||||
Progressive disclosure, one consistent "following the plan" signal at every level, plain language throughout:
|
||||
|
||||
```
|
||||
Overview / -> fleet scan; an executing strategy shows a "real trades" sub-line
|
||||
| + a "X% behind plan" pill (REAL PAPER badge on the row)
|
||||
|
|
||||
Paper books /paper -> BOTH books as clickable cards: Crypto 4-edge (PAPER, simulated)
|
||||
| + IBKR paper account (REAL PAPER, real). Same layout for each.
|
||||
|-- Book detail -> click a book: WHAT IT'S HOLDING (assets + shares + value + %),
|
||||
| recent trades (cost-to-trade %), value-over-time (real vs plan),
|
||||
| and — IBKR book — the per-strategy breakdown (1 row now, N later)
|
||||
|
|
||||
Backtests /paper/sim -> both venues' honest-cost curves in one place (human book names)
|
||||
```
|
||||
|
||||
UX principles (professional ops UI):
|
||||
- **At-a-glance first:** the overview must answer "which strats are actually paper-trading, and is execution
|
||||
tracking the model?" from badges/chips alone — no drill-in required.
|
||||
- **Progressive disclosure:** overview (fleet) → strategy detail (one slice) → account view (the pot). Each
|
||||
level adds resolution, none duplicates another's job.
|
||||
- **One divergence signal, rendered consistently** at all three levels (chip on overview, headline on detail,
|
||||
breakdown on account) — the same computation, the same color language.
|
||||
- **Reuse the existing visual system:** the gate/health badge palette + `exec_badge` macro; do NOT invent a new
|
||||
palette. The EXEC badge and divergence chip are new *states* in the existing badge vocabulary.
|
||||
- **Scale by row, not redesign:** every view renders a *set* of executing strats (size 1 today); adding a strat
|
||||
never changes the layout, only adds a row/chip.
|
||||
- **Honest empty/degraded states:** no captured account yet, or ib-gateway unreachable, renders a clear "no
|
||||
executed data yet" state — never a 500, never a fabricated number. Consistent with the health axis (a stalled
|
||||
exec track already surfaces as STALE).
|
||||
|
||||
## Data model
|
||||
|
||||
- **Reuse** `exec_fills` (`ExecFillRow`) — populate it (currently empty). It already carries `rebalance_id`;
|
||||
ensure each rebalance's `rebalance_id` maps to its `strategy_id` (a `strategy_id` column or a rebalance→strat
|
||||
mapping) so fills are attributable per strategy for D4's breakdown. No breaking schema change (add the tag).
|
||||
- **New** `ibkr_account_nav` (per-run real account snapshot): `run_date` PK, `nlv`, `cash`, `gross`,
|
||||
`positions_json`, `at`. Account-level (the whole DU pot). Idempotent create + replace-per-run; migrate follows
|
||||
the existing `forward_nav`/`paper_repo`/`strategy_health` pattern (no heavy startup DDL).
|
||||
- **Per-strategy executed return** already exists as the `*_exec` observed forward tracks (`multistrat_exec`
|
||||
today; each future executor gets its own, same pattern) — the account view's per-strategy breakdown reads
|
||||
these + the strategy-tagged `exec_fills`. The account NAV rolls up the account; per-strat return stays on the
|
||||
`*_exec` tracks. No new per-strat NAV store.
|
||||
- **No change** to `forward_summary`, `backtest_summary`, `strategy_health`, gate/promotion/allocation.
|
||||
|
||||
## Architecture / components
|
||||
|
||||
- `adapters/broker/ibkr.py` — add fill collection from placed trades; expose fills + the account snapshot to the
|
||||
caller (keep the broker a thin adapter; no persistence here).
|
||||
- `application/multistrat_exec_record.py` (or a small sibling) — persist the captured fills → `exec_fills` and the
|
||||
account snapshot → `ibkr_account_nav`. Additive to `plan_and_record`; gated behind `account_is_paper` (matches
|
||||
the existing paper-only guard). Does not alter the observed-return basis.
|
||||
- `adapters/persistence/` — the `ibkr_account_nav` repo (create/replace/read a NAV series + latest positions),
|
||||
following `forward_nav`/`paper_repo` conventions.
|
||||
- `adapters/web/app.py` + templates — D1 sim book-list/curve-source dispatch; D2 the multistrat-detail "IBKR
|
||||
paper account" section (real NAV curve + positions + fills + reconciliation headline); D3 the overview EXEC
|
||||
badge + divergence chip + account-rollup card (via `dashboard_service.fleet()`/`overview()` + the badge
|
||||
macros); D4 the `/paper/ibkr` account view (total NAV + all positions/fills + per-strategy breakdown table).
|
||||
- `application/dashboard_service.py` — compute the per-strategy exec status + real-vs-sim divergence (one shared
|
||||
function feeding overview chip, detail headline, account breakdown — the "one signal, three places" rule) and
|
||||
the account rollup.
|
||||
- `application/backtest_refs*` / the nightly asset — persist the IBKR recompute-replay curve where the sim can
|
||||
read it cheaply (mirrors `bybit_sleeve_ret`).
|
||||
|
||||
## Error handling
|
||||
|
||||
- Capture is BEST-EFFORT and paper-only: a fills/snapshot persistence failure logs (does not fail the rebalance
|
||||
or block execution) — the rebalance already succeeded; the capture is observability. Follows the existing
|
||||
"ref persist is best-effort" pattern.
|
||||
- Empty/absent data degrades to an empty section (never 500), matching the Bybit `/paper` "missing precomputed
|
||||
table → []" guard.
|
||||
- ib-gateway unreachable at ingest → no snapshot that run; the health axis already surfaces a stalled exec track.
|
||||
|
||||
## Testing
|
||||
|
||||
- **D1:** `/paper/sim` book list includes `multistrat`; `_sim_returns("multistrat")` sources the recompute-replay
|
||||
curve (not `bybit_sleeve_ret`); the sim curve + stats render for an IBKR book (TestClient); Bybit sim
|
||||
unchanged.
|
||||
- **D2.1:** captured fills persist to `exec_fills` with correct fields incl. `shortfall_bps`; the account snapshot
|
||||
persists to `ibkr_account_nav`; capture is paper-only + best-effort (a persistence error does not fail the
|
||||
rebalance; verified via a raising fake).
|
||||
- **D2.2/2.3:** the multistrat detail page renders the real NAV curve, positions, fills; the sim-vs-executed
|
||||
reconciliation computes the divergence on a fixture (known sim series vs known executed fills → expected
|
||||
divergence + median slippage); empty data → empty section, no 500.
|
||||
- **D3:** the shared divergence function returns the expected value on a fixture; the overview renders the EXEC
|
||||
badge + divergence chip for an executing strat and NOT for a pure-sim strat; the account-rollup card shows
|
||||
total NAV + strat count; no-executed-data → no badge/empty card, no 500.
|
||||
- **D4:** `/paper/ibkr` renders total NAV + all positions/fills + a per-strategy breakdown row for `multistrat`;
|
||||
the breakdown is a SET (assert it renders correctly for one strat and would render N — e.g. a two-strat
|
||||
fixture yields two rows); empty account → clean empty state, no 500.
|
||||
- **Scale test:** a fixture with a second (synthetic) executing strat proves overview/account views render both
|
||||
without layout change — the "new row not redesign" invariant.
|
||||
|
||||
## Constraints
|
||||
|
||||
- **Plain language, no dev jargon** in any user-facing copy (see UX rules). A single `display_name` mapping
|
||||
(internal `strategy_id`/book id → human name) is the source of every rendered name — no raw ids, no `bps`,
|
||||
no "sim"/"recompute-replay". `REAL PAPER`/`PAPER` badges as specified.
|
||||
- Paper only — no live/real-capital path, no arm, no gate/promotion/allocation change.
|
||||
- The consolidated `/paper` page shows BOTH books; the Bybit book's underlying data/behavior is untouched
|
||||
(only its landing is now shared with the IBKR book). Bybit stays a simulated book.
|
||||
- DB migrations idempotent + Postgres-safe (no heavy startup DDL — crashloops the cockpit).
|
||||
- Verify the cockpit LOCALLY in a real browser before deploy (read the rendered numbers; never claim "works"
|
||||
off synthetic events).
|
||||
- No silent degradation: capture failures log loudly; a stalled exec track already surfaces via the health axis.
|
||||
|
||||
## Sequencing
|
||||
|
||||
1. **D1** — sim curve for IBKR (low-risk, backtests exist, immediate parity).
|
||||
2. **D2.1** — capture the real account (fills → `exec_fills` tagged by strategy; snapshot → `ibkr_account_nav`).
|
||||
The data foundation everything else reads.
|
||||
3. **D2.2 + D2.3** — the multistrat detail "IBKR paper account" section + the sim-vs-executed reconciliation
|
||||
(build the ONE shared divergence function here).
|
||||
4. **D3** — overview EXEC badge + divergence chip (reusing the shared function) + account-rollup card.
|
||||
5. **D4** — the `/paper/ibkr` account view (total NAV + all positions/fills + per-strategy breakdown).
|
||||
|
||||
Each step is independently testable and shippable; D2.1 is the pivot (it fills `exec_fills` + the account
|
||||
snapshot that D2.2/D3/D4 all render). The shared divergence function (built in step 3) is reused by D3 and D4
|
||||
so the signal is computed once and rendered three places.
|
||||
@@ -0,0 +1,148 @@
|
||||
# Capacity-Aware Capital Allocator — Design
|
||||
|
||||
**Date:** 2026-07-15
|
||||
**Status:** Design. A focused UPGRADE of the existing B2a allocation engine — not a new system.
|
||||
**Author:** jgrusewski (+ Claude)
|
||||
**Builds on:** `docs/superpowers/specs/2026-07-15-crypto-honest-reference-pipeline-design.md` (merged 20437d7).
|
||||
|
||||
## 1. Problem
|
||||
|
||||
We are running a small, growing multi-edge fund across two venues (Bybit crypto sleeves + an IBKR equity
|
||||
book). The honest-reference pipeline (merged) proved the edges are real at low capital but revealed hard
|
||||
**capacity ceilings** — e.g. `xsfunding` is strong at $35k (Sh 3.5) but **breaks by ~$350k–$1M** (Sh → −2.5).
|
||||
The existing allocation engine (`allocation_engine.compute_allocation`, sub-project B2a) is **capacity-blind**:
|
||||
it sizes edges equal-weight (capped at a flat `max_strategy_weight`) + an ex-ante vol-target, with no knowledge
|
||||
that an edge stops working above a dollar figure. A growing fund would keep pouring capital into a
|
||||
capacity-limited edge past its break.
|
||||
|
||||
## 2. Goal
|
||||
|
||||
Make the allocation **capacity-aware and risk-parity weighted**, so the fund allocates its pot correctly across
|
||||
edges within each edge's honest capacity — and, as the pot grows past an edge's ceiling, shifts automatically to
|
||||
the capacity-robust edges. Minimal, SSOT-respecting upgrade of the existing engine; **no new allocation
|
||||
system**. Scope stays PAPER (the `capital_ceiling` human gate + forward-gate promotion already enforce this).
|
||||
|
||||
## 3. What already exists (verified 2026-07-15) — reuse, do not rebuild
|
||||
|
||||
`allocation_ingest.record_allocation(dsn, equity)` already:
|
||||
- allocates over **deploy-tier + forward-promoted** strategies (`evaluate_promotions` promotes on gate PASS) —
|
||||
**this IS the forward-gating**; a not-yet-passed edge gets $0;
|
||||
- is **cross-venue** — `returns = {sid: {date: ret}}` keyed by strategy id spans crypto + IBKR uniformly;
|
||||
- **de-funds decaying edges** (B3 `compute_funding`), applies the **rare drawdown killswitch**, and an **ex-ante
|
||||
vol-target** leverage `K` (`AllocationPolicy`: target_vol 0.12, kelly_fraction 0.5, max_leverage 1.5,
|
||||
cvar_alpha 0.10, `max_strategy_weight` 0.5, `capital_ceiling` $35k) sized from **full-history** vol (never
|
||||
trailing — the anti-reactive invariant);
|
||||
- writes `StrategyAllocationRow(target_weight, target_dollars)` which the rebalancers (`execute-multistrat`,
|
||||
`execute-bybit`) consume to size.
|
||||
|
||||
So the allocator's plumbing, gating, cross-venue combination, decay, killswitch, and $ conversion are DONE.
|
||||
|
||||
## 3.5 Current gate reality (investigated 2026-07-15) — why the sizing basis must change
|
||||
|
||||
Live state of the paper edges:
|
||||
- **Deploy-tier = only `bybit_4edge` (book) + `positioning`** (both crypto, both currently `WAIT`, building
|
||||
8/21 and 8/14 days). `multistrat` (IBKR) is tier=research, `WAIT` 6/20 — it joins the allocation only when
|
||||
promoted. `unlock` is `PASS` but tier=research with no `promote_on_pass`, so it is not in the deploy set
|
||||
(it lives inside the `bybit_4edge` book).
|
||||
- **`strategy_allocation` is currently EMPTY (0 rows).** Root cause: `compute_allocation` requires
|
||||
`min_history_days = 60` FORWARD observations for a stable ex-ante vol, but the 2026-07-14 re-warm reset the
|
||||
forward tracks to ~8 days. 8 < 60 → not eligible → empty allocation. Under the current design the allocation
|
||||
would not populate until ~60 forward days (~mid-September) — far later than the forward gates themselves
|
||||
(14–21 days, late July / early August).
|
||||
|
||||
**Consequence:** sizing off the 8-day forward record is both impossible (too short) and, after any future
|
||||
re-inception, self-defeating. But the honest-reference has 1,290–1,992 days of stable, capacity-adjusted
|
||||
history. So the sizing basis must move to the backtest (§4c).
|
||||
|
||||
## 4. The three changes
|
||||
|
||||
### 4a. Weighting: equal-weight → inverse-vol (risk-parity), capped
|
||||
In `compute_allocation`, replace the equal-weight `base` with **inverse-full-history-vol** weights, still capped
|
||||
at `max_strategy_weight` (the concentration prudence cap) and still NO-renormalise when the cap binds. Rationale
|
||||
(decided with the user): risk-parity gives better risk-adjusted return and is adaptive/not-tuned (reuses the
|
||||
existing `full_history_vol`), but a **firm concentration cap (~0.4–0.5)** is kept because this is a young fund
|
||||
whose edges are not yet forward-proven — do not bet the fund on one edge (the honest-reference's unconstrained
|
||||
risk-parity put 0.89 in `xsfunding`; that is a backtest weight, not a fund weight). Diversify across edges,
|
||||
concentrate only as the forward tracks confirm which edge is real.
|
||||
|
||||
### 4b. Capacity: a per-strategy dollar ceiling from the honest-reference
|
||||
Add a per-strategy `capacity_ceiling[sid]` and cap each edge's dollars at `min(target_dollars,
|
||||
capacity_ceiling[sid])` in `target_capital` (or as a post-step). The ceiling per edge = the AUM at which the
|
||||
honest-reference Sharpe crosses a viability threshold (e.g. the largest AUM on the capacity curve with honest
|
||||
Sharpe ≥ `capacity_min_sharpe`, default 1.0). This makes growth automatic: below the ceiling the cap is
|
||||
non-binding; as equity grows into it, the capped edge holds cash / the allocation shifts to capacity-robust
|
||||
edges. Un-capped (no honest curve) → no ceiling (unbounded), logged.
|
||||
|
||||
### 4c. Sizing basis: the honest-reference backtest, not the 8-day forward record
|
||||
`compute_allocation`'s ex-ante vol (and the new inverse-vol weighting) must be computed from the
|
||||
**honest-reference backtest return series** (capacity-adjusted, at the target AUM — 1,290–1,992 days, stable),
|
||||
NOT from `nav_history` (the forward record, ~8 days post-re-warm → fails `min_history_days`). The **forward
|
||||
gate stays the LIVE-DEPLOY switch**: `record_allocation` still allocates real (paper) dollars only to deploy +
|
||||
forward-promoted edges. So the honest target allocation is computable and visible NOW (backtest-sized +
|
||||
capacity-capped), while capital is only deployed to an edge once its forward gate passes — exactly "honest
|
||||
weights now, live per gate". Implementation: `record_allocation` builds `returns` from the honest-reference
|
||||
series (via a provider) rather than `nav_history`; the `min_history_days` eligibility is then trivially met by
|
||||
the long backtest. Keep `nav_history`-based sizing available behind the provider so a mature forward track
|
||||
(≥ `min_history_days` real days) can later blend/replace the backtest (a documented follow-on, not v1).
|
||||
|
||||
## 5. Granularity & cross-asset capacity (the load-bearing design decision)
|
||||
|
||||
The allocation runs over deploy **strategies** (`bybit_4edge` book, `positioning`, `multistrat`), but the
|
||||
honest-reference capacity is per **sleeve** (`xsfunding`/`positioning`/`unlock`) and is **crypto-specific**
|
||||
(Bybit `turnover` ADV). Resolution for v1:
|
||||
- **Crypto book (`bybit_4edge`):** extend the honest-reference to emit a **book-level** capacity ceiling (run
|
||||
the honest-cost pipeline over the book's constituent sleeves, take the book's honest capacity curve). Its
|
||||
ceiling is the min of the constituents' effective ceilings, roughly (the book breaks when its biggest
|
||||
capacity-limited sleeve does).
|
||||
- **`positioning`** (deploy sleeve, tracked directly): use its own sleeve capacity curve (ceiling >> $10M in the
|
||||
tested range — effectively unbounded at fund scale).
|
||||
- **`multistrat` (IBKR equity):** the crypto ADV model does NOT apply (equities, different data). US-ETF ADV is
|
||||
deep, so v1 assigns a **high default ceiling** (a documented constant, e.g. $10M) and flags that a proper
|
||||
equity-ADV capacity model is a follow-on. Do NOT fabricate a crypto-style number for it.
|
||||
|
||||
This mapping (deploy-strategy → capacity ceiling) is the crux; it must be explicit and auditable, not implicit.
|
||||
|
||||
## 6. Forward-gating (user decision: "honest weights now, live switch per gate")
|
||||
|
||||
Two separate surfaces, already aligned:
|
||||
- **Honest target for ALL edges (now, paper):** the honest-reference report shows the full honest picture across
|
||||
every edge (already built).
|
||||
- **Deployed allocation (live/paper $):** `record_allocation` only allocates to deploy + forward-**promoted**
|
||||
edges. As each edge's forward gate PASSes (`evaluate_promotions`), it enters the allocation and earns capital.
|
||||
Real money follows proven forward performance; nothing here changes that.
|
||||
|
||||
## 7. Scope
|
||||
|
||||
**In:** the two `compute_allocation`/`target_capital` changes; the honest-reference emitting per-deploy-strategy
|
||||
capacity ceilings; wiring the ceilings into `record_allocation`; policy knobs (`capacity_min_sharpe`, the
|
||||
equity default ceiling); tests. **Out (follow-on):** a real equity-ADV capacity model for IBKR; raising the
|
||||
`capital_ceiling` above $35k / going live (a deliberate human gate, per-edge on gate PASS + creds); any new
|
||||
execution path.
|
||||
|
||||
## 8. Critical review — where this could go wrong
|
||||
|
||||
- **Sharpe-4 is not yet forward-proven.** The honest backtest is strong but the forward tracks just re-warmed.
|
||||
The concentration cap (§4a) and forward-gating (§6) are the two guards; do not remove them to chase the
|
||||
backtest weight.
|
||||
- **Capacity ceiling is threshold-sensitive.** "Sharpe ≥ 1.0" is a choice; the curve is steep for `xsfunding`
|
||||
(3.5 → 0.28 → −2.5 across $35k/$350k/$1M), so the ceiling lands ~$350k either way, but report the ceiling
|
||||
next to the curve so the choice is visible, not buried.
|
||||
- **Book-vs-sleeve capacity is an approximation.** The `bybit_4edge` book capacity ≠ a clean sum of sleeve
|
||||
capacities; state the derivation and its limits.
|
||||
- **Equity capacity is a stub in v1** (high default). Be honest that the IBKR ceiling is not yet modeled.
|
||||
- **Sizing from the BACKTEST, not the live record (§4c).** Necessary today (8-day forward is too short), but it
|
||||
means the ex-ante vol/weights reflect the honest backtest, not live performance — the two can diverge. The
|
||||
forward gate (live-deploy switch) + the funding-decay layer are the guards that catch a live divergence; the
|
||||
sizing basis should switch to live vol once a forward track is long enough (documented follow-on). Do not
|
||||
read the backtest-sized weights as "what the live book is doing".
|
||||
- **Still a backtest-derived ceiling on a live allocation.** The capacity cap protects against over-scaling a
|
||||
known-limited edge; it cannot protect against an edge decaying — that is the funding-decay layer's job (which
|
||||
already exists). Don't conflate the two.
|
||||
|
||||
## 9. Decisions locked (from brainstorming)
|
||||
|
||||
- Weighting: **inverse-vol (risk-parity) capped at `max_strategy_weight`**. Sizing basis: **the honest-reference
|
||||
backtest series (§4c), NOT the 8-day forward record**; forward gate = live-deploy switch only. Forward-gating:
|
||||
**honest weights shown now for all edges; deployed $ only to forward-promoted edges** (existing). Capacity:
|
||||
**per-deploy-strategy ceiling from the honest-reference (crypto), high default for equity (v1)**. Scope:
|
||||
**paper**, an upgrade of the B2a engine, no new system.
|
||||
@@ -0,0 +1,172 @@
|
||||
# Honest-Reference Pipeline for the Bybit Crypto Book — Design
|
||||
|
||||
**Date:** 2026-07-15
|
||||
**Status:** Design (research/validation scope). Production gate-replacement is an explicit follow-on, NOT this spec.
|
||||
**Author:** jgrusewski (+ Claude)
|
||||
|
||||
## 1. Problem
|
||||
|
||||
The Bybit 4-edge book is graded (recon gate) and sized against a backtest that is **optimistic on
|
||||
exactly the days that drive its numbers**. Three linked modeling problems, all quantified 2026-07-15:
|
||||
|
||||
1. **Tail/capacity inflation.** Return is concentrated in a handful of microcap-pump days. Top-10 days
|
||||
(of ~1,300–2,000) drive **58%** of `unlock`'s return, **29%** of `positioning`'s. Under a P98 upside
|
||||
winsorize, `unlock` collapses (Sharpe 1.03 → **0.19**), `positioning` holds a real core (2.62 → 2.02),
|
||||
`xsfunding` is untouched (3.61 → 3.05). The equal-weight book drops from Sh 2.17 → 1.71.
|
||||
2. **Leverage modeling.** The "levered" backtest shows a *lower* maxDD (−11.8%) than the naive book
|
||||
(−15.4%) — impossible from leverage; it is `carry-static` **re-weighting** flattering the curve, plus an
|
||||
implausibly low modeled vol (13.7% for a 1.5× crypto book). The smooth modeled drawdown understates the
|
||||
real fat-tail path.
|
||||
3. **Cost/slippage.** A flat 5.5 bps applied to all history is a *current-liquidity snapshot* on 2021–22
|
||||
thinner markets, and — critically — is most wrong on the pump days (blown-out spreads), i.e. exactly the
|
||||
return-driving days.
|
||||
|
||||
All three inflate the **same** thing: the tail days. "The forward reconciles with the backtest" is only
|
||||
meaningful if the backtest is honest on those days. It is not.
|
||||
|
||||
## 2. Goal
|
||||
|
||||
Build a **capacity-, cost-, and leverage-honest reference** for the crypto book at the coin level, and use
|
||||
it to: (a) decide the honest sleeve **allocation**; (b) set an **ex-ante risk-management** layer; (c)
|
||||
**falsification-test** candidate edges — including an "anomaly/pump-detection" edge — through the same
|
||||
honest-cost machinery. **No live cockpit change in this spec** (see §9 scope).
|
||||
|
||||
Success = we can state, per sleeve and for the book, at a target AUM: honest **net** Sharpe / CAGR / vol /
|
||||
maxDD, the honest allocation, the **capacity curve** (edge vs AUM), the **leverage ceiling**, and a
|
||||
pass/fail verdict on the candidate pump edge — all reproducible from one pipeline.
|
||||
|
||||
## 3. Data feasibility (verified 2026-07-15)
|
||||
|
||||
- **Volume/ADV:** `bybit_features` feature `turnover` = USD volume per coin per day, **2021-01-01 →
|
||||
2026-07-14, 818 coins** (BTCUSDT ~$5.2 bn/day sanity-checks). → real ADV capacity + market-impact model.
|
||||
- **Spread:** `bybit_real_spread(coin, spread_bps, fetched_at)` is a **single current snapshot**, NOT a
|
||||
daily history. Usable as a per-coin slippage floor; the pump-day spread must be **inferred**, not measured
|
||||
(§8 — the largest honesty gap).
|
||||
- **Sleeve returns (aggregated):** `bybit_sleeve_ret(run_date, sleeve, ret)` — the *input* we are replacing
|
||||
with a coin-level recompute.
|
||||
- **Coin features:** `bybit_features` also has `close/high/low/funding/open_interest/long_ratio/worst_basis`
|
||||
per coin — enough to reconstruct sleeve coin-weights.
|
||||
|
||||
## 4. Architecture
|
||||
|
||||
Instead of taking the aggregated `bybit_sleeve_ret` as given, **recompute each sleeve at the coin level**,
|
||||
apply per-coin capacity + impact, re-aggregate, then risk-parity into a book, then grade against it.
|
||||
|
||||
```
|
||||
coin features ──▶ [0a sleeve coin-weights] ──▶ [0b ADV capacity cap] ──▶ [0c impact slippage]
|
||||
│
|
||||
honest daily sleeve returns ◀────────────────────────┘
|
||||
│
|
||||
[2 risk-parity weights] ──▶ honest book ──▶ [1 falsification gates]
|
||||
│ │
|
||||
[3 ex-ante risk layer] ┘ [4 candidate edge #4 through the same pipeline]
|
||||
```
|
||||
|
||||
## 5. Phase 0 — Honest-reference pipeline
|
||||
|
||||
- **0a — Coin-level sleeve recompute.** Surface each sleeve's *daily target coin weights* from the existing
|
||||
sleeve logic (`sleeve_returns_from_store` / the per-sleeve strategy). **Validation gate:** the recompute,
|
||||
*before* any haircut, must reproduce the stored `bybit_sleeve_ret` within tolerance (else the honest
|
||||
numbers are apples-to-oranges). If a sleeve's logic is not cleanly separable, that is a task blocker to
|
||||
surface, not paper over.
|
||||
- **0b — Capacity model.** For each coin-day, cap the position at `min(target_notional,
|
||||
participation_cap × turnover_coin_day)`. **Parameterize by AUM** → output the **capacity curve** (honest
|
||||
Sharpe vs AUM). `participation_cap` = **3%** (a single global knob, documented; not per-sleeve tuned).
|
||||
**AUM series = {$35k, $100k, $350k}** (the realistic growth path) **+ {$1M, $10M}** (anchor points to
|
||||
*locate the ceiling* — at $35k–350k the 3% cap likely never binds on liquid coins, so the curve may be
|
||||
flat there; the high anchors reveal where the edge actually breaks). Report the full curve.
|
||||
- **0c — Slippage model.** Market-impact per coin-position: `slippage_bps = k · sqrt(order_notional /
|
||||
turnover_coin_day)` (square-root impact law), floored at that coin's quoted spread from
|
||||
`bybit_real_spread`. `k` is one global constant, calibrated to the snapshot spreads (not per-sleeve).
|
||||
Applied to each coin-position's traded notional.
|
||||
- **0d — Re-aggregate** to honest daily sleeve returns → honest sleeve stats (Sharpe/CAGR/vol/maxDD, net of
|
||||
capacity + cost, at the target AUM).
|
||||
|
||||
## 6. Phase 1 — Falsification gates (predetermined, before looking)
|
||||
|
||||
- **G1 (edge survives):** each sleeve's honest net Sharpe > **1.0** at the target AUM. (`unlock` expected to
|
||||
fail — do not rescue it.)
|
||||
- **G2 (honest LOO):** each sleeve adds marginal Sharpe to the honest **risk-parity** book (drop-one test).
|
||||
- **G3 (beats concentration):** honest book Sharpe > the best single sleeve (diversification is real, not a
|
||||
label). Prior estimate: xsf+pos ≈ 3.14 > xsfunding-alone 3.06 — must survive the coin-level model.
|
||||
- **G4 (anti-reactive risk-management):** any proposed *de-risk* rule must beat "do nothing" in an honest
|
||||
A/B, else it is excluded. This gate exists specifically to keep reactive de-sizing out (§7).
|
||||
|
||||
## 7. Phase 2+3 — Allocation & ex-ante risk-management layer
|
||||
|
||||
**Phase 2 (allocation)** = the risk-parity/inverse-vol row below: sleeve weights are set by honest vol, not
|
||||
by an in-sample Sharpe grid; `unlock` receives near-zero weight naturally if G1/G2 reject it.
|
||||
|
||||
Risk management is **baked into sizing**, not a reactive overlay bolted on top. Reactive de-sizing
|
||||
(vol-targeting, drawdown-triggered cuts) has been **A/B-rejected 3× on this book** (pro-cyclical, cuts
|
||||
before the bounce, vol-shrinkage not return) — see `pearl_fxhnt_reactive_risk_controllers_rejected`,
|
||||
`pearl_fxhnt_voltarget_overlay_is_reactive_desizing_rejected`. That is out of scope by construction; G4
|
||||
enforces it.
|
||||
|
||||
| Layer | Ex-ante hendel | Source |
|
||||
|---|---|---|
|
||||
| Position | per-coin **ADV cap** (hard backstop) **and** impact-slippage penalty (soft — sizing self-adjusts) | 0b/0c |
|
||||
| Sleeve | **risk-parity / inverse-vol** on honest vol (adaptive, not tuned) | §2 decision |
|
||||
| Book | **leverage ceiling anchored to the honest real DD** — O3: −9.7% @1×, ~linear → ~**1.5–1.75×** with a 2× stress buffer under a −35% hard limit. This is ex-ante K(D*). | O3 |
|
||||
| Tail | `unlock` dropped/minimized per honest LOO (dead-weight fat-downside tail) | G1/G2 |
|
||||
| Regime | **rare killswitch** for genuine structural breaks only (existing accepted design) | existing |
|
||||
|
||||
## 8. Phase 4 — Candidate edge #4: anomaly / pump-detection
|
||||
|
||||
The pump days that inflate the backtest are also a *hypothesised* edge: detect the anomaly, profit from it.
|
||||
**Recognition ≠ profit** — the gap is the slippage/tail we just measured, and the recent precedent is
|
||||
sobering: `unlock` *is* an explicit "detect the scheduled anomaly and short it" edge, and it collapses under
|
||||
the honest haircut. P&D is a harder version (less predictable, worse execution, adversarial,
|
||||
short-borrow-constrained). It is the **maximally tail-exposed** strategy — the opposite of §7.
|
||||
|
||||
Therefore: run it through the **same honest-cost pipeline** as a candidate, with predetermined skepticism:
|
||||
- **G-detection:** a detectable, **out-of-sample** pump signal (volume/price/OI anomaly), not in-sample fit
|
||||
and not look-ahead (the signal must be knowable *before* the move it trades).
|
||||
- **G-execution (the killer):** the edge survives the **real pump-day slippage** (§0c) + any borrow cost. ~90%
|
||||
of such ideas die here.
|
||||
- **G-marginal:** it adds Sharpe *beyond* what `positioning` already harvests diffusely (same source →
|
||||
otherwise duplicate risk).
|
||||
|
||||
Verdict rule: passes all three → a **separate, hard-capped** sleeve (never risk-parity-core weight; too
|
||||
tail-heavy). Fails → excluded, documented. Expectation is high bar / likely fail — but cheap to falsify on
|
||||
the pipeline we build anyway.
|
||||
|
||||
## 9. Scope & deliverables
|
||||
|
||||
**In scope (this spec):** the Phase-0 pipeline, the capacity curve, the honest allocation recommendation,
|
||||
gate verdicts G1–G4, the leverage-ceiling number, and the candidate-edge-#4 verdict. Pure research output.
|
||||
|
||||
**Out of scope (follow-on spec):** replacing the cockpit `backtest_summary` reference / recon-gate,
|
||||
reweighting the live book, changing any live sizing. This spec must not alter live behavior.
|
||||
|
||||
**Deliverables:** a reproducible report (numbers + gate verdicts) + the pipeline code + tests. No cockpit
|
||||
diff.
|
||||
|
||||
## 10. Critical review — where this could still fool us (be skeptical)
|
||||
|
||||
- **Recompute fidelity (0a).** If the coin-weight reconstruction doesn't reproduce `bybit_sleeve_ret`
|
||||
pre-haircut, every honest number is invalid. This is the load-bearing validation; treat a mismatch as a
|
||||
hard stop.
|
||||
- **Slippage is modeled, not measured.** We have no historical spread — only a snapshot. The pump-day
|
||||
slippage (the crux of all three problems) is *inferred* via the impact law + snapshot floor. This is the
|
||||
biggest remaining honesty gap; the spec must state the slippage results as *model-dependent* and run a
|
||||
sensitivity (k × {1, 2, 4}) rather than a single number.
|
||||
- **Survivorship.** If delisted microcaps (pumped-then-died coins) are absent from `bybit_features`, the
|
||||
pump edge is survivorship-inflated even after haircut. **Verify** `bybit_features` is survivorship-free
|
||||
(memory `project_fxhnt_bybit_ingestion_and_book` claims yes — confirm, don't assume).
|
||||
- **AUM relevance.** At $35–100k the ADV cap barely binds — the honest picture is dominated by *slippage +
|
||||
repeatability*, not capacity. The capacity model's real value is the **future-growth ceiling**, not
|
||||
today's haircut. Say this plainly; don't oversell capacity at current size.
|
||||
- **Still a backtest.** The honest reference de-inflates *known* problems; it cannot de-inflate unknown
|
||||
future regime change. The **forward gate remains the real arbiter** — this improves the reference it
|
||||
grades against, nothing more.
|
||||
- **In-sample weighting risk.** Risk-parity uses realized vol; on the same sample it can still overfit.
|
||||
Weights must be set on a rule and validated on a held-out window, not grid-searched on Sharpe.
|
||||
|
||||
## 11. Decisions locked (from brainstorming)
|
||||
|
||||
- Capacity: **B** (coin-level ADV/impact), `participation_cap` **3%**, AUM curve **{35k, 100k, 350k, 1M,
|
||||
10M}**. Weighting: **risk-parity/inverse-vol**. Scope: **research now**.
|
||||
- Exit-liquidity: **hard ADV cap + slippage penalty** (backstop + adaptive). `unlock`: **not hard-cut** —
|
||||
let G1/G2 decide (expected ≈ 0). Edge #4: **candidate, gated**, separate capped sleeve if it survives.
|
||||
- Anti-reactive risk-management enforced by **G4**.
|
||||
@@ -0,0 +1,213 @@
|
||||
# Deploy-the-Fund Knobs — CLI-configurable capital + risk, per-sleeve allocation — Design
|
||||
|
||||
**Date:** 2026-07-16
|
||||
**Status:** Design. A focused UPGRADE of the merged capacity-aware allocator — not a new system.
|
||||
**Author:** jgrusewski (+ Claude)
|
||||
**Builds on:** `docs/superpowers/specs/2026-07-15-capacity-aware-capital-allocator-design.md` (merged) and
|
||||
`docs/superpowers/specs/2026-07-15-crypto-honest-reference-pipeline-design.md` (merged 20437d7).
|
||||
|
||||
## 1. Problem
|
||||
|
||||
The nightly allocation deploys only ~$8,840 of the $100k paper equity (≈9% — 91% cash). Two structural
|
||||
causes, both confirmed by an in-cluster smoke:
|
||||
|
||||
1. **The marginal-prune (leave-one-out) gate keeps only the volatile `positioning` sleeve** and drops the
|
||||
diversified `bybit_4edge` book: `survivors_after_marginal_prune:[positioning]` (the book adds no marginal
|
||||
Sharpe — it *contains* positioning plus lower-Sharpe drag sleeves).
|
||||
2. **The conservative 12% vol-target de-levers `positioning` hard**: its honest full-history vol is ~34% →
|
||||
`K = kelly 0.5 × target_vol 0.12 / 0.34 ≈ 0.18`, base capped at `max_strategy_weight 0.5` → deployed
|
||||
weight ≈ 0.088.
|
||||
|
||||
Compounding this: the fund's **capital and risk knobs are hardcoded defaults** — `paper_capital` in
|
||||
`config.py` and `target_vol` / `kelly_fraction` in `allocation_policy.py`. Changing them requires an env-var
|
||||
edit + redeploy. A small, actively-managed, growing fund needs to (a) tune its capital + risk appetite live
|
||||
via CLI, and (b) allocate across the individual capacity-bounded crypto sleeves rather than double-counting a
|
||||
book that overlaps a sleeve it already deploys.
|
||||
|
||||
## 2. Goal
|
||||
|
||||
Two coupled changes, paper scope:
|
||||
|
||||
- **Fund-config knobs (CLI-settable, single SSOT):** the fund's `capital`, `target_vol`, and `kelly_fraction`
|
||||
become a single persisted DB record, set via `fxhnt fund-config`, read by the nightly allocation — live, no
|
||||
redeploy, not hand-editable.
|
||||
- **Per-sleeve crypto allocation:** the allocation set switches from the `bybit_4edge` book (double-counted
|
||||
with the `positioning` sleeve) to the individual crypto sleeves (`positioning`, `xsfunding`, `unlock`), each
|
||||
sized within its own honest capacity ceiling. The book stays a tracked *reference* forward track, not an
|
||||
allocation target.
|
||||
|
||||
The forward gate (live-deploy switch) and the `capital_ceiling` human gate are unchanged. This is a minimal,
|
||||
SSOT-respecting upgrade — **no new allocation system**.
|
||||
|
||||
## 3. What already exists (verified 2026-07-16) — reuse, do not rebuild
|
||||
|
||||
- `allocation_ingest.record_allocation(dsn, equity, *, store, spread_bps, unlock_events)` allocates over
|
||||
`deploy_sids = {sid : tier == "deploy"} | repo.promoted_strategy_ids()`, honest-sizes via
|
||||
`allocation_honest_inputs.honest_allocation_inputs` when `store` is injected (else nav fallback), applies the
|
||||
killswitch on the forward record, de-funds decayers, and writes `StrategyAllocationRow`.
|
||||
- `allocation_engine.compute_allocation` does eligibility → LOO marginal-prune → inverse-vol (risk-parity)
|
||||
base capped at `max_strategy_weight` → killswitch → ex-ante vol-target `K` (kelly × target_vol /
|
||||
full-history-vol, CVaR-floored, max_leverage-capped). `target_capital` converts to dollars with per-strategy
|
||||
capacity ceilings + the `capital_ceiling` gross cap.
|
||||
- `allocation_honest_inputs.honest_allocation_inputs` maps each deploy sid → its honest-reference series +
|
||||
capacity ceiling. Today `_CRYPTO_DEPLOY_SIDS = {"positioning", "bybit_4edge"}`; `_deploy_series` special-cases
|
||||
those two; every other sid gets `equity_default_ceiling` and no series. `_sleeve_series` already computes ANY
|
||||
sleeve's honest series (it is called per-sleeve inside `_book_series`).
|
||||
- `config.py`: `paper_capital = 100_000`, `capital_ceiling_paper = 1_000_000` (the $35k figure is
|
||||
`capital_ceiling_live` — the LIVE gate, not paper). `allocation_policy.py`: `AllocationPolicy(target_vol
|
||||
0.12, kelly_fraction 0.5, max_leverage 1.5, max_strategy_weight 0.5, capital_ceiling 35_000 default,
|
||||
min_history_days 60, ...)`; `make_policy()` picks the paper/live ceiling from settings;
|
||||
`ALLOCATION_POLICY_VERSION = 3`.
|
||||
- `cockpit_forward` (Dagster asset, `assets.py:1046`) is where the nightly allocation runs:
|
||||
`equity = get_settings().paper_capital` → `record_allocation(...)`, wrapped in a graceful try/except that
|
||||
falls back to the nav path on any store failure.
|
||||
- The registry (`registry.py`) drives both the allocation set (`tier`) and the cockpit grouping. `bybit_4edge`
|
||||
is `tier: deploy`; `positioning` is `tier: deploy`; `xsfunding` / `unlock` are `tier: research`;
|
||||
`bybit_4edge_exec` / `bybit_4edge_levered` are already `tier: research`. `multistrat` uses
|
||||
`promote_on_pass: True` to auto-flip research→deploy on gate PASS (`forward_ingest.evaluate_promotions`).
|
||||
|
||||
So the plumbing, honest-sizing seam, killswitch, capacity ceilings, and $ conversion are DONE. This spec adds a
|
||||
config SSOT + read path, and re-points the crypto allocation from the book to its sleeves.
|
||||
|
||||
## 4. Design
|
||||
|
||||
### 4a. `fund_config` — the single SSOT for the tunable knobs
|
||||
|
||||
A **single-row** table (id fixed to 1), migration-created:
|
||||
|
||||
```sql
|
||||
CREATE TABLE IF NOT EXISTS fund_config (
|
||||
id INTEGER PRIMARY KEY CHECK (id = 1),
|
||||
capital DOUBLE PRECISION NOT NULL,
|
||||
target_vol DOUBLE PRECISION NOT NULL,
|
||||
kelly_fraction DOUBLE PRECISION NOT NULL,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
```
|
||||
|
||||
A small module `application/fund_config.py`:
|
||||
|
||||
- `@dataclass(frozen=True) FundConfig(capital: float, target_vol: float, kelly_fraction: float)`.
|
||||
- `read_fund_config(dsn) -> FundConfig` — reads row id=1. **Graceful fallback**: if the table or row is
|
||||
absent, returns `FundConfig(capital=settings.paper_capital, target_vol=AllocationPolicy().target_vol,
|
||||
kelly_fraction=AllocationPolicy().kelly_fraction)` — byte-identical to today's behaviour, so the nightly
|
||||
never breaks on a fresh DB (mirrors the existing store-fallback pattern in `cockpit_forward`).
|
||||
- `write_fund_config(dsn, *, capital=None, target_vol=None, kelly_fraction=None)` — **upsert with partial
|
||||
override**: reads the current effective config, overlays only the provided fields, writes row id=1. So
|
||||
`set --capital 250000` changes only capital and leaves vol/kelly as they were.
|
||||
- **Validation** (fail loud on nonsense — every input validated at the boundary): `capital > 0`,
|
||||
`0 < target_vol <= 1.0`, `0 < kelly_fraction <= 1.0`. Reject with a clear message, do not silently clamp.
|
||||
|
||||
### 4b. `fxhnt fund-config` CLI (typer, `cli.py`)
|
||||
|
||||
- `fxhnt fund-config show` — prints the effective config (capital, target_vol, kelly_fraction) and whether it
|
||||
came from the DB row or the settings fallback (so the operator can see "is this the default or a set value").
|
||||
- `fxhnt fund-config set [--capital FLOAT] [--target-vol FLOAT] [--kelly FLOAT]` — validates, upserts, prints
|
||||
the new effective config. At least one flag required (a no-op `set` errors).
|
||||
|
||||
### 4c. Wire the nightly to read `fund_config`
|
||||
|
||||
In `cockpit_forward` (`assets.py`), replace `equity = get_settings().paper_capital` with
|
||||
`cfg = read_fund_config(dsn); equity = cfg.capital`, and pass the risk knobs down so `record_allocation`
|
||||
builds its `AllocationPolicy` with `target_vol = cfg.target_vol`, `kelly_fraction = cfg.kelly_fraction` (all
|
||||
other policy fields unchanged, via `make_policy()` then a dataclass `replace`). `record_allocation` gains an
|
||||
optional `policy: AllocationPolicy | None = None` parameter (default `None` → `make_policy()`, byte-identical
|
||||
to today); `cockpit_forward` passes the fund-config-derived policy. Both the honest path and the nav-fallback
|
||||
path use the same policy.
|
||||
|
||||
### 4d. Per-sleeve allocation — resolve the book/sleeve overlap
|
||||
|
||||
Three coordinated changes:
|
||||
|
||||
1. **Registry (`registry.py`):**
|
||||
- `xsfunding`: `tier: research` → **`promote_on_pass: True`** (NOT a hard flip to deploy). It enters the
|
||||
allocation when its reconciliation gate PASSes — respecting the forward gate as the deploy switch, exactly
|
||||
as `multistrat` does. Same for **`unlock`**.
|
||||
- `bybit_4edge`: `tier: deploy` → **`tier: research`**. The book becomes a tracked *reference* aggregate
|
||||
(it already sits alongside its `bybit_4edge_exec` / `bybit_4edge_levered` research twins), no longer an
|
||||
allocation target. This removes the positioning/book double-count. `positioning` stays `tier: deploy`.
|
||||
- Net deploy set today: `{positioning}` + any of `{unlock, xsfunding}` whose gate has PASSed. As each
|
||||
sleeve gate passes, the allocation gains that sleeve — a gated per-sleeve rollout.
|
||||
|
||||
2. **Provider (`allocation_honest_inputs.py`):** generalize the crypto mapping so ANY crypto sleeve is
|
||||
honest-sized (not just `positioning`):
|
||||
- `_CRYPTO_DEPLOY_SIDS = frozenset({"positioning", "xsfunding", "unlock"})` (drop `bybit_4edge` — it is no
|
||||
longer in the deploy set; `_book_series` / `_BOOK_SLEEVES` stay for the reference report, unused by the
|
||||
allocation path).
|
||||
- `_deploy_series(store, sid, ...)`: for `sid in {"positioning", "xsfunding", "unlock"}` →
|
||||
`_sleeve_series(store, sid, ...)` (the existing per-sleeve engine); else `{}`. The `sid == "bybit_4edge"`
|
||||
book branch is removed from the allocation path.
|
||||
|
||||
3. **No execution change today.** Both rebalancers are SUSPENDED (no live crypto creds). The per-sleeve
|
||||
allocation writes per-sleeve `StrategyAllocationRow`s (paper/display sizing). When crypto execution is later
|
||||
armed, `execute-bybit` must size the book as `Σ(sleeve target_dollars)` over the deploy sleeves — a
|
||||
documented **follow-on**, out of scope here (nothing executes today, so no live risk).
|
||||
|
||||
### 4e. Seed the initial `fund_config` row (migration data step)
|
||||
|
||||
Insert row id=1: `capital = 100_000` (current), `target_vol = 0.20` (the chosen risk appetite — moderately
|
||||
conservative), `kelly_fraction = 0.5` (unchanged). Idempotent (`ON CONFLICT (id) DO NOTHING`) so a re-run does
|
||||
not clobber a value the operator has since set via CLI.
|
||||
|
||||
## 5. Falsification gate — does per-sleeve actually deploy more? (the honest check)
|
||||
|
||||
Per-sleeve is **not guaranteed** to raise deployment. The LOO marginal-prune gate — correctly — drops sleeves
|
||||
that do not add marginal Sharpe, and the diversifier-hunt (2026-07-15) already found **`unlock` fails the LOO
|
||||
gate** (co-crash / immaterial). So the deploy set may honestly collapse toward `{positioning}` or
|
||||
`{positioning, xsfunding}`. This spec does NOT weaken or remove the marginal gate to force diversification —
|
||||
co-crashers *should* be pruned.
|
||||
|
||||
**Predetermined gate (run in-cluster after implementation, before any go-live):**
|
||||
|
||||
- **G1 (survival):** report `survivors_after_marginal_prune` for the per-sleeve deploy set on the honest
|
||||
series. Expected: `{positioning}` and, if it adds marginal Sharpe, `{xsfunding}`; `unlock` expected pruned.
|
||||
- **G2 (deployment):** report the resulting deployment % of equity at `target_vol = 0.20`. Compare against the
|
||||
positioning-alone baseline (8.84% at 12% vol). The honest lever ranking is: **vol-target 0.12→0.20 raises
|
||||
deployment regardless of survival; per-sleeve raises it further ONLY if xsfunding survives LOO.**
|
||||
- **G3 (no reactive de-sizing):** confirm the ex-ante `K` is still computed from full-history vol (never
|
||||
trailing) — the anti-reactive invariant is untouched.
|
||||
- **Verdict rule:** if the deploy set collapses to `{positioning}` alone, the honest conclusion is
|
||||
*concentrate + use the vol-target as the deployment lever* (per-sleeve added nothing) — report that plainly,
|
||||
do NOT override the gate. If `{positioning, xsfunding}` survives, per-sleeve is the diversified win.
|
||||
|
||||
## 6. Scope
|
||||
|
||||
**In:** the `fund_config` table + migration + seed row; `application/fund_config.py` (read/write/fallback +
|
||||
validation); `fxhnt fund-config show|set` CLI; `record_allocation` optional `policy` param + `cockpit_forward`
|
||||
wiring; the registry tier/promote changes; the provider crypto-sleeve generalization; the falsification-gate
|
||||
in-cluster run + report; tests; local cockpit verification that `bybit_4edge`-as-reference renders sanely.
|
||||
|
||||
**Out (follow-on):** `execute-bybit` per-sleeve book sizing (execution is suspended); a real equity-ADV
|
||||
capacity model for IBKR; raising `capital_ceiling_live` above $35k / going live (the deliberate human gate);
|
||||
any new execution path; making `max_strategy_weight` / `max_leverage` / `capital_ceiling` CLI-tunable (only
|
||||
capital + target_vol + kelly are in scope — the three the user is actively choosing).
|
||||
|
||||
## 7. Critical review — where this could go wrong
|
||||
|
||||
- **Per-sleeve may not diversify** — §5 is the guard. `unlock` is a known LOO failure; do not force it in. The
|
||||
gate reports the truth; accept it.
|
||||
- **`bybit_4edge` → research is a cockpit change.** The fund's crypto headline book moves from the deploy
|
||||
section to research (with its exec/levered twins). Verify locally that the cockpit renders this without a
|
||||
broken headline or an empty deploy section before merge (the deploy section then shows the sleeves).
|
||||
- **Promoting sleeves via `promote_on_pass`, not a hard tier flip.** This respects the forward gate. It means
|
||||
the per-sleeve allocation is *gradual* (sleeves join on gate PASS), not instant — a feature, not a bug, but
|
||||
state it so "why is xsfunding not allocated yet" is answerable (its gate is WAIT 8/14).
|
||||
- **20% vol is a real risk increase** over 12%. At risk-parity the low-vol `xsfunding` (if it survives)
|
||||
dominates the base weight (capped 0.5) and `K` can run toward `max_leverage 1.5`. Report the actual
|
||||
deployed gross + implied book vol in the G2 run so the number is visible, not assumed.
|
||||
- **`fund_config` fallback must be byte-identical** to today when the table/row is absent — a fresh DB or a
|
||||
failed read must not change the allocation. Test this explicitly (fallback == settings default).
|
||||
- **Partial-override upsert** must not reset unspecified fields to defaults (a classic upsert bug):
|
||||
`set --capital X` must leave target_vol/kelly at their current values. Test it.
|
||||
- **Single-writer / idempotent seed:** the seed uses `ON CONFLICT DO NOTHING`; a re-applied migration must not
|
||||
overwrite an operator-set value.
|
||||
|
||||
## 8. Decisions locked (from the design dialogue)
|
||||
|
||||
- Knobs: **capital + target_vol + kelly_fraction**, one persisted `fund_config` DB row, CLI-set only (never
|
||||
hand-edited), read by the nightly with a settings fallback. Initial: **capital $100k, target_vol 0.20, kelly
|
||||
0.5**.
|
||||
- Per-sleeve: **promote `xsfunding` + `unlock` via `promote_on_pass`; demote `bybit_4edge` to a research
|
||||
reference track; generalize the provider to honest-size any crypto sleeve.** Marginal gate KEPT.
|
||||
- A **predetermined falsification gate** (§5) decides whether per-sleeve actually deploys more; the honest
|
||||
verdict is accepted either way. Scope stays **paper**; execution wiring is a follow-on.
|
||||
@@ -0,0 +1,81 @@
|
||||
# Equity Leg Deployable — Design
|
||||
|
||||
**Date:** 2026-07-16
|
||||
**Status:** Design. Makes the IBKR US-ETF `multistrat` book an honest-sized, capacity-bounded deploy edge — the growth vehicle above the crypto book's ~$350k break.
|
||||
**Author:** jgrusewski (+ Claude)
|
||||
**Builds on:** the capacity-aware allocator + honest-reference pipeline (crypto) + the load-once/reprice provider (merged 2026-07-16).
|
||||
|
||||
## 1. Problem
|
||||
|
||||
The fund's crypto book caps at ~$350k (measured; xsfunding-driven). To grow past that toward $1M the **IBKR
|
||||
US-ETF `multistrat` book** must carry the additional capital. But `multistrat` is NOT deployable today:
|
||||
1. **It can't be honest-sized.** The crypto provider (`honest_allocation_inputs`) gives equity sids no series,
|
||||
so `record_allocation` falls back to `multistrat`'s ~7-day forward NAV record, which is < `min_history_days`
|
||||
(60) → it is dropped from the allocation entirely (gets $0).
|
||||
2. **Its capacity is a $10M stub** (`equity_default_ceiling`) — not a real, data-derived ceiling.
|
||||
|
||||
User decision (2026-07-16): build it the rigorous way — **ingest real ETF volume** for a data-driven ADV
|
||||
capacity model, and charge **real trading costs** (not the inflated 15bp research stub).
|
||||
|
||||
## 2. Goal
|
||||
|
||||
An **equity honest-reference** analog (mirroring the crypto one) that produces, for `multistrat`:
|
||||
- a **cost-netted honest backtest sizing series** (~441 days on Yahoo PIT → passes `min_history_days`), and
|
||||
- a **real ADV-derived capacity ceiling** (from ingested ETF volume),
|
||||
merged into `record_allocation` so the equity book is honest-sized + capacity-bounded exactly like crypto.
|
||||
Scope stays PAPER; the forward gate (`multistrat` promote_on_pass, currently 7/20) remains the live-deploy
|
||||
switch. As equity grows into it, the capacity-aware allocator already shifts capital correctly.
|
||||
|
||||
## 3. What exists (verified 2026-07-16) — reuse
|
||||
|
||||
- `multistrat.book_series(closes, ...)` → the GROSS unlevered daily book return (~441d). `multistrat.target_weights(closes, ...)` → the per-instrument position vector per day (for turnover). `FUND_INSTRUMENTS = [SPY, IEF, GLD, PDBC, DBMF]`.
|
||||
- `equity_backtest_runner._turnover(prev, cur)` (two-way round-trip turnover) + its `cost_bps_per_turnover` idiom — the cost-charging pattern to mirror (but with REAL, per-instrument costs, not a flat 15bp).
|
||||
- Yahoo ingestion: `YahooDailyClient.adj_closes(sym)` extracts `ind["quote"][0].close`/`adjclose`; the SAME Yahoo response already carries `ind["quote"][0].volume` — volume is fetchable by extending the extraction, no new data source. The nightly asset (`assets.py:408-421`) snapshots `adj_closes` into `YahooPitCloseRow` via `snapshot_yahoo_closes`; `YahooPitClient.adj_closes` reads it back PIT.
|
||||
- `honest_reference.honest_sleeve_returns` (turnover-basis cost, sqrt-impact on ADV) + `capacity_curve` + `allocation_engine.capacity_ceiling_from_curve` — the crypto templates to mirror for equity.
|
||||
- `record_allocation` merges `honest_returns` (provider) with a nav fallback; the load-once provider pattern.
|
||||
|
||||
## 4. Design
|
||||
|
||||
### 4a. Ingest ETF volume (the real ADV input)
|
||||
- `YahooDailyClient.volumes(symbol) -> dict[str, float]`: extract `ind["quote"][0]["volume"]` (raw daily share volume), same fetch/retry path as `adj_closes`.
|
||||
- `YahooPitVolumeRow` (cockpit_models, `yahoo_pit_volume`, cols: symbol, date, volume, first_seen_at) + `ForwardNavRepo.snapshot_yahoo_volumes` / `yahoo_pit_volumes(symbol)` — PIT store parallel to closes (append-only, first-seen frozen — same PIT discipline).
|
||||
- Nightly asset (`assets.py`, next to the close snapshot): also `snapshot_yahoo_volumes(sym, live.volumes(sym), at)`.
|
||||
- **Backfill**: a `fxhnt backfill-etf-volume` CLI — one Yahoo fetch per FUND_INSTRUMENT (25y range), snapshot the volume history, so ADV is available immediately (not only forward). PIT: first-seen frozen.
|
||||
- **Dollar-ADV**: `adv_usd[sym][day] = adj_close[sym][day] * volume[sym][day]`, then a trailing **median** over a window (e.g. 63d) for a stable, outlier-robust ADV (median, not mean — a single volume spike must not inflate capacity). Missing volume → that day excluded (no fabricated ADV).
|
||||
|
||||
### 4b. Real equity cost model (`equity_honest.py` — the equity analog of honest_reference)
|
||||
Per instrument, per rebalance day, charge on TRADED weight (turnover-basis, mirroring the crypto model):
|
||||
- **Market impact** = `IMPACT_COEFF_BPS * impact_k * sqrt(traded_notional / adv_usd)` bps — REAL, from ingested ADV. This is what makes capacity bite: as AUM grows, `traded_notional` rises → impact rises → Sharpe degrades.
|
||||
- **Spread** = a realistic PER-ETF half-spread constant (bps), documented from published typical ETF spreads (SPY ~0.3, IEF ~1.0, GLD ~1.0, PDBC ~4.0, DBMF ~8.0 bps). This is the ONE estimated input (no ETF bid/ask feed); it is a documented constant, NOT the inflated flat 15bp. `slip_bps = max(impact, spread_bps[sym])` (same max-with-spread-floor as crypto).
|
||||
- **Commission** = IBKR real per-share (tiered ~ $0.0035/share, min/max) converted to bps at the day's price: `commission_bps = 100 * per_share / price`. Charged per traded share.
|
||||
- `equity_honest_returns(book_ret_by_day, weights_by_day, adv_usd, spread_bps, *, aum, participation_cap, impact_k, commission_per_share) -> dict[str,float]`: for each day, net = `book_ret[day] - Σ_sym traded_w * (slip_bps + commission_bps)/1e4`, where `traded_w = |w[sym,day] - w[sym,day-1]|`. Also apply the **participation cap** to positions (a position that would exceed `participation_cap * adv_usd` is scaled down — capacity, not just cost), consistent with `honest_sleeve_returns`.
|
||||
|
||||
### 4c. Equity capacity curve + ceiling
|
||||
- `equity_capacity_curve(book_ret, weights, adv_usd, spread_bps, *, aums, ...)` → `{aum: equity_honest_returns(...at aum)}` (load-once: book_ret/weights/adv computed once, repriced per AUM — mirrors the crypto load-once).
|
||||
- Ceiling = `capacity_ceiling_from_curve({aum: sharpe_of(series)}, min_sharpe)` — the largest AUM whose net Sharpe stays viable. For these liquid ETFs the ceiling is high but REAL (DBMF/PDBC ADV-limited); report it next to the curve.
|
||||
|
||||
### 4d. Equity provider + wiring
|
||||
- `equity_allocation_inputs(repo, deploy_sids, *, target_aum, aums, participation_cap, impact_k) -> (returns_by_strategy, ceilings)`: for each equity deploy sid (venue `ibkr-equity`, i.e. `multistrat`), build closes+volume from the PIT store, compute `book_series`/`target_weights`, then the cost-netted series at `target_aum` (ISO-keyed) + the ADV capacity ceiling. Non-equity sids ignored (crypto provider handles them).
|
||||
- `record_allocation`: when `store is not None`, ALSO call `equity_allocation_inputs` for the equity deploy sids and MERGE: `honest_returns |= eq_returns`, `ceilings |= eq_ceilings`. Then the existing `returns = {sid: honest_returns.get(sid) or nav_fallback}` picks up `multistrat`'s 441-day cost-netted series → eligible (>60) → sized; `target_capital` caps it at the real ADV ceiling. `store=None`/no-equity → byte-identical to today.
|
||||
|
||||
## 5. Falsification / validation gates (in-cluster, after build)
|
||||
- **G1 (sizes):** `multistrat` now has ≥ `min_history_days` in the allocation returns (not dropped). Report its cost-netted Sharpe/vol vs the gross book_series (costs must REDUCE Sharpe, not increase it).
|
||||
- **G2 (real capacity):** the ADV ceiling is finite and data-derived (report it + the tightest-ETF binding). Sanity: it must be well above the fund's current $100k and reflect DBMF/PDBC being the tightest.
|
||||
- **G3 (cost realism):** the equity cost must be materially BELOW the old 15bp stub for the liquid ETFs (SPY/IEF/GLD) — report the effective bps/turnover; if it lands ~15bp we mis-modeled.
|
||||
- **G4 (combined marginal gate):** with `multistrat` sized, re-run the marginal gate for {crypto book, multistrat}. Report whether the uncorrelated equity book SURVIVES alongside the crypto book (it should add marginal Sharpe vs the crypto-book-alone, unlike vs volatile positioning-alone) and the combined deployment. Accept the honest verdict.
|
||||
- **PIT integrity:** the volume snapshot is first-seen-frozen (a re-fetch must not rewrite history).
|
||||
|
||||
## 6. Scope
|
||||
**In:** volume extraction + PIT store + snapshot + backfill CLI; dollar-ADV (median); `equity_honest.py` (real cost model + capacity curve); `equity_allocation_inputs` + `record_allocation` merge; tests; in-cluster gates.
|
||||
**Out (follow-on):** live intraday spread/quote feed for ETFs (spread stays a documented per-ETF constant); options/VRP equity; raising the live `capital_ceiling` / going live (human gate); execute-side IBKR sizing beyond what already exists.
|
||||
|
||||
## 7. Critical review — where this could go wrong
|
||||
- **Spread is the one estimated input.** No ETF bid/ask feed → per-ETF published constants. Document them; they dominate cost for liquid ETFs at fund scale (impact ~0 there), so a wrong spread mis-sizes. Keep them conservative-realistic and visible, not buried.
|
||||
- **book_series is gross + unlevered.** Confirm the cost netting uses the SAME weights `book_series` applies (via `target_weights` = `L·tw·scale`), else cost/return bases mismatch. The turnover must be the ACTUAL traded weight, not the raw signal.
|
||||
- **ADV median window + PIT.** A too-short ADV window is noisy; too-long lags real liquidity. 63d median is a choice — report it. Volume PIT-frozen so a Yahoo revision can't retroactively inflate past capacity.
|
||||
- **Capacity for liquid ETFs is high → the ceiling may not bind until >$1M.** That is correct (deep ETFs), but it means the equity ceiling won't shape sizing at $100k–$350k — it is the GROWTH guard, not a near-term constraint. State it.
|
||||
- **Combined Sharpe dilution (§5 G4).** Equity is lower-Sharpe; the blended book Sharpe drops as equity's weight grows. The marginal gate + risk-parity handle WHETHER to include it; do not force it if it fails the gate against the crypto book.
|
||||
- **Equity turnover cost realism.** multistrat is adaptive (daily reweight) → non-trivial turnover; the cost model must charge it honestly (two-way turnover) or it flatters the equity Sharpe.
|
||||
|
||||
## 8. Decisions locked
|
||||
- **Ingest real ETF volume** (extend the Yahoo extraction + PIT snapshot + backfill) for a data-driven dollar-ADV (63d median). **Real costs**: sqrt-impact on real ADV + per-ETF documented spread + real IBKR commission (NOT the flat 15bp). Sizing series = `book_series` NET of these. Provider merged into `record_allocation`; forward gate unchanged; PAPER scope. Live spread feed = follow-on.
|
||||
@@ -0,0 +1,185 @@
|
||||
# Foxhunt → Bizworx Platform Consolidation — Design
|
||||
|
||||
**Date:** 2026-07-18
|
||||
**Status:** DRAFT (awaiting operator review)
|
||||
**Goal:** Retire the separate **foxhunt** Scaleway Kapsule cluster by moving the live fxhnt fund onto the
|
||||
**bizworx** Kapsule cluster and consolidating to a **single gitea**, then destroying the foxhunt Kapsule —
|
||||
eliminating a whole managed cluster + its nodes (cost saving).
|
||||
|
||||
> Written durably to git because this session suffered heavy context loss (a stale 2026-07-13 replay + a
|
||||
> dev-machine move). Ground truth in this doc was verified live against both clusters on 2026-07-18.
|
||||
|
||||
---
|
||||
|
||||
## 1. Current state (verified live 2026-07-18)
|
||||
|
||||
Two Scaleway Kapsule clusters, **same org (`a55098a3`), different projects**, both `fr-par`, k8s 1.34.6, cilium:
|
||||
|
||||
| Platform | Kapsule | Project | Nodes | Role |
|
||||
|---|---|---|---|---|
|
||||
| **bizworx** (survivor) | `05e1dae3` | `96d0717d` | 3× gp1_xs (+large) | MSP platform (msp-portal, tacticalrmm, argocd, vaultwarden, gitea…) |
|
||||
| **foxhunt** (to retire) | `34a1e3c4` | `c293eb98` | 3× dev1_l (+ci 0-4) | the fxhnt fund (ns `foxhunt`) + lots of legacy |
|
||||
|
||||
**Access note (resolved):** the foxhunt Kapsule API has an IP allowlist (`acl_available: true`). After the
|
||||
dev-machine move, this box's egress `51.158.111.225` was not allowed → HTTP 000. Fixed by appending it to the
|
||||
ACL (`scw k8s acl add … bizworx-devbox-v4`; existing tailscale-cgnat + admin entries preserved). kubeconfig
|
||||
re-installed via `scw k8s kubeconfig install 34a1e3c4…`. Both contexts now in `~/.kube/config`.
|
||||
|
||||
**xps-ubnt** (tailnet `100.91.132.125`) = the OLD dev machine. k3s inactive; runs some Docker (a dev
|
||||
`foxhunt-postgres` Timescale, a buildx builder, unrelated lexpdf/PDF tools). NOT the fund host — just where
|
||||
the foxhunt kubeconfig still lived. Not part of this migration except as a source of the old kubeconfig.
|
||||
|
||||
### 1a. What actually runs in ns `foxhunt` (empirical)
|
||||
|
||||
**Live (READY=1) — the fund's real footprint:** `postgres`, `dagster`, `ib-gateway`, `fxhnt-dashboard`
|
||||
(cockpit), `gitea`, `argo` (workflows + events + gitea push/deploy sensors), plus `minio`, `redis`,
|
||||
`grafana`/`prometheus` (the last four are NOT fund dependencies — see below).
|
||||
|
||||
**Scaled to zero / dead (READY none) — legacy foxhunt, Phase-2 triage:** `api`, `backtesting-service`,
|
||||
`broker-gateway`, `data-acquisition-service`, `ml-training-service` (old Rust microservice architecture),
|
||||
`kanidm` (SSO), `mattermost`, `stalwart` (mail), `netbird-*`, `questdb`, `loki`, `pushgateway`.
|
||||
|
||||
### 1b. Stateful data — what migrates vs drops (empirical: mounted-by-running-pod)
|
||||
|
||||
| PVC | Size | Mounted by | Decision |
|
||||
|---|---|---|---|
|
||||
| `postgres-pvc` | 100Gi | postgres | **MIGRATE** (operational DB + Timescale warehouse = the SSOT) |
|
||||
| `ib-gateway-settings` | 1Gi | ib-gateway | migrate OR re-auth fresh (trivial) |
|
||||
| `fxhnt-surfer-data` | 5Gi | dagster | holds only `unlock_calendar.json` (DB-first + JSON fallback) → copy the file or let it rebuild |
|
||||
| `gitea-shared-storage` | 5Gi | gitea | migrates via the gitea repo-migration (§3), not as a raw PVC |
|
||||
| `feature-cache-pvc` | 100Gi | — (orphan) | DROP (rebuildable cache) |
|
||||
| `fxhnt-backtest-data` | 20Gi | — (orphan) | DROP (B3 DuckDB warehouse, parked in 0a) |
|
||||
| `questdb-pvc` | 10Gi | — (questdb dead) | DROP |
|
||||
| `multistrat-state` | 1Gi | — (orphan) | DROP (vestigial state_file; forward state is DB-SSOT) |
|
||||
| `minio-data-new` | 150Gi | minio | Phase-2 triage (NOT a fund dep — `grep` of fxhnt src/manifests finds no minio/s3/redis runtime use) |
|
||||
| `training-data-pvc` | 500Gi | — | Phase-2 (decommissioned Rust ML) → DROP |
|
||||
| `repo-data-gitlab-gitaly-0` | 50Gi | — | Phase-2 (legacy GitLab, replaced by gitea) → DROP |
|
||||
| `kanidm/mattermost/stalwart/netbird/test-data/tempo/…` | ~120Gi | — | Phase-2 triage (drop unless a non-fund need surfaces) |
|
||||
|
||||
**Net fund data to physically migrate ≈ 100Gi (one Postgres), everything else is drop-or-rebuild.**
|
||||
"fxhnt heeft minder nodig" — confirmed.
|
||||
|
||||
### 1c. Gitea reality
|
||||
|
||||
- `git.fxhnt.ai` → `100.95.225.27` (tailnet node `foxhunt-gitlab`) → the **foxhunt-cluster gitea** (repos
|
||||
`gitadmin/fxhnt`, `gitadmin/foxhunt`). Canonical today.
|
||||
- **bizworx** already runs its own gitea (ns `gitea`, 50Gi, ClusterIP, no ingress).
|
||||
- The fund's in-cluster git-sync clones from `gitea-sshd.foxhunt.svc.cluster.local:22`.
|
||||
|
||||
### 1d. IaC form
|
||||
|
||||
Two layers, both in the fxhnt repo: **Terragrunt/Terraform** (`infra/live/production/{kapsule,
|
||||
public-gateway,block-storage}`, scw S3 tfstate backend) for the cloud infra (the Kapsule itself), and
|
||||
**k8s manifests** (`infra/k8s/{argo,databases,gitea,jobs,monitoring,…}`) applied via **argo** (git-sync from
|
||||
gitea; `reference_fxhnt_prod_deploy_mechanics`: `argo submit … -p mode=deploy`, NOT image-baked).
|
||||
|
||||
---
|
||||
|
||||
## 2. Target architecture (after Phase 1)
|
||||
|
||||
The fund runs in **ns `foxhunt` on the bizworx Kapsule** (name reused → minimal manifest churn):
|
||||
- Workloads: `postgres` (Timescale), `dagster`, `ib-gateway`, `fxhnt-dashboard`, the rebalancer/backtest
|
||||
CronJobs, git-sync from the **bizworx gitea**.
|
||||
- **Reuse bizworx's existing `argo` + observability** (do NOT migrate prometheus/grafana/loki/tempo).
|
||||
- Single **bizworx gitea** holds `gitadmin/fxhnt` + `gitadmin/foxhunt` (archive); `git.fxhnt.ai` re-points to it.
|
||||
- `dashboard.fxhnt.ai` ingress → bizworx (ACME TLS, per `reference_fxhnt_dashboard_proxy`).
|
||||
|
||||
Decisions locked with the operator (2026-07-18):
|
||||
- **Survivor gitea = bizworx**; migrate both `fxhnt` + `foxhunt` (archive); keep hostname `git.fxhnt.ai`,
|
||||
re-point to bizworx (no remote-URL changes in repos/CI).
|
||||
- **ib-gateway:** no 2FA, no IP allowlist on the IBKR side → re-auth = IBKR paper creds in a secret + reconnect.
|
||||
- **Downtime:** flexible (fund runs ~once/day) — schedule a convenient quiet window; no tight SLA.
|
||||
- Reuse bizworx argo + observability.
|
||||
|
||||
---
|
||||
|
||||
## 3. Phase 1 — migration mechanics
|
||||
|
||||
### 3.1 Prep bizworx (no downtime)
|
||||
- Create ns `foxhunt` on bizworx; confirm a suitable StorageClass (bizworx uses `sbs-5k`/`sbs-default`;
|
||||
foxhunt used `sbs-default`/`scw-bssd`).
|
||||
- Recreate secrets on bizworx (source via bizworx external-secrets + openbao where possible): `db-credentials`,
|
||||
`databento-credentials`, IBKR/IB-gateway creds, `scw-registry` (image pull), `argo-git-ssh-key` /
|
||||
`git-ssh-key`, tailscale auth. **Never** commit secrets; use `! kubectl`/openbao, never the transcript.
|
||||
- Migrate gitea repos: create `gitadmin/fxhnt` + `gitadmin/foxhunt` on the bizworx gitea and
|
||||
`git push --mirror` (or gitea's built-in migration) all refs/LFS. Verify commit counts + tags match.
|
||||
- Re-target the k8s manifests' cluster/namespace/gitea endpoint to bizworx; `kubectl apply --dry-run` on the
|
||||
bizworx `foxhunt` ns to validate before cutover.
|
||||
|
||||
### 3.2 Cutover (the downtime window)
|
||||
1. **Freeze foxhunt:** `kubectl patch cronjob … suspend:true` for all fund CronJobs; scale dagster/cockpit
|
||||
writers down so Postgres is quiescent.
|
||||
2. **Migrate the DB:** `pg_dump` the foxhunt Postgres (`fxhnt` DB, all schemas incl. TimescaleDB hypertables —
|
||||
use `pg_dump`-with-timescaledb guidance / `timescaledb_pre_restore`/`post_restore`) → `pg_restore` into the
|
||||
bizworx Postgres. Verify row counts on the key tables (`forward_nav`, `backtest_summary`, `bybit_sleeve_ret`,
|
||||
`xsp_option_bars`, the feature tables).
|
||||
3. **Stand up the fund on bizworx:** apply the re-targeted manifests; bring up postgres (restored),
|
||||
dagster, cockpit, ib-gateway (re-auth the IBKR paper session).
|
||||
4. **Re-point DNS:** `git.fxhnt.ai` and `dashboard.fxhnt.ai` → bizworx (tailnet node / ingress). Fund git-sync
|
||||
now resolves the bizworx gitea `gitea-sshd`.
|
||||
5. **Copy the small bits:** `unlock_calendar.json` (or let it rebuild from the DB-first loader);
|
||||
ib-gateway-settings if not re-authing fresh.
|
||||
|
||||
### 3.3 Verify (before decommissioning anything)
|
||||
- Cockpit renders real data — **locally in a real browser** (`reference_fxhnt_local_cockpit_dev_loop`) AND on
|
||||
`dashboard.fxhnt.ai`; read the numbers (`feedback_verify_cockpit_locally_not_prod`).
|
||||
- A `fxhnt-ucits-rebalancer` **dry-run** on bizworx fills/plans correctly against IBKR (EU hours).
|
||||
- Reconciliation gate + forward tracks intact (the migrated `forward_nav`/`backtest_summary` reconcile;
|
||||
no false re-inception — `forward_engine` recompute is deterministic from the DB anchor).
|
||||
- Dagster nightly asset graph runs green on bizworx.
|
||||
- Run **in parallel/observe** for a short window (foxhunt suspended, bizworx live) before teardown.
|
||||
|
||||
### 3.4 Rollback
|
||||
Until the foxhunt Kapsule is destroyed (Phase 3), rollback = re-point DNS back + un-suspend foxhunt CronJobs
|
||||
(its Postgres still holds the pre-cutover state). The foxhunt cluster is left intact-but-suspended through the
|
||||
observation window precisely to keep this cheap.
|
||||
|
||||
---
|
||||
|
||||
## 4. Phase 2 — triage the rest of ns `foxhunt`
|
||||
|
||||
Produce a definitive **keep/drop** list for every non-fund service + orphan PVC, so the teardown is
|
||||
accountable (nothing silently lost). Expected outcome (from §1a/1b, mostly DROP):
|
||||
- **DROP (confirmed dead):** the Rust-era microservices (api/backtesting-service/broker-gateway/
|
||||
data-acquisition-service/ml-training-service), `questdb`, `loki`, `pushgateway`, `training-data-pvc` (500Gi),
|
||||
`repo-data-gitlab-gitaly-0` (50Gi), `feature-cache`, `fxhnt-backtest-data`, `multistrat-state`, `test-data`.
|
||||
- **DECIDE (non-fund, may still be wanted somewhere):** `minio` (150Gi), `redis`, `kanidm` (SSO),
|
||||
`mattermost`, `stalwart` (mail), `netbird`. For each: drop, or migrate to bizworx / keep external. The
|
||||
operator confirms per item before its data is deleted (`feedback` — look before deleting).
|
||||
|
||||
## 5. Phase 3 — teardown (cost saving)
|
||||
|
||||
After Phase-1 verification holds and the Phase-2 list is resolved:
|
||||
- Remove the foxhunt k8s manifests; delete the foxhunt Kapsule via **Terragrunt** (`infra/live/production/
|
||||
kapsule` for project `c293eb98`) — control plane + the 3 dev1_l nodes + the ci pool.
|
||||
- Release orphaned block volumes (retain-class PVCs: `gitea-shared-storage`, `feature-cache` are
|
||||
`*-retain` → must be deleted explicitly), the load balancer, the private network, and the foxhunt-project
|
||||
scw resources.
|
||||
- Remove the temporary foxhunt API ACL entry for this box (or keep if still useful).
|
||||
- Update memory: mark the foxhunt Kapsule retired; the fund now lives on bizworx.
|
||||
|
||||
---
|
||||
|
||||
## 6. Risks / open items
|
||||
- **TimescaleDB dump/restore** needs the hypertable-aware procedure (`timescaledb_pre_restore()` /
|
||||
`timescaledb_post_restore()`), not a naive `pg_dump`. Verify the extension version matches on bizworx.
|
||||
- **ib-gateway** must re-establish its IBKR paper session on bizworx (no 2FA/IP-limit per operator, so
|
||||
expected simple — but confirm it connects + a whatIf order is accepted before arming the rebalancer).
|
||||
- **Secrets provenance:** confirm every secret can be re-sourced on bizworx (openbao/external-secrets) before
|
||||
cutover — a missing secret wedges a workload.
|
||||
- **Cross-project scw:** LB/DNS/tailnet re-pointing spans the two scw projects; confirm the tailnet ACLs +
|
||||
ingress + ACME cert issuance work from the bizworx side.
|
||||
- **Deploy pipeline:** the fund deploys via argo git-sync from gitea (`mode=deploy`); confirm the bizworx argo
|
||||
+ the migrated gitea + the eventsource/sensor wiring reproduce the deploy flow.
|
||||
|
||||
## 7. Success criteria
|
||||
- The fxhnt fund runs entirely on the bizworx Kapsule (cockpit, dagster, ib-gateway, rebalancers) against the
|
||||
migrated Postgres, verified in a real browser + a live rebalancer dry-run.
|
||||
- One gitea (bizworx) serves `git.fxhnt.ai` with both repos; fund git-sync + CI unchanged.
|
||||
- The foxhunt Kapsule + its nodes are destroyed; no fund data or wanted service lost (Phase-2 list resolved).
|
||||
- Monthly scw cost drops by the foxhunt cluster's control-plane + 3 dev1_l nodes + its block storage.
|
||||
|
||||
---
|
||||
*Related memory:* `project_phase2_gitlab_to_gitea` (GitLab→Gitea + foxhunt→fxhnt IaC), `netbird-mesh`,
|
||||
`reference_fxhnt_prod_deploy_mechanics`, `reference_fxhnt_dashboard_proxy`,
|
||||
`reference_fxhnt_local_cockpit_dev_loop`, `reference_ci_pool_and_tfstate`, `infrastructure`.
|
||||
85
docs/superpowers/specs/2026-07-20-eur-fund-ledger-design.md
Normal file
85
docs/superpowers/specs/2026-07-20-eur-fund-ledger-design.md
Normal file
@@ -0,0 +1,85 @@
|
||||
# EUR Fund Ledger — Design Spec (2026-07-20)
|
||||
|
||||
**Status:** design approved (scope + cost-basis decided with operator). Implementation plan follows separately.
|
||||
|
||||
**> NOT tax advice.** This is an engineering design for a bookkeeping system. The tax *treatment* it records
|
||||
(Box 3 passive capital, FIFO cost basis, EUR-functional) reflects the operator's resolved structure and must be
|
||||
confirmed by a Dutch belastingadviseur before it is relied on for a filing. See
|
||||
`docs/superpowers/2026-07-20-fund-tax-accounting-research-brief.md` and memory
|
||||
`project_fxhnt_fund_tax_accounting_research`.
|
||||
|
||||
## Goal
|
||||
|
||||
A tax-ready, append-only, double-entry **EUR ledger** for the fxhnt fund's **passive Box-3 investment book**,
|
||||
living in the existing Postgres/Timescale warehouse, fed by an automated **IBKR Flex** ingestion Dagster asset.
|
||||
Phase 1 books **the fund only** — the passive IBKR investment book. The tech-BV bookkeeping and the arm's-length
|
||||
BV→fund service stay in the operator's existing BV administration (explicitly out of scope).
|
||||
|
||||
## Why this shape (context)
|
||||
|
||||
- The operator's structure separates **arbeid** (tech-BV, taxed there) from **kapitaal** (private fund, Box 3).
|
||||
Phase 1 books only the kapitaal side → a clean Box-3 record with no BV entanglement.
|
||||
- **EUR is the functional currency**, not just presentation. IBKR DU9600528 is EUR-native; if crypto ever goes
|
||||
live it is Bybit-EU (EUR, not USDT). There is **no USD leg**, so there is **no conversion pipeline** — only a
|
||||
dormant `currency` + `fx_rate_to_eur` seam (default EUR/1.0) so a stray non-EUR line is representable and
|
||||
correct rather than silently mis-booked. A pure-EUR book has **no FX P&L** — only trading P&L.
|
||||
- Reuses existing patterns: SQLAlchemy 2.0 ORM (`Base`/`Mapped`, Postgres+SQLite), a `*Repo` with `migrate()`,
|
||||
a Dagster asset alongside the nightly ETL. No new infrastructure.
|
||||
|
||||
## Global Constraints
|
||||
|
||||
1. **EUR-functional.** Every posting stored in EUR. `currency` defaults `"EUR"`, `fx_rate_to_eur` defaults
|
||||
`1.0`. Build NO conversion pipeline — the FX columns are dormant insurance only.
|
||||
2. **Append-only.** `journal_entries` / `journal_postings` are immutable. Corrections are reversing entries
|
||||
(never UPDATE/DELETE of a posted row). Ingestion is idempotent (re-running a Flex pull must not double-book).
|
||||
3. **Balanced in EUR.** A DB-level trigger (or repo-enforced invariant on both Postgres and SQLite) rejects any
|
||||
entry whose postings do not sum to zero in `eur_amount`.
|
||||
4. **FIFO** cost basis, applied consistently. Realized gains come from lot-matching, not cash-flow deltas.
|
||||
5. **Scope = fund only.** No tech-BV, dividend, or arm's-length-service postings in Phase 1.
|
||||
6. **Runs on paper now.** The IBKR Flex asset must run against the *paper* account (DU9600528) — the schema and
|
||||
the EUR-sum-to-zero invariant are validated on paper data before any real capital exists.
|
||||
7. **Same test discipline as the codebase:** models work on SQLite (unit tests) and Postgres (prod); full suite
|
||||
green before finishing.
|
||||
|
||||
## Schema (SQLAlchemy 2.0 ORM, in a new `ledger` module)
|
||||
|
||||
- **`accounts`** — chart of accounts. `account_id` (str PK, e.g. `IBKR-EUR-cash`, `IBKR-securities`,
|
||||
`contributed-capital`, `trading-pnl`, `fees`), `kind` (asset|liability|equity|income|expense), `venue`,
|
||||
`currency` (default EUR). Per-venue sub-ledgers each balance standalone.
|
||||
- **`journal_entries`** — header. `entry_id` (str PK), `entry_date` (date), `description`, `source_venue`,
|
||||
`strategy_tag` (nullable — the Dagster attribution dimension), `source_ref` (the idempotency key: the IBKR
|
||||
Flex trade/cash id, so re-ingest is a no-op). Immutable.
|
||||
- **`journal_postings`** — lines. `posting_id` (PK), `entry_id` (FK), `account_id` (FK), `side` (debit|credit),
|
||||
`amount`, `currency` (default EUR), `fx_rate_to_eur` (default 1.0), `eur_amount`. Invariant: per `entry_id`,
|
||||
Σ `eur_amount`(debit) == Σ `eur_amount`(credit).
|
||||
- **`cost_basis_lots`** — `lot_id` (PK), `account_id`, `asset`, `open_date`, `qty_open`, `qty_remaining`,
|
||||
`unit_cost_eur`, and on close a link to the realizing entry. FIFO consumption.
|
||||
- **`fx_rates`** — `value_date` + `pair` → `rate`. **Dormant** — populated only if a non-EUR asset appears.
|
||||
Present so the seam exists; no ingestion job in Phase 1.
|
||||
|
||||
## IBKR Flex ingestion (new adapter surface + Dagster asset)
|
||||
|
||||
The existing `IbkrBroker` is a live `ib_insync` socket connection — NOT the Flex API. Flex is a separate
|
||||
**Flex Web Service**: a per-query token + query id, a 2-step `SendRequest` → `GetStatement` poll (~1 req/sec),
|
||||
returning an XML statement. Add a read-only `IbkrFlexClient` (new file) that fetches and parses:
|
||||
trades, cash transactions, dividends, corporate actions, positions, NAV.
|
||||
|
||||
- Land raw XML/rows in an append-only `raw` schema (mirrors the DB-anchor-SSOT pattern), then a transform
|
||||
normalizes into `journal_entries`/`journal_postings` (idempotent on `source_ref`).
|
||||
- One Dagster asset (`fund_ledger_ingest` or similar) alongside the nightly ETL; reuses monitoring patterns.
|
||||
- W-8BEN note: as a non-US person the account gets a 1042-S, never a Consolidated 1099 — irrelevant to the
|
||||
EUR ledger math, but archive the annual Activity Statement as an audit doc.
|
||||
|
||||
## Out of scope (Phase 1) — deliberately deferred
|
||||
|
||||
- Bybit ingestion (no live crypto book; Bybit-EU is EUR anyway → add later, same schema).
|
||||
- The tech-BV → fund arm's-length service postings, dividend, capital-contribution accounting (stays in BV admin).
|
||||
- The Box-3 *werkelijk-rendement* parallel tax-basis calc (unrealized + fee-disallow) — only needed if the
|
||||
tegenbewijsregeling is elected; add when the advisor confirms it's worth it.
|
||||
- Any FX conversion pipeline (there is no non-EUR leg).
|
||||
|
||||
## First build step
|
||||
|
||||
The EUR-functional ledger core (models + repo + the EUR-sum-to-zero invariant) **plus** the IBKR Flex ingestion
|
||||
asset — validated on the paper account — before anything else. This exercises every load-bearing decision:
|
||||
append-only immutability, the EUR-balanced invariant, the `strategy_tag` dimension, FIFO lots.
|
||||
134
docs/superpowers/specs/2026-07-21-fxhnt-cicd-pipeline-design.md
Normal file
134
docs/superpowers/specs/2026-07-21-fxhnt-cicd-pipeline-design.md
Normal file
@@ -0,0 +1,134 @@
|
||||
# fxhnt CI/CD Pipeline on bizworx — Design Spec (2026-07-21)
|
||||
|
||||
**Status:** design approved (approach + build-trigger decided with operator). Implementation plan follows.
|
||||
|
||||
## Goal
|
||||
|
||||
Restore automated build+deploy for fxhnt on the bizworx Kapsule so a push to `master` on
|
||||
`gitadmin/fxhnt` (Gitea) lands live without a manual `kubectl rollout restart` — the exact gap that left the
|
||||
cockpit on 2.5-day-old code (the CBU0/CSBGU0 fix). **Git-sync-first, image-on-demand:** code changes trigger a
|
||||
fast rollout (git-sync re-pulls src); only dependency/Dockerfile changes trigger a Kaniko image rebuild.
|
||||
|
||||
## Why this shape
|
||||
|
||||
The operator's insight: **code changes often, the image changes rarely.** git-sync already pulls fresh `src/`
|
||||
on every (re)start, so a code-only push needs only a rollout — not a 5-minute Kaniko build. A rebuild is needed
|
||||
only when `pyproject.toml` / `fxhnt.Dockerfile` change. This is exactly the `mode=build` vs `mode=deploy` split
|
||||
the retired `infra/argo/cockpit-build-deploy.yaml` already encoded — we reanimate and fix it, rather than adopt
|
||||
the generic `build-image` template (which lacks the split and fxhnt's deploy specifics).
|
||||
|
||||
## Governing principle: shared engine, separate projects
|
||||
|
||||
fxhnt and bizworx are **different projects**. fxhnt should *use* bizworx's heavy shared platform infra but not
|
||||
*pollute* the platform namespaces with fxhnt-specific objects or secrets:
|
||||
|
||||
- **SHARE (bizworx platform, use as-is):** the one Gitea (webhook source), the Argo Events eventbus +
|
||||
`gitea-webhook` eventsource, the Argo Workflows controller (the build *engine*), external-secrets/openbao,
|
||||
the `large` node-pool, the Scaleway registry.
|
||||
- **SEPARATE (fxhnt-owned, live in `foxhunt`):** the `WorkflowTemplate`, all build+deploy pods, fxhnt's
|
||||
secrets (`argo-git-ssh-key`, `scw-registry` — already in `foxhunt`), and fxhnt's RBAC.
|
||||
- **The ONE unavoidable object in a platform ns:** the Sensor. A Sensor must live in `argo-events` to hear the
|
||||
shared eventbus. Keep its footprint minimal — it only matches fxhnt pushes and *submits* a Workflow that runs
|
||||
in `foxhunt`. No fxhnt secrets, no fxhnt logic beyond the trigger, live in `argo-events`.
|
||||
|
||||
This is NOT the old "everything in foxhunt because the secrets were there" (path of least resistance) — it's the
|
||||
same placement for the *right* reason: respecting the project boundary. It also means **no cross-namespace RBAC**
|
||||
(the deploy runs in `foxhunt`, where the app + its permissions already are) and **no fxhnt secrets replicated
|
||||
into `argo`**.
|
||||
|
||||
## Verified current state (all components already exist on bizworx)
|
||||
|
||||
- **One Gitea** (ns `gitea`): `git.fxhnt.ai` and `gitea-http.gitea.svc` are the SAME instance; it holds
|
||||
`gitadmin/fxhnt` (code) alongside `bizworx-ops/infrastructure` (the msp GitOps monorepo). No second Gitea.
|
||||
- **Argo Events** (`gitea-webhook` eventsource, `/push`, repo-agnostic) + **Argo Workflows** (Kaniko) — already
|
||||
driving `build-msp-*` via path-filtered sensors. Reusable.
|
||||
- **Registry:** fxhnt's image is `rg.fr-par.scw.cloud/bizworx/fxhnt-cockpit` (Scaleway) — DIFFERENT from msp's
|
||||
`registry.bizworx.nl/bizworx-ops/`. Secret `scw-registry` (ns foxhunt) holds the push creds.
|
||||
- **Dockerfile EXISTS:** `infra/docker/fxhnt.Dockerfile` (python:3.11-slim, `fxhnt serve` :8080). Reusable as-is.
|
||||
- **SAs exist:** `argo-workflow` (ns argo), `argo-events-sa` (ns argo-events); `argo-git-ssh-key` (ns foxhunt).
|
||||
- **What's broken in the old pipeline:** (1) `nodeSelector: ci-compile-cpu` — that pool is DECOMMISSIONED
|
||||
(only `large` + `sfs` pools exist now) → build/deploy pods won't schedule; (2) the Sensor that triggered it
|
||||
is gone; (3) no webhook on `gitadmin/fxhnt` → the eventsource never hears fxhnt pushes.
|
||||
- **No fxhnt ArgoCD app** — fxhnt manifests are applied by the workflow's `kubectl apply`, not GitOps-synced.
|
||||
We keep it that way (see Non-goals).
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
git push → gitadmin/fxhnt (master) [SHARED: Gitea]
|
||||
│ Gitea webhook ──────────────► argo-events `gitea-webhook` eventsource [SHARED: platform eventbus]
|
||||
▼
|
||||
Sensor `build-fxhnt` (ns argo-events — the ONE fxhnt object in a platform ns, kept minimal)
|
||||
│ filters: ref==refs/heads/master, pusher != argocd-image-updater (SINGLE trigger, like the msp sensors)
|
||||
▼ submits ONE Workflow into ns `foxhunt`, injecting commit-sha ─────── [FXHNT-OWNED from here down]
|
||||
Argo Workflow (WorkflowTemplate `fxhnt-cockpit`, ns foxhunt, SA argo-workflow, node-pool `large`) — DAG:
|
||||
├─ decide-mode: git-clone(sha) → `git diff --name-only sha~1 sha` →
|
||||
│ touched pyproject.toml OR infra/docker/fxhnt.Dockerfile → mode=build ; else mode=deploy
|
||||
│ (mode decided IN the workflow, not a fragile sensor filter — the CRD schema is open, so a
|
||||
│ sensor-side path-negation can't be validated; a git-diff step is robust + testable. No lock file
|
||||
│ exists → those two paths are the complete rebuild-trigger set.)
|
||||
├─ mode=build: Kaniko → rg.fr-par.scw.cloud/bizworx/fxhnt-cockpit:<sha7>+:latest
|
||||
└─ (build skipped when mode=deploy)
|
||||
→ deploy step (in foxhunt, no cross-ns RBAC): kubectl apply manifests + refresh daemons (git-sync re-pulls src)
|
||||
```
|
||||
|
||||
## Components to build/fix
|
||||
|
||||
### 1. WorkflowTemplate `fxhnt-cockpit` (reanimate `infra/argo/cockpit-build-deploy.yaml`, ns `foxhunt`)
|
||||
- Fix `nodeSelector` `ci-compile-cpu` → `large` (both build + deploy templates — `ci-compile-cpu` pool is gone;
|
||||
only `large`/`sfs` exist).
|
||||
- Keep the `mode` DAG: `build` (Kaniko rebuild + deploy) vs `deploy` (skip build, deploy only).
|
||||
- Keep the Kaniko dest `rg.fr-par.scw.cloud/bizworx/fxhnt-cockpit` but ADD an immutable `:<sha7>` tag next to
|
||||
`:latest` (matches the msp pattern). Cache stays on.
|
||||
- Clone via the in-cluster Gitea using `argo-git-ssh-key`; push via `scw-registry`. Both already exist in
|
||||
`foxhunt` — the workflow runs there, so they resolve locally (no replication into a platform ns).
|
||||
- **Reproducibility despite git-sync:** the deploy step stamps the deployed commit sha onto the daemon
|
||||
Deployments as an annotation (`fxhnt.io/deployed-sha=<sha7>`), so "which commit is this pod running" is always
|
||||
answerable — closing the one real gap of git-sync-first without adopting full image-based deploys.
|
||||
|
||||
### 2. Sensor `build-fxhnt` (new, model on `build-msp-portal`)
|
||||
- Dependency: `gitea-webhook` push, filter `body.ref == refs/heads/master` AND
|
||||
`body.pusher.username != argocd-image-updater` (loop-guard; harmless here since no image-updater writes fxhnt,
|
||||
but kept for parity/safety).
|
||||
- **Mode decision via path-filter on `body.commits.#.modified`:** if any modified path matches
|
||||
`pyproject.toml` | `infra/docker/fxhnt.Dockerfile` → submit workflow with `mode=build`; else `mode=deploy`.
|
||||
(No lock file exists, so those two paths are the complete rebuild-trigger set.) Mechanism: two triggers with
|
||||
complementary data-filters (one matching the dep-paths → build, one matching their negation → deploy), or one
|
||||
trigger whose param is computed — the plan picks whichever Argo Events supports cleanly.
|
||||
- Inject `commit-sha` from `body.after`.
|
||||
|
||||
### 3. Gitea webhook on `gitadmin/fxhnt`
|
||||
- Add a repo (or org) webhook POSTing push events to the `gitea-webhook` eventsource endpoint (same target the
|
||||
infrastructure repo already uses). This is the one genuinely-missing wire.
|
||||
|
||||
### 4. The deploy step — VERIFY, don't assume (open design point)
|
||||
The old deploy used **`scale 0→1` for fxhnt-dashboard, NOT `rollout restart`**, because the tailscale sidecar
|
||||
served `dashboard.fxhnt.ai` and a rolling restart overlapped two tailscale sessions → external proxy read
|
||||
"down". BUT: the dashboard now also has a `fxhnt-ingress`, and a manual `rollout restart` on 2026-07-21
|
||||
succeeded cleanly. **The plan must verify which external route is live** before choosing:
|
||||
- If `dashboard.fxhnt.ai` still resolves through the tailscale sidecar → keep `scale 0→1` (the race is real).
|
||||
- If it now resolves through `fxhnt-ingress` (tailscale sidecar vestigial) → a plain `rollout restart` is
|
||||
simpler and correct; drop the scale-dance.
|
||||
Dagster has no external-proxy race → `rollout restart deploy/dagster` either way.
|
||||
|
||||
## Non-goals (YAGNI) — deliberately NOT doing, with the reason
|
||||
|
||||
- **No ArgoCD GitOps app for fxhnt.** Chosen over declarative GitOps: it would add a second source of truth for
|
||||
the pod's src (ArgoCD-synced manifests vs git-sync'd code) and needs a git.fxhnt.ai repo registration. The
|
||||
imperative `kubectl apply` from the foxhunt workflow fits git-sync-first and keeps manifests in the fxhnt repo.
|
||||
- **No registry unification.** fxhnt stays on Scaleway `rg.fr-par.scw.cloud/bizworx` (not the shared
|
||||
`registry.bizworx.nl/bizworx-ops`). Migrating would mean re-pointing the dashboard + reconciling old images
|
||||
for zero pipeline benefit — a separate, riskier task.
|
||||
- **No CI workloads in a platform namespace.** Only the Sensor lives in `argo-events` (unavoidable). The
|
||||
WorkflowTemplate + all build/deploy pods run in `foxhunt` — no fxhnt secrets in `argo`, no cross-ns RBAC.
|
||||
- **No image-updater for fxhnt.** The sensor writes the `:sha7`+`:latest` tags; the deploy step points the
|
||||
daemons at the fresh image. No git-write-back loop needed.
|
||||
- **No change to the execution CronJobs** (ucits/bybit/ibkr rebalancers) — they git-sync per run already.
|
||||
|
||||
## Success criteria
|
||||
|
||||
1. A code-only push to `master` → cockpit reflects it within ~1–2 min WITHOUT a manual restart, NO Kaniko build.
|
||||
2. A `pyproject.toml`/Dockerfile push → a Kaniko rebuild produces `fxhnt-cockpit:<sha7>`+`:latest`, then deploys.
|
||||
3. The dashboard external route (`dashboard.fxhnt.ai`) stays UP across a deploy (no stale-proxy "down").
|
||||
4. No build-loop, no false trigger on unrelated pushes.
|
||||
5. Real-money safety unaffected — the pipeline only builds/deploys the cockpit+dagster daemons, never arms exec.
|
||||
@@ -0,0 +1,90 @@
|
||||
# Portfolio Backtester — Design Spec (2026-07-21)
|
||||
|
||||
**Status:** design approved (all decisions made with operator). Implementation plan follows in a fresh session.
|
||||
Slice 1 of the "cockpit = the platform" sequence (agreed order: portfolio-backtester → trade register →
|
||||
cards-to-tables/IA polish).
|
||||
|
||||
## Goal
|
||||
|
||||
A new cockpit surface where you assemble an arbitrary set of edges, choose a weighting method + date range, and
|
||||
get portfolio-level Sharpe/CAGR/maxDD **plus** the diversification view the research actually cares about — the
|
||||
pairwise correlation matrix (full AND left-tail) and each edge's marginal-Sharpe contribution. Reuses the
|
||||
existing, production-tested combiner/metrics/diversification engine end-to-end; new code is glue + a page.
|
||||
|
||||
## Why this shape
|
||||
|
||||
The three-explorer map (2026-07-21) established: the portfolio-backtest ENGINE already exists and is battle-tested
|
||||
(`combine_book`, `combine_returns`, `compute_stats`, `marginal_sharpe`, `full_correlation`,
|
||||
`left_tail_correlation`), and per-edge daily return series are already MATERIALIZED and queryable (three tables).
|
||||
So a portfolio backtester is glue + surface, not a new engine. It exposes the exact machinery the research arc
|
||||
already trusts (inverse-vol book, marginal-Sharpe LOO gate, co-crash / left-tail lens) as an interactive tool.
|
||||
Pure additive + read-only — cannot break the live fund.
|
||||
|
||||
## Decisions (all made with the operator)
|
||||
|
||||
1. **Both combiners, default `combine_book`.** Default = the live inverse-vol → cap → optional marginal-prune →
|
||||
vol-target + Kelly method (`book_allocator.combine_book`, book_allocator.py:117) — "what does my real
|
||||
allocation do with these edges?". Optional **explicit-weights what-if** mode via `combine_returns`
|
||||
(domain/portfolio/combine.py:10) — "what if positioning were 40%?".
|
||||
2. **Three-part report:** (a) equity curve + headline metrics (Sharpe/CAGR/maxDD/vol/worst-year) via
|
||||
`compute_stats`; (b) **correlation heatmap showing BOTH full and left-tail** correlation per pair
|
||||
(`full_correlation` + `left_tail_correlation`, tail_frac 0.10) — the honest co-crash lens; (c) per-edge
|
||||
**marginal-Sharpe** table (leave-one-out contribution, `marginal_sharpe`) — the same accept/reject gate.
|
||||
Correlation + marginal-Sharpe are computed on the RAW edge series (properties of the edges, weight-independent);
|
||||
the curve + metrics reflect the chosen weighting. Both lenses shown.
|
||||
3. **New page `/paper/portfolio`** — a dedicated first-class cockpit section (the first IA step toward
|
||||
cockpit-as-platform), NOT an extension of `/paper/sim` (which stays "the 4 canonical books"). Reuses existing
|
||||
dashboard styling + the table pattern (with `data-label` mobile reflow).
|
||||
4. **Charting = Plotly via CDN.** Adopt Plotly as the new platform charting layer, loaded from CDN (operator's
|
||||
choice; correctly noted the cockpit renders in the operator's browser over tailscale serve, so browser fetches
|
||||
are NOT blocked by the pod NetworkPolicy — the earlier "CDN is blocked" reasoning was wrong). Accepted
|
||||
trade-off: CDN is a browser-side dependency (small SPOF/telemetry cost) for a private single-user cockpit.
|
||||
The existing server-SVG charts (charts.py sparklines/equity/overlay) migrate to Plotly LATER, in a dedicated
|
||||
charting-migration slice — NOT in this slice (keeps slice 1 focused).
|
||||
|
||||
## Available edges (the multiselect options)
|
||||
|
||||
- **Bybit 4 sleeves** from `bybit_sleeve_ret` (paper_repo.bybit_sleeve_returns() → {sleeve:{epoch_day:ret}}):
|
||||
`crypto_tstrend`, `unlock`, `xsfunding`, `positioning`.
|
||||
- **IBKR recompute-replay books** from `sim_curve_ret` (paper_repo.sim_curve_returns(sid) → {epoch_day:ret}):
|
||||
`multistrat` (+ `multistrat_levered`).
|
||||
- (Optionally `paper_sleeve_ret` live-paper series — a later addition; slice 1 covers the two stored-backtest sources.)
|
||||
|
||||
## New code (glue + surface only)
|
||||
|
||||
1. **Cross-store series aggregator** — a function that unions the three stores into one
|
||||
`series_by_edge: dict[str, dict[int,float]]` for an arbitrary selected edge set, regardless of venue. This is
|
||||
the ONE missing data primitive; `combine_book`/`combine_returns` both already consume exactly this shape.
|
||||
2. **Report bundler** — takes selected edges + weighting method (+ optional explicit weights) + date range →
|
||||
slices the series to the range → calls `combine_book` (or `combine_returns`) for the curve, `compute_stats`
|
||||
for metrics, `full_correlation`+`left_tail_correlation` for the N×N matrix, `marginal_sharpe` per edge → one
|
||||
report object. Never called together on a user-defined portfolio today (only em_asia_marginal_eval does a
|
||||
fixed candidate-vs-book version) — this is the new assembly.
|
||||
3. **Route + template + Plotly rendering** — `GET /paper/portfolio` (page) + `GET /paper/portfolio/run` (HTMX
|
||||
partial: edge-multiselect + weighting toggle + date range → the three-part report). Plotly `<script>` from
|
||||
CDN in base.html (or the portfolio template). Equity curve + heatmap = Plotly; marginal-Sharpe = an HTML
|
||||
table (seeds the interactive-datatable pattern the register slice will reuse).
|
||||
|
||||
## Reused (exists, production-tested — do NOT rebuild)
|
||||
|
||||
`align_edges` + `combine_book` (book_allocator.py), `combine_returns` (domain/portfolio/combine.py),
|
||||
`compute_stats` (domain/backtest.py), `marginal_sharpe` (domain/diversification/marginal.py),
|
||||
`full_correlation`/`left_tail_correlation` (domain/diversification/correlation.py), the stored return series
|
||||
(`bybit_sleeve_returns`, `sim_curve_returns`), the existing DashboardService wiring + dark design system.
|
||||
|
||||
## Out of scope (this slice)
|
||||
|
||||
- Migrating the existing server-SVG charts to Plotly (own slice later).
|
||||
- Bybit/IBKR real-fill ingestion, the trade register, the EUR ledger (slice 2).
|
||||
- Live-paper `paper_sleeve_ret` as a selectable source (a small later add).
|
||||
- Persisting/saving named portfolios (could add; not needed for v1).
|
||||
|
||||
## Success criteria
|
||||
|
||||
1. Select any subset of the available edges + a date range → get a portfolio equity curve + Sharpe/CAGR/maxDD
|
||||
in the cockpit, using `combine_book` by default, instantly (series are pre-materialized — no recompute).
|
||||
2. Explicit-weights mode lets you supply a weight per selected edge → curve/metrics reflect those weights.
|
||||
3. The correlation heatmap shows full AND left-tail correlation per pair; the marginal-Sharpe table shows each
|
||||
edge's LOO contribution — matching the numbers the research pipeline uses to accept/reject edges.
|
||||
4. Read-only + additive: no live fund path touched; `/paper/sim` and everything else unchanged.
|
||||
5. Plotly charts render over the tailscale-served cockpit in the browser.
|
||||
@@ -1,126 +0,0 @@
|
||||
# Argo WorkflowTemplate — build + deploy the fxhnt cockpit, fully in-cluster.
|
||||
#
|
||||
# Reuses cluster primitives: the `argo-workflow` ServiceAccount, the `argo-git-ssh-key` deploy key (clone
|
||||
# the fxhnt repo over in-cluster Gitea SSH, gitea-sshd:22), and the `scw-registry` dockerconfig (push to
|
||||
# the external Scaleway Container Registry rg.fr-par.scw.cloud/bizworx). Build runs in-cluster.
|
||||
# (Phase 2B: migrated off the in-cluster GitLab shell + registry.)
|
||||
#
|
||||
# Submit: argo submit -n foxhunt --from=wftmpl/fxhnt-cockpit -p commit-sha=<sha> --watch
|
||||
# (or use scripts/argo-deploy-cockpit.sh)
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: WorkflowTemplate
|
||||
metadata:
|
||||
name: fxhnt-cockpit
|
||||
namespace: foxhunt
|
||||
labels: { app.kubernetes.io/name: fxhnt-cockpit, app.kubernetes.io/part-of: foxhunt }
|
||||
spec:
|
||||
serviceAccountName: argo-workflow
|
||||
entrypoint: pipeline
|
||||
archiveLogs: true # archive build/deploy logs to MinIO (argo-logs bucket). Reachable now that the
|
||||
# pods carry app.kubernetes.io/part-of: argo-workflows (MinIO ingress netpol allow).
|
||||
activeDeadlineSeconds: 1800
|
||||
ttlStrategy: { secondsAfterCompletion: 3600 }
|
||||
podGC: { strategy: OnPodCompletion }
|
||||
arguments:
|
||||
parameters:
|
||||
- { name: commit-sha, value: HEAD }
|
||||
templates:
|
||||
- name: pipeline
|
||||
dag:
|
||||
tasks:
|
||||
- { name: build, template: kaniko-build }
|
||||
- { name: deploy, template: kubectl-deploy, depends: build }
|
||||
|
||||
# --- build: Kaniko builds infra/docker/fxhnt.Dockerfile -> internal registry ---
|
||||
- name: kaniko-build
|
||||
metadata:
|
||||
labels: { app.kubernetes.io/part-of: argo-workflows } # MinIO ingress allows part-of in {foxhunt,foxhunt-ci,argo-workflows} → log archival works. NOT 'foxhunt' (that adds argo-base-egress which blocks registry:5000/git:2222).
|
||||
nodeSelector: { k8s.scaleway.com/pool-name: ci-compile-cpu, topology.kubernetes.io/zone: fr-par-2 }
|
||||
tolerations: [{ effect: NoSchedule, key: node.cilium.io/agent-not-ready, operator: Exists }]
|
||||
initContainers:
|
||||
- name: git-clone
|
||||
image: alpine/git:latest
|
||||
command: [/bin/sh, -c]
|
||||
args:
|
||||
- |
|
||||
set -ex
|
||||
mkdir -p /root/.ssh
|
||||
cp /etc/git-ssh/ssh-privatekey /root/.ssh/id_ed25519
|
||||
chmod 600 /root/.ssh/id_ed25519
|
||||
printf 'Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n' > /root/.ssh/config
|
||||
git clone --no-checkout --filter=blob:none \
|
||||
"ssh://git@gitea-sshd.foxhunt.svc.cluster.local:22/gitadmin/fxhnt.git" /workspace/src
|
||||
cd /workspace/src && git checkout "{{workflow.parameters.commit-sha}}"
|
||||
echo "checked out $(git rev-parse --short HEAD)"
|
||||
volumeMounts:
|
||||
- { name: git-ssh-key, mountPath: /etc/git-ssh, readOnly: true }
|
||||
- { name: workspace, mountPath: /workspace }
|
||||
container:
|
||||
image: gcr.io/kaniko-project/executor:debug
|
||||
command: [/busybox/sh, -c]
|
||||
args:
|
||||
- |
|
||||
/kaniko/executor \
|
||||
--context=/workspace/src \
|
||||
--dockerfile=/workspace/src/infra/docker/fxhnt.Dockerfile \
|
||||
--destination=rg.fr-par.scw.cloud/bizworx/fxhnt-cockpit:latest \
|
||||
--cache=true \
|
||||
--cache-repo=rg.fr-par.scw.cloud/bizworx/cache \
|
||||
--snapshot-mode=redo
|
||||
env:
|
||||
- { name: DOCKER_CONFIG, value: /kaniko/.docker }
|
||||
resources: { requests: { cpu: "2", memory: 4Gi }, limits: { cpu: "6", memory: 12Gi } }
|
||||
volumeMounts:
|
||||
- { name: registry-auth, mountPath: /kaniko/.docker, readOnly: true }
|
||||
- { name: workspace, mountPath: /workspace }
|
||||
volumes:
|
||||
- { name: workspace, emptyDir: {} }
|
||||
- { name: git-ssh-key, secret: { secretName: argo-git-ssh-key, defaultMode: 256 } }
|
||||
- name: registry-auth
|
||||
secret:
|
||||
secretName: scw-registry
|
||||
items: [{ key: .dockerconfigjson, path: config.json }]
|
||||
|
||||
# --- deploy: apply manifests from the repo + roll the dashboard to the fresh image ---
|
||||
- name: kubectl-deploy
|
||||
metadata:
|
||||
labels: { app.kubernetes.io/part-of: argo-workflows } # MinIO ingress allow → log archival works (see kaniko-build note)
|
||||
nodeSelector: { k8s.scaleway.com/pool-name: platform, topology.kubernetes.io/zone: fr-par-2 }
|
||||
serviceAccountName: argo-workflow
|
||||
initContainers:
|
||||
- name: git-clone
|
||||
image: alpine/git:latest
|
||||
command: [/bin/sh, -c]
|
||||
args:
|
||||
- |
|
||||
set -ex
|
||||
mkdir -p /root/.ssh
|
||||
cp /etc/git-ssh/ssh-privatekey /root/.ssh/id_ed25519
|
||||
chmod 600 /root/.ssh/id_ed25519
|
||||
printf 'Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n' > /root/.ssh/config
|
||||
git clone --no-checkout --filter=blob:none \
|
||||
"ssh://git@gitea-sshd.foxhunt.svc.cluster.local:22/gitadmin/fxhnt.git" /workspace/src
|
||||
cd /workspace/src && git checkout "{{workflow.parameters.commit-sha}}"
|
||||
volumeMounts:
|
||||
- { name: git-ssh-key, mountPath: /etc/git-ssh, readOnly: true }
|
||||
- { name: workspace, mountPath: /workspace }
|
||||
container:
|
||||
image: alpine/k8s:1.31.0 # Alpine-based (has /bin/sh + kubectl). NOT registry.k8s.io/kubectl — that's distroless (no shell) and the deploy runs a shell script. bitnami/kubectl was removed from Docker Hub (2025).
|
||||
command: [/bin/sh, -c]
|
||||
args:
|
||||
- |
|
||||
set -e
|
||||
cd /workspace/src
|
||||
kubectl apply -f infra/k8s/services/fxhnt-dashboard.yaml
|
||||
kubectl apply -f infra/k8s/jobs/fxhnt-forward-cronjob.yaml
|
||||
kubectl apply -f infra/k8s/orchestration/dagster.yaml
|
||||
kubectl apply -f infra/k8s/network-policies/fxhnt-cockpit.yaml
|
||||
kubectl -n foxhunt rollout restart deploy/fxhnt-dashboard
|
||||
kubectl -n foxhunt rollout restart deploy/dagster
|
||||
kubectl -n foxhunt rollout status deploy/fxhnt-dashboard --timeout=180s
|
||||
kubectl -n foxhunt rollout status deploy/dagster --timeout=300s
|
||||
volumeMounts:
|
||||
- { name: workspace, mountPath: /workspace }
|
||||
volumes:
|
||||
- { name: workspace, emptyDir: {} }
|
||||
- { name: git-ssh-key, secret: { secretName: argo-git-ssh-key, defaultMode: 256 } }
|
||||
57
infra/argo/fxhnt-build-sensor.yaml
Normal file
57
infra/argo/fxhnt-build-sensor.yaml
Normal file
@@ -0,0 +1,57 @@
|
||||
# Argo Events Sensor — hears the shared Gitea webhook for pushes to gitadmin/fxhnt master, and submits the
|
||||
# fxhnt-cockpit WorkflowTemplate (ns foxhunt). Structurally identical to build-msp-portal (single dependency,
|
||||
# single trigger) — the build-vs-deploy decision is made INSIDE the workflow via `git diff` (decide-mode step in
|
||||
# fxhnt-cockpit-workflowtemplate.yaml), not via a sensor-side path filter. See
|
||||
# docs/superpowers/plans/2026-07-21-fxhnt-cicd-pipeline.md Task 3 design note: the Sensor CRD here has an open
|
||||
# schema, so trigger-level path-filter negation can't be reliably validated — keeping the sensor simple is the
|
||||
# robust choice.
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Sensor
|
||||
metadata:
|
||||
name: build-fxhnt
|
||||
namespace: argo-events
|
||||
labels: { app.kubernetes.io/instance: build-pipeline }
|
||||
spec:
|
||||
dependencies:
|
||||
- name: push-to-master
|
||||
eventSourceName: gitea-webhook
|
||||
eventName: push
|
||||
filters:
|
||||
dataLogicalOperator: and
|
||||
data:
|
||||
- path: body.ref
|
||||
type: string
|
||||
comparator: "="
|
||||
value: ["refs/heads/master"]
|
||||
- path: body.pusher.username
|
||||
type: string
|
||||
comparator: "!="
|
||||
value: ["argocd-image-updater"]
|
||||
template:
|
||||
serviceAccountName: argo-events-sa
|
||||
triggers:
|
||||
- template:
|
||||
name: submit-build
|
||||
argoWorkflow:
|
||||
operation: submit
|
||||
source:
|
||||
resource:
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Workflow
|
||||
metadata:
|
||||
generateName: build-fxhnt-
|
||||
namespace: foxhunt
|
||||
spec:
|
||||
workflowTemplateRef:
|
||||
name: fxhnt-cockpit
|
||||
arguments:
|
||||
parameters:
|
||||
- name: commit-sha
|
||||
value: ""
|
||||
- name: mode
|
||||
value: auto
|
||||
parameters:
|
||||
- src:
|
||||
dependencyName: push-to-master
|
||||
dataKey: body.after
|
||||
dest: spec.arguments.parameters.0.value
|
||||
124
infra/argo/fxhnt-ci-rbac.yaml
Normal file
124
infra/argo/fxhnt-ci-rbac.yaml
Normal file
@@ -0,0 +1,124 @@
|
||||
# RBAC for the fxhnt CI pipeline — all fxhnt-owned, in the foxhunt namespace (project boundary).
|
||||
#
|
||||
# Fills the two gaps the first real workflow run surfaced (2026-07-21):
|
||||
# 1. The fxhnt-cockpit WorkflowTemplate references SA `argo-workflow`, which existed in `argo` (where msp
|
||||
# builds) but NOT in `foxhunt` (where fxhnt's project boundary places its build/deploy pods).
|
||||
# 2. The build-fxhnt Sensor (argo-events-sa) submits Workflows into `foxhunt`; the bizworx `workflow-submitter`
|
||||
# Role only granted that in `argo`, so a cross-namespace submit into foxhunt was forbidden.
|
||||
#
|
||||
# Least privilege: the workflow SA gets exactly what Argo needs to run pods + what the kubectl-deploy step does
|
||||
# (apply/scale/annotate/rollout-status the two daemon Deployments and their Services/NetworkPolicies), nothing more.
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: argo-workflow
|
||||
namespace: foxhunt
|
||||
labels: { app.kubernetes.io/part-of: fxhnt-ci }
|
||||
---
|
||||
# Argo needs the workflow SA to write task results (the minimum the workflow-controller requires per step).
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: fxhnt-argo-workflow
|
||||
namespace: foxhunt
|
||||
labels: { app.kubernetes.io/part-of: fxhnt-ci }
|
||||
rules:
|
||||
- apiGroups: ["argoproj.io"]
|
||||
resources: ["workflowtaskresults"]
|
||||
verbs: ["create", "patch"]
|
||||
---
|
||||
# The kubectl-deploy step's exact actions on the foxhunt daemons — scoped to what it runs, no wildcards.
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: fxhnt-ci-deployer
|
||||
namespace: foxhunt
|
||||
labels: { app.kubernetes.io/part-of: fxhnt-ci }
|
||||
rules:
|
||||
# apply + scale + annotate + rollout-status the daemon Deployments (dashboard, dagster).
|
||||
- apiGroups: ["apps"]
|
||||
resources: ["deployments"]
|
||||
verbs: ["get", "list", "watch", "create", "update", "patch"]
|
||||
- apiGroups: ["apps"]
|
||||
resources: ["deployments/scale"]
|
||||
verbs: ["get", "update", "patch"]
|
||||
- apiGroups: ["apps"]
|
||||
resources: ["replicasets"]
|
||||
verbs: ["get", "list", "watch"] # rollout status reads the RS
|
||||
# apply the Services the manifests contain, and `wait --for=delete pod` + rollout status read pods.
|
||||
- apiGroups: [""]
|
||||
resources: ["services"]
|
||||
verbs: ["get", "list", "watch", "create", "update", "patch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["pods"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
# the fxhnt-dashboard + dagster manifests also bundle ConfigMaps (the tailscale sidecar serve-config) —
|
||||
# kubectl apply of the whole file must be able to read/apply them. NOTE (CI review M1): the tailscale
|
||||
# ServiceAccount + Role + RoleBinding objects were MOVED out of these app manifests into
|
||||
# infra/k8s/rbac/tailscale-sidecars.yaml (applied once, out-of-band, by a human) specifically so this
|
||||
# deployer SA does not need roles/rolebindings/serviceaccounts create-update-patch — that combination on a
|
||||
# webhook-triggered SA is a privilege-escalation primitive. Neither manifest applied by CI creates a
|
||||
# ServiceAccount anymore, so `serviceaccounts` is dropped from this grant too.
|
||||
- apiGroups: [""]
|
||||
resources: ["configmaps"]
|
||||
verbs: ["get", "list", "watch", "create", "update", "patch"]
|
||||
# the manifests apply NetworkPolicies (fxhnt-cockpit.yaml + dagster's).
|
||||
- apiGroups: ["networking.k8s.io"]
|
||||
resources: ["networkpolicies"]
|
||||
verbs: ["get", "list", "watch", "create", "update", "patch"]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: fxhnt-argo-workflow
|
||||
namespace: foxhunt
|
||||
labels: { app.kubernetes.io/part-of: fxhnt-ci }
|
||||
subjects:
|
||||
- { kind: ServiceAccount, name: argo-workflow, namespace: foxhunt }
|
||||
roleRef:
|
||||
kind: Role
|
||||
name: fxhnt-argo-workflow
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: fxhnt-ci-deployer
|
||||
namespace: foxhunt
|
||||
labels: { app.kubernetes.io/part-of: fxhnt-ci }
|
||||
subjects:
|
||||
- { kind: ServiceAccount, name: argo-workflow, namespace: foxhunt }
|
||||
roleRef:
|
||||
kind: Role
|
||||
name: fxhnt-ci-deployer
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
---
|
||||
# Cross-namespace: let the shared argo-events-sa submit Workflows into foxhunt (the sensor lives in argo-events
|
||||
# but its Workflow runs here). Mirrors the bizworx `workflow-submitter` Role, scoped to foxhunt.
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: fxhnt-workflow-submitter
|
||||
namespace: foxhunt
|
||||
labels: { app.kubernetes.io/part-of: fxhnt-ci }
|
||||
rules:
|
||||
- apiGroups: ["argoproj.io"]
|
||||
resources: ["workflows"]
|
||||
verbs: ["create", "get", "list", "watch"]
|
||||
- apiGroups: ["argoproj.io"]
|
||||
resources: ["workflowtemplates"]
|
||||
verbs: ["get", "list"]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: argo-events-sa-fxhnt-submitter
|
||||
namespace: foxhunt
|
||||
labels: { app.kubernetes.io/part-of: fxhnt-ci }
|
||||
subjects:
|
||||
- { kind: ServiceAccount, name: argo-events-sa, namespace: argo-events }
|
||||
roleRef:
|
||||
kind: Role
|
||||
name: fxhnt-workflow-submitter
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user