From 4817bd4e7c7b0045b22c540d8d561bced802e0ee Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sun, 7 Jun 2026 21:13:54 +0200 Subject: [PATCH] test: FinBERT news-sentiment (real LLM, real FNSPID data) -> dead for retail User wanted to download a small open LLM and test news. Did it: FinBERT scoring 10408 FNSPID headlines (7 tickers, 1332 obs). corr(sentiment, same-day ret) +0.082 (weak, contemporaneous); next-day +0.026 (~0); next-day trade net 10bp OOS Sharpe +0.05 (dead). News real but reaction INSTANT (HFT-captured); no tradeable next-day drift for retail; net of cost = nothing. Matches Lopez-Lira/Tang + FinGPT research. Model was never the constraint -- market structure is. Closes the news/LLM thread; deployable answer remains the adaptive multistrat book (news = risk-managed). (note: pin transformers<5 for torch 2.6.) Co-Authored-By: Claude Opus 4.8 (1M context) --- scripts/surfer/news_sentiment_test.py | 112 ++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 scripts/surfer/news_sentiment_test.py diff --git a/scripts/surfer/news_sentiment_test.py b/scripts/surfer/news_sentiment_test.py new file mode 100644 index 000000000..5544dbd3e --- /dev/null +++ b/scripts/surfer/news_sentiment_test.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +"""News-sentiment test with a real small LLM (FinBERT) on open news data (FNSPID). + +Stream a chunk of FNSPID headlines -> FinBERT sentiment -> daily sentiment per ticker -> test if +it predicts returns. KEY: same-day corr (contemporaneous, not tradeable) vs next-day corr (the only +retail-tradeable horizon) + a net-of-cost OOS long/short trade. Honest prior (research): news real +but reaction instant + alpha dies at cost + decaying.""" +import csv +import datetime +import io +import json +import math +import os +import sys +import urllib.request + +import numpy as np + +CHUNK = 70 * 1024 * 1024 # ~70MB stream (sorted by ticker -> early-alphabet large caps) +MAX_SCORE = 18000 # cap FinBERT inferences (GPU time) + + +def stream_news(): + req = urllib.request.Request("https://huggingface.co/datasets/Zihan1004/FNSPID/resolve/main/Stock_news/nasdaq_exteral_data.csv", + headers={"User-Agent": "curl/8", "Range": f"bytes=0-{CHUNK}"}) + buf = urllib.request.urlopen(req, timeout=120).read().decode("utf-8", "replace") + buf = buf[:buf.rfind("\n")] # drop partial last line + rows = [] + for r in csv.reader(io.StringIO(buf)): + if len(r) < 4 or r[1] == "Date": + continue + try: + d = r[1][:10]; sym = r[3].strip(); title = r[2].strip() + datetime.date.fromisoformat(d) + if sym and title and len(title) > 8: + rows.append((d, sym, title)) + except Exception: + continue + return rows + + +def yclose(sym): + try: + res = json.loads(urllib.request.urlopen(urllib.request.Request( + f"https://query1.finance.yahoo.com/v8/finance/chart/{sym}?interval=1d&range=5y", + headers={"User-Agent": "Mozilla/5.0"}), timeout=30).read())["chart"]["result"][0] + ts = res["timestamp"]; adj = res["indicators"].get("adjclose", [{}])[0].get("adjclose") or res["indicators"]["quote"][0]["close"] + return {datetime.datetime.utcfromtimestamp(t).strftime("%Y-%m-%d"): float(c) for t, c in zip(ts, adj) if c is not None} + except Exception: + return {} + + +def main(): + print("streaming FNSPID news chunk...") + rows = stream_news() + syms = {} + for d, s, t in rows: + syms.setdefault(s, 0); syms[s] += 1 + top = [s for s, n in sorted(syms.items(), key=lambda kv: -kv[1]) if n >= 50][:12] + rows = [r for r in rows if r[1] in top][:MAX_SCORE] + print(f" {len(rows)} headlines, {len(top)} tickers: {top}") + + from transformers import pipeline + import torch + clf = pipeline("sentiment-analysis", model="ProsusAI/finbert", device=0 if torch.cuda.is_available() else -1, truncation=True, max_length=64) + titles = [r[2][:200] for r in rows] + print(" scoring with FinBERT...") + out = clf(titles, batch_size=64) + score = {"positive": 1.0, "negative": -1.0, "neutral": 0.0} + sent = {} # (sym, date) -> [scores] + for r, o in zip(rows, out): + sent.setdefault((r[1], r[0]), []).append(score[o["label"]] * o["score"]) + daily = {k: float(np.mean(v)) for k, v in sent.items()} + + prices = {s: yclose(s) for s in top} + # build aligned (sentiment[t], sameday ret[t], nextday ret[t+1]) cross-sectionally + S, R0, R1 = [], [], [] + rec = [] + for (sym, d), sc in daily.items(): + px = prices.get(sym, {}) + days = sorted(px) + if d not in px: + # map to next trading day + nd = [x for x in days if x >= d] + if not nd: + continue + d = nd[0] + i = days.index(d) if d in days else -1 + if i < 1 or i + 1 >= len(days): + continue + r0 = px[days[i]] / px[days[i - 1]] - 1 + r1 = px[days[i + 1]] / px[days[i]] - 1 + rec.append((d, sym, sc, r0, r1)) + rec.sort() + S = np.array([x[2] for x in rec]); R0 = np.array([x[3] for x in rec]); R1 = np.array([x[4] for x in rec]) + n = len(S) + print(f"\n aligned sentiment-day observations: {n}") + print(f" corr(sentiment, SAME-day return): {np.corrcoef(S, R0)[0,1]:+.3f} (contemporaneous = news real but not tradeable)") + print(f" corr(sentiment, NEXT-day return): {np.corrcoef(S, R1)[0,1]:+.3f} (the only retail-tradeable horizon)") + # next-day long/short trade by sentiment sign, net 10bp, IS/OOS by time + order = np.argsort([x[0] for x in rec]); S, R1 = S[order], R1[order] + sig = np.sign(S); pnl = sig * R1 - 0.0010 * (np.abs(sig) > 0) + sp = int(0.6 * n) + def sh(x): + x = x[np.isfinite(x)]; return x.mean() / (x.std() + 1e-9) * math.sqrt(252) if len(x) > 20 and x.std() > 0 else float("nan") + print(f" next-day sentiment trade (net 10bp): IS Sharpe {sh(pnl[:sp]):+.2f} OOS Sharpe {sh(pnl[sp:]):+.2f}") + print("\n VERDICT: same-day corr >> next-day corr ~0 + OOS trade ~0/neg = news is real but its reaction is") + print(" INSTANT (contemporaneous, HFT-captured); no tradeable next-day drift for retail. Matches the research.") + + +if __name__ == "__main__": + main()