0b59e8cabe
Binance `exchangeInfo` can return junk meme symbols with non-ASCII (e.g. Chinese) characters (`币安人生USDT`) that crash the klines GET with "'ascii' codec can't encode". Filter the universe to ASCII-only symbols (still PERPETUAL / USDT-quote / TRADING as before). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
97 lines
4.5 KiB
Python
97 lines
4.5 KiB
Python
"""Crypto fetch: offline (injected GET) — universe selection, npz keys/shape, <200-bar skip, refresh."""
|
|
from __future__ import annotations
|
|
|
|
import numpy as np
|
|
|
|
from fxhnt.adapters.data.crypto_fetch import fetch_crypto_pit
|
|
|
|
|
|
def _fake_get_factory(n_bars: int):
|
|
"""Return a `get(url)` that serves canned Binance fapi JSON for any symbol."""
|
|
def get(url: str):
|
|
if "exchangeInfo" in url:
|
|
return {"symbols": [
|
|
{"symbol": "AAAUSDT", "contractType": "PERPETUAL", "quoteAsset": "USDT", "status": "TRADING"},
|
|
{"symbol": "BBBUSDT", "contractType": "PERPETUAL", "quoteAsset": "USDT", "status": "TRADING"},
|
|
]}
|
|
if "ticker/24hr" in url:
|
|
return [{"symbol": "AAAUSDT", "quoteVolume": "9999"},
|
|
{"symbol": "BBBUSDT", "quoteVolume": "10"}]
|
|
if "klines" in url:
|
|
# n_bars daily klines: [openTime, o,h,l, close, vol, closeTime, quoteVol, ...]
|
|
# high = close+5, low = close-5 so high>close>low (real intraday range)
|
|
day_ms = 86_400_000
|
|
return [[i * day_ms, "1", str(105 + i), str(95 + i), str(100 + i), "1",
|
|
i * day_ms + 1, "1000", 0, 0, 0, 0]
|
|
for i in range(n_bars)]
|
|
if "fundingRate" in url:
|
|
return []
|
|
return None
|
|
return get
|
|
|
|
|
|
def test_writes_npz_with_day_and_close(tmp_path) -> None:
|
|
n, total = fetch_crypto_pit(str(tmp_path), top_n=2, get=_fake_get_factory(300), sleep=lambda s: None, dead=[])
|
|
assert n == 2 and total == 2
|
|
z = np.load(tmp_path / "crypto_pit" / "AAAUSDT.npz")
|
|
assert "day" in z and "close" in z
|
|
assert len(z["day"]) == 300 and z["close"][0] == 100.0
|
|
assert len(z["funding"]) == len(z["day"]) # funding is day-aligned
|
|
|
|
|
|
def test_writes_npz_with_real_high_low(tmp_path) -> None:
|
|
"""The fetcher must capture the kline HIGH (idx2) and LOW (idx3), not discard them."""
|
|
fetch_crypto_pit(str(tmp_path), top_n=2, get=_fake_get_factory(300), sleep=lambda s: None, dead=[])
|
|
z = np.load(tmp_path / "crypto_pit" / "AAAUSDT.npz")
|
|
assert "high" in z and "low" in z
|
|
assert len(z["high"]) == len(z["low"]) == len(z["day"]) == 300
|
|
# fake kline row 0: high=105, low=95, close=100 -> a real (non-degenerate) range
|
|
assert z["high"][0] == 105.0 and z["low"][0] == 95.0 and z["close"][0] == 100.0
|
|
assert (z["high"] > z["close"]).all() and (z["low"] < z["close"]).all()
|
|
|
|
|
|
def test_klines_parses_high_low_from_payload() -> None:
|
|
"""_klines returns (day, close, high, low, qvol) parsed from the Binance kline rows."""
|
|
from fxhnt.adapters.data.crypto_fetch import _klines
|
|
days, cl, hi, lo, qv = _klines("AAAUSDT", _fake_get_factory(3))
|
|
assert len(days) == len(cl) == len(hi) == len(lo) == len(qv) == 3
|
|
assert list(cl) == [100.0, 101.0, 102.0]
|
|
assert list(hi) == [105.0, 106.0, 107.0]
|
|
assert list(lo) == [95.0, 96.0, 97.0]
|
|
|
|
|
|
def test_skips_symbols_with_too_few_bars(tmp_path) -> None:
|
|
n, total = fetch_crypto_pit(str(tmp_path), top_n=2, get=_fake_get_factory(50), sleep=lambda s: None, dead=[])
|
|
assert n == 0 and total == 2 # <200 bars -> skipped
|
|
|
|
|
|
def test_universe_excludes_non_ascii_symbols() -> None:
|
|
"""A junk meme symbol with non-ASCII chars (e.g. Chinese) would crash the klines GET with an
|
|
'ascii' codec error — it must be filtered out of the universe entirely."""
|
|
from fxhnt.adapters.data.crypto_fetch import _universe
|
|
|
|
def get(url: str):
|
|
if "exchangeInfo" in url:
|
|
return {"symbols": [
|
|
{"symbol": "BTCUSDT", "contractType": "PERPETUAL", "quoteAsset": "USDT", "status": "TRADING"},
|
|
{"symbol": "币安人生USDT", "contractType": "PERPETUAL",
|
|
"quoteAsset": "USDT", "status": "TRADING"},
|
|
]}
|
|
if "ticker/24hr" in url:
|
|
return [{"symbol": "BTCUSDT", "quoteVolume": "9999"},
|
|
{"symbol": "币安人生USDT", "quoteVolume": "5000"}]
|
|
return None
|
|
|
|
uni = _universe(10, get)
|
|
assert "BTCUSDT" in uni
|
|
assert "币安人生USDT" not in uni
|
|
assert all(s.isascii() for s in uni)
|
|
|
|
|
|
def test_refresh_overwrites_existing(tmp_path) -> None:
|
|
fetch_crypto_pit(str(tmp_path), top_n=1, get=_fake_get_factory(300), sleep=lambda s: None, dead=[])
|
|
# second run with longer series must overwrite (refresh), not skip
|
|
fetch_crypto_pit(str(tmp_path), top_n=1, get=_fake_get_factory(400), sleep=lambda s: None, dead=[])
|
|
z = np.load(tmp_path / "crypto_pit" / "AAAUSDT.npz")
|
|
assert len(z["day"]) == 400
|