"""Bounded-concurrency fetch helper + rate-limit backoff for the Bybit ingests.

The Bybit ingests are NETWORK-bound: each symbol paginates the public API, and the ~794-symbol
survivorship-free universe takes ~1–2 hr processed sequentially (klines = perp+spot = 2× funding). The DB
writes are already batched and cheap, so the fetch is the bottleneck.

Two pieces, both fully unit-testable with NO network / NO real sleeps:

  * fetch_symbols_concurrently(symbols, fetch_one, *, workers, on_result)
        Runs `fetch_one(symbol)` (the per-symbol network fetch) across a bounded ThreadPoolExecutor, and
        hands each result to `on_result(symbol, result)` AS IT COMPLETES — in the MAIN thread, serialized.
        This captures ~all the win (concurrent fetch) while sidestepping store thread-safety questions
        (writes are single-threaded). Memory stays ~one symbol: results are drained as completed, never
        collected up front — preserving the per-symbol bounded-memory property of the OOM fix.

  * with_rate_limit_backoff(fetch, *, sleep, base, tries)
        Wraps a `fetch(url)->dict` so a Bybit rate-limit response (retCode 10006 "too many visits" / 10018
        "ip rate limit") OR an HTTP-429 raised by the fetch triggers exponential backoff + retry
        (base, 2·base, 4·base, …) up to `tries`, then surfaces a RateLimitError. Non-rate-limit errors are
        NOT retried (they surface immediately). The `sleep` is injectable so tests don't actually wait.
        This lets higher concurrency stay polite to Bybit's shared public limits.
"""
from __future__ import annotations

import time
from collections.abc import Callable
from concurrent.futures import ThreadPoolExecutor, as_completed
from typing import Any, TypeVar

_T = TypeVar("_T")

# Bybit v5 rate-limit retCodes (public market endpoints). 10006 = "Too many visits", 10018 = IP rate limit.
BYBIT_RATE_LIMIT_RETCODES = frozenset({10006, 10018})

# Default worker count: brisk but polite to Bybit's shared public limits (no API key on market data, so the
# IP limit is what matters). 12 keeps ~12 paginating symbols in flight — a ~5–10× speedup over sequential —
# while the backoff absorbs the occasional 429 instead of hammering.
DEFAULT_WORKERS = 12


def resume_floor_ms(last_day: dict[str, int], symbol: str, default_start_ms: int | None) -> int | None:
    """Per-symbol backward-pagination floor for a DATE-RESUMING ingest: resume from the symbol's last persisted
    day (`last_day[symbol] * 86_400_000`) so the adapter pulls only NEW days (the idempotent upsert overwrites
    the boundary day in place). A symbol with no persisted day yet falls back to `default_start_ms` (the
    caller's explicit floor, or None → the adapter's own backstop → full back-fill).

    This is the shared primitive behind the resume-from-last-date contract: it REPLACES the old symbol-skip
    resume that, once a symbol carried ANY row, never fetched it again — which froze the feature at the last
    ingest date and stalled the forward gate (see `last_day_by_symbol` on the feature store)."""
    d = last_day.get(symbol)
    return d * 86_400_000 if d is not None else default_start_ms


class RateLimitError(RuntimeError):
    """Raised when a fetch stays rate-limited after exhausting the backoff retries."""


def _is_http_429(exc: BaseException) -> bool:
    """True if `exc` looks like an HTTP 429 (urllib.error.HTTPError exposes `.code`; others may too)."""
    return getattr(exc, "code", None) == 429


def with_rate_limit_backoff(
    fetch: Callable[[str], Any],
    *,
    sleep: Callable[[float], None] = time.sleep,
    base: float = 0.5,
    tries: int = 5,
) -> Callable[[str], Any]:
    """Wrap `fetch(url)->dict` with exponential backoff on Bybit rate limiting.

    A response carrying a rate-limit retCode, or an HTTP-429 exception raised by `fetch`, is retried after
    `base * 2**attempt` seconds, up to `tries` attempts; the final exhaustion raises RateLimitError. Any
    other exception propagates immediately (no retry). `sleep` is injectable (tests pass a no-op recorder)."""
    def wrapped(url: str) -> Any:
        for attempt in range(tries):
            try:
                data = fetch(url)
            except Exception as exc:  # noqa: BLE001 — only RETRY on 429; re-raise everything else
                if not _is_http_429(exc):
                    raise
                if attempt == tries - 1:
                    raise RateLimitError(f"rate-limited after {tries} tries: {url}") from exc
                sleep(base * (2 ** attempt))
                continue
            if isinstance(data, dict) and data.get("retCode") in BYBIT_RATE_LIMIT_RETCODES:
                if attempt == tries - 1:
                    raise RateLimitError(
                        f"rate-limited (retCode={data.get('retCode')}) after {tries} tries: {url}")
                sleep(base * (2 ** attempt))
                continue
            return data
        raise RateLimitError("unreachable")  # pragma: no cover — loop always returns or raises above
    return wrapped


def fetch_symbols_concurrently(
    symbols: list[str],
    fetch_one: Callable[[str], _T],
    *,
    workers: int = DEFAULT_WORKERS,
    on_result: Callable[[str, _T], None] | None = None,
) -> None:
    """Fetch each symbol concurrently (bounded by `workers`); hand each result to `on_result(symbol, result)`
    as it completes, in the calling thread (serialized writes — no store thread-safety needed).

    Memory-bounded: results are consumed as-completed, NOT collected into a list first, so peak in-memory
    payload is ~one symbol's result at a time. The first fetch exception aborts (surfaces to the caller).
    A `workers` ≤ 1 still works (effectively sequential). `on_result` is optional (fetch-only / discard)."""
    if not symbols:
        return
    max_workers = max(1, workers)
    with ThreadPoolExecutor(max_workers=max_workers) as pool:
        futures = {pool.submit(fetch_one, sym): sym for sym in symbols}
        for fut in as_completed(futures):
            sym = futures[fut]
            result = fut.result()                    # raises here if the fetch raised → propagate
            if on_result is not None:
                on_result(sym, result)               # main-thread, one-at-a-time → writes are serialized
