Coverage for src/polars_yahoo/_http.py: 100%
83 statements
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-17 15:16 +0000
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-17 15:16 +0000
1"""HTTP access to Yahoo's chart endpoint.
3A browser-impersonating ``curl_cffi`` session is used because Yahoo
4rate-limits clients that don't look like a browser. Sessions live in a
5module-level pool and outlast a single call, so their connections are reused.
6A request takes a session out of the pool and puts it back afterwards, which
7means a ``curl_cffi`` session is never used by two threads at once.
8"""
10from __future__ import annotations
12import queue
13import random
14import time
15from collections.abc import Iterator
16from contextlib import contextmanager
17from typing import Any, Protocol
18from urllib.parse import quote
20from curl_cffi import requests as _requests
22from .exceptions import NetworkError, RateLimitError, ResponseError, TickerNotFoundError
24CHART_URL = "https://query2.finance.yahoo.com/v8/finance/chart/{ticker}"
26# Request defaults, shared with Ticker and download so the three cannot drift apart.
27TIMEOUT = 30.0
28RETRIES = 3
29BACKOFF = 1.0
32def check_request_settings(timeout: float, retries: int, backoff: float) -> None:
33 """Raise ValueError unless ``timeout > 0``, ``retries >= 0`` and ``backoff >= 0``."""
34 if not timeout > 0:
35 raise ValueError(f"timeout must be positive, got {timeout!r}") # noqa: TRY003 - argument validation; a ValueError with context is the idiom
36 if retries < 0:
37 raise ValueError(f"retries must not be negative, got {retries!r}") # noqa: TRY003 - argument validation; a ValueError with context is the idiom
38 if not backoff >= 0:
39 raise ValueError(f"backoff must not be negative, got {backoff!r}") # noqa: TRY003 - argument validation; a ValueError with context is the idiom
42class Response(Protocol):
43 """The subset of a response object this package relies on."""
45 status_code: int
47 def json(self) -> Any:
48 """Decode the body as JSON."""
49 ...
52class Session(Protocol):
53 """Anything with a ``requests``-style ``get`` — lets tests inject a fake."""
55 def get(self, url: str, *, params: dict[str, Any], timeout: float) -> Response:
56 """Issue a GET request."""
57 ...
60def _new_session() -> Session:
61 """Create a browser-impersonating session."""
62 return _requests.Session(impersonate="chrome")
65# Idle sessions. It grows to at most the number of requests that ran at the same time.
66_pool: queue.SimpleQueue[Session] = queue.SimpleQueue()
69@contextmanager
70def pooled_session() -> Iterator[Session]:
71 """Lend an idle session from the pool, or a new one if none is idle, and return it afterwards."""
72 try:
73 session = _pool.get_nowait()
74 except queue.Empty:
75 session = _new_session()
76 try:
77 yield session
78 finally:
79 _pool.put(session)
82def _jitter(cap: float) -> float:
83 """Return a random wait between 0 and ``cap`` seconds ("full jitter").
85 Randomising the wait keeps concurrent requests that failed together from all
86 retrying at the same moment. This is the one source of randomness, so tests can pin it.
87 """
88 # Not security-relevant: the randomness only spreads retries out in time.
89 return random.uniform(0, cap) # noqa: S311 # nosec B311
92def _send(
93 session: Session, url: str, params: dict[str, Any], ticker: str, timeout: float, retries: int, backoff: float
94) -> Response:
95 """Issue the GET, retrying HTTP 429, 5xx and transport failures with jittered exponential backoff.
97 Before retry ``n`` (from 1) it waits a random time between 0 and ``backoff * 2**(n - 1)``
98 seconds. Once the retries are spent, a 429 raises RateLimitError, a transport failure
99 raises NetworkError, and a 5xx response is returned for ``_decode`` to report.
100 """
101 for attempt in range(retries + 1):
102 if attempt:
103 time.sleep(_jitter(backoff * 2 ** (attempt - 1)))
104 last = attempt == retries
105 try:
106 response = session.get(url, params=params, timeout=timeout)
107 except OSError as exc: # curl_cffi's RequestException is an OSError
108 if last:
109 raise NetworkError(ticker, str(exc)) from exc
110 continue
111 if response.status_code != 429 and (response.status_code < 500 or last):
112 return response
113 raise RateLimitError(ticker, retries)
116_MALFORMED = "malformed chart payload"
119def _first_result(ticker: str, status: int, results: Any) -> dict[str, Any]:
120 """Return the first chart ``result``, or raise ResponseError when it is not an object."""
121 result = results[0] if isinstance(results, list) else None
122 if not isinstance(result, dict):
123 raise ResponseError(ticker, status, _MALFORMED)
124 return result
127def _decode(ticker: str, response: Response) -> dict[str, Any]:
128 """Return the single chart ``result`` from ``response``, or raise the matching YahooError."""
129 try:
130 payload = response.json()
131 except ValueError as exc:
132 raise ResponseError(ticker, response.status_code, "body is not JSON") from exc
134 chart = payload.get("chart") if isinstance(payload, dict) else None
135 if not isinstance(chart, dict):
136 raise ResponseError(ticker, response.status_code, _MALFORMED)
137 error = chart.get("error")
138 results = chart.get("result")
139 if results and not error:
140 return _first_result(ticker, response.status_code, results)
141 reason = error.get("description") if isinstance(error, dict) else None
142 if response.status_code in (200, 404):
143 raise TickerNotFoundError(ticker, reason)
144 raise ResponseError(ticker, response.status_code, reason or "unexpected response")
147def get_chart(
148 ticker: str,
149 params: dict[str, Any],
150 *,
151 session: Session | None = None,
152 timeout: float = TIMEOUT,
153 retries: int = RETRIES,
154 backoff: float = BACKOFF,
155) -> dict[str, Any]:
156 """Fetch ``/v8/finance/chart/{ticker}`` and return its single ``result`` object.
158 Retries HTTP 429, 5xx and transport failures with exponential backoff: the wait
159 before each retry is random, capped at ``backoff`` and doubling each time. Once the
160 retries are spent, a 429 raises :class:`RateLimitError`, a 5xx :class:`ResponseError`,
161 and a transport failure :class:`NetworkError`. Raises :class:`TickerNotFoundError`
162 when Yahoo reports no data for the symbol.
163 """
164 # Escape every reserved character so the symbol stays one path segment.
165 url = CHART_URL.format(ticker=quote(ticker, safe=""))
166 if session is None:
167 with pooled_session() as pooled:
168 response = _send(pooled, url, params, ticker, timeout, retries, backoff)
169 else:
170 response = _send(session, url, params, ticker, timeout, retries, backoff)
171 return _decode(ticker, response)