Coverage for src/polars_yahoo/_http.py: 100%

83 statements  

« prev     ^ index     » next       coverage.py v7.17.3, created at 2026-11-17 15:16 +0000

1"""HTTP access to Yahoo's chart endpoint. 

2 

3A browser-impersonating ``curl_cffi`` session is used because Yahoo 

4rate-limits clients that don't look like a browser. Sessions live in a 

5module-level pool and outlast a single call, so their connections are reused. 

6A request takes a session out of the pool and puts it back afterwards, which 

7means a ``curl_cffi`` session is never used by two threads at once. 

8""" 

9 

10from __future__ import annotations 

11 

12import queue 

13import random 

14import time 

15from collections.abc import Iterator 

16from contextlib import contextmanager 

17from typing import Any, Protocol 

18from urllib.parse import quote 

19 

20from curl_cffi import requests as _requests 

21 

22from .exceptions import NetworkError, RateLimitError, ResponseError, TickerNotFoundError 

23 

24CHART_URL = "https://query2.finance.yahoo.com/v8/finance/chart/{ticker}" 

25 

26# Request defaults, shared with Ticker and download so the three cannot drift apart. 

27TIMEOUT = 30.0 

28RETRIES = 3 

29BACKOFF = 1.0 

30 

31 

32def check_request_settings(timeout: float, retries: int, backoff: float) -> None: 

33 """Raise ValueError unless ``timeout > 0``, ``retries >= 0`` and ``backoff >= 0``.""" 

34 if not timeout > 0: 

35 raise ValueError(f"timeout must be positive, got {timeout!r}") # noqa: TRY003 - argument validation; a ValueError with context is the idiom 

36 if retries < 0: 

37 raise ValueError(f"retries must not be negative, got {retries!r}") # noqa: TRY003 - argument validation; a ValueError with context is the idiom 

38 if not backoff >= 0: 

39 raise ValueError(f"backoff must not be negative, got {backoff!r}") # noqa: TRY003 - argument validation; a ValueError with context is the idiom 

40 

41 

42class Response(Protocol): 

43 """The subset of a response object this package relies on.""" 

44 

45 status_code: int 

46 

47 def json(self) -> Any: 

48 """Decode the body as JSON.""" 

49 ... 

50 

51 

52class Session(Protocol): 

53 """Anything with a ``requests``-style ``get`` — lets tests inject a fake.""" 

54 

55 def get(self, url: str, *, params: dict[str, Any], timeout: float) -> Response: 

56 """Issue a GET request.""" 

57 ... 

58 

59 

60def _new_session() -> Session: 

61 """Create a browser-impersonating session.""" 

62 return _requests.Session(impersonate="chrome") 

63 

64 

65# Idle sessions. It grows to at most the number of requests that ran at the same time. 

66_pool: queue.SimpleQueue[Session] = queue.SimpleQueue() 

67 

68 

69@contextmanager 

70def pooled_session() -> Iterator[Session]: 

71 """Lend an idle session from the pool, or a new one if none is idle, and return it afterwards.""" 

72 try: 

73 session = _pool.get_nowait() 

74 except queue.Empty: 

75 session = _new_session() 

76 try: 

77 yield session 

78 finally: 

79 _pool.put(session) 

80 

81 

82def _jitter(cap: float) -> float: 

83 """Return a random wait between 0 and ``cap`` seconds ("full jitter"). 

84 

85 Randomising the wait keeps concurrent requests that failed together from all 

86 retrying at the same moment. This is the one source of randomness, so tests can pin it. 

87 """ 

88 # Not security-relevant: the randomness only spreads retries out in time. 

89 return random.uniform(0, cap) # noqa: S311 # nosec B311 

90 

91 

92def _send( 

93 session: Session, url: str, params: dict[str, Any], ticker: str, timeout: float, retries: int, backoff: float 

94) -> Response: 

95 """Issue the GET, retrying HTTP 429, 5xx and transport failures with jittered exponential backoff. 

96 

97 Before retry ``n`` (from 1) it waits a random time between 0 and ``backoff * 2**(n - 1)`` 

98 seconds. Once the retries are spent, a 429 raises RateLimitError, a transport failure 

99 raises NetworkError, and a 5xx response is returned for ``_decode`` to report. 

100 """ 

101 for attempt in range(retries + 1): 

102 if attempt: 

103 time.sleep(_jitter(backoff * 2 ** (attempt - 1))) 

104 last = attempt == retries 

105 try: 

106 response = session.get(url, params=params, timeout=timeout) 

107 except OSError as exc: # curl_cffi's RequestException is an OSError 

108 if last: 

109 raise NetworkError(ticker, str(exc)) from exc 

110 continue 

111 if response.status_code != 429 and (response.status_code < 500 or last): 

112 return response 

113 raise RateLimitError(ticker, retries) 

114 

115 

116_MALFORMED = "malformed chart payload" 

117 

118 

119def _first_result(ticker: str, status: int, results: Any) -> dict[str, Any]: 

120 """Return the first chart ``result``, or raise ResponseError when it is not an object.""" 

121 result = results[0] if isinstance(results, list) else None 

122 if not isinstance(result, dict): 

123 raise ResponseError(ticker, status, _MALFORMED) 

124 return result 

125 

126 

127def _decode(ticker: str, response: Response) -> dict[str, Any]: 

128 """Return the single chart ``result`` from ``response``, or raise the matching YahooError.""" 

129 try: 

130 payload = response.json() 

131 except ValueError as exc: 

132 raise ResponseError(ticker, response.status_code, "body is not JSON") from exc 

133 

134 chart = payload.get("chart") if isinstance(payload, dict) else None 

135 if not isinstance(chart, dict): 

136 raise ResponseError(ticker, response.status_code, _MALFORMED) 

137 error = chart.get("error") 

138 results = chart.get("result") 

139 if results and not error: 

140 return _first_result(ticker, response.status_code, results) 

141 reason = error.get("description") if isinstance(error, dict) else None 

142 if response.status_code in (200, 404): 

143 raise TickerNotFoundError(ticker, reason) 

144 raise ResponseError(ticker, response.status_code, reason or "unexpected response") 

145 

146 

147def get_chart( 

148 ticker: str, 

149 params: dict[str, Any], 

150 *, 

151 session: Session | None = None, 

152 timeout: float = TIMEOUT, 

153 retries: int = RETRIES, 

154 backoff: float = BACKOFF, 

155) -> dict[str, Any]: 

156 """Fetch ``/v8/finance/chart/{ticker}`` and return its single ``result`` object. 

157 

158 Retries HTTP 429, 5xx and transport failures with exponential backoff: the wait 

159 before each retry is random, capped at ``backoff`` and doubling each time. Once the 

160 retries are spent, a 429 raises :class:`RateLimitError`, a 5xx :class:`ResponseError`, 

161 and a transport failure :class:`NetworkError`. Raises :class:`TickerNotFoundError` 

162 when Yahoo reports no data for the symbol. 

163 """ 

164 # Escape every reserved character so the symbol stays one path segment. 

165 url = CHART_URL.format(ticker=quote(ticker, safe="")) 

166 if session is None: 

167 with pooled_session() as pooled: 

168 response = _send(pooled, url, params, ticker, timeout, retries, backoff) 

169 else: 

170 response = _send(session, url, params, ticker, timeout, retries, backoff) 

171 return _decode(ticker, response)