diff --git a/src/ycli/yandex/base.py b/src/ycli/yandex/base.py index da8f8f5d..f1b3a54c 100644 --- a/src/ycli/yandex/base.py +++ b/src/ycli/yandex/base.py @@ -13,20 +13,23 @@ from __future__ import annotations +import copy from abc import ABC, abstractmethod from typing import TYPE_CHECKING, ClassVar, Self from pydantic import SecretStr from ycli.settings import CLOUD_ORGANIZATION_ID_ENV, ORGANIZATION_ID_ENV, HTTPConfig +from ycli.yandex.errors import YandexInvalidRequestError if TYPE_CHECKING: + from collections.abc import Callable from types import TracebackType import httpx2 from ycli.yandex.core.endpoint import Endpoint, Paged - from ycli.yandex.core.guard import Guard + from ycli.yandex.core.guard import Guard, PlannedRequest from ycli.yandex.core.listing import Listing from ycli.yandex.core.profile import ServiceProfile from ycli.yandex.core.session import BeforeSend, SyncSession @@ -89,6 +92,8 @@ def __init__( guard=guard, ) self._wire(self._session) + #: A view made by :meth:`with_options` shares the session of the client it came from. + self._owns_session = True def __enter__(self) -> Self: return self @@ -102,8 +107,80 @@ def __exit__( self.close() def close(self) -> None: - """Close the core session's connection pool.""" - self._session.close() + """Close the core session's connection pool; a view of a client has none of its own.""" + if self._owns_session: + self._session.close() + + def with_options( + self, + *, + timeout_seconds: float | None = None, + retries: int | None = None, + dry_run: bool | None = None, + ) -> Self: + """This client with other options, for the calls made through what is returned. + + A view of this client: the same connections and the same credentials, so nothing is + opened and there is nothing of its own to close. + + Args: + timeout_seconds: The timeout of each request; ``None`` keeps this client's. + retries: How many times an idempotent request is tried again; ``None`` keeps it. + dry_run: Whether a write is stopped and raised as + :class:`~ycli.yandex.core.guard.RequestPlanned` in place of being sent + (:meth:`plan` returns it as a value); ``None`` keeps what this client does. + + Returns: + The client to make those calls through. + + Examples: + >>> patient = tracker.with_options(timeout_seconds=120, retries=5) + >>> patient.issues.get("DE-7").key + 'DE-7' + """ + view = copy.copy(self) + view._session = self._session.with_options( + timeout_seconds=timeout_seconds, retries=retries, dry_run=dry_run + ) + view._wire(view._session) + view._owns_session = False + return view + + def plan(self, call: Callable[[Self], object]) -> PlannedRequest: + """The first write ``call`` would send, not sent: ``client -> the request``. + + ``call`` is given this client under ``dry_run`` and makes its calls through it. What it + reads before its first write is read for real; the write is stopped and returned, and + nothing after it runs. The plan says whether the request would grant access + (``grants_access``). + + Args: + call: What to plan, as a function of the client: ``lambda tracker: ...``. + + Returns: + The request that would have gone out: its method, its URL and its body, with + secrets masked. + + Raises: + YandexInvalidRequestError: ``call`` sent no write: there is nothing to plan. + + Examples: + >>> planned = tracker.plan(lambda tracker: tracker.boards.delete(31)) + >>> planned.method, planned.url.endswith("/v3/boards/31"), planned.grants_access + ('DELETE', True, False) + >>> tracker.plan(lambda tracker: tracker.boards.get(31)) + Traceback (most recent call last): + ycli.yandex.errors.YandexInvalidRequestError: the call sends no write: nothing to plan + """ + # Imported here, like the rest of the core: httpx2 is paid for once a client is built. + from ycli.yandex.core.guard import RequestPlanned + + try: + call(self.with_options(dry_run=True)) + except RequestPlanned as planned: + return planned.plan + # violation(arch-9): the caller asked for the plan of a call that writes nothing + raise YandexInvalidRequestError("the call sends no write: nothing to plan") def send[T](self, endpoint: Endpoint[T]) -> T: """Call any ``endpoint`` of this service through its session: auth, retries, errors, logs. diff --git a/src/ycli/yandex/core/guard.py b/src/ycli/yandex/core/guard.py index b3b01dfb..b5b36802 100644 --- a/src/ycli/yandex/core/guard.py +++ b/src/ycli/yandex/core/guard.py @@ -126,6 +126,17 @@ def __init__(self, plan: PlannedRequest) -> None: self.plan = plan +def _plan(endpoint: Endpoint[Any], request: httpx2.Request) -> PlannedRequest: + """The plan of ``request``, told whether its operation grants access. + + Told of a plan that is only shown as well as of one that is asked about: the one who is + shown it may be the one asked next. + """ + plan = PlannedRequest.of(request) + plan._grants_access = endpoint.grants_access + return plan + + @dataclass(frozen=True) class Guard: """The rule every write goes through, the same for every surface. @@ -154,12 +165,11 @@ def check(self, endpoint: Endpoint[Any], request: httpx2.Request) -> None: if endpoint.effect is Effect.READ: return if self.dry_run: - raise RequestPlanned(PlannedRequest.of(request)) + raise RequestPlanned(_plan(endpoint, request)) asked_about = endpoint.effect is Effect.DESTRUCTIVE or endpoint.grants_access if not asked_about or self.confirm is None: return - plan = PlannedRequest.of(request) - plan._grants_access = endpoint.grants_access + plan = _plan(endpoint, request) if not self.confirm(plan): raise YandexDeclinedError( f"{plan.method} {plan.url} was not confirmed; nothing was sent", url=plan.url diff --git a/src/ycli/yandex/core/session.py b/src/ycli/yandex/core/session.py index 929b027f..b39025ca 100644 --- a/src/ycli/yandex/core/session.py +++ b/src/ycli/yandex/core/session.py @@ -34,9 +34,11 @@ from __future__ import annotations +import copy import logging import math import time +from dataclasses import replace from typing import TYPE_CHECKING import httpx2 @@ -190,6 +192,33 @@ def _ended(walk: Walk, pages: int, max_pages: int) -> None: walk.cut() +def _timed(request: httpx2.Request, timeout_seconds: float | None) -> None: + """Give ``request`` a timeout of its own, where the session was given one for its calls.""" + if timeout_seconds is not None: + request.extensions["timeout"] = httpx2.Timeout(timeout_seconds).as_dict() + + +def _with_options[S: (SyncSession, AsyncSession)]( + session: S, *, timeout_seconds: float | None, retries: int | None, dry_run: bool | None +) -> S: + """A copy of ``session`` over the same client, with the options that were given.""" + # Here and not above: the guard masks a URL with this module's `shown`, so it imports it. + from ycli.yandex.core.guard import Guard + + other = copy.copy(session) + given = {"timeout_seconds": timeout_seconds, "retries": retries} + changed = {name: value for name, value in given.items() if value is not None} + # Validated like the configuration it stands in for: no negative retries, no zero timeout. + other._http = HTTPConfig.model_validate({**session._http.model_dump(), **changed}) + other._attempts = other._http.retries + 1 + other._timeout_seconds = ( + session._timeout_seconds if timeout_seconds is None else timeout_seconds + ) + if dry_run is not None: + other._guard = replace(session._guard or Guard(), dry_run=dry_run) + return other + + def _first_page(paged: Paged, client: httpx2.Client | httpx2.AsyncClient) -> httpx2.Request: """The request for a listing's first page; it and the pages after it carry ``paged``.""" request = paged.pagination.first(paged.endpoint.request(client)) @@ -216,6 +245,8 @@ def __init__( self._attempts = self._http.retries + 1 self._before_send = before_send self._guard = guard + #: The timeout of each request where it is not the client's own (:meth:`with_options`). + self._timeout_seconds: float | None = None def _send( self, request: httpx2.Request, *, idempotent: bool, follow_redirects: bool = True @@ -234,6 +265,7 @@ def _send( _log_retry(request, attempt.num, self._attempts) started = time.perf_counter() try: + _timed(request, self._timeout_seconds) response = self._client.send(request, follow_redirects=follow_redirects) except httpx2.RequestError as exc: url = shown(request.url) @@ -303,6 +335,31 @@ def pages() -> Iterator[I]: return Listing(walk, pages) + def with_options( + self, + *, + timeout_seconds: float | None = None, + retries: int | None = None, + dry_run: bool | None = None, + ) -> SyncSession: + """This session with other options for the calls made through what is returned. + + The connection pool is this session's own: nothing is opened, and closing either + closes both. + + Args: + timeout_seconds: The timeout of each request; ``None`` keeps this session's. + retries: How many times an idempotent request is tried again; ``None`` keeps it. + dry_run: Whether a write is stopped and shown in place of being sent; ``None`` + keeps what this session does. + + Returns: + A session over the same connections, with the options given. + """ + return _with_options( + self, timeout_seconds=timeout_seconds, retries=retries, dry_run=dry_run + ) + def way_of(self, token: str) -> str: """How the operation that gave ``token`` pages: the name of its pagination. @@ -350,6 +407,8 @@ def __init__( self._attempts = self._http.retries + 1 self._before_send = before_send self._guard = guard + #: The timeout of each request where it is not the client's own (:meth:`with_options`). + self._timeout_seconds: float | None = None async def _send( self, request: httpx2.Request, *, idempotent: bool, follow_redirects: bool = True @@ -368,6 +427,7 @@ async def _send( _log_retry(request, attempt.num, self._attempts) started = time.perf_counter() try: + _timed(request, self._timeout_seconds) response = await self._client.send(request, follow_redirects=follow_redirects) except httpx2.RequestError as exc: url = shown(request.url) @@ -438,6 +498,31 @@ async def pages() -> AsyncIterator[I]: return AsyncListing(walk, pages) + def with_options( + self, + *, + timeout_seconds: float | None = None, + retries: int | None = None, + dry_run: bool | None = None, + ) -> AsyncSession: + """This session with other options for the calls made through what is returned. + + The connection pool is this session's own: nothing is opened, and closing either + closes both. + + Args: + timeout_seconds: The timeout of each request; ``None`` keeps this session's. + retries: How many times an idempotent request is tried again; ``None`` keeps it. + dry_run: Whether a write is stopped and shown in place of being sent; ``None`` + keeps what this session does. + + Returns: + A session over the same connections, with the options given. + """ + return _with_options( + self, timeout_seconds=timeout_seconds, retries=retries, dry_run=dry_run + ) + def way_of(self, token: str) -> str: """How the operation that gave ``token`` pages: the name of its pagination. diff --git a/tests/contract/test_sdk_plan.py b/tests/contract/test_sdk_plan.py new file mode 100644 index 00000000..2fe22b0a --- /dev/null +++ b/tests/contract/test_sdk_plan.py @@ -0,0 +1,61 @@ +"""Every operation that writes, planned through the SDK: `client.plan(...)` sends no write. + +Its own file beside ``test_contract.py``: the cases and the way a case is served are that +module's. +""" + +from __future__ import annotations + +import json +import re +from typing import TYPE_CHECKING + +import pytest +from typer.testing import CliRunner + +from tests.contract import Case, Sibling +from tests.contract.test_contract import CASES, SERVICE_BY_NAME, _client, _serve +from ycli.cli.app import app +from ycli.yandex.core.endpoint import ENDPOINT_EXTENSION, Effect +from ycli.yandex.core.listing import Listing + +if TYPE_CHECKING: + from ycli.yandex.base import DomainClient + + +def _one_boundary(plan: object) -> object: + """``plan`` with the boundary of a file upload, drawn anew for every request, made one.""" + return json.loads(re.sub(r"boundary=[0-9a-f]+", "boundary=x", json.dumps(plan))) + + +SDK_WRITE_CASES = [case for case in CASES if case.expected_effect != "read"] + + +@pytest.mark.parametrize("case", SDK_WRITE_CASES, ids=[case.id for case in SDK_WRITE_CASES]) +def test_the_sdk_plans_every_write_and_sends_none(case: Case, monkeypatch): + """`client.plan(...)` of every operation that writes: no write reaches the service. + + Whatever resource the call goes through, however deep, it talks through the view of the + plan. Where the case has a command, the plan is the one `--dry-run -o json` prints. + """ + for name, value in case.env.items(): + monkeypatch.setenv(name, value) + api = _serve(monkeypatch, case) + domain, resource, method = case.operation.split(".") + + def call(client: DomainClient) -> None: + args = [getattr(client, a.resource) if isinstance(a, Sibling) else a for a in case.args] + result = getattr(getattr(client, resource), method)(*args, **case.kwargs) + if isinstance(result, Listing): + result.collect() # lazy: a listing that writes writes when it is read + + with _client(SERVICE_BY_NAME[domain]) as client: + planned = client.plan(call) + for sent in api.calls: + assert sent.extensions[ENDPOINT_EXTENSION].effect is Effect.READ, f"sent {sent.method}" + if case.cli is not None: + _serve(monkeypatch, case) + printed = CliRunner().invoke(app, ["--format", "json", "--dry-run", *case.cli]) + assert printed.exit_code == 0, printed.output + by_sdk = planned.model_dump(by_alias=True, mode="json") + assert _one_boundary(by_sdk) == _one_boundary(json.loads(printed.stdout_bytes)) diff --git a/tests/unit/yandex/core/test_session.py b/tests/unit/yandex/core/test_session.py index 81893b4f..14cad1b8 100644 --- a/tests/unit/yandex/core/test_session.py +++ b/tests/unit/yandex/core/test_session.py @@ -11,6 +11,7 @@ from ycli.settings import HTTPConfig from ycli.yandex.core.auth import OAuthTokenAuth from ycli.yandex.core.endpoint import ENDPOINT_EXTENSION, PAGED_EXTENSION, Endpoint, Paged +from ycli.yandex.core.guard import RequestPlanned from ycli.yandex.core.pagination import CursorPagination, PageNumberPagination from ycli.yandex.core.profile import ServiceProfile from ycli.yandex.core.session import ( @@ -481,3 +482,23 @@ def test_a_session_is_given_its_organization_as_a_header_and_an_id_or_not_at_all AsyncSession(httpx2.AsyncClient(), organization=organization) assert SyncSession(client, organization="X-Org-Id: o 7/x") # any id, under a header's name assert SyncSession(client) # a service that names none + + +async def test_an_asynchronous_session_takes_the_same_options(): + api = MockAPI() + api.add("DELETE", f"{URL}/1", status=204) + api.add("GET", URL, json=[1]) + session = connect_async( + PROFILE, + auth=OAuthTokenAuth(SecretStr("t")), + http=HTTPConfig(retries=1), + transport=api.transport(), + ) + view = session.with_options(timeout_seconds=5, retries=0, dry_run=True) + assert view._client is session._client and (view._attempts, session._attempts) == (1, 2) + with pytest.raises(RequestPlanned): + await view.send(Endpoint(HTTPMethod.DELETE, "items/1")) + await view.send(Endpoint(HTTPMethod.GET, "items", list[int])) + assert [call.method for call in api.calls] == ["GET"] + assert api.calls[0].extensions["timeout"]["connect"] == 5 + await session.aclose() diff --git a/tests/unit/yandex/test_base.py b/tests/unit/yandex/test_base.py index d17d616a..0b7f9dbc 100644 --- a/tests/unit/yandex/test_base.py +++ b/tests/unit/yandex/test_base.py @@ -4,14 +4,21 @@ import httpx2 import pytest -from pydantic import SecretStr +from pydantic import SecretStr, ValidationError from tests.hosts import WIKI_BASE from tests.mock_api import MockAPI +from ycli.yandex.base import DomainClient from ycli.yandex.core.auth import IAMTokenAuth from ycli.yandex.core.endpoint import Endpoint, Paged +from ycli.yandex.core.guard import RequestPlanned +from ycli.yandex.core.session import SyncSession +from ycli.yandex.errors import YandexError, YandexInvalidRequestError, YandexNotFoundError from ycli.yandex.forms.client import FormsClient +from ycli.yandex.registry import SERVICES +from ycli.yandex.tracker.boards.models import BoardUpdate from ycli.yandex.tracker.client import TrackerClient +from ycli.yandex.wiki.access.models import PageAccessCreate from ycli.yandex.wiki.client import WikiClient from ycli.yandex.wiki.cursor import WIKI_CURSOR @@ -75,3 +82,203 @@ def test_a_client_walks_any_listing_through_its_session(api): paged = Paged(Endpoint(HTTPMethod.GET, "x", dict), WIKI_CURSOR, lambda page: page["results"]) with WikiClient(oauth_token="t", organization_id="o") as client: assert list(client.iterate(paged, limit=3)) == [1, 2, 3] + + +BOARD = "https://api.tracker.yandex.net/v3/boards/31" + + +def test_a_plan_is_the_first_write_of_a_call_and_nothing_of_it_is_sent(api): + """`plan`: what the call reads before its first write is read; the write is given back.""" + api.add("GET", BOARD, json={"id": 31, "name": "Old"}) + api.add("PATCH", BOARD, json={"id": 31}) + api.add("DELETE", BOARD, status=204) + + def rename_then_delete(tracker: TrackerClient) -> None: + name = tracker.boards.get(31).name + tracker.boards.update(31, BoardUpdate(name=f"{name}!")) + tracker.boards.delete(31) # never reached: the plan is of the first write + + with TrackerClient(oauth_token="t", organization_id="o") as tracker: + planned = tracker.plan(rename_then_delete) + assert (planned.method, planned.url, planned.body) == ("PATCH", BOARD, {"name": "Old!"}) + assert planned.grants_access is False + assert [call.method for call in api.calls] == ["GET"] + # The client itself is as it was: the same call through it is sent. + tracker.boards.delete(31) + assert [call.method for call in api.calls] == ["GET", "DELETE"] + + +def test_a_plan_of_a_call_that_writes_nothing_is_an_error_and_a_grant_says_what_it_is(api): + api.add("GET", BOARD, json={"id": 31}) + tracker = TrackerClient(oauth_token="t", organization_id="o") + refused = pytest.raises(YandexInvalidRequestError, match="sends no write: nothing to plan") + with tracker, refused: + tracker.plan(lambda tracker: tracker.boards.get(31)) + body = PageAccessCreate.model_validate({"role": "editor", "user": {"uid": "9001"}}) + with WikiClient(oauth_token="t", organization_id="o") as wiki: + planned = wiki.plan(lambda wiki: wiki.access.create(7, body)) + assert planned.grants_access is True and planned.method == "POST" + assert [call.method for call in api.calls] == ["GET"] + + +def test_a_view_with_other_options_shares_the_connections_and_changes_only_its_calls(api): + """`with_options`: a timeout and a number of retries for the calls made through the view.""" + api.add("GET", BOARD, status=500, json={}) + with TrackerClient(oauth_token="t", organization_id="o") as tracker: + view = tracker.with_options(timeout_seconds=120, retries=0) + assert type(view) is TrackerClient and view is not tracker + assert view.boards._session._client is tracker.boards._session._client + with pytest.raises(YandexError): + view.boards.get(31) + assert len(api.calls) == 1 # not tried again + assert api.calls[0].extensions["timeout"]["read"] == 120 + # The client it came from keeps its own. + assert tracker._session._attempts == 4 and tracker._session._timeout_seconds is None + # One option changed keeps the other: a view of a view. + again = view.with_options(retries=2) + assert (again._session._timeout_seconds, again._session._attempts) == (120, 3) + with pytest.raises(ValidationError): + tracker.with_options(retries=-1) + with pytest.raises(ValidationError): + tracker.with_options(timeout_seconds=0) + # A view has no connections of its own to close. + with view: + pass + assert not tracker.boards._session._client.is_closed + assert tracker.boards._session._client.is_closed + + +def test_dry_run_can_be_turned_on_and_off_for_a_view(api): + api.add("DELETE", BOARD, status=204) + with TrackerClient(oauth_token="t", organization_id="o") as tracker: + careful = tracker.with_options(dry_run=True) + with pytest.raises(RequestPlanned): + careful.boards.delete(31) + assert api.calls == [] + careful.with_options(dry_run=False).boards.delete(31) + assert [call.method for call in api.calls] == ["DELETE"] + + +def _clients() -> list[DomainClient]: + """One client of every service, signed in the way the service takes.""" + made: list[DomainClient] = [] + for service in SERVICES: + client_class = service.client_class() + auth = None if service.profile.oauth_token else IAMTokenAuth(SecretStr("t")) + token = "t" if service.profile.oauth_token else None + made.append( + client_class( + oauth_token=token, auth=auth, organization_id="o", cloud_organization_id="c" + ) + ) + return made + + +def _session_holders(client: DomainClient) -> dict[str, tuple[object, SyncSession]]: + """Every object reachable from ``client`` that holds a session: its path, itself, the session. + + Walked through attributes, and through lists, tuples and dicts of them, to any depth: a + resource of a resource, or a helper that was handed the session, is found as well. + """ + found: dict[str, tuple[object, SyncSession]] = {} + seen: set[int] = set() + queue: list[tuple[str, object]] = [("client", client)] + while queue: + path, thing = queue.pop() + if id(thing) in seen or isinstance(thing, SyncSession | str | bytes | int | type): + continue + seen.add(id(thing)) + parts: list[tuple[str, object]] = [] + if isinstance(thing, dict): + parts = [(f"{path}[{key!r}]", value) for key, value in thing.items()] + elif isinstance(thing, list | tuple | set | frozenset): + parts = [(f"{path}[{index}]", value) for index, value in enumerate(thing)] + elif hasattr(thing, "__dict__"): + parts = [(f"{path}.{name}", value) for name, value in vars(thing).items()] + for name, value in parts: + if isinstance(value, SyncSession): + found[name] = (thing, value) + elif type(value).__module__.startswith("ycli.") or isinstance( + value, dict | list | tuple | set | frozenset + ): + queue.append((name, value)) + return found + + +@pytest.mark.parametrize("client", _clients(), ids=lambda client: type(client).__name__) +def test_everything_reachable_from_a_view_talks_through_the_session_of_the_view(client): + """`plan` is safe only if no part of a view still holds the session of its client. + + A sub-client that `with_options` did not rebuild would send a real write from inside + `plan`. Each holder of a session, at any depth, is looked at and named. + """ + with client: + view = client.with_options(dry_run=True) + of_client, of_view = _session_holders(client), _session_holders(view) + assert of_view.keys() == of_client.keys() and len(of_view) > 5 + assert view._session is not client._session + wrong = [ + path + for path, (holder, session) in of_view.items() + if session is not view._session + or (holder is of_client[path][0] and holder is not client) + ] + assert not wrong + assert all(session is client._session for _, session in of_client.values()) + + +def test_the_walk_over_a_view_sees_a_resource_that_was_not_rebuilt(): + """What the walk is for, on a client made to do it: the stale resource is named.""" + + class Forgetful(TrackerClient): + def _wire(self, session: SyncSession) -> None: + super()._wire(session) + if not hasattr(self, "kept"): + self.kept = self.boards # set once, outside what a view rebuilds + self.nested = [{"deep": self.kept}] + + with Forgetful(oauth_token="t", organization_id="o") as client: + view = client.with_options(dry_run=True) + stale = [ + path + for path, (_, session) in _session_holders(view).items() + if session is not view._session + ] + # One stale object, named by the first path it was reached by, however deep. + assert stale in (["client.kept._session"], ["client.nested[0]['deep']._session"]) + + +def test_a_plan_and_a_view_leave_the_client_as_it_was(api): + """The other direction: after a plan, and beside a dry-run view, the client still sends.""" + api.add("DELETE", BOARD, status=204) + with TrackerClient(oauth_token="t", organization_id="o") as tracker: + tracker.plan(lambda tracker: tracker.boards.delete(31)) + careful = tracker.with_options(dry_run=True) + with pytest.raises(RequestPlanned): + careful.boards.delete(31) + assert api.calls == [] + tracker.boards.delete(31) # the client was not turned into a dry run + careful.with_options(dry_run=False).boards.delete(31) # nor is a view of it one for good + assert [call.method for call in api.calls] == ["DELETE", "DELETE"] + assert tracker._session._guard is None + + +def test_what_a_call_raises_before_its_first_write_is_raised_as_itself(api): + """A read inside the call that fails is that failure, not "nothing to plan".""" + api.add("GET", BOARD, status=404, json={"errorMessages": ["no such board"]}) + api.add("DELETE", BOARD, status=204) + + def read_then_delete(tracker: TrackerClient) -> None: + tracker.boards.get(31) + tracker.boards.delete(31) + + with TrackerClient(oauth_token="t", organization_id="o") as tracker: + with pytest.raises(YandexNotFoundError): + tracker.plan(read_then_delete) + + def fails(tracker: TrackerClient) -> None: + raise KeyError("a mistake of the caller's own") + + with pytest.raises(KeyError): + tracker.plan(fails) + assert [call.method for call in api.calls] == ["GET"]