From 4757a271e3280e3c3e1bce3066eed2a20567b902 Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Fri, 22 May 2020 11:58:13 +0000 Subject: [PATCH 1/6] Add ScopeVar, like contextvars.ContextVar but inherited from the parent nursery rather than from the spawner --- docs/source/reference-core.rst | 2 + docs/source/reference-lowlevel.rst | 10 ++- trio/_core/__init__.py | 2 +- trio/_core/_local.py | 110 +++++++++++++++++++++++++++-- trio/_core/_run.py | 14 +++- trio/_core/tests/test_local.py | 88 +++++++++++++++++++++++ trio/lowlevel.py | 1 + 7 files changed, 218 insertions(+), 9 deletions(-) diff --git a/docs/source/reference-core.rst b/docs/source/reference-core.rst index b5f6d7c4d8..b82faa7418 100644 --- a/docs/source/reference-core.rst +++ b/docs/source/reference-core.rst @@ -656,6 +656,8 @@ finished. This code will wait 5 seconds (for the child task to finish), and then return. +.. _child-tasks-and-cancellation: + Child tasks and cancellation ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/source/reference-lowlevel.rst b/docs/source/reference-lowlevel.rst index 53e5cf8a8c..b513081100 100644 --- a/docs/source/reference-lowlevel.rst +++ b/docs/source/reference-lowlevel.rst @@ -226,11 +226,11 @@ Windows-specific API .. function:: WaitForSingleObject(handle) :async: - + Async and cancellable variant of `WaitForSingleObject `__. Windows only. - + :arg handle: A Win32 object handle, as a Python integer. :raises OSError: @@ -261,6 +261,12 @@ Global state: system tasks and run-local variables .. autofunction:: spawn_system_task +Scope variables +=============== + +.. autoclass:: ScopeVar + + Trio tokens =========== diff --git a/trio/_core/__init__.py b/trio/_core/__init__.py index c28b7f4078..0ca5ae8c7d 100644 --- a/trio/_core/__init__.py +++ b/trio/_core/__init__.py @@ -66,7 +66,7 @@ from ._unbounded_queue import UnboundedQueue -from ._local import RunVar +from ._local import RunVar, ScopeVar # Kqueue imports try: diff --git a/trio/_core/_local.py b/trio/_core/_local.py index 352caa5682..5650c136e9 100644 --- a/trio/_core/_local.py +++ b/trio/_core/_local.py @@ -1,7 +1,9 @@ -# Runvar implementations +# Implementations of RunVar and ScopeVar +import contextvars +from contextlib import contextmanager from . import _run -from .._util import SubclassingDeprecatedIn_v0_15_0 +from .._util import Final, SubclassingDeprecatedIn_v0_15_0 class _RunVarToken: @@ -19,6 +21,9 @@ def __init__(self, var, value): self.redeemed = False +_NO_DEFAULT = object() + + class RunVar(metaclass=SubclassingDeprecatedIn_v0_15_0): """The run-local variant of a context variable. @@ -28,7 +33,6 @@ class RunVar(metaclass=SubclassingDeprecatedIn_v0_15_0): """ - _NO_DEFAULT = object() __slots__ = ("_name", "_default") def __init__(self, name, default=_NO_DEFAULT): @@ -43,10 +47,10 @@ def get(self, default=_NO_DEFAULT): raise RuntimeError("Cannot be used outside of a run context") from None except KeyError: # contextvars consistency - if default is not self._NO_DEFAULT: + if default is not _NO_DEFAULT: return default - if self._default is not self._NO_DEFAULT: + if self._default is not _NO_DEFAULT: return self._default raise LookupError(self) from None @@ -95,3 +99,99 @@ def reset(self, token): def __repr__(self): return "".format(self._name) + + +class ScopeVar(metaclass=Final): + """A "scope variable": like a context variable except that its value + is inherited by new tasks in a different way. + + In simple terms, `ScopeVar` lets you define your own custom state + that's inherited in the same way that :ref:`cancel scopes are + `. Ordinary context variables are + inherited by new tasks based on the environment surrounding the + :meth:`~trio.Nursery.start_soon` call that created the task. By + contrast, scope variables are inherited based on the environment + surrounding the nursery into which the new task was spawned. This + difference makes scope variables a better fit than context + variables for state that naturally propagates down the task tree. + + Some example uses of a `ScopeVar`: + + * Provide access to a resource that is only usable in a certain + scope. This might be a nursery, network connection, or anything + else whose lifetime is bound to a context manager. Tasks that + run entirely within the resource's lifetime can use it; tasks + that might keep running past the resource being destroyed won't see + it at all. + + * Constrain a function's caller-visible behavior, such as what exceptions + it might throw. The `ScopeVar`\\'s value will be inherited by every + task whose exceptions might propagate to the point where the value was + set. + + `ScopeVar` objects support all the same methods as `~contextvars.ContextVar` + objects, plus the additional methods :meth:`being` and :meth:`get_in`. + + .. note:: `ScopeVar` values are not directly stored in the + `contextvars.Context`, so you can't use `Context.get() + ` to access them; use :meth:`get_in` + instead, if you need the value in a context other than your own. + """ + + __slots__ = ("_cvar",) + + def __init__(self, name, **default): + self._cvar = contextvars.ContextVar(name, **default) + + @property + def name(self): + """The name of the variable, as passed during construction. Read-only.""" + return self._cvar.name + + def get(self, default=_NO_DEFAULT): + """Gets the value of this :class:`ScopeVar` for the current task.""" + # This is effectively an inlining for efficiency of: + # return _run.current_task()._scope_context.run(self._cvar.get, default) + try: + return _run.GLOBAL_RUN_CONTEXT.task._scope_context[self._cvar] + except AttributeError: + raise RuntimeError("must be called from async context") from None + except KeyError: + pass + # This will always return the default or raise, because we never give + # self._cvar a value in any context in which we run user code. + if default is _NO_DEFAULT: + return self._cvar.get() + else: + return self._cvar.get(default) + + def set(self, value): + """Sets the value of this :class:`ScopeVar` for the current task and + any tasks that run in child nurseries that it later creates. + """ + return _run.current_task()._scope_context.run(self._cvar.set, value) + + def reset(self, token): + """Resets the value of this :class:`ScopeVar` to what it was + previously, as specified by the token. + """ + _run.current_task()._scope_context.run(self._cvar.reset, token) + + @contextmanager + def being(self, value): + """Returns a context manager which sets the value of this `ScopeVar` to + *value* upon entry and restores its previous value upon exit. + """ + token = self.set(value) + try: + yield + finally: + self.reset(token) + + def get_in(self, task_or_nursery, default=_NO_DEFAULT): + """Gets the value of this :class:`ScopeVar` for the given task or nursery.""" + defarg = () if default is _NO_DEFAULT else (default,) + return task_or_nursery._scope_context.run(self._cvar.get, *defarg) + + def __repr__(self): + return f"" diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 4de4e9a2bf..64d4ee98fd 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -1,3 +1,5 @@ +# coding: utf-8 + import functools import itertools import logging @@ -10,7 +12,7 @@ import collections.abc from contextlib import contextmanager, closing -from contextvars import copy_context +from contextvars import copy_context, Context from math import inf from time import perf_counter @@ -771,6 +773,7 @@ class Nursery(metaclass=NoPublicConstructor): def __init__(self, parent_task, cancel_scope): self._parent_task = parent_task + self._scope_context = parent_task._scope_context.copy() parent_task._child_nurseries.append(self) # the cancel status that children inherit - we take a snapshot, so it # won't be affected by any changes in the parent. @@ -974,6 +977,9 @@ class Task(metaclass=NoPublicConstructor): context = attr.ib() _counter = attr.ib(init=False, factory=itertools.count().__next__) + # Contextvars context that contains ScopeVar values + _scope_context = attr.ib(init=False) + # Invariant: # - for unscheduled tasks, _next_send_fn and _next_send are both None # - for scheduled tasks, _next_send_fn(_next_send) resumes the task; @@ -999,6 +1005,12 @@ class Task(metaclass=NoPublicConstructor): _cancel_points = attr.ib(default=0) _schedule_points = attr.ib(default=0) + def __attrs_post_init__(self): + if self._parent_nursery is None: + self._scope_context = Context() + else: + self._scope_context = self._parent_nursery._scope_context.copy() + def __repr__(self): return "".format(self.name, id(self)) diff --git a/trio/_core/tests/test_local.py b/trio/_core/tests/test_local.py index 7f403168ea..f47e88132d 100644 --- a/trio/_core/tests/test_local.py +++ b/trio/_core/tests/test_local.py @@ -1,4 +1,5 @@ import pytest +from functools import partial from ... import _core @@ -113,3 +114,90 @@ async def get_token(): with pytest.raises(RuntimeError): t1.reset(token) + + +async def test_scopevar(): + sv1 = _core.ScopeVar("sv1") + sv2 = _core.ScopeVar("sv2", default=None) + with pytest.raises(LookupError): + sv1.get() + assert sv2.get() is None + assert sv1.get(42) == 42 + assert sv2.get(42) == 42 + + NOTHING = object() + + async def should_be(val1, val2, new1=NOTHING): + assert sv1.get(NOTHING) == val1 + assert sv2.get(NOTHING) == val2 + if new1 is not NOTHING: + sv1.set(new1) + + tok1 = sv1.set(10) + async with _core.open_nursery() as outer: + tok2 = sv1.set(15) + with sv2.being(20): + assert sv2.get_in(_core.current_task()) == 20 + async with _core.open_nursery() as inner: + sv1.reset(tok2) + outer.start_soon(should_be, 10, NOTHING, 100) + inner.start_soon(should_be, 15, 20, 200) + await _core.wait_all_tasks_blocked() + assert sv1.get_in(_core.current_task()) == 10 + await should_be(10, 20, 300) + assert sv1.get_in(inner) == 15 + assert sv1.get_in(outer) == 10 + assert sv1.get_in(_core.current_task()) == 300 + assert sv2.get_in(inner) == 20 + assert sv2.get_in(outer) is None + assert sv2.get_in(_core.current_task()) == 20 + sv1.reset(tok1) + await should_be(NOTHING, 20) + assert sv1.get_in(inner) == 15 + assert sv1.get_in(outer) == 10 + with pytest.raises(LookupError): + assert sv1.get_in(_core.current_task()) + assert sv2.get() is None + assert sv2.get_in(_core.current_task()) is None + + +async def test_scopevar_token_bound_to_task_that_obtained_it(): + sv1 = _core.ScopeVar("sv1") + token = None + + async def get_token(): + nonlocal token + token = sv1.set(10) + try: + await _core.wait_task_rescheduled(lambda _: _core.Abort.SUCCEEDED) + finally: + sv1.reset(token) + with pytest.raises(LookupError): + sv1.get() + with pytest.raises(LookupError): + sv1.get_in(_core.current_task()) + + async with _core.open_nursery() as nursery: + nursery.start_soon(get_token) + await _core.wait_all_tasks_blocked() + assert token is not None + with pytest.raises(ValueError, match="different Context"): + sv1.reset(token) + assert sv1.get_in(list(nursery.child_tasks)[0]) == 10 + nursery.cancel_scope.cancel() + + +def test_scopevar_outside_run(): + async def run_sync(fn, *args): + return fn(*args) + + sv1 = _core.ScopeVar("sv1", default=10) + for operation in ( + sv1.get, + partial(sv1.get, 20), + partial(sv1.set, 30), + lambda: sv1.reset(_core.run(run_sync, sv1.set, 10)), + sv1.being(40).__enter__, + ): + with pytest.raises(RuntimeError, match="must be called from async context"): + operation() diff --git a/trio/lowlevel.py b/trio/lowlevel.py index 21ec0597df..6442607390 100644 --- a/trio/lowlevel.py +++ b/trio/lowlevel.py @@ -25,6 +25,7 @@ ParkingLot, UnboundedQueue, RunVar, + ScopeVar, TrioToken, current_trio_token, temporarily_detach_coroutine_object, From d3569cb3d37d2e8178720a09131b34cc48942863 Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Sat, 23 May 2020 02:44:50 +0000 Subject: [PATCH 2/6] Add newsfragment --- newsfragments/1523.feature.rst | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 newsfragments/1523.feature.rst diff --git a/newsfragments/1523.feature.rst b/newsfragments/1523.feature.rst new file mode 100644 index 0000000000..21b0fd1e97 --- /dev/null +++ b/newsfragments/1523.feature.rst @@ -0,0 +1,7 @@ +Added the concept of a "scope variable" (`trio.lowlevel.ScopeVar`), which is +like a context variable except that its value in a new task is determined +differently. A context variable is inherited by a new task based on its value +at the location where the task was spawned, while a scope variable is inherited +based on its value where the task's parent nursery was opened. This distinction +makes scope variables useful for anything that's naturally inherited along +parent/child task relationships. From 2a555b9c1069fe442d8f94be77d606c0f5a4de41 Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Sat, 23 May 2020 03:11:14 +0000 Subject: [PATCH 3/6] Document individual methods --- docs/source/reference-lowlevel.rst | 16 +++++++-- trio/_core/_local.py | 53 ++++++++++++++++++++++-------- 2 files changed, 54 insertions(+), 15 deletions(-) diff --git a/docs/source/reference-lowlevel.rst b/docs/source/reference-lowlevel.rst index b513081100..c65fc14742 100644 --- a/docs/source/reference-lowlevel.rst +++ b/docs/source/reference-lowlevel.rst @@ -256,7 +256,11 @@ anything real. See `#26 Global state: system tasks and run-local variables ================================================== -.. autoclass:: RunVar +.. autoclass:: RunVar(name, [default]) + + .. automethod:: get([default]) + .. automethod:: set + .. automethod:: reset .. autofunction:: spawn_system_task @@ -264,7 +268,15 @@ Global state: system tasks and run-local variables Scope variables =============== -.. autoclass:: ScopeVar +.. autoclass:: ScopeVar(name, [*, default]) + + .. autoattribute:: name + .. automethod:: get([default]) + .. automethod:: set + .. automethod:: reset + .. automethod:: being + :with: + .. automethod:: get_in(task_or_nursery, [default]) Trio tokens diff --git a/trio/_core/_local.py b/trio/_core/_local.py index 5650c136e9..63e9f37ea0 100644 --- a/trio/_core/_local.py +++ b/trio/_core/_local.py @@ -30,7 +30,6 @@ class RunVar(metaclass=SubclassingDeprecatedIn_v0_15_0): :class:`RunVar` objects are similar to context variable objects, except that they are shared across a single call to :func:`trio.run` rather than a single task. - """ __slots__ = ("_name", "_default") @@ -40,7 +39,8 @@ def __init__(self, name, default=_NO_DEFAULT): self._default = default def get(self, default=_NO_DEFAULT): - """Gets the value of this :class:`RunVar` for the current run call.""" + """Gets the value of this `RunVar` for the current call + to :func:`trio.run`.""" try: return _run.GLOBAL_RUN_CONTEXT.runner._locals[self] except AttributeError: @@ -56,9 +56,11 @@ def get(self, default=_NO_DEFAULT): raise LookupError(self) from None def set(self, value): - """Sets the value of this :class:`RunVar` for this current run - call. + """Sets the value of this `RunVar` for the current call to + :func:`trio.run`. + Returns a token which may be passed to :meth:`reset` to restore + the previous value. """ try: old_value = self.get() @@ -73,9 +75,8 @@ def set(self, value): return token def reset(self, token): - """Resets the value of this :class:`RunVar` to what it was - previously specified by the token. - + """Resets the value of this `RunVar` to the value it had + before the call to :meth:`set` that returned the given *token*. """ if token is None: raise TypeError("token must not be none") @@ -149,7 +150,15 @@ def name(self): return self._cvar.name def get(self, default=_NO_DEFAULT): - """Gets the value of this :class:`ScopeVar` for the current task.""" + """Gets the value of this `ScopeVar` for the current task. + + If this `ScopeVar` has no value in the current task, then + :meth:`get` returns the *default* specified as argument to + :meth:`get`, or else the *default* specified when constructing + the `ScopeVar`, or else raises `LookupError`. See the + documentation of :meth:`contextvars.ContextVar.get` for more + details. + """ # This is effectively an inlining for efficiency of: # return _run.current_task()._scope_context.run(self._cvar.get, default) try: @@ -166,14 +175,24 @@ def get(self, default=_NO_DEFAULT): return self._cvar.get(default) def set(self, value): - """Sets the value of this :class:`ScopeVar` for the current task and - any tasks that run in child nurseries that it later creates. + """Sets the value of this `ScopeVar` for the current task. The new + value will be inherited by nurseries that are later opened in + this task, so that new tasks can inherit whatever value was + set when their parent nursery was created. + + Returns a token which may be passed to :meth:`reset` to restore + the previous value. """ return _run.current_task()._scope_context.run(self._cvar.set, value) def reset(self, token): - """Resets the value of this :class:`ScopeVar` to what it was - previously, as specified by the token. + """Resets the value of this `ScopeVar` to the value it had + before the call to :meth:`set` that returned the given *token*. + + The *token* must have been obtained from a call to :meth:`set` on + this same `ScopeVar` and in the same task that is now calling + :meth:`reset`. Also, each *token* may only be used in one call to + :meth:`reset`. Violating these conditions will raise `ValueError`. """ _run.current_task()._scope_context.run(self._cvar.reset, token) @@ -189,7 +208,15 @@ def being(self, value): self.reset(token) def get_in(self, task_or_nursery, default=_NO_DEFAULT): - """Gets the value of this :class:`ScopeVar` for the given task or nursery.""" + """Gets the value of this :class:`ScopeVar` for the given task or nursery. + + The value in a task is the value that would be returned by a call to + :meth:`get` in that task. The value in a nursery is the value that would + be returned by :meth:`get` at the beginning of a new child task started + in that nursery. The *default* argument has the same semantics as it does + for :meth:`get`. + """ + defarg = () if default is _NO_DEFAULT else (default,) return task_or_nursery._scope_context.run(self._cvar.get, *defarg) From 3670071c523db08437a4a1ddf739ed9ad22c6bfe Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Sat, 23 May 2020 03:12:59 +0000 Subject: [PATCH 4/6] Improve coverage --- trio/_core/tests/test_local.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/trio/_core/tests/test_local.py b/trio/_core/tests/test_local.py index f47e88132d..83db5a5cbe 100644 --- a/trio/_core/tests/test_local.py +++ b/trio/_core/tests/test_local.py @@ -119,6 +119,9 @@ async def get_token(): async def test_scopevar(): sv1 = _core.ScopeVar("sv1") sv2 = _core.ScopeVar("sv2", default=None) + assert sv1.name == "sv1" + assert "ScopeVar name='sv2'" in repr(sv2) + with pytest.raises(LookupError): sv1.get() assert sv2.get() is None From c1e27d06fbf4970f5b4c2453f52066f6c9225ee0 Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Mon, 1 Jun 2020 07:39:35 +0000 Subject: [PATCH 5/6] Rename to TreeVar, move out of lowlevel, substantially improve docs --- docs/source/reference-core.rst | 99 +++++++++++++++++++++- docs/source/reference-lowlevel.rst | 14 ---- newsfragments/1523.feature.rst | 15 ++-- trio/__init__.py | 1 + trio/_core/__init__.py | 2 +- trio/_core/_local.py | 103 +++++++++++------------ trio/_core/_run.py | 26 +++--- trio/_core/tests/test_local.py | 128 ++++++++++++++++++----------- trio/lowlevel.py | 1 - 9 files changed, 252 insertions(+), 137 deletions(-) diff --git a/docs/source/reference-core.rst b/docs/source/reference-core.rst index b82faa7418..1ff5bde8fe 100644 --- a/docs/source/reference-core.rst +++ b/docs/source/reference-core.rst @@ -679,6 +679,13 @@ nothing at all:: with move_on_after(TIMEOUT): # don't do this! nursery.start_soon(child) +This is because a cancel scope can only cancel code that runs entirely +within the body of its ``with`` block. Once you exit the ``with`` +block, no further cancellation of that scope can have any effect, +because there's nowhere to catch the `~trio.Cancelled` exception. The +only way to keep the ``with`` block open as long as the child task runs +is to put it outside the child task's parent nursery. + Errors in multiple child tasks ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -930,8 +937,8 @@ Will properly log the inner exceptions: .. _task-local-storage: -Task-local storage ------------------- +Context variables support task-local storage +-------------------------------------------- Suppose you're writing a server that responds to network requests, and you log some information about each request as you process it. If the @@ -1017,6 +1024,94 @@ For more information, read the `contextvar docs `__. +More on context variable inheritance +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +As you can see from the example above, a task's context variables are +automatically inherited by any other tasks it starts. To be precise, +each new task gets a *shallow copy* of the context in the task that +spawned it. That means: + +* If the new task changes one of its context variables using `ContextVar.set() + `, then that change is not visible in the + task that started it. (This would hardly be "task-local" storage otherwise!) + +* But if the context variable referred to a mutable object (such as a + list or dictionary), and the new task makes a change to that object + (such as by calling ``some_contextvar.get().append(42)``), then that + change *is* visible in the task that started it, as well as in any other + tasks it started. Since that's rather confusing, it's often best to + limit yourself to immutable values (strings, integers, tuples, and so + on) when working with context variables. + +A new task's context is set up as a copy of the context that existed +at the time of the call to :meth:`~Nursery.start_soon` or +:meth:`~Nursery.start` that created it. For example, this code: + +.. code-block:: python3 + + some_cvar = contextvars.ContextVar() + + async def print_in_child(tag): + print("In child", tag, "some_cvar has value", some_cvar.get()) + + some_cvar.set(1) + async with trio.open_nursery() as nursery: + nursery.start_soon(print_in_child, 1) + some_cvar.set(2) + nursery.start_soon(print_in_child, 2) + some_cvar.set(3) + print("In parent some_cvar has value", some_cvar.get()) + +will produce output like:: + + In parent some_cvar has value 3 + In child 1 some_cvar has value 1 + In child 2 some_cvar has value 2 + +(If you run it yourself, you might find that the "child 2" line comes +before "child 1", but it will still be the case that child 1 sees value 1 +while child 2 sees value 2.) + +You might wonder why this differs from the behavior of cancel scopes, +which only apply to a new task if they surround the new task's entire +nursery (as explained above in the section on +:ref:`child-tasks-and-cancellation`). The difference is that a cancel +scope has a limited lifetime (it can't cancel anything once you exit +its ``with`` block), while a context variable's value is just a value +(request #42 can keep being request #42 for as long as it likes, +without any cooperation from the task that created it). + +In specialized cases, you might want to provide a task-local value +that's inherited only from the parent nursery, like cancel scopes are. +(For example, maybe you're trying to provide child tasks with access +to a limited-lifetime resource such as a nursery or network +connection, and you only want a task to be able to use the resource if +it's going to remain available for the task's entire lifetime.) Trio +supports this using `TreeVar`, which is like `contextvars.ContextVar` +except for the way that it's inherited by new tasks. (It's a "tree" +variable because it's inherited along the parent-child links that +form the Trio task tree.) + +If the above example used `TreeVar`, then its output would be: + +.. code-block:: none + :emphasize-lines: 3 + + In parent some_cvar has value 3 + In child 1 some_cvar has value 1 + In child 2 some_cvar has value 1 + +because child 2 would inherit the value from its parent nursery, rather than +from the environment of the ``start_soon()`` call that creates it. + +.. autoclass:: trio.TreeVar(name, [*, default]) + + .. automethod:: being + :with: + .. automethod:: get_in(task_or_nursery, [default]) + + .. _synchronization: Synchronizing and communicating between tasks diff --git a/docs/source/reference-lowlevel.rst b/docs/source/reference-lowlevel.rst index f2648e94a4..cf9e29b1b6 100644 --- a/docs/source/reference-lowlevel.rst +++ b/docs/source/reference-lowlevel.rst @@ -265,20 +265,6 @@ Global state: system tasks and run-local variables .. autofunction:: spawn_system_task -Scope variables -=============== - -.. autoclass:: ScopeVar(name, [*, default]) - - .. autoattribute:: name - .. automethod:: get([default]) - .. automethod:: set - .. automethod:: reset - .. automethod:: being - :with: - .. automethod:: get_in(task_or_nursery, [default]) - - Trio tokens =========== diff --git a/newsfragments/1523.feature.rst b/newsfragments/1523.feature.rst index 21b0fd1e97..bc11952474 100644 --- a/newsfragments/1523.feature.rst +++ b/newsfragments/1523.feature.rst @@ -1,7 +1,8 @@ -Added the concept of a "scope variable" (`trio.lowlevel.ScopeVar`), which is -like a context variable except that its value in a new task is determined -differently. A context variable is inherited by a new task based on its value -at the location where the task was spawned, while a scope variable is inherited -based on its value where the task's parent nursery was opened. This distinction -makes scope variables useful for anything that's naturally inherited along -parent/child task relationships. +Added the concept of a "tree variable" (`trio.TreeVar`), which is like +a context variable except that its value in a new task is inherited +from the new task's parent nursery rather than from the task that +spawned it. (The `~trio.TreeVar` behavior matches the existing +:ref:`behavior of cancel scopes `.) +This distinction makes tree variables useful for anything that's +naturally inherited along parent/child task relationships, such as a +reference to a resource that has a limited lifetime. diff --git a/trio/__init__.py b/trio/__init__.py index 5339d107eb..a537e1d19e 100644 --- a/trio/__init__.py +++ b/trio/__init__.py @@ -32,6 +32,7 @@ BrokenResourceError, EndOfChannel, Nursery, + TreeVar, ) from ._timeouts import ( diff --git a/trio/_core/__init__.py b/trio/_core/__init__.py index 2717585517..50eefaaf2f 100644 --- a/trio/_core/__init__.py +++ b/trio/_core/__init__.py @@ -66,7 +66,7 @@ from ._unbounded_queue import UnboundedQueue -from ._local import RunVar, ScopeVar +from ._local import RunVar, TreeVar from ._thread_cache import start_thread_soon diff --git a/trio/_core/_local.py b/trio/_core/_local.py index 63e9f37ea0..f32ddeb029 100644 --- a/trio/_core/_local.py +++ b/trio/_core/_local.py @@ -102,41 +102,30 @@ def __repr__(self): return "".format(self._name) -class ScopeVar(metaclass=Final): - """A "scope variable": like a context variable except that its value - is inherited by new tasks in a different way. - - In simple terms, `ScopeVar` lets you define your own custom state - that's inherited in the same way that :ref:`cancel scopes are - `. Ordinary context variables are - inherited by new tasks based on the environment surrounding the - :meth:`~trio.Nursery.start_soon` call that created the task. By - contrast, scope variables are inherited based on the environment - surrounding the nursery into which the new task was spawned. This - difference makes scope variables a better fit than context - variables for state that naturally propagates down the task tree. - - Some example uses of a `ScopeVar`: - - * Provide access to a resource that is only usable in a certain - scope. This might be a nursery, network connection, or anything - else whose lifetime is bound to a context manager. Tasks that - run entirely within the resource's lifetime can use it; tasks - that might keep running past the resource being destroyed won't see - it at all. - - * Constrain a function's caller-visible behavior, such as what exceptions - it might throw. The `ScopeVar`\\'s value will be inherited by every - task whose exceptions might propagate to the point where the value was - set. - - `ScopeVar` objects support all the same methods as `~contextvars.ContextVar` - objects, plus the additional methods :meth:`being` and :meth:`get_in`. - - .. note:: `ScopeVar` values are not directly stored in the +class TreeVar(metaclass=Final): + """A "tree variable": like a context variable except that its value + in a new task is inherited from the new task's parent nursery rather + than from the new task's spawner. + + `TreeVar` objects support all the same methods and attributes as + `~contextvars.ContextVar` objects + (:meth:`~contextvars.ContextVar.get`, + :meth:`~contextvars.ContextVar.set`, + :meth:`~contextvars.ContextVar.reset`, and + `~contextvars.ContextVar.name`), and they are constructed the same + way. They also provide the additional methods :meth:`being` and + :meth:`get_in`, documented below. + + Accessing or changing the value of a `TreeVar` outside of a Trio + task will raise `RuntimeError`. (Exception: :meth:`get_in` still + works outside of a task, as long as you have a reference to the + task or nursery of interest.) + + .. note:: `TreeVar` values are not directly stored in the `contextvars.Context`, so you can't use `Context.get() - ` to access them; use :meth:`get_in` - instead, if you need the value in a context other than your own. + ` to access them. If you need the value + in a context other than your own, use :meth:`get_in`. + """ __slots__ = ("_cvar",) @@ -150,19 +139,19 @@ def name(self): return self._cvar.name def get(self, default=_NO_DEFAULT): - """Gets the value of this `ScopeVar` for the current task. + """Gets the value of this `TreeVar` for the current task. - If this `ScopeVar` has no value in the current task, then + If this `TreeVar` has no value in the current task, then :meth:`get` returns the *default* specified as argument to :meth:`get`, or else the *default* specified when constructing - the `ScopeVar`, or else raises `LookupError`. See the + the `TreeVar`, or else raises `LookupError`. See the documentation of :meth:`contextvars.ContextVar.get` for more details. """ # This is effectively an inlining for efficiency of: - # return _run.current_task()._scope_context.run(self._cvar.get, default) + # return _run.current_task()._tree_context.run(self._cvar.get, default) try: - return _run.GLOBAL_RUN_CONTEXT.task._scope_context[self._cvar] + return _run.GLOBAL_RUN_CONTEXT.task._tree_context[self._cvar] except AttributeError: raise RuntimeError("must be called from async context") from None except KeyError: @@ -175,7 +164,7 @@ def get(self, default=_NO_DEFAULT): return self._cvar.get(default) def set(self, value): - """Sets the value of this `ScopeVar` for the current task. The new + """Sets the value of this `TreeVar` for the current task. The new value will be inherited by nurseries that are later opened in this task, so that new tasks can inherit whatever value was set when their parent nursery was created. @@ -183,22 +172,22 @@ def set(self, value): Returns a token which may be passed to :meth:`reset` to restore the previous value. """ - return _run.current_task()._scope_context.run(self._cvar.set, value) + return _run.current_task()._tree_context.run(self._cvar.set, value) def reset(self, token): - """Resets the value of this `ScopeVar` to the value it had + """Resets the value of this `TreeVar` to the value it had before the call to :meth:`set` that returned the given *token*. The *token* must have been obtained from a call to :meth:`set` on - this same `ScopeVar` and in the same task that is now calling + this same `TreeVar` and in the same task that is now calling :meth:`reset`. Also, each *token* may only be used in one call to :meth:`reset`. Violating these conditions will raise `ValueError`. """ - _run.current_task()._scope_context.run(self._cvar.reset, token) + _run.current_task()._tree_context.run(self._cvar.reset, token) @contextmanager def being(self, value): - """Returns a context manager which sets the value of this `ScopeVar` to + """Returns a context manager which sets the value of this `TreeVar` to *value* upon entry and restores its previous value upon exit. """ token = self.set(value) @@ -208,17 +197,21 @@ def being(self, value): self.reset(token) def get_in(self, task_or_nursery, default=_NO_DEFAULT): - """Gets the value of this :class:`ScopeVar` for the given task or nursery. - - The value in a task is the value that would be returned by a call to - :meth:`get` in that task. The value in a nursery is the value that would - be returned by :meth:`get` at the beginning of a new child task started - in that nursery. The *default* argument has the same semantics as it does - for :meth:`get`. + """Gets the value of this `TreeVar` in the given + `~trio.lowlevel.Task` or `~trio.Nursery`. + + The value in a task is the value that would be returned by a + call to :meth:`~contextvars.ContextVar.get` in that task. The + value in a nursery is the value that would be returned by + :meth:`~contextvars.ContextVar.get` at the beginning of a new + child task started in that nursery. The *default* argument has + the same semantics as it does for :meth:`~contextvars.ContextVar.get`. """ - + # copy() so this works from a different thread too. It's a + # cheap and thread-safe operation (just copying one reference) + # since the underlying context data is immutable. defarg = () if default is _NO_DEFAULT else (default,) - return task_or_nursery._scope_context.run(self._cvar.get, *defarg) + return task_or_nursery._tree_context.copy().run(self._cvar.get, *defarg) def __repr__(self): - return f"" + return f"" diff --git a/trio/_core/_run.py b/trio/_core/_run.py index e5ab371134..54e27ff326 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -774,7 +774,7 @@ class Nursery(metaclass=NoPublicConstructor): def __init__(self, parent_task, cancel_scope): self._parent_task = parent_task - self._scope_context = parent_task._scope_context.copy() + self._tree_context = parent_task._tree_context.copy() parent_task._child_nurseries.append(self) # the cancel status that children inherit - we take a snapshot, so it # won't be affected by any changes in the parent. @@ -946,10 +946,9 @@ async def async_fn(arg1, arg2, \*, task_status=trio.TASK_STATUS_IGNORED): async with open_nursery() as old_nursery: task_status = _TaskStatus(old_nursery, self) thunk = functools.partial(async_fn, task_status=task_status) - task = GLOBAL_RUN_CONTEXT.runner.spawn_impl( - thunk, args, old_nursery, name + GLOBAL_RUN_CONTEXT.runner.spawn_impl( + thunk, args, old_nursery, name, eventual_nursery=self, ) - task._eventual_parent_nursery = self # Wait for either _TaskStatus.started or an exception to # cancel this nursery: # If we get here, then the child either got reparented or exited @@ -982,7 +981,7 @@ class Task(metaclass=NoPublicConstructor): _counter = attr.ib(init=False, factory=itertools.count().__next__) # Contextvars context that contains ScopeVar values - _scope_context = attr.ib(init=False) + _tree_context = attr.ib(init=False) # Invariant: # - for unscheduled tasks, _next_send_fn and _next_send are both None @@ -1012,9 +1011,10 @@ class Task(metaclass=NoPublicConstructor): def __attrs_post_init__(self): if self._parent_nursery is None: - self._scope_context = Context() + self._tree_context = Context() else: - self._scope_context = self._parent_nursery._scope_context.copy() + parent = self._eventual_parent_nursery or self._parent_nursery + self._tree_context = parent._tree_context.copy() def __repr__(self): return "".format(self.name, id(self)) @@ -1255,8 +1255,9 @@ def reschedule(self, task, next_send=_NO_SEND): if self.instruments: self.instrument("task_scheduled", task) - def spawn_impl(self, async_fn, args, nursery, name, *, system_task=False): - + def spawn_impl( + self, async_fn, args, nursery, name, *, eventual_nursery=None, system_task=False + ): ###### # Make sure the nursery is in working order ###### @@ -1302,7 +1303,12 @@ async def python_wrapper(orig_coro): # Set up the Task object ###### task = Task._create( - coro=coro, parent_nursery=nursery, runner=self, name=name, context=context, + coro=coro, + parent_nursery=nursery, + runner=self, + name=name, + context=context, + eventual_parent_nursery=eventual_nursery, ) self.tasks.add(task) diff --git a/trio/_core/tests/test_local.py b/trio/_core/tests/test_local.py index 83db5a5cbe..c097730ce2 100644 --- a/trio/_core/tests/test_local.py +++ b/trio/_core/tests/test_local.py @@ -116,91 +116,125 @@ async def get_token(): t1.reset(token) -async def test_scopevar(): - sv1 = _core.ScopeVar("sv1") - sv2 = _core.ScopeVar("sv2", default=None) - assert sv1.name == "sv1" - assert "ScopeVar name='sv2'" in repr(sv2) +async def test_treevar(): + tv1 = _core.TreeVar("tv1") + tv2 = _core.TreeVar("tv2", default=None) + assert tv1.name == "tv1" + assert "TreeVar name='tv2'" in repr(tv2) with pytest.raises(LookupError): - sv1.get() - assert sv2.get() is None - assert sv1.get(42) == 42 - assert sv2.get(42) == 42 + tv1.get() + assert tv2.get() is None + assert tv1.get(42) == 42 + assert tv2.get(42) == 42 NOTHING = object() async def should_be(val1, val2, new1=NOTHING): - assert sv1.get(NOTHING) == val1 - assert sv2.get(NOTHING) == val2 + assert tv1.get(NOTHING) == val1 + assert tv2.get(NOTHING) == val2 if new1 is not NOTHING: - sv1.set(new1) + tv1.set(new1) - tok1 = sv1.set(10) + tok1 = tv1.set(10) async with _core.open_nursery() as outer: - tok2 = sv1.set(15) - with sv2.being(20): - assert sv2.get_in(_core.current_task()) == 20 + tok2 = tv1.set(15) + with tv2.being(20): + assert tv2.get_in(_core.current_task()) == 20 async with _core.open_nursery() as inner: - sv1.reset(tok2) + tv1.reset(tok2) outer.start_soon(should_be, 10, NOTHING, 100) inner.start_soon(should_be, 15, 20, 200) await _core.wait_all_tasks_blocked() - assert sv1.get_in(_core.current_task()) == 10 + assert tv1.get_in(_core.current_task()) == 10 await should_be(10, 20, 300) - assert sv1.get_in(inner) == 15 - assert sv1.get_in(outer) == 10 - assert sv1.get_in(_core.current_task()) == 300 - assert sv2.get_in(inner) == 20 - assert sv2.get_in(outer) is None - assert sv2.get_in(_core.current_task()) == 20 - sv1.reset(tok1) + assert tv1.get_in(inner) == 15 + assert tv1.get_in(outer) == 10 + assert tv1.get_in(_core.current_task()) == 300 + assert tv2.get_in(inner) == 20 + assert tv2.get_in(outer) is None + assert tv2.get_in(_core.current_task()) == 20 + tv1.reset(tok1) await should_be(NOTHING, 20) - assert sv1.get_in(inner) == 15 - assert sv1.get_in(outer) == 10 + assert tv1.get_in(inner) == 15 + assert tv1.get_in(outer) == 10 with pytest.raises(LookupError): - assert sv1.get_in(_core.current_task()) - assert sv2.get() is None - assert sv2.get_in(_core.current_task()) is None - - -async def test_scopevar_token_bound_to_task_that_obtained_it(): - sv1 = _core.ScopeVar("sv1") + assert tv1.get_in(_core.current_task()) + assert tv2.get() is None + assert tv2.get_in(_core.current_task()) is None + + +async def test_treevar_follows_eventual_parent(): + tv1 = _core.TreeVar("tv1") + + async def manage_target(task_status): + assert tv1.get() == "source nursery" + with tv1.being("target nursery"): + assert tv1.get() == "target nursery" + async with _core.open_nursery() as target_nursery: + with tv1.being("target nested child"): + assert tv1.get() == "target nested child" + task_status.started(target_nursery) + await _core.wait_task_rescheduled(lambda _: _core.Abort.SUCCEEDED) + assert tv1.get() == "target nested child" + assert tv1.get() == "target nursery" + assert tv1.get() == "target nursery" + assert tv1.get() == "source nursery" + + async def verify(value, *, task_status=_core.TASK_STATUS_IGNORED): + assert tv1.get() == value + task_status.started() + assert tv1.get() == value + + with tv1.being("source nursery"): + async with _core.open_nursery() as source_nursery: + with tv1.being("source->target start call"): + target_nursery = await source_nursery.start(manage_target) + with tv1.being("verify task"): + source_nursery.start_soon(verify, "source nursery") + target_nursery.start_soon(verify, "target nursery") + await source_nursery.start(verify, "source nursery") + await target_nursery.start(verify, "target nursery") + _core.reschedule(target_nursery.parent_task) + + +async def test_treevar_token_bound_to_task_that_obtained_it(): + tv1 = _core.TreeVar("tv1") token = None async def get_token(): nonlocal token - token = sv1.set(10) + token = tv1.set(10) try: await _core.wait_task_rescheduled(lambda _: _core.Abort.SUCCEEDED) finally: - sv1.reset(token) + tv1.reset(token) with pytest.raises(LookupError): - sv1.get() + tv1.get() with pytest.raises(LookupError): - sv1.get_in(_core.current_task()) + tv1.get_in(_core.current_task()) async with _core.open_nursery() as nursery: nursery.start_soon(get_token) await _core.wait_all_tasks_blocked() assert token is not None with pytest.raises(ValueError, match="different Context"): - sv1.reset(token) - assert sv1.get_in(list(nursery.child_tasks)[0]) == 10 + tv1.reset(token) + assert tv1.get_in(list(nursery.child_tasks)[0]) == 10 nursery.cancel_scope.cancel() -def test_scopevar_outside_run(): +def test_treevar_outside_run(): async def run_sync(fn, *args): return fn(*args) - sv1 = _core.ScopeVar("sv1", default=10) + tv1 = _core.TreeVar("tv1", default=10) for operation in ( - sv1.get, - partial(sv1.get, 20), - partial(sv1.set, 30), - lambda: sv1.reset(_core.run(run_sync, sv1.set, 10)), - sv1.being(40).__enter__, + tv1.get, + partial(tv1.get, 20), + partial(tv1.set, 30), + lambda: tv1.reset(_core.run(run_sync, tv1.set, 10)), + tv1.being(40).__enter__, ): with pytest.raises(RuntimeError, match="must be called from async context"): operation() diff --git a/trio/lowlevel.py b/trio/lowlevel.py index ba425d5d0e..3ce3e741ba 100644 --- a/trio/lowlevel.py +++ b/trio/lowlevel.py @@ -25,7 +25,6 @@ ParkingLot, UnboundedQueue, RunVar, - ScopeVar, TrioToken, current_trio_token, temporarily_detach_coroutine_object, From 2f81d47e9aaa116c073e3b6628b23eb329279a90 Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Mon, 1 Jun 2020 07:55:44 +0000 Subject: [PATCH 6/6] Fix coverage --- trio/_core/tests/test_local.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/trio/_core/tests/test_local.py b/trio/_core/tests/test_local.py index c097730ce2..b43f8406fe 100644 --- a/trio/_core/tests/test_local.py +++ b/trio/_core/tests/test_local.py @@ -167,6 +167,9 @@ async def should_be(val1, val2, new1=NOTHING): async def test_treevar_follows_eventual_parent(): tv1 = _core.TreeVar("tv1") + def trivial_abort(_): + return _core.Abort.SUCCEEDED # pragma: no cover + async def manage_target(task_status): assert tv1.get() == "source nursery" with tv1.being("target nursery"): @@ -175,7 +178,7 @@ async def manage_target(task_status): with tv1.being("target nested child"): assert tv1.get() == "target nested child" task_status.started(target_nursery) - await _core.wait_task_rescheduled(lambda _: _core.Abort.SUCCEEDED) + await _core.wait_task_rescheduled(trivial_abort) assert tv1.get() == "target nested child" assert tv1.get() == "target nursery" assert tv1.get() == "target nursery"