[mypyc] Generate __getattr__ wrapper (#19909)

Fixes mypyc/mypyc#887

Generate a wrapper function for `__getattr__` in classes where it's
defined and put it into the `tp_getattro` slot.

At runtime, this wrapper function is called for every attribute access,
so to match behavior of interpreted python it needs to first check if
the given name is present in the type dictionary and only call the
user-defined `__getattr__` when it's not present.

Checking the type dictionary is implemented using a cpython function
`_PyObject_GenericGetAttrWithDict` which is also used in the default
attribute access handler in
[cpython](https://github.com/python/cpython/blob/dd45179fa0f5ad2fd169cdd35065df2c3bce85bc/Objects/typeobject.c#L10676).

In compiled code, the wrapper will only be called when the attribute
name cannot be statically resolved. When it can be resolved, the
attribute will be accessed directly in the underlying C struct, or the
generated function will be directly called in case of resolving method
names. No change from existing behavior.

When the name cannot be statically resolved, mypyc generates calls to
`PyObject_GetAttr` which internally calls `tp_getattro`.

In interpreted code that uses compiled classes, attribute access will
always result in calls to `PyObject_GetAttr` so there's always a dict
look-up in the wrapper to find the attribute. But that's the case
already, the dict look-up happens in the default attribute access
handler in cpython. So the wrapper should not bring any negative
performance impact.

Author: p-sawicki

mypyc/codegen/emitclass.py

@@ -39,6 +39,12 @@ def native_slot(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str:
     return f"{NATIVE_PREFIX}{fn.cname(emitter.names)}"
 
 
+def dunder_attr_slot(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str:
+    wrapper_fn = cl.get_method(fn.name + "__wrapper")
+    assert wrapper_fn
+    return f"{NATIVE_PREFIX}{wrapper_fn.cname(emitter.names)}"
+
+
 # We maintain a table from dunder function names to struct slots they
 # correspond to and functions that generate a wrapper (if necessary)
 # and return the function name to stick in the slot.
@@ -55,6 +61,7 @@ def native_slot(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str:
     "__iter__": ("tp_iter", native_slot),
     "__hash__": ("tp_hash", generate_hash_wrapper),
     "__get__": ("tp_descr_get", generate_get_wrapper),
+    "__getattr__": ("tp_getattro", dunder_attr_slot),
 }
 
 AS_MAPPING_SLOT_DEFS: SlotTable = {

mypyc/ir/ops.py

@@ -1221,6 +1221,7 @@ def __init__(
         var_arg_idx: int = -1,
         *,
         is_pure: bool = False,
+        returns_null: bool = False,
     ) -> None:
         self.error_kind = error_kind
         super().__init__(line)
@@ -1235,7 +1236,10 @@ def __init__(
         # and all the arguments are immutable. Pure functions support
         # additional optimizations. Pure functions never fail.
         self.is_pure = is_pure
-        if is_pure:
+        # The function might return a null value that does not indicate
+        # an error.
+        self.returns_null = returns_null
+        if is_pure or returns_null:
             assert error_kind == ERR_NEVER
 
     def sources(self) -> list[Value]:

mypyc/irbuild/builder.py

@@ -1241,6 +1241,7 @@ def enter_method(
         ret_type: RType,
         fn_info: FuncInfo | str = "",
         self_type: RType | None = None,
+        internal: bool = False,
     ) -> Iterator[None]:
         """Generate IR for a method.
 
@@ -1268,7 +1269,7 @@ def enter_method(
             sig = FuncSignature(args, ret_type)
             name = self.function_name_stack.pop()
             class_ir = self.class_ir_stack.pop()
-            decl = FuncDecl(name, class_ir.name, self.module_name, sig)
+            decl = FuncDecl(name, class_ir.name, self.module_name, sig, internal=internal)
             ir = FuncIR(decl, arg_regs, blocks)
             class_ir.methods[name] = ir
             class_ir.method_decls[name] = ir.decl

mypyc/irbuild/function.py

@@ -42,6 +42,7 @@
 )
 from mypyc.ir.ops import (
     BasicBlock,
+    ComparisonOp,
     GetAttr,
     Integer,
     LoadAddress,
@@ -81,7 +82,7 @@
     dict_new_op,
     exact_dict_set_item_op,
 )
-from mypyc.primitives.generic_ops import py_setattr_op
+from mypyc.primitives.generic_ops import generic_getattr, py_setattr_op
 from mypyc.primitives.misc_ops import register_function
 from mypyc.primitives.registry import builtin_names
 from mypyc.sametype import is_same_method_signature, is_same_type
@@ -364,6 +365,56 @@ def gen_func_ir(
     return (func_ir, func_reg)
 
 
+def generate_getattr_wrapper(builder: IRBuilder, cdef: ClassDef, getattr: FuncDef) -> None:
+    """
+    Generate a wrapper function for __getattr__ that can be put into the tp_getattro slot.
+    The wrapper takes one argument besides self which is the attribute name.
+    It first checks if the name matches any of the attributes of this class.
+    If it does, it returns that attribute. If none match, it calls __getattr__.
+
+    __getattr__ is not supported in classes that allow interpreted subclasses because the
+    tp_getattro slot is inherited by subclasses and if the subclass overrides __getattr__,
+    the override would be ignored in our wrapper. TODO: To support this, the wrapper would
+    have to check type of self and if it's not the compiled class, resolve "__getattr__" against
+    the type at runtime and call the returned method, like _Py_slot_tp_getattr_hook in cpython.
+
+    __getattr__ is not supported in classes which inherit from non-native classes because those
+    have __dict__ which currently has some strange interactions when class attributes and
+    variables are assigned through __dict__ vs. through regular attribute access. Allowing
+    __getattr__ on top of that could be problematic.
+    """
+    name = getattr.name + "__wrapper"
+    ir = builder.mapper.type_to_ir[cdef.info]
+    line = getattr.line
+
+    error_base = f'"__getattr__" not supported in class "{cdef.name}" because '
+    if ir.allow_interpreted_subclasses:
+        builder.error(error_base + "it allows interpreted subclasses", line)
+    if ir.inherits_python:
+        builder.error(error_base + "it inherits from a non-native class", line)
+
+    with builder.enter_method(ir, name, object_rprimitive, internal=True):
+        attr_arg = builder.add_argument("attr", object_rprimitive)
+        generic_getattr_result = builder.call_c(generic_getattr, [builder.self(), attr_arg], line)
+
+        return_generic, call_getattr = BasicBlock(), BasicBlock()
+        null = Integer(0, object_rprimitive, line)
+        got_generic = builder.add(
+            ComparisonOp(generic_getattr_result, null, ComparisonOp.NEQ, line)
+        )
+        builder.add_bool_branch(got_generic, return_generic, call_getattr)
+
+        builder.activate_block(return_generic)
+        builder.add(Return(generic_getattr_result, line))
+
+        builder.activate_block(call_getattr)
+        # No attribute matched so call user-provided __getattr__.
+        getattr_result = builder.gen_method_call(
+            builder.self(), getattr.name, [attr_arg], object_rprimitive, line
+        )
+        builder.add(Return(getattr_result, line))
+
+
 def handle_ext_method(builder: IRBuilder, cdef: ClassDef, fdef: FuncDef) -> None:
     # Perform the function of visit_method for methods inside extension classes.
     name = fdef.name
@@ -430,6 +481,9 @@ def handle_ext_method(builder: IRBuilder, cdef: ClassDef, fdef: FuncDef) -> None
         class_ir.glue_methods[(class_ir, name)] = f
         builder.functions.append(f)
 
+    if fdef.name == "__getattr__":
+        generate_getattr_wrapper(builder, cdef, fdef)
+
 
 def handle_non_ext_method(
     builder: IRBuilder, non_ext: NonExtClassInfo, cdef: ClassDef, fdef: FuncDef

mypyc/irbuild/ll_builder.py

@@ -2048,6 +2048,7 @@ def call_c(
                 line,
                 var_arg_idx,
                 is_pure=desc.is_pure,
+                returns_null=desc.returns_null,
             )
         )
         if desc.is_borrowed:
@@ -2131,6 +2132,7 @@ def primitive_op(
                 desc.extra_int_constants,
                 desc.priority,
                 is_pure=desc.is_pure,
+                returns_null=False,
             )
             return self.call_c(c_desc, args, line, result_type=result_type)
 

mypyc/lib-rt/CPy.h

@@ -949,6 +949,10 @@ PyObject *CPy_GetANext(PyObject *aiter);
 void CPy_SetTypeAliasTypeComputeFunction(PyObject *alias, PyObject *compute_value);
 void CPyTrace_LogEvent(const char *location, const char *line, const char *op, const char *details);
 
+static inline PyObject *CPyObject_GenericGetAttr(PyObject *self, PyObject *name) {
+    return _PyObject_GenericGetAttrWithDict(self, name, NULL, 1);
+}
+
 #if CPY_3_11_FEATURES
 PyObject *CPy_GetName(PyObject *obj);
 #endif

mypyc/primitives/generic_ops.py

@@ -401,3 +401,12 @@
     c_function_name="CPy_GetName",
     error_kind=ERR_MAGIC,
 )
+
+# look-up name in tp_dict but don't raise AttributeError on failure
+generic_getattr = custom_op(
+    arg_types=[object_rprimitive, object_rprimitive],
+    return_type=object_rprimitive,
+    c_function_name="CPyObject_GenericGetAttr",
+    error_kind=ERR_NEVER,
+    returns_null=True,
+)

mypyc/primitives/registry.py

@@ -61,6 +61,7 @@ class CFunctionDescription(NamedTuple):
     extra_int_constants: list[tuple[int, RType]]
     priority: int
     is_pure: bool
+    returns_null: bool
 
 
 # A description for C load operations including LoadGlobal and LoadAddress
@@ -253,6 +254,7 @@ def custom_op(
     is_borrowed: bool = False,
     *,
     is_pure: bool = False,
+    returns_null: bool = False,
 ) -> CFunctionDescription:
     """Create a one-off CallC op that can't be automatically generated from the AST.
 
@@ -274,6 +276,7 @@ def custom_op(
         extra_int_constants,
         0,
         is_pure=is_pure,
+        returns_null=returns_null,
     )
 
 

mypyc/test-data/irbuild-classes.test

@@ -2088,3 +2088,128 @@ class NonNative:
 @mypyc_attr(free_list_len=1, allow_interpreted_subclasses=True)  # E: "free_list_len" can't be used in a class that allows interpreted subclasses
 class InterpSub:
     pass
+
+[case testUnsupportedGetAttr]
+from mypy_extensions import mypyc_attr
+
+@mypyc_attr(allow_interpreted_subclasses=True)
+class AllowsInterpreted:
+    def __getattr__(self, attr: str) -> object:  # E: "__getattr__" not supported in class "AllowsInterpreted" because it allows interpreted subclasses
+        return 0
+
+class InheritsInterpreted(dict):
+    def __getattr__(self, attr: str) -> object:  # E: "__getattr__" not supported in class "InheritsInterpreted" because it inherits from a non-native class
+        return 0
+
+@mypyc_attr(native_class=False)
+class NonNative:
+    pass
+
+class InheritsNonNative(NonNative):
+    def __getattr__(self, attr: str) -> object:  # E: "__getattr__" not supported in class "InheritsNonNative" because it inherits from a non-native class
+        return 0
+
+[case testGetAttr]
+from typing import ClassVar
+
+class GetAttr:
+    class_var = "x"
+    class_var_annotated: ClassVar[int] = 99
+
+    def __init__(self, regular_attr: int):
+        self.regular_attr = regular_attr
+
+    def __getattr__(self, attr: str) -> object:
+        return attr
+
+    def method(self) -> int:
+        return 0
+
+def test_getattr() -> list[object]:
+    i = GetAttr(42)
+    one = i.one
+    two = i.regular_attr
+    three = i.class_var
+    four = i.class_var_annotated
+    five = i.method()
+    return [one, two, three, four, five]
+
+[typing fixtures/typing-full.pyi]
+[out]
+def GetAttr.__init__(self, regular_attr):
+    self :: __main__.GetAttr
+    regular_attr :: int
+L0:
+    self.regular_attr = regular_attr
+    return 1
+def GetAttr.__getattr__(self, attr):
+    self :: __main__.GetAttr
+    attr :: str
+L0:
+    return attr
+def GetAttr.__getattr____wrapper(__mypyc_self__, attr):
+    __mypyc_self__ :: __main__.GetAttr
+    attr, r0 :: object
+    r1 :: bit
+    r2 :: str
+    r3 :: object
+L0:
+    r0 = CPyObject_GenericGetAttr(__mypyc_self__, attr)
+    r1 = r0 != 0
+    if r1 goto L1 else goto L2 :: bool
+L1:
+    return r0
+L2:
+    r2 = cast(str, attr)
+    r3 = __mypyc_self__.__getattr__(r2)
+    return r3
+def GetAttr.method(self):
+    self :: __main__.GetAttr
+L0:
+    return 0
+def GetAttr.__mypyc_defaults_setup(__mypyc_self__):
+    __mypyc_self__ :: __main__.GetAttr
+    r0 :: str
+L0:
+    r0 = 'x'
+    __mypyc_self__.class_var = r0
+    return 1
+def test_getattr():
+    r0, i :: __main__.GetAttr
+    r1 :: str
+    r2, one :: object
+    r3, two :: int
+    r4, three, r5 :: str
+    r6 :: object
+    r7, four, r8, five :: int
+    r9 :: list
+    r10, r11, r12 :: object
+    r13 :: ptr
+L0:
+    r0 = GetAttr(84)
+    i = r0
+    r1 = 'one'
+    r2 = CPyObject_GetAttr(i, r1)
+    one = r2
+    r3 = i.regular_attr
+    two = r3
+    r4 = i.class_var
+    three = r4
+    r5 = 'class_var_annotated'
+    r6 = CPyObject_GetAttr(i, r5)
+    r7 = unbox(int, r6)
+    four = r7
+    r8 = i.method()
+    five = r8
+    r9 = PyList_New(5)
+    r10 = box(int, two)
+    r11 = box(int, four)
+    r12 = box(int, five)
+    r13 = list_items r9
+    buf_init_item r13, 0, one
+    buf_init_item r13, 1, r10
+    buf_init_item r13, 2, three
+    buf_init_item r13, 3, r11
+    buf_init_item r13, 4, r12
+    keep_alive r9
+    return r9