feat: Reference schema support (#800) (#1307)

This PR adds support for reference schemas
(#800).

The idea with this change is that during parsing of the schemas, when
the parser encounters a `Reference` schema, rather than aborting it will
attempt to fully resolve that reference by following the chain of
references until it encounters a `Schema`. When it does, it will attempt
to fetch that parsed `Schema` from the dictionary of available
components, which should include all possible referent schemas. If it
exists, it will parse the current/original reference Schema as if it was
its ultimate referent. If it reaches a reference that does not exist, it
logs an error and moves on: it won't be able to parse this `Reference`.

The result of this is that every reference schema in the API doc will
result in a separate-but-identical class, aside from the class name.
This feels like a safe, basic way to handle the situation, since
collapsing identical classes might break compatibility between the API
and the codegen at some point. I suppose we could instead have
references be subclasses or some trick like that to cut down on code
duplication and let identical API objects be equivalent? If that would
be preferable it is probably doable downstream of this change, since
with this parsing the `Schema` objects added as references in the
`Schemas` class should be identical objects which could be identified
and collapsed in some fashion during code templating. Perhaps it would
be useful to record the direction of references during parsing to better
choose what a base class would be... That would require some more
thought.

In any case, this works both for single references as well as
arbitrarily deep nested references, something useful for the API I'm
developing for.

Also includes some functional tests which include both a valid and
invalid (circular reference) case for reference schemas and adjusts an
existing unit test which assumed that the reference schema would be
skipped to one that tests whether it has been parsed.

Author: goodsonjr

end_to_end_tests/functional_tests/generated_code_execution/test_properties.py

@@ -12,7 +12,7 @@
 
 
 @with_generated_client_fixture(
-"""
+    """
 components:
   schemas:
     MyModel:
@@ -29,7 +29,8 @@
           properties:
             req3: {"type": "string"}
           required: ["req3"]
-""")
+"""
+)
 @with_generated_code_imports(
     ".models.MyModel",
     ".models.DerivedModel",
@@ -74,7 +75,7 @@ def test_type_hints(self, MyModel, Unset):
 
 
 @with_generated_client_fixture(
-"""
+    """
 components:
   schemas:
     MyModel:
@@ -89,7 +90,8 @@ def test_type_hints(self, MyModel, Unset):
         anyProp: {}
     AnyObject:
         type: object
-""")
+"""
+)
 @with_generated_code_imports(
     ".models.MyModel",
     ".models.AnyObject",
@@ -104,7 +106,7 @@ def test_decode_encode(self, MyModel, AnyObject):
             "intProp": 2,
             "anyObjectProp": {"d": 3},
             "nullProp": None,
-            "anyProp": "e"
+            "anyProp": "e",
         }
         expected_any_object = AnyObject()
         expected_any_object.additional_properties = {"d": 3}
@@ -116,10 +118,10 @@ def test_decode_encode(self, MyModel, AnyObject):
                 string_prop="a",
                 number_prop=1.5,
                 int_prop=2,
-                any_object_prop = expected_any_object,
+                any_object_prop=expected_any_object,
                 null_prop=None,
                 any_prop="e",
-            )
+            ),
         )
 
     @pytest.mark.parametrize(
@@ -144,7 +146,7 @@ def test_type_hints(self, MyModel, Unset):
 
 
 @with_generated_client_fixture(
-"""
+    """
 components:
   schemas:
     MyModel:
@@ -154,7 +156,8 @@ def test_type_hints(self, MyModel, Unset):
         dateTimeProp: {"type": "string", "format": "date-time"}
         uuidProp: {"type": "string", "format": "uuid"}
         unknownFormatProp: {"type": "string", "format": "weird"}
-""")
+"""
+)
 @with_generated_code_imports(
     ".models.MyModel",
     ".types.Unset",
@@ -184,3 +187,59 @@ def test_type_hints(self, MyModel, Unset):
         assert_model_property_type_hint(MyModel, "date_time_prop", Union[datetime.datetime, Unset])
         assert_model_property_type_hint(MyModel, "uuid_prop", Union[uuid.UUID, Unset])
         assert_model_property_type_hint(MyModel, "unknown_format_prop", Union[str, Unset])
+
+
+@with_generated_client_fixture(
+    """
+components:
+  schemas:
+    MyModel:
+      type: object
+      properties:
+        booleanProp: {"type": "boolean"}
+        stringProp: {"type": "string"}
+        numberProp: {"type": "number"}
+        intProp: {"type": "integer"}
+        anyObjectProp: {"$ref": "#/components/schemas/AnyObject"}
+        nullProp: {"type": "null"}
+        anyProp: {}
+    AnyObject:
+      $ref: "#/components/schemas/OtherObject"
+    OtherObject:
+      $ref: "#/components/schemas/AnotherObject"
+    AnotherObject:
+      type: object
+      properties:
+        booleanProp: {"type": "boolean"}
+    
+"""
+)
+@with_generated_code_imports(
+    ".models.MyModel",
+    ".models.AnyObject",
+    ".types.Unset",
+)
+class TestReferenceSchemaProperties:
+    def test_decode_encode(self, MyModel, AnyObject):
+        json_data = {
+            "booleanProp": True,
+            "stringProp": "a",
+            "numberProp": 1.5,
+            "intProp": 2,
+            "anyObjectProp": {"booleanProp": False},
+            "nullProp": None,
+            "anyProp": "e",
+        }
+        assert_model_decode_encode(
+            MyModel,
+            json_data,
+            MyModel(
+                boolean_prop=True,
+                string_prop="a",
+                number_prop=1.5,
+                int_prop=2,
+                any_object_prop=AnyObject(boolean_prop=False),
+                null_prop=None,
+                any_prop="e",
+            ),
+        )

end_to_end_tests/functional_tests/generator_failure_cases/test_invalid_references.py

@@ -0,0 +1,29 @@
+import pytest
+
+from end_to_end_tests.functional_tests.helpers import assert_bad_schema, with_generated_client_fixture
+
+
+@with_generated_client_fixture(
+    """
+components:
+  schemas:
+    MyModel:
+      type: object
+      properties:
+        booleanProp: {"type": "boolean"}
+        stringProp: {"type": "string"}
+        numberProp: {"type": "number"}
+        intProp: {"type": "integer"}
+        anyObjectProp: {"$ref": "#/components/schemas/AnyObject"}
+        nullProp: {"type": "null"}
+        anyProp: {}
+    AnyObject:
+      $ref: "#/components/schemas/OtherObject"
+    OtherObject:
+      $ref: "#/components/schemas/AnyObject"
+    
+"""
+)
+class TestReferenceSchemaProperties:
+    def test_decode_encode(self, generated_client):
+        assert "Circular schema references found" in generated_client.generator_result.stdout

openapi_python_client/parser/properties/__init__.py

@@ -40,6 +40,7 @@
     Parameters,
     ReferencePath,
     Schemas,
+    get_reference_simple_name,
     parse_reference_path,
     update_parameters_with_data,
     update_schemas_with_data,
@@ -324,17 +325,30 @@ def _create_schemas(
     while still_making_progress:
         still_making_progress = False
         errors = []
-        next_round = []
+        next_round: list[tuple[str, oai.Reference | oai.Schema]] = []
         # Only accumulate errors from the last round, since we might fix some along the way
         for name, data in to_process:
-            if isinstance(data, oai.Reference):
-                schemas.errors.append(PropertyError(data=data, detail="Reference schemas are not supported."))
-                continue
+            schema_data: oai.Reference | oai.Schema | None = data
             ref_path = parse_reference_path(f"#/components/schemas/{name}")
             if isinstance(ref_path, ParseError):
                 schemas.errors.append(PropertyError(detail=ref_path.detail, data=data))
                 continue
-            schemas_or_err = update_schemas_with_data(ref_path=ref_path, data=data, schemas=schemas, config=config)
+            if isinstance(data, oai.Reference):
+                # Fully dereference reference schemas
+                seen = [name]
+                while isinstance(schema_data, oai.Reference):
+                    data_ref_schema = get_reference_simple_name(schema_data.ref)
+                    if data_ref_schema in seen:
+                        schemas.errors.append(PropertyError(detail="Circular schema references found", data=data))
+                        break
+                    # use derefenced schema definition for this schema
+                    schema_data = components.get(data_ref_schema)
+            if isinstance(schema_data, oai.Schema):
+                schemas_or_err = update_schemas_with_data(
+                    ref_path=ref_path, data=schema_data, schemas=schemas, config=config
+                )
+            else:
+                schemas.errors.append(PropertyError(detail="Referent schema not found", data=data))
             if isinstance(schemas_or_err, PropertyError):
                 next_round.append((name, data))
                 errors.append(schemas_or_err)

tests/test_parser/test_properties/test_init.py

@@ -235,22 +235,22 @@ def test__string_based_property_binary_format(self, file_property_factory, confi
 
 
 class TestCreateSchemas:
-    def test_skips_references_and_keeps_going(self, mocker, config):
-        components = {"a_ref": Reference.model_construct(), "a_schema": Schema.model_construct()}
+    def test_dereference_references(self, mocker, config):
+        components = {"a_ref": Reference(ref="#/components/schemas/a_schema"), "a_schema": Schema.model_construct()}
         update_schemas_with_data = mocker.patch(f"{MODULE_NAME}.update_schemas_with_data")
         parse_reference_path = mocker.patch(f"{MODULE_NAME}.parse_reference_path")
         schemas = Schemas()
 
         result = _create_schemas(components=components, schemas=schemas, config=config)
-        # Should not even try to parse a path for the Reference
-        parse_reference_path.assert_called_once_with("#/components/schemas/a_schema")
-        update_schemas_with_data.assert_called_once_with(
+
+        parse_reference_path.assert_has_calls(
+            [call("#/components/schemas/a_ref"), call("#/components/schemas/a_schema")]
+        )
+        update_schemas_with_data.assert_called_with(
             ref_path=parse_reference_path.return_value,
             config=config,
             data=components["a_schema"],
-            schemas=Schemas(
-                errors=[PropertyError(detail="Reference schemas are not supported.", data=components["a_ref"])]
-            ),
+            schemas=result,
         )
         assert result == update_schemas_with_data.return_value