Giters

wyfo / apischema

JSON (de)serialization, GraphQL and JSON schema generation using Python typing.

Home Page:https://wyfo.github.io/apischema/

Geek Repo:Geek Repo

Github PK Tool:Github PK Tool

Creating deserialization_schema for recursive Generic fails

thomascobb opened this issue · comments

commented

I'm trying to make a deserialization_schema for a recursive Generic type, but it is failing. If I remove the Genericness of the class Group it appears to work fine. Am I missing something, or is this a bug?

from __future__ import annotations
from dataclasses import dataclass
from typing import Generic, Sequence, TypeVar, Union
from apischema.json_schema import deserialization_schema

T = TypeVar("T")

@dataclass
class Foo:
    bar: int

@dataclass
class Group(Generic[T]):
    children: Tree[T]

Tree = Sequence[Union[T, Group[T]]]

def test_schema():
    assert deserialization_schema(Tree[Foo])

Fails with:

./tests/test_group.py::test_schema Failed: [undefined]TypeError: Recursive type typing.Sequence[typing.Union[test_group.Foo, test_group.Group[test_group.Foo]]] need a ref
Traceback (most recent call last):
  File "/dls/science/users/tmc43/common/python/pvi/tests/test_group.py", line 25, in test_schema
    assert deserialization_schema(Tree[Foo])
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/utils.py", line 424, in wrapper
    return wrapped(*args, **kwargs)
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/json_schema/schema.py", line 569, in deserialization_schema
    additional_properties,
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/json_schema/schema.py", line 517, in _schema
    refs = _extract_refs([(tp, conversion)], default_conversion, builder, all_refs)
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/json_schema/schema.py", line 476, in _extract_refs
    builder.RefsExtractor(default_conversion, refs).visit_with_conv(tp, conversion)
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/conversions/visitor.py", line 100, in visit_with_conv
    return self.visit(tp)
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/conversions/visitor.py", line 135, in visit
    return self.visit_conversion(tp, conversion, dynamic, next_conversion)
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/json_schema/refs.py", line 131, in visit_conversion
    super().visit_conversion(tp, conversion, dynamic, next_conversion)
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/conversions/visitor.py", line 122, in visit_conversion
    return super().visit(tp)
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/visitor.py", line 177, in visit
    return self.collection(origin, args[0])
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/json_schema/refs.py", line 84, in collection
    self.visit(value_type)
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/conversions/visitor.py", line 126, in visit
    return self.visit_conversion(tp, None, False, self._conversion)
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/json_schema/refs.py", line 131, in visit_conversion
    super().visit_conversion(tp, conversion, dynamic, next_conversion)
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/conversions/visitor.py", line 122, in visit_conversion
    return super().visit(tp)
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/visitor.py", line 172, in visit
    return self.union(args[0]) if len(args) == 1 else self.union(args)
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/conversions/visitor.py", line 88, in union
    return self._visited_union(self._union_results(alternatives))
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/conversions/visitor.py", line 79, in _union_results
    results.append(self.visit(alt))
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/conversions/visitor.py", line 135, in visit
    return self.visit_conversion(tp, conversion, dynamic, next_conversion)
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/json_schema/refs.py", line 131, in visit_conversion
    super().visit_conversion(tp, conversion, dynamic, next_conversion)
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/conversions/visitor.py", line 122, in visit_conversion
    return super().visit(tp)
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/visitor.py", line 185, in visit
    return self.dataclass(tp, *dataclass_types_and_fields(tp))  # type: ignore
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/objects/visitor.py", line 103, in dataclass
    return self._object(tp, self._override_fields(tp, object_fields))
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/objects/visitor.py", line 84, in _object
    return self.object(tp, fields)
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/json_schema/refs.py", line 98, in object
    self.visit_with_conv(field.type, self._field_conversion(field))
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/conversions/visitor.py", line 100, in visit_with_conv
    return self.visit(tp)
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/conversions/visitor.py", line 135, in visit
    return self.visit_conversion(tp, conversion, dynamic, next_conversion)
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/json_schema/refs.py", line 131, in visit_conversion
    super().visit_conversion(tp, conversion, dynamic, next_conversion)
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/conversions/visitor.py", line 122, in visit_conversion
    return super().visit(tp)
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/visitor.py", line 177, in visit
    return self.collection(origin, args[0])
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/json_schema/refs.py", line 84, in collection
    self.visit(value_type)
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/conversions/visitor.py", line 126, in visit
    return self.visit_conversion(tp, None, False, self._conversion)
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/json_schema/refs.py", line 131, in visit_conversion
    super().visit_conversion(tp, conversion, dynamic, next_conversion)
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/conversions/visitor.py", line 122, in visit_conversion
    return super().visit(tp)
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/visitor.py", line 172, in visit
    return self.union(args[0]) if len(args) == 1 else self.union(args)
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/conversions/visitor.py", line 88, in union
    return self._visited_union(self._union_results(alternatives))
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/conversions/visitor.py", line 79, in _union_results
    results.append(self.visit(alt))
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/conversions/visitor.py", line 135, in visit
    return self.visit_conversion(tp, conversion, dynamic, next_conversion)
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/json_schema/refs.py", line 131, in visit_conversion
    super().visit_conversion(tp, conversion, dynamic, next_conversion)
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/conversions/visitor.py", line 122, in visit_conversion
    return super().visit(tp)
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/visitor.py", line 185, in visit
    return self.dataclass(tp, *dataclass_types_and_fields(tp))  # type: ignore
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/objects/visitor.py", line 103, in dataclass
    return self._object(tp, self._override_fields(tp, object_fields))
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/objects/visitor.py", line 84, in _object
    return self.object(tp, fields)
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/json_schema/refs.py", line 98, in object
    self.visit_with_conv(field.type, self._field_conversion(field))
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/conversions/visitor.py", line 100, in visit_with_conv
    return self.visit(tp)
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/conversions/visitor.py", line 135, in visit
    return self.visit_conversion(tp, conversion, dynamic, next_conversion)
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/json_schema/refs.py", line 131, in visit_conversion
    super().visit_conversion(tp, conversion, dynamic, next_conversion)
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/conversions/visitor.py", line 122, in visit_conversion
    return super().visit(tp)
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/visitor.py", line 177, in visit
    return self.collection(origin, args[0])
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/json_schema/refs.py", line 84, in collection
    self.visit(value_type)
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/conversions/visitor.py", line 126, in visit
    return self.visit_conversion(tp, None, False, self._conversion)
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/json_schema/refs.py", line 131, in visit_conversion
    super().visit_conversion(tp, conversion, dynamic, next_conversion)
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/conversions/visitor.py", line 122, in visit_conversion
    return super().visit(tp)
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/visitor.py", line 172, in visit
    return self.union(args[0]) if len(args) == 1 else self.union(args)
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/conversions/visitor.py", line 88, in union
    return self._visited_union(self._union_results(alternatives))
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/conversions/visitor.py", line 79, in _union_results
    results.append(self.visit(alt))
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/conversions/visitor.py", line 135, in visit
    return self.visit_conversion(tp, conversion, dynamic, next_conversion)
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/json_schema/refs.py", line 131, in visit_conversion
    super().visit_conversion(tp, conversion, dynamic, next_conversion)
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/conversions/visitor.py", line 122, in visit_conversion
    return super().visit(tp)
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/visitor.py", line 185, in visit
    return self.dataclass(tp, *dataclass_types_and_fields(tp))  # type: ignore
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/objects/visitor.py", line 103, in dataclass
    return self._object(tp, self._override_fields(tp, object_fields))
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/objects/visitor.py", line 84, in _object
    return self.object(tp, fields)
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/json_schema/refs.py", line 98, in object
    self.visit_with_conv(field.type, self._field_conversion(field))
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/conversions/visitor.py", line 100, in visit_with_conv
    return self.visit(tp)
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/conversions/visitor.py", line 135, in visit
    return self.visit_conversion(tp, conversion, dynamic, next_conversion)
  File "/scratch/tmc43/pipenv/pvi-8WvQsKFD/lib/python3.7/site-packages/apischema/json_schema/refs.py", line 128, in visit_conversion
    raise TypeError(f"Recursive type {tp} need a ref")
TypeError: Recursive type typing.Sequence[typing.Union[test_group.Foo, test_group.Group[test_group.Foo]]] need a ref
commented

In fact, this is the expected behavior; reason is given in the error message: f"Recursive type {tp} need a ref".
As you know, JSON schema needs $ref to handle recursion. In the non-generic case, classes have a default type_name (their name), which is used in the JSON schema.

However, generic classes doesn't have a default type name — mainly because that's a matter of taste, for example, in your case, should it be FooGroup, GroupFoo or something else? — so schema generation cannot use a $ref to stop the recursion.

Solution can be to assign a type_name:

from __future__ import annotations
from dataclasses import dataclass
from typing import Generic, Sequence, TypeVar, Union

from apischema import type_name
from apischema.json_schema import deserialization_schema

T = TypeVar("T")


@dataclass
class Foo:
    bar: int


@type_name(lambda tp, arg: f"{arg.__name__}{tp.__name__}")
@dataclass
class Group(Generic[T]):
    children: Tree[T]


Tree = Sequence[Union[T, Group[T]]]

assert deserialization_schema(Tree[Foo]) == {
    "type": "array",
    "items": {"anyOf": [{"$ref": "#/$defs/Foo"}, {"$ref": "#/$defs/FooGroup"}]},
    "$defs": {
        "Foo": {
            "type": "object",
            "properties": {"bar": {"type": "integer"}},
            "required": ["bar"],
            "additionalProperties": False,
        },
        "FooGroup": {
            "type": "object",
            "properties": {
                "children": {
                    "type": "array",
                    "items": {
                        "anyOf": [{"$ref": "#/$defs/Foo"}, {"$ref": "#/$defs/FooGroup"}]
                    },
                }
            },
            "required": ["children"],
            "additionalProperties": False,
        },
    },
    "$schema": "http://json-schema.org/draft/2020-12/schema#",
}

I admit the documentation is not clear about this use case, I should add a warning somewhere about lack of default "$ref" for generic type.

commented

Thanks, that fixes it!