From 7150d62949cdb35bcc931b092225e3e3bef7541d Mon Sep 17 00:00:00 2001 From: Mauricio Villegas <5780272+mauvilsa@users.noreply.github.com> Date: Mon, 31 Aug 2026 07:18:14 +0200 Subject: [PATCH] Support for NewType, LiteralString, NamedTuple, TypedDict ReadOnly keys and subscripted generic type aliases --- CHANGELOG.rst | 26 ++ DOCUMENTATION.rst | 62 ++- jsonargparse/_actions.py | 35 +- jsonargparse/_common.py | 7 + jsonargparse/_completions.py | 14 +- jsonargparse/_completions_jsonschema.py | 25 +- jsonargparse/_namespace.py | 22 +- jsonargparse/_optionals.py | 58 ++- jsonargparse/_parameter_resolvers.py | 30 +- jsonargparse/_signatures.py | 6 +- jsonargparse/_typehints.py | 172 ++++++- jsonargparse/_util.py | 8 + .../test_completions_jsonschema.py | 58 ++- jsonargparse_tests/test_dataclasses.py | 51 +++ jsonargparse_tests/test_namespace.py | 12 + jsonargparse_tests/test_shtab.py | 31 +- jsonargparse_tests/test_signatures.py | 2 + jsonargparse_tests/test_typehints.py | 429 +++++++++++++++++- 18 files changed, 959 insertions(+), 89 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0f9e2738..e1bc219d 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -46,6 +46,19 @@ Added which raises an exception for all parameters that don't have a type annotation, not only the required ones (`#965 `__). +- Support for ``NamedTuple`` as a type. The value is an object with the fields + as keys or an array of positional values, parsing gives an instance of the + named tuple and dumping gives an object. It has a ``--*.help`` option that + shows the accepted fields, is accepted by ``add_class_arguments`` and works + subscripted when generic, see :ref:`type-hints` (`#967 + `__). +- ``TypedDict`` now accepts ``ReadOnly`` for its keys (`#967 + `__). +- Support for ``NewType`` and ``LiteralString`` as types. Previously they were + not validated, i.e. any value was accepted. Now a ``NewType`` is validated as + the supertype it stands for and a ``LiteralString`` as a ``str``, in both + cases the help showing the name as in the source code, see :ref:`type-hints` + (`#967 `__). Fixed ^^^^^ @@ -90,6 +103,19 @@ Fixed subscripted, e.g. ``os.PathLike[str]`` for a registered ``PathLike``. Now the registration of the unsubscripted type is used (`#966 `__). +- A subscripted generic ``TypeAliasType``, e.g. ``Alias[int]`` for ``type + Alias[T] = list[T]``, raised ``Unsupported type hint``. Unsubscripted, its + type parameters now also stand for their default, constraints or bound, the + same as any other ``TypeVar`` (`#967 + `__). +- ``Namespace.as_dict`` did not convert the namespaces nested in a ``dict`` or + ``list`` that also holds values which are not namespaces, e.g. a ``TypedDict`` + with one key of a class type and another of a simple type. Dumping such a + config as json failed with ``Object of type Namespace is not JSON + serializable`` (`#967 `__). +- ``add_class_arguments`` given a subscripted generic class, e.g. + ``SomeClass[int]``, did not instantiate it, giving a ``Namespace`` instead of + an instance (`#967 `__). Changed ^^^^^^^ diff --git a/DOCUMENTATION.rst b/DOCUMENTATION.rst index 09de782b..c75de94d 100644 --- a/DOCUMENTATION.rst +++ b/DOCUMENTATION.rst @@ -448,23 +448,26 @@ Types can be nested with any complexity. Notes about the support: class that owns the method, e.g. a nested class referred to without qualifying it. -- Fully supported types are: ``str``, ``bool`` (see :ref:`boolean-arguments`), - ``int``, ``float``, ``Decimal``, ``complex``, ``bytes``/``bytearray`` (Base64 - encoding), ``range``, ``list`` (see :ref:`list-append`), ``Deque``, - ``Iterable``, ``Sequence``, ``MutableSequence``, ``Collection``, - ``Container``, ``Reversible``, ``Any``/``object``, ``Union``/``Optional`` (see - :ref:`union-types`), ``Literal``, ``Type``, ``Enum``, ``PathLike``, ``UUID``, - ``timedelta``, the restricted types of :ref:`restricted-numbers` and - :ref:`restricted-strings`, and the path and URL types of :ref:`parsing-paths` - and :ref:`parsing-urls`. +- Fully supported types are: ``str``/``LiteralString``, ``bool`` (see + :ref:`boolean-arguments`), ``int``, ``float``, ``Decimal``, ``complex``, + ``bytes``/``bytearray`` (Base64 encoding), ``range``, ``list`` (see + :ref:`list-append`), ``Deque``, ``Iterable``, ``Sequence``, + ``MutableSequence``, ``Collection``, ``Container``, ``Reversible``, + ``Any``/``object``, ``Union``/``Optional`` (see :ref:`union-types`), + ``Literal``, ``Type``, ``Enum``, ``PathLike``, ``UUID``, ``timedelta``, the + restricted types of :ref:`restricted-numbers` and :ref:`restricted-strings`, + and the path and URL types of :ref:`parsing-paths` and :ref:`parsing-urls`. - ``dict``, ``Mapping``, ``MutableMapping``, ``MappingProxyType``, ``OrderedDict`` and ``TypedDict`` are supported, but only with ``str`` or ``int`` keys, see :ref:`dict-items`. - ``TypedDict`` accepts ``Required`` and ``NotRequired`` to mark single keys as - required or optional, and ``Unpack`` to type ``**kwargs`` precisely, see PEP - `692 `__. A ``--*.help`` option, e.g. + required or optional, ``ReadOnly`` (PEP `705 + `__) which only marks a key as not mutable + and thus changes neither its type nor its requiredness, and ``Unpack`` to + type ``**kwargs`` precisely, see PEP `692 + `__. A ``--*.help`` option, e.g. ``--data.help``, shows the accepted keys. It takes no value, unless the ``TypedDict`` is in a union with other types that have their own help, in which case the value is the name of the typed dict, e.g. ``--data.help @@ -491,6 +494,15 @@ Types can be nested with any complexity. Notes about the support: a list when parsing, since subclass specs are not hashable, and becomes a set on :meth:`instantiate <.ArgumentParser.instantiate>`. +- ``NamedTuple`` is supported. The value is either an object with the fields as + keys or an array of positional values, fields with a default can be omitted, + and parsing gives an instance of the named tuple. It is always dumped as an + object, so that the fields are named. A ``--*.help`` option shows the accepted + fields, and :meth:`add_class_arguments <.ArgumentParser.add_class_arguments>` + accepts a ``NamedTuple``, both the same as for a ``TypedDict``. A generic + ``NamedTuple`` works unsubscripted and subscripted, e.g. ``SomeTuple[int]``. A + field of an untyped ``collections.namedtuple`` accepts any value. + - ``None`` is written as ``null``, as JSON/YAML define it. For the same reason the help shows ``NoneType`` as ``null``, e.g. a parameter with type and default ``Optional[str] = None`` is shown as ``type: Union[str, null], @@ -558,7 +570,15 @@ Types can be nested with any complexity. Notes about the support: - ``TypeAliasType`` is supported. Values are parsed as the aliased type and the help shows the alias as the argument type. This includes aliases defined with the `PEP 695 `__ ``type X = ...`` statement - (Python 3.12+) and aliases created with ``typing_extensions.TypeAliasType``. + (Python 3.12+) and aliases created with ``typing_extensions.TypeAliasType``. A + generic alias, e.g. ``type X[T] = list[T]``, is parsed as its target with the + type parameters substituted by what it is subscripted with, e.g. ``X[int]`` + behaves as ``list[int]``. Unsubscripted, its type parameters stand for their + default, constraints or bound, the same as any other ``TypeVar``. + +- ``NewType`` is supported. Values are parsed as the supertype it stands for, + including a ``NewType`` of a ``NewType``, and the help shows the name given in + the source code. .. _union-types: @@ -635,13 +655,13 @@ an argument of type ``Union[int, list[int]]``, ``--val=1`` gives ``1``, while Unvalidated types ----------------- -A :ref:`signature parameter ` or a ``TypedDict`` key -can have a type that jsonargparse can't validate. The argument is still added, -with only the parts of the type that can't be validated replaced by a type that -accepts any value. The help shows these parts as ``Unvalidated<...>``, keeping -the name used in the source code. For example, a class with a parameter -``items: list[SomeType] = []`` for which ``SomeType`` can't be validated is -shown in the help as: +A :ref:`signature parameter `, a ``TypedDict`` key or +a ``NamedTuple`` field can have a type that jsonargparse can't validate. The +argument is still added, with only the parts of the type that can't be validated +replaced by a type that accepts any value. The help shows these parts as +``Unvalidated<...>``, keeping the name used in the source code. For example, a +class with a parameter ``items: list[SomeType] = []`` for which ``SomeType`` +can't be validated is shown in the help as: .. code-block:: text @@ -3441,8 +3461,8 @@ which subclasses accept each one. For example: $ example.py --cls other.module.SubclassA --cls.param2 Expected type: int; Accepted by subclasses: SubclassA -Analogously, for subclasses-disabled types and ``TypedDict``, the fields or keys -are completed, as well as the values that they accept, e.g.: +Analogously, for subclasses-disabled types, ``TypedDict`` and ``NamedTuple``, +the fields or keys are completed, as well as the values that they accept, e.g.: .. code-block:: bash diff --git a/jsonargparse/_actions.py b/jsonargparse/_actions.py index ef3f5339..33ff1d44 100644 --- a/jsonargparse/_actions.py +++ b/jsonargparse/_actions.py @@ -31,6 +31,7 @@ get_import_path, import_object, indent_text, + iter_to_or_str, iter_to_set_str, load_config_path_context, merge_config, @@ -353,15 +354,15 @@ def __init__(self, typehint=None, **kwargs): super().__init__(**kwargs) def update_init_kwargs(self, kwargs): - from ._typehints import get_help_types, is_protocol, is_typed_dict + from ._typehints import get_help_types, is_namedtuple, is_protocol, is_structured_value_type, is_typed_dict self._typehint = kwargs.pop("_typehint") self._help_types = get_help_types(self._typehint) - typed_dicts = [t for t in self._help_types if is_typed_dict(t)] - # a subscripted generic typed dict is a generic alias instead of a class, see get_typed_dict_type - assert self._help_types and all(isinstance(b, type) for b in self._help_types if b not in typed_dicts) + # a subscripted generic typed dict or named tuple is a generic alias instead of a class + structured = [t for t in self._help_types if is_structured_value_type(t)] + assert self._help_types and all(isinstance(b, type) for b in self._help_types if b not in structured) # a single type means that the help refers to it, so no value is expected - single_type = len(self._help_types) == 1 and (is_subclasses_disabled(self._help_types[0]) or bool(typed_dicts)) + single_type = len(self._help_types) == 1 and (is_subclasses_disabled(self._help_types[0]) or bool(structured)) self._basename = iter_to_set_str(t.__name__ for t in self._help_types) if len(self._help_types) == 1: @@ -374,13 +375,15 @@ def update_init_kwargs(self, kwargs): self._kind = "subclass of" if any(is_protocol(b) for b in self._help_types): self._kind = "subclass or implementer of protocol" - if typed_dicts: - # a typed dict is given by name, since it doesn't accept a class path - if len(typed_dicts) == len(self._help_types): + if structured: + # a typed dict or named tuple is given by name, since it doesn't accept a class path + kinds = ["typed dict"] if any(is_typed_dict(t) for t in structured) else [] + kinds += ["named tuple"] if any(is_namedtuple(t) for t in structured) else [] + if len(structured) == len(self._help_types): kwargs["metavar"] = "NAME" - self._kind = "typed dict" else: - self._kind = "class or typed dict" + kinds.insert(0, "class") + self._kind = iter_to_or_str(kinds) msg = f"the given {self._kind} " kwargs["default"] = SUPPRESS @@ -393,15 +396,15 @@ def __call__(self, *args, **kwargs): return self.print_help(args) def resolve_help_type(self, value, option_string): - from ._typehints import implements_protocol, is_typed_dict, resolve_class_path_by_name + from ._typehints import implements_protocol, is_structured_value_type, resolve_class_path_by_name if self.nargs == 0 or (self.nargs == "?" and value is None): return self._help_types[0] - typed_dict = next((t for t in self._help_types if is_typed_dict(t) and t.__name__ == value), None) - if typed_dict: - return typed_dict - # typed dicts excluded since they don't have subclasses that a class path could refer to - class_types = tuple(t for t in self._help_types if not is_typed_dict(t)) + structured = next((t for t in self._help_types if is_structured_value_type(t) and t.__name__ == value), None) + if structured: + return structured + # structured value types excluded, they don't have subclasses that a class path could refer to + class_types = tuple(t for t in self._help_types if not is_structured_value_type(t)) val_class = None if class_types: try: diff --git a/jsonargparse/_common.py b/jsonargparse/_common.py index 60aabb04..883aa165 100644 --- a/jsonargparse/_common.py +++ b/jsonargparse/_common.py @@ -21,9 +21,12 @@ capture_typing_extension_shadows, get_alias_target, get_annotated_base_type, + get_new_type_supertype, is_alias_type, is_annotated, is_attrs_class, + is_literal_string, + is_new_type, is_pydantic_model, typing_extensions_import, ) @@ -637,6 +640,10 @@ def get_unaliased_type(cls): new_cls = get_annotated_base_type(new_cls) if is_alias_type(new_cls): new_cls = get_alias_target(new_cls) + if is_new_type(new_cls): + new_cls = get_new_type_supertype(new_cls) + if is_literal_string(new_cls): + new_cls = str origin = get_unsubscripted_alias_origin(new_cls) if origin is not None: new_cls = origin diff --git a/jsonargparse/_completions.py b/jsonargparse/_completions.py index 0d45936e..3e1c9b67 100644 --- a/jsonargparse/_completions.py +++ b/jsonargparse/_completions.py @@ -31,8 +31,8 @@ get_typed_dict_key_type, get_typehint_origin, is_single_subclass_or_closed_type, + is_structured_value_type, is_subclass, - is_typed_dict, type_to_str, ) from ._util import NoneType, Path, import_object, merge_config, unique @@ -370,11 +370,12 @@ def get_choices_state(typehint) -> tuple[list[str], bool, bool]: choices = add_subactions_and_get_subclass_choices(typehint, prefix, parser, skip, added_subclasses) return choices, True, False - if is_typed_dict(typehint) or ( + if is_structured_value_type(typehint) or ( is_single_subclass_or_closed_type(typehint, origin) and is_subclasses_disabled(typehint) ): - # a dataclass-like type is only inlined as a group when not in a union and a typed - # dict never is, so their init args or keys need to be added as options to complete them + # a dataclass-like type is only inlined as a group when not in a union and a typed dict + # or named tuple never is, so their init args, keys or fields need to be added as + # options to complete them added_subclasses.add(typehint) add_subactions_and_get_subclass_choices(typehint, prefix, parser, skip, added_subclasses, closed_type=True) return [], False, True @@ -451,8 +452,9 @@ def add_subactions_and_get_subclass_choices( def get_help_class_choices(typehint) -> list[str]: choices: list[str] = [] for help_type in get_help_types(typehint) or []: - if is_typed_dict(help_type): - choices.append(help_type.__name__) # typed dicts don't accept a class path, only their name + if is_structured_value_type(help_type): + # a typed dict or named tuple doesn't accept a class path, only its name + choices.append(help_type.__name__) else: choices += [p for p in get_all_subclass_paths(help_type) if p not in choices] return choices diff --git a/jsonargparse/_completions_jsonschema.py b/jsonargparse/_completions_jsonschema.py index dae756ab..35c48b4b 100644 --- a/jsonargparse/_completions_jsonschema.py +++ b/jsonargparse/_completions_jsonschema.py @@ -27,16 +27,18 @@ callable_origin_types, get_all_subclass_paths, get_callable_return_type, + get_namedtuple_annotations, get_typed_dict_annotations, get_typed_dict_required_keys, get_typehint_origin, + is_namedtuple, is_single_subclass_or_closed_type, is_single_subclass_type, literal_types, mapping_origin_types, - not_required_required_types, sequence_origin_types, tuple_set_origin_types, + typed_dict_key_qualifiers, typed_dict_meta_types, ) from ._util import NoneType, get_import_path, import_object @@ -396,7 +398,7 @@ def typehint_schema(self, typehint, action) -> dict: if typehint in {Any, object}: return {} - if origin in not_required_required_types: # requiredness comes from the TypedDict, not the type + if origin in typed_dict_key_qualifiers: # requiredness and mutability come from the TypedDict return self.typehint_schema(typehint.__args__[0], action) if typehint is uuid.UUID: return dict(uuid_schema) @@ -409,6 +411,8 @@ def typehint_schema(self, typehint, action) -> dict: return {"enum": list(typehint.__members__)} if type(typehint) in typed_dict_meta_types: return self.typed_dict_schema(typehint, action) + if is_namedtuple(typehint): + return self.namedtuple_schema(typehint, action) if root in literal_types: return {"enum": [json_value(arg) for arg in typehint.__args__]} if origin is Union: @@ -475,6 +479,23 @@ def typed_dict_schema(self, typehint, action) -> dict: add_required(schema, key) return schema + def namedtuple_schema(self, typehint, action) -> dict: + """Describes both forms accepted for a NamedTuple: an object of fields or an array of values.""" + annotations = get_namedtuple_annotations(typehint) + defaults = typehint._field_defaults + obj_schema = new_object(get_doc_short_description(typehint), schema_key=False) + for field, annotation in annotations.items(): + obj_schema["properties"][field] = self.typehint_schema(annotation, action) + if field not in defaults: + add_required(obj_schema, field) + array_schema = { + "type": "array", + "prefixItems": [obj_schema["properties"][f] for f in annotations], + "items": False, + "minItems": len(annotations) - len(defaults), + } + return anyof_schema([obj_schema, array_schema]) + # classes def class_ref(self, class_type, action, subclass: bool) -> dict: diff --git a/jsonargparse/_namespace.py b/jsonargparse/_namespace.py index 25e89caf..90af3dd8 100644 --- a/jsonargparse/_namespace.py +++ b/jsonargparse/_namespace.py @@ -169,16 +169,7 @@ def __bool__(self) -> bool: def as_dict(self) -> dict[str, Any]: """Converts the nested namespaces into nested dictionaries.""" - dic = {} - for key, val in vars(self).items(): - if isinstance(val, Namespace): - val = val.as_dict() - elif isinstance(val, dict) and val != {} and all(isinstance(v, Namespace) for v in val.values()): - val = {k: v.as_dict() for k, v in val.items()} - elif isinstance(val, list) and val != [] and all(isinstance(v, Namespace) for v in val): - val = [v.as_dict() for v in val] - dic[del_clash_mark(key)] = val - return dic + return {del_clash_mark(key): _value_as_dict(val) for key, val in vars(self).items()} def as_flat(self) -> argparse.Namespace: """Converts the nested namespaces into a single argparse flat namespace.""" @@ -260,6 +251,17 @@ def pop(self, key: str, default: Any = None) -> Any: clash_mark = "\u200b" +def _value_as_dict(val): + """Converts the namespaces nested in a value into dictionaries, including in containers.""" + if isinstance(val, Namespace): + return val.as_dict() + if isinstance(val, dict): + return {k: _value_as_dict(v) for k, v in val.items()} + if isinstance(val, list): + return [_value_as_dict(v) for v in val] + return val + + def add_clash_mark(key: str) -> str: if key in clash_names: key = clash_mark + key diff --git a/jsonargparse/_optionals.py b/jsonargparse/_optionals.py index 13288751..d8e6bce0 100644 --- a/jsonargparse/_optionals.py +++ b/jsonargparse/_optionals.py @@ -413,14 +413,62 @@ def get_annotated_base_type(typehint: type) -> type: typing_type_alias_type = None -def is_alias_type(typehint: Any) -> bool: - return (type_alias_type and isinstance(typehint, type_alias_type)) or ( - typing_type_alias_type and isinstance(typehint, typing_type_alias_type) # type: ignore[truthy-function] +def _is_alias_type(typehint: Any) -> bool: + return bool( + (type_alias_type and isinstance(typehint, type_alias_type)) + or (typing_type_alias_type and isinstance(typehint, typing_type_alias_type)) # type: ignore[truthy-function] ) -def get_alias_target(typehint: type) -> bool: - return typehint.__value__ # type: ignore[attr-defined] +def is_alias_type(typehint: Any) -> bool: + """Whether a type hint is a TypeAliasType, including a subscripted generic one. + + A subscripted generic alias, e.g. ``Alias[int]`` for ``type Alias[T] = list[T]``, + is a generic alias whose origin is the TypeAliasType. + """ + return _is_alias_type(typehint) or _is_alias_type(getattr(typehint, "__origin__", None)) + + +def get_alias_target(typehint: Any) -> Any: + """Returns what an alias stands for, with the type parameters of a generic alias resolved. + + A subscripted generic alias substitutes its type parameters with the types that + it is subscripted with. An unsubscripted one replaces them by what they stand + for, i.e. their default, constraints or bound, the same as any other TypeVar. + """ + alias = typehint if _is_alias_type(typehint) else typehint.__origin__ + target = alias.__value__ + type_params = getattr(alias, "__type_params__", None) + if not type_params: + return target + from ._typehints import replace_type_vars, substitute_type_vars + + args = getattr(typehint, "__args__", None) or () + return replace_type_vars(substitute_type_vars(target, dict(zip(type_params, args)))) + + +def is_new_type(typehint: Any) -> bool: + """Whether a type hint is a ``NewType``, which stands for its supertype.""" + # NewType is a class since python 3.10, thus an instance check identifies one + return isinstance(typehint, __import__("typing").NewType) + + +def get_new_type_supertype(typehint: Any) -> Any: + """Returns the type that a ``NewType`` stands for, i.e. the second argument given to it.""" + return typehint.__supertype__ + + +LiteralString = typing_extensions_import("LiteralString") +literal_string_types = {LiteralString} +capture_typing_extension_shadows(LiteralString, "LiteralString", literal_string_types) + + +def is_literal_string(typehint: Any) -> bool: + """Whether a type hint is ``LiteralString``, which stands for ``str`` at runtime. + + Compared by identity, since a special form is not necessarily hashable. + """ + return any(typehint is literal_string for literal_string in literal_string_types) def get_pydantic_support() -> int: diff --git a/jsonargparse/_parameter_resolvers.py b/jsonargparse/_parameter_resolvers.py index ac483b38..13a083fe 100644 --- a/jsonargparse/_parameter_resolvers.py +++ b/jsonargparse/_parameter_resolvers.py @@ -331,6 +331,7 @@ def get_typed_dict_params(typed_dict, logger=None, **param_kwargs) -> ParamList: get_typed_dict_annotations, get_typed_dict_required_keys, not_required_types, + strip_read_only, ) annotations = get_typed_dict_annotations(typed_dict, logger) @@ -338,7 +339,7 @@ def get_typed_dict_params(typed_dict, logger=None, **param_kwargs) -> ParamList: doc_params = parse_docs(typed_dict, None, logger) params = [] for name, annotation in annotations.items(): - if name not in required_keys and get_typehint_origin(annotation) not in not_required_types: + if name not in required_keys and get_typehint_origin(strip_read_only(annotation)) not in not_required_types: # Mark optional keys (e.g. from total=False) as NotRequired so that they # are added as non-required arguments. annotation = NotRequired[annotation] @@ -355,6 +356,27 @@ def get_typed_dict_params(typed_dict, logger=None, **param_kwargs) -> ParamList: return params +def get_namedtuple_params(namedtuple, logger=None, **param_kwargs) -> ParamList: + """Parameters that correspond to the fields of a NamedTuple.""" + from ._typehints import get_namedtuple_annotations, get_namedtuple_type + + annotations = get_namedtuple_annotations(namedtuple, logger) + namedtuple = get_namedtuple_type(namedtuple) + defaults = namedtuple._field_defaults + doc_params = parse_docs(namedtuple, None, logger) + return [ + ParamData( + name=name, + annotation=annotation, + default=defaults.get(name, inspect._empty), + kind=inspect._ParameterKind.KEYWORD_ONLY, + doc=doc_params.get(name), + **param_kwargs, + ) + for name, annotation in annotations.items() + ] + + def unpack_typed_dict_kwargs(params: ParamList, kwargs_idx: int, logger=None) -> int: kwargs = params[kwargs_idx] annotation = kwargs.annotation @@ -1224,12 +1246,16 @@ def get_signature_parameters( the parameters for ``__init__``. logger: Useful for debugging. Only logs at ``DEBUG`` level. """ - from ._typehints import is_typed_dict + from ._typehints import is_namedtuple, is_typed_dict logger = parse_logger(logger, "get_signature_parameters") if method_or_property is None and is_typed_dict(function_or_class): # a typed dict has no signature to inspect, its parameters correspond to its keys return get_typed_dict_params(function_or_class, logger, component=function_or_class) + if method_or_property is None and is_namedtuple(function_or_class): + # the generated __new__ of a named tuple doesn't keep the annotations as written, and a + # subscripted generic one has no signature, so its parameters come from its fields + return get_namedtuple_params(function_or_class, logger, component=function_or_class) get_component_and_parent(function_or_class, method_or_property) # verify input params = None for get_parameters in [ diff --git a/jsonargparse/_signatures.py b/jsonargparse/_signatures.py index 8e106510..eae2b9e8 100644 --- a/jsonargparse/_signatures.py +++ b/jsonargparse/_signatures.py @@ -661,9 +661,11 @@ def _create_group_if_requested( if config_load_type is None and inspect.isclass(obj): config_load_type = obj group.add_argument("--" + nested_key, action=_ActionConfigLoad(basetype=config_load_type)) - if inspect.isclass(obj) and nested_key is not None and instantiate: + # a subscripted generic is instantiated as its origin class, since the subscript + # only says what its type parameters stand for + if inspect.isclass(get_generic_origin(obj)) and nested_key is not None and instantiate: group.dest = nested_key.replace("-", "_") - group.group_class = obj + group.group_class = get_generic_origin(obj) group.instantiate_class = group_instantiate_class return group diff --git a/jsonargparse/_typehints.py b/jsonargparse/_typehints.py index 13e74e1c..ed278974 100644 --- a/jsonargparse/_typehints.py +++ b/jsonargparse/_typehints.py @@ -85,9 +85,12 @@ from ._optionals import ( capture_typing_extension_shadows, get_alias_target, + get_new_type_supertype, is_alias_type, is_annotated, is_annotated_validator, + is_literal_string, + is_new_type, typing_extensions_import, validate_annotated, ) @@ -111,6 +114,7 @@ from .typing import _LazyInitBaseClass, get_registered_type, is_pydantic_type NotRequired = typing_extensions_import("NotRequired") +ReadOnly = typing_extensions_import("ReadOnly") Required = typing_extensions_import("Required") _TypedDictMeta = typing_extensions_import("_TypedDictMeta") Unpack = typing_extensions_import("Unpack") @@ -177,6 +181,7 @@ def _capture_typing_extension_shadows(name: str, *collections) -> None: UnionType, GenericAlias, NotRequired, + ReadOnly, Required, Unpack, } @@ -233,6 +238,10 @@ def _capture_typing_extension_shadows(name: str, *collections) -> None: _capture_typing_extension_shadows("Required", root_types, required_types) not_required_required_types = not_required_types.union(required_types) +read_only_types = {ReadOnly} +_capture_typing_extension_shadows("ReadOnly", root_types, read_only_types) +typed_dict_key_qualifiers = not_required_required_types.union(read_only_types) + typed_dict_types = {TypedDict} _capture_typing_extension_shadows("TypedDict", typed_dict_types) @@ -313,12 +322,26 @@ def cached_get_class_parser(*, val_class, sub_add_kwargs, skip_args, parent_pars return parser +def strip_read_only(typehint): + """Removes a top level ReadOnly wrapper, which only marks a TypedDict key as not mutable. + + Nothing is ever written back into a parsed TypedDict, so ReadOnly imposes no + restriction and changes neither the type of a key nor its requiredness. + """ + if get_typehint_origin(typehint) in read_only_types: + assert len(typehint.__args__) == 1, "ReadOnly requires a single type argument" + return typehint.__args__[0] + return typehint + + def strip_required_typehint(typehint, is_required: bool, source: str): - """Removes a top level Required/NotRequired wrapper, failing if it disagrees with the requiredness. + """Removes top level Required/NotRequired and ReadOnly wrappers, checking the requiredness. - The requiredness of an argument is already given by the argument itself, thus the wrappers are - only accepted as a redundant specification and not included in the type shown in the help. + The requiredness of an argument is already given by the argument itself, thus Required and + NotRequired are only accepted as a redundant specification, failing when they disagree, and + neither they nor ReadOnly are included in the type shown in the help. """ + typehint = strip_read_only(typehint) typehint_origin = get_typehint_origin(typehint) if typehint_origin not in not_required_required_types: return typehint @@ -331,7 +354,7 @@ def strip_required_typehint(typehint, is_required: bool, source: str): f"argument is {'required' if expect_required else 'not required'}." ) assert len(typehint.__args__) == 1, "(Not)Required requires a single type argument" - return typehint.__args__[0] + return strip_read_only(typehint.__args__[0]) class ActionTypeHint(Action): @@ -446,6 +469,7 @@ def is_supported_typehint(typehint, full=False): or is_subclass(typehint, Enum) or is_subclasses_disabled(typehint) or is_typed_dict(typehint) + or is_namedtuple(typehint) or ActionTypeHint.is_subclass_typehint(typehint) ) if full and supported: @@ -1160,11 +1184,11 @@ def get_typed_dict_annotations(typed_dict, logger=None) -> dict: global_vars = {} update_module_global_vars(module, global_vars, logger) annotations.update(resolve_module_annotations(module, module_annotations, global_vars, logger)) - return {k: resolve_typed_dict_key_type_vars(annotations[k], type_var_maps[k]) for k in typed_dict.__annotations__} + return {k: resolve_annotation_type_vars(annotations[k], type_var_maps[k]) for k in typed_dict.__annotations__} -def resolve_typed_dict_key_type_vars(annotation, type_var_map: dict): - """Returns the annotation of a TypedDict key with the TypeVars in it resolved. +def resolve_annotation_type_vars(annotation, type_var_map: dict): + """Returns the annotation of a TypedDict key or a NamedTuple field with the TypeVars in it resolved. First the TypeVars that the subscript binds are substituted, then the ones that it doesn't are replaced by what they stand for, i.e. their default, @@ -1181,18 +1205,85 @@ def get_typed_dict_required_keys(typed_dict, annotations: dict) -> set: # reflected there (e.g. below Python 3.11 or with postponed annotations), so they are # adjusted based on the resolved annotations. required_keys = set(getattr(typed_dict, "__required_keys__", set(annotations))) - required_keys.update({k for k, v in annotations.items() if get_typehint_origin(v) in required_types}) - required_keys.difference_update({k for k, v in annotations.items() if get_typehint_origin(v) in not_required_types}) + unqualified = {k: strip_read_only(v) for k, v in annotations.items()} + required_keys.update({k for k, v in unqualified.items() if get_typehint_origin(v) in required_types}) + required_keys.difference_update({k for k, v in unqualified.items() if get_typehint_origin(v) in not_required_types}) return required_keys def get_typed_dict_key_type(annotation): - # Required and NotRequired only change the requiredness of a key, not its type + # Required and NotRequired only change the requiredness of a key and ReadOnly only marks it as + # not mutable, so none of them change its type. ReadOnly can wrap or be wrapped by the others. + annotation = strip_read_only(annotation) if get_typehint_origin(annotation) in not_required_required_types: - return annotation.__args__[0] + annotation = strip_read_only(annotation.__args__[0]) return annotation +def _is_namedtuple_class(typehint) -> bool: + return inspect.isclass(typehint) and issubclass(typehint, tuple) and hasattr(typehint, "_fields") + + +def get_namedtuple_type(typehint): + """Returns the NamedTuple that a subscripted generic NamedTuple stands for, or the type hint unchanged. + + Its type parameters don't change which fields there are, only the types of + the ones annotated with a TypeVar. + """ + origin = getattr(typehint, "__origin__", None) + return origin if _is_namedtuple_class(origin) else typehint + + +def is_namedtuple(typehint) -> bool: + """Whether a type hint is a NamedTuple, i.e. a tuple subclass that has named fields. + + Includes a subscripted generic one, see get_namedtuple_type. + """ + return _is_namedtuple_class(get_namedtuple_type(typehint)) + + +def is_structured_value_type(typehint) -> bool: + """Whether a type hint is a structure of named keys or fields, i.e. a TypedDict or a NamedTuple. + + Their value is not a class to instantiate, so it is never given as a class + path and a ``--*.help`` option refers to them by name. + """ + return is_typed_dict(typehint) or is_namedtuple(typehint) + + +def get_namedtuple_annotations(namedtuple, logger=None) -> dict: + """Returns the resolved annotation of each field of a NamedTuple. + + A field without an annotation, i.e. from an untyped ``collections.namedtuple``, + accepts any value, the same as an unparameterized ``dict``. + """ + from ._postponed_annotations import get_global_vars + + typehint = namedtuple + namedtuple = get_namedtuple_type(typehint) + type_var_map = get_type_var_map(typehint, namedtuple) + annotations: dict = {} + for cls in reversed(namedtuple.__mro__): + annotations.update(getattr(cls, "__annotations__", None) or {}) + annotations = {f: annotations[f] for f in namedtuple._fields if f in annotations} + global_vars = get_global_vars(namedtuple, logger) + resolved = resolve_module_annotations(namedtuple.__module__, annotations, global_vars, logger) + return {f: resolve_annotation_type_vars(resolved.get(f, Any), type_var_map) for f in namedtuple._fields} + + +def get_namedtuple_value_as_dict(val, namedtuple, fields) -> dict: + """Returns the fields given for a NamedTuple as a dict, from any of its accepted spellings.""" + if isinstance(val, namedtuple): + return val._asdict() + if isinstance(val, (list, tuple)): + if len(val) > len(fields): + raise_unexpected_value(f"Expected at most {len(fields)} values", val) + return dict(zip(fields, val)) + if not isinstance(val, dict): + raise_unexpected_value(f"Expected a NamedTuple {namedtuple.__name__}, given as an array or an object", val) + return val.copy() + + def is_typed_dict_subtype(subtype, typed_dict, logger=None) -> bool: # TypedDicts don't support issubclass, so as specified in PEP 589 the check is done # structurally, i.e. the subtype must have all keys of the typed dict, with the same @@ -1545,11 +1636,43 @@ def adapt_typehints( elif typehint_origin is OrderedDict: val = dict(val) if serialize else OrderedDict(val) - # TypedDict NotRequired and Required - elif typehint_origin in not_required_required_types: - assert len(subtypehints) == 1, "(Not)Required requires a single type argument" + # TypedDict Required, NotRequired and ReadOnly + elif typehint_origin in typed_dict_key_qualifiers: + assert len(subtypehints) == 1, "A TypedDict key qualifier requires a single type argument" val = adapt_typehints(val, subtypehints[0], **adapt_kwargs) + # NamedTuple + elif is_namedtuple(typehint): + annotations = get_namedtuple_annotations(typehint, logger) + # a subscripted generic NamedTuple is built as its unsubscripted form, see get_namedtuple_type + typehint = get_namedtuple_type(typehint) + fields = typehint._fields + if isinstance(val, NestedArg): + prev = prev_val._asdict() if isinstance(prev_val, typehint) else prev_val + field, field_val = val.key, val.val + if isinstance(field, str) and "." in field: + # kept as a NestedArg, so that the field merges it with its own previous value + field, sub_key = field.split(".", 1) + field_val = NestedArg(key=sub_key, val=field_val) + val = {**prev, field: field_val} if isinstance(prev, dict) else {field: field_val} + val = get_namedtuple_value_as_dict(val, typehint, fields) + extra_fields = val.keys() - set(fields) + if extra_fields: + raise_unexpected_value(f"Unexpected fields: {extra_fields}", val) + missing_fields = set(fields) - typehint._field_defaults.keys() - val.keys() + if missing_fields: + raise_unexpected_value(f"Missing required fields: {missing_fields}", val) + for k, v in val.items(): + kwargs = adapt_kwargs.copy() + if kwargs.get("prev_val"): + prev_field = kwargs["prev_val"] + prev_field = prev_field._asdict() if isinstance(prev_field, typehint) else prev_field + kwargs["prev_val"] = prev_field.get(k) if isinstance(prev_field, dict) else None + # what can't be validated accepts any value, as the help shows it + val[k] = adapt_typehints(v, replace_unvalidatable_typehints(annotations[k]), **kwargs) + if not serialize: + val = typehint(**val) + # Callable elif ( typehint_origin in callable_origin_types @@ -1726,6 +1849,14 @@ def adapt_typehints( elif is_alias_type(typehint): return adapt_typehints(val, get_alias_target(typehint), **adapt_kwargs) + # NewType -- validated as the supertype that it stands for + elif is_new_type(typehint): + return adapt_typehints(val, get_new_type_supertype(typehint), **adapt_kwargs) + + # LiteralString -- at runtime there is no way to tell it apart from a str + elif is_literal_string(typehint): + return adapt_typehints(val, str, **adapt_kwargs) + else: raise RuntimeError(f"The code should never reach here: typehint={typehint}") # pragma: no cover @@ -2082,6 +2213,7 @@ def is_single_class_type(typehint, typehint_origin, closed_class): ) and typehint not in leaf_or_root_types and not is_typed_dict(typehint) + and not is_namedtuple(typehint) and not get_registered_type(typehint) and not is_pydantic_type(typehint) and not is_subclass(typehint, (Path, Enum)) @@ -2126,9 +2258,9 @@ def yield_class_types(typehint, is_single, also_lists=False, also_containers=Fal if subtype is not Ellipsis: yield from yield_class_types(subtype, **kwargs) if is_single(typehint, typehint_origin): - if is_typed_dict(typehint): - # a subscripted generic TypedDict is yielded as is, since its keys are - # resolved from it, substituting what it is subscripted with + if is_structured_value_type(typehint): + # a subscripted generic TypedDict or NamedTuple is yielded as is, since its keys or + # fields are resolved from it, substituting what it is subscripted with yield typehint else: # a subscripted user defined generic, e.g. Strategy[T], is yielded as its origin @@ -2162,7 +2294,7 @@ def get_subclass_or_closed_types(typehint, also_lists=False, callable_return=Fal def is_single_help_type(typehint, typehint_origin): - return is_typed_dict(typehint) or is_single_subclass_or_closed_type(typehint, typehint_origin) + return is_structured_value_type(typehint) or is_single_subclass_or_closed_type(typehint, typehint_origin) def get_help_types(typehint): @@ -2458,7 +2590,9 @@ def subclasses_disabled_remove_class_path(value): elif isinstance(val, list): value[key] = [subclasses_disabled_remove_class_path(item) for item in val] elif isinstance(val, tuple): - value[key] = tuple(subclasses_disabled_remove_class_path(item) for item in val) + items = [subclasses_disabled_remove_class_path(item) for item in val] + # a NamedTuple is rebuilt as itself, since it is not constructed from an iterable + value[key] = type(val)(*items) if is_namedtuple(type(val)) else tuple(items) if value.pop(subclasses_disabled_meta_key, False): init_args = Namespace({**value.get("init_args", {}), **value.get("dict_kwargs", {})}) diff --git a/jsonargparse/_util.py b/jsonargparse/_util.py index 59c2605a..b02a49cf 100644 --- a/jsonargparse/_util.py +++ b/jsonargparse/_util.py @@ -406,6 +406,14 @@ def unique(iterable): return unique_items +def iter_to_or_str(val) -> str: + """Joins the given strings into an enumeration, e.g. "a, b or c".""" + val = unique(val) + if len(val) == 1: + return str(val[0]) + return ", ".join(str(x) for x in val[:-1]) + f" or {val[-1]}" + + def iter_to_set_str(val, sep=","): val = unique(val) if len(val) == 1: diff --git a/jsonargparse_tests/test_completions_jsonschema.py b/jsonargparse_tests/test_completions_jsonschema.py index b3024817..0ed0a6ac 100644 --- a/jsonargparse_tests/test_completions_jsonschema.py +++ b/jsonargparse_tests/test_completions_jsonschema.py @@ -10,7 +10,21 @@ from calendar import Calendar from enum import Enum from importlib.util import find_spec -from typing import Any, Callable, Dict, List, Literal, Optional, Set, Tuple, Type, TypedDict, TypeVar, Union +from typing import ( + Any, + Callable, + Dict, + List, + Literal, + NamedTuple, + Optional, + Set, + Tuple, + Type, + TypedDict, + TypeVar, + Union, +) from unittest.mock import patch import pytest @@ -24,7 +38,7 @@ lazy_instance, set_parsing_settings, ) -from jsonargparse._typehints import NotRequired +from jsonargparse._typehints import NotRequired, ReadOnly from jsonargparse.typing import ( ClosedUnitInterval, Email, @@ -783,6 +797,46 @@ def test_typed_dict_not_required(parser): assert schema["required"] == ["title"] +@pytest.mark.skipif(not ReadOnly, reason="ReadOnly introduced in python 3.13 or backported in typing_extensions") +def test_typed_dict_read_only(parser): + parser.add_argument("--movie", type=TypedDict("Movie", {"title": ReadOnly[str], "year": NotRequired[int]})) + schema = get_schema(parser)["properties"]["movie"] + assert schema["properties"] == {"title": {"type": "string"}, "year": {"type": "integer"}} + assert schema["required"] == ["title"] + + +class Coordinate(NamedTuple): + """A coordinate.""" + + x: int + y: int = 3 + + +def test_namedtuple(parser): + parser.add_argument("--coord", type=Coordinate) + object_schema, array_schema = get_schema(parser)["properties"]["coord"]["anyOf"] + object_schema.pop("description", None) # only when docstring_parser is available, see below + assert object_schema == { + "type": "object", + "additionalProperties": False, + "properties": {"x": {"type": "integer"}, "y": {"type": "integer"}}, + "required": ["x"], + } + assert array_schema == { + "type": "array", + "prefixItems": [{"type": "integer"}, {"type": "integer"}], + "items": False, + "minItems": 1, + } + + +@skip_if_docstring_parser_unavailable +def test_namedtuple_description_from_docstring(parser): + parser.add_argument("--coord", type=Coordinate) + object_schema = get_schema(parser)["properties"]["coord"]["anyOf"][0] + assert object_schema["description"] == "A coordinate." + + # unions mixing structured and unvalidated types diff --git a/jsonargparse_tests/test_dataclasses.py b/jsonargparse_tests/test_dataclasses.py index 1b7457f9..8de2cd26 100644 --- a/jsonargparse_tests/test_dataclasses.py +++ b/jsonargparse_tests/test_dataclasses.py @@ -771,13 +771,23 @@ def test_deeply_nested_dataclass_in_union(parser): assert cfg.parent.path == Namespace(folder="/tmp", file=Namespace(name="data.txt")) +AliasVar = TypeVar("AliasVar") +BoundAliasVar = TypeVar("BoundAliasVar", bound=int) + if type_alias_type: IntOrString = type_alias_type("IntOrString", Union[int, str]) + ListOfVar = type_alias_type("ListOfVar", List[AliasVar], type_params=(AliasVar,)) # type: ignore[valid-type] + DictOfVar = type_alias_type("DictOfVar", Dict[str, ListOfVar[AliasVar]], type_params=(AliasVar,)) # type: ignore[valid-type] + ListOfBoundVar = type_alias_type("ListOfBoundVar", List[BoundAliasVar], type_params=(BoundAliasVar,)) # type: ignore[valid-type] @dataclasses.dataclass class DataClassWithAliasType: p1: IntOrString # type: ignore[valid-type] + @dataclasses.dataclass + class DataClassWithGenericAliasType: + p1: ListOfVar[int] # type: ignore[valid-type] + if annotated: @dataclasses.dataclass @@ -821,6 +831,47 @@ def test_annotated_alias_type(self, parser): cfg = parser.parse_args(["--data=3"]) assert cfg.data == 3 + def test_subscripted_generic_alias_type(self, parser): + parser.add_argument("--data", type=ListOfVar[int]) + help_str = get_parser_help(parser) + assert "type: ListOfVar[int]" in help_str + cfg = parser.parse_args(["--data=[1, 2]"]) + assert cfg.data == [1, 2] + assert json_or_yaml_load(parser.dump(cfg)) == {"data": [1, 2]} + with pytest.raises(ArgumentError) as ctx: + parser.parse_args(['--data=["x"]']) + ctx.match("Expected a ") + + def test_nested_subscripted_generic_alias_type(self, parser): + parser.add_argument("--data", type=DictOfVar[int]) + cfg = parser.parse_args(['--data={"a": [1, 2]}']) + assert cfg.data == {"a": [1, 2]} + with pytest.raises(ArgumentError) as ctx: + parser.parse_args(['--data={"a": ["x"]}']) + ctx.match("Expected a ") + + def test_unsubscripted_generic_alias_type_with_bound(self, parser): + parser.add_argument("--data", type=ListOfBoundVar) + assert "type: ListOfBoundVar" in get_parser_help(parser) + assert parser.parse_args(["--data=[1, 2]"]).data == [1, 2] + with pytest.raises(ArgumentError) as ctx: + parser.parse_args(['--data=["x"]']) + ctx.match("Expected a ") + + def test_unsubscripted_generic_alias_type_unbound(self, parser): + with pytest.raises(ValueError) as ctx: + parser.add_argument("--data", type=ListOfVar) + ctx.match("Unsupported type hint ListOfVar") + + def test_dataclass_with_generic_alias_type(self, parser): + parser.add_argument("--data", type=DataClassWithGenericAliasType) + assert "type: ListOfVar[int]" in get_parser_help(parser) + cfg = parser.parse_args(["--data.p1=[1, 2]"]) + assert cfg.data.p1 == [1, 2] + with pytest.raises(ArgumentError) as ctx: + parser.parse_args(['--data.p1=["x"]']) + ctx.match("Expected a ") + @pytest.mark.skipif(not annotated, reason="Annotated is required") def test_dataclass_with_annotated_alias_type(self, parser): parser.add_argument("--data", type=DataClassWithAnnotatedAliasType) diff --git a/jsonargparse_tests/test_namespace.py b/jsonargparse_tests/test_namespace.py index 4271c93d..9ea2fbe6 100644 --- a/jsonargparse_tests/test_namespace.py +++ b/jsonargparse_tests/test_namespace.py @@ -173,6 +173,18 @@ def test_as_dict(): assert Namespace().as_dict() == {} +def test_as_dict_namespaces_mixed_with_other_values(): + ns = Namespace() + ns["a"] = {"n": Namespace(r=1), "s": "str"} + ns["b"] = [Namespace(r=2), "str"] + ns["c"] = {"deep": [{"n": Namespace(r=3)}]} + assert ns.as_dict() == { + "a": {"n": {"r": 1}, "s": "str"}, + "b": [{"r": 2}, "str"], + "c": {"deep": [{"n": {"r": 3}}]}, + } + + def test_as_flat(): ns = Namespace() ns["w"] = 1 diff --git a/jsonargparse_tests/test_shtab.py b/jsonargparse_tests/test_shtab.py index 4c9d2e6d..7f5ccd6f 100644 --- a/jsonargparse_tests/test_shtab.py +++ b/jsonargparse_tests/test_shtab.py @@ -13,7 +13,7 @@ from importlib.util import find_spec from os import PathLike from pathlib import Path -from typing import Any, Callable, Dict, Generic, List, Literal, Optional, TypedDict, TypeVar, Union +from typing import Any, Callable, Dict, Generic, List, Literal, NamedTuple, Optional, TypedDict, TypeVar, Union from unittest.mock import patch import pytest @@ -631,6 +631,18 @@ def test_bash_typed_dict_help_choices(parser): assert choices == ["AreaDict", f"{__name__}.Base", f"{__name__}.SubA", f"{__name__}.SubB"] +class AreaTuple(NamedTuple): + latitude: float + longitude: float + + +def test_bash_namedtuple_help_choices(parser): + parser.add_argument("--area", type=Union[AreaTuple, Base]) + shtab_script = get_shtab_script(parser, "bash") + choices = get_bash_array(shtab_script, "_shtab_tool___area_help_choices") + assert choices == ["AreaTuple", f"{__name__}.Base", f"{__name__}.SubA", f"{__name__}.SubB"] + + PointVar = TypeVar("PointVar") if sys.version_info >= (3, 11): # a generic TypedDict requires python 3.11 or later @@ -675,6 +687,23 @@ def test_bash_typed_dict_key_types(parser, subtests): ) +class OptionsTuple(NamedTuple): + verbose: bool = False + mode: AXEnum = AXEnum.XY + + +def test_bash_namedtuple_field_types(parser, subtests): + parser.add_argument("--opts", type=OptionsTuple) + assert_bash_typehint_completions( + subtests, + parser, + [ + ("opts.verbose", bool, "", ["true", "false"], "2/2"), + ("opts.mode", AXEnum, "X", ["XY", "XZ"], "2/3"), + ], + ) + + def test_bash_typed_dict_in_union_key_types(parser, subtests): parser.add_argument("--opts", type=Union[OptionsDict, Base]) assert_bash_typehint_completions( diff --git a/jsonargparse_tests/test_signatures.py b/jsonargparse_tests/test_signatures.py index a1938bfd..0d5b472f 100644 --- a/jsonargparse_tests/test_signatures.py +++ b/jsonargparse_tests/test_signatures.py @@ -508,6 +508,8 @@ def test_add_class_generics(parser): parser.add_class_arguments(WithGenerics[int, complex], "p") cfg = parser.parse_args(["--p.a=5", "--p.b=(6+7j)"]) assert cfg.p == Namespace(a=5, b=6 + 7j) + # a subscripted generic is instantiated as the class that it stands for + assert isinstance(parser.instantiate(cfg).p, WithGenerics) class WithGenericsDocstring(Generic[X]): diff --git a/jsonargparse_tests/test_typehints.py b/jsonargparse_tests/test_typehints.py index 89951394..8fc05fe3 100644 --- a/jsonargparse_tests/test_typehints.py +++ b/jsonargparse_tests/test_typehints.py @@ -8,7 +8,7 @@ import sys import time import uuid -from collections import OrderedDict, abc, deque +from collections import OrderedDict, abc, deque, namedtuple from contextlib import contextmanager from dataclasses import dataclass, field from datetime import date @@ -35,6 +35,8 @@ Mapping, MutableMapping, MutableSequence, + NamedTuple, + NewType, NoReturn, Optional, Protocol, @@ -54,10 +56,11 @@ import pytest from jsonargparse import ArgumentError, Namespace, lazy_instance -from jsonargparse._optionals import pyyaml_available, typing_extensions_support +from jsonargparse._optionals import LiteralString, pyyaml_available, typing_extensions_support from jsonargparse._typehints import ( ActionTypeHint, NotRequired, + ReadOnly, Required, Unpack, UnvalidatedType, @@ -1172,6 +1175,359 @@ def test_required_support(): assert ActionTypeHint.is_supported_typehint(Required[Any]) +skip_if_no_read_only = pytest.mark.skipif( + not ReadOnly, reason="ReadOnly introduced in python 3.13 or backported in typing_extensions" +) + +if ReadOnly: + # both nestings are valid, see PEP 705 + read_only_not_required = [ReadOnly[NotRequired[int]], NotRequired[ReadOnly[int]]] + read_only_required = [ReadOnly[Required[int]], Required[ReadOnly[int]]] +else: + read_only_not_required = read_only_required = [None] # the tests that use them are skipped + + +@skip_if_no_read_only +def test_read_only_support(): + assert ActionTypeHint.is_supported_typehint(ReadOnly[Any]) + + +@skip_if_no_read_only +def test_typeddict_with_read_only_arg(parser): + parser.add_argument("--typeddict", type=TypedDict("MyDict", {"a": ReadOnly[int], "b": int})) + assert {"a": 1, "b": 2} == parser.parse_args(['--typeddict={"a": 1, "b": 2}'])["typeddict"] + with pytest.raises(ArgumentError) as ctx: + parser.parse_args(['--typeddict={"a": "x", "b": 2}']) + ctx.match("Expected a ") + with pytest.raises(ArgumentError) as ctx: + parser.parse_args(['--typeddict={"b": 2}']) + ctx.match("Missing required keys") + + +@skip_if_no_read_only +@pytest.mark.parametrize("annotation", read_only_not_required, ids=type_to_str) +def test_typeddict_read_only_not_required(parser, annotation): + parser.add_argument("--typeddict", type=TypedDict("MyDict", {"a": annotation, "b": int})) + assert {"b": 2} == parser.parse_args(['--typeddict={"b": 2}'])["typeddict"] + assert {"a": 1, "b": 2} == parser.parse_args(['--typeddict={"a": 1, "b": 2}'])["typeddict"] + with pytest.raises(ArgumentError) as ctx: + parser.parse_args(['--typeddict={"a": "x", "b": 2}']) + ctx.match("Expected a ") + + +@skip_if_no_read_only +@pytest.mark.parametrize("annotation", read_only_required, ids=type_to_str) +def test_typeddict_read_only_required(parser, annotation): + parser.add_argument("--typeddict", type=TypedDict("MyDict", {"a": annotation}, total=False)) + assert {"a": 1} == parser.parse_args(['--typeddict={"a": 1}'])["typeddict"] + with pytest.raises(ArgumentError) as ctx: + parser.parse_args(["--typeddict={}"]) + ctx.match("Missing required keys") + + +@skip_if_no_read_only +def test_typeddict_read_only_add_class_arguments(parser): + parser.add_class_arguments(TypedDict("MyDict", {"a": ReadOnly[int]}), "data") + assert "--data.a A (required, type: int)" in get_parser_help(parser) + assert {"a": 1} == parser.instantiate(parser.parse_args(["--data.a=1"])).data + with pytest.raises(ArgumentError) as ctx: + parser.parse_args(["--data.a=x"]) + ctx.match("Expected a ") + + +@skip_if_no_read_only +def test_typeddict_read_only_subtype(): + read_only = TypedDict("MyDict", {"a": ReadOnly[int]}) + read_write = TypedDict("MyDict", {"a": int}) + assert is_typed_dict_subtype(read_write, read_only) + assert is_typed_dict_subtype(read_only, read_write) + assert not is_typed_dict_subtype(TypedDict("Other", {"a": ReadOnly[str]}), read_only) + + +# NamedTuple tests + + +class Coord(NamedTuple): + """A coordinate. + + Args: + x: the x + y: the y + """ + + x: int + y: int = 3 + label: Optional[str] = None + + +class NestedCoord(NamedTuple): + name: str = "n" + coord: Coord = Coord(1, 2) + + +UntypedCoord = namedtuple("UntypedCoord", ["x", "y"]) + + +class CoordHolder: + def __init__(self, coord: Coord = Coord(1, 2)): + self.coord = coord + + +@parser_modes +def test_namedtuple_from_object(parser): + parser.add_argument("--coord", type=Coord) + cfg = parser.parse_args(['--coord={"x": 1, "y": 2, "label": "a"}']) + assert cfg.coord == Coord(1, 2, "a") + assert isinstance(cfg.coord, Coord) + assert parser.parse_args(['--coord={"x": 1}']).coord == Coord(1, 3, None) + + +def test_namedtuple_from_array(parser): + parser.add_argument("--coord", type=Coord) + assert parser.parse_args(["--coord=[1, 2]"]).coord == Coord(1, 2, None) + assert parser.parse_args(["--coord=[1]"]).coord == Coord(1, 3, None) + + +def test_namedtuple_instance_as_default_and_dump(parser): + parser.add_argument("--coord", type=Coord, default=Coord(1, 2)) + cfg = parser.parse_args([]) + assert cfg.coord == Coord(1, 2, None) + # always dumped as an object, so that the fields are named + assert json_or_yaml_load(parser.dump(cfg)) == {"coord": {"x": 1, "y": 2, "label": None}} + cfg = parser.parse_args(["--coord=[4, 5]"]) + assert json_or_yaml_load(parser.dump(cfg)) == {"coord": {"x": 4, "y": 5, "label": None}} + + +def test_namedtuple_invalid_values(parser): + parser.add_argument("--coord", type=Coord) + with pytest.raises(ArgumentError, match="Missing required fields: {'x'}"): + parser.parse_args(['--coord={"y": 2}']) + with pytest.raises(ArgumentError, match="Unexpected fields: {'z'}"): + parser.parse_args(['--coord={"x": 1, "z": 2}']) + with pytest.raises(ArgumentError, match="Expected at most 3 values"): + parser.parse_args(['--coord=[1, 2, "a", 4]']) + with pytest.raises(ArgumentError, match="Expected a "): + parser.parse_args(['--coord={"x": "not an int"}']) + with pytest.raises(ArgumentError, match="Expected a NamedTuple Coord"): + parser.parse_args(["--coord=1"]) + + +def test_namedtuple_nested_arg(parser): + parser.add_argument("--coord", type=Coord) + assert parser.parse_args(["--coord.x=1", "--coord.y=2"]).coord == Coord(1, 2, None) + + +def test_namedtuple_nested_arg_over_default(parser): + parser.add_argument("--nested", type=NestedCoord, default=NestedCoord()) + # the fields not given keep the value they have in the default + cfg = parser.parse_args(["--nested.name=z", "--nested.coord.y=7"]) + assert cfg.nested == NestedCoord("z", Coord(1, 7, None)) + assert json_or_yaml_load(parser.dump(cfg))["nested"]["coord"] == {"x": 1, "y": 7, "label": None} + + +def test_namedtuple_in_list(parser): + parser.add_argument("--coords", type=List[Coord]) + cfg = parser.parse_args(['--coords=[{"x": 1}, [2, 4]]']) + assert cfg.coords == [Coord(1, 3, None), Coord(2, 4, None)] + assert json_or_yaml_load(parser.dump(cfg))["coords"][1] == {"x": 2, "y": 4, "label": None} + + +def test_namedtuple_in_union(parser): + parser.add_argument("--val", type=Optional[Union[Coord, int]], default=None) + assert parser.parse_args([]).val is None + assert parser.parse_args(["--val=3"]).val == 3 + assert parser.parse_args(['--val={"x": 1}']).val == Coord(1, 3, None) + + +def test_namedtuple_untyped_fields_accept_any(parser): + parser.add_argument("--coord", type=UntypedCoord) + assert parser.parse_args(['--coord={"x": "a", "y": 2}']).coord == UntypedCoord("a", 2) + with pytest.raises(ArgumentError) as ctx: + parser.parse_args(['--coord={"x": 1}']) + ctx.match("Missing required fields: {'y'}") + + +def test_namedtuple_signature_parameter(parser): + parser.add_class_arguments(CoordHolder, "holder") + assert "--holder.coord.help" in get_parser_help(parser) + cfg = parser.parse_args([]) + assert cfg.holder.coord == Coord(1, 2, None) + assert json_or_yaml_load(parser.dump(cfg)) == {"holder": {"coord": {"x": 1, "y": 2, "label": None}}} + cfg = parser.parse_args(['--holder.coord={"x": 5, "y": 6}']) + assert cfg.holder.coord == Coord(5, 6, None) + assert isinstance(parser.instantiate(cfg).holder, CoordHolder) + + +def test_namedtuple_add_class_arguments(parser): + parser.add_class_arguments(Coord, "coord") + cfg = parser.parse_args(["--coord.x=1"]) + assert cfg.coord == Namespace(x=1, y=3, label=None) + assert isinstance(parser.instantiate(cfg).coord, Coord) + + +class CoordEngine: + def __init__(self, power: int = 10): + self.power = power + + +class PoweredCoord(NamedTuple): + name: str + engine: CoordEngine = CoordEngine() + + +def test_namedtuple_class_field_instantiate(parser): + parser.add_argument("--car", type=PoweredCoord) + engine = f'{{"class_path": "{__name__}.CoordEngine", "init_args": {{"power": 5}}}}' + cfg = parser.parse_args([f'--car={{"name": "a", "engine": {engine}}}']) + assert cfg.car.engine == Namespace(class_path=f"{__name__}.CoordEngine", init_args=Namespace(power=5)) + assert json.loads(parser.dump(cfg, format="json"))["car"]["engine"]["init_args"] == {"power": 5} + init = parser.instantiate(cfg) + assert isinstance(init.car, PoweredCoord) + assert isinstance(init.car.engine, CoordEngine) + assert init.car.engine.power == 5 + + +# NewType tests. A NewType stands for its supertype, so it is validated as the +# supertype while the help keeps the name given in the source code. + + +UserId = NewType("UserId", int) +Vector = NewType("Vector", List[float]) +NestedUserId = NewType("NestedUserId", UserId) +CalendarType = NewType("CalendarType", calendar.Calendar) +UnsupportedNewType = NewType("UnsupportedNewType", Iterator[int]) # type: ignore[misc] + + +def test_new_type(parser): + parser.add_argument("--x", type=UserId) + assert "(type: UserId, default: null)" in get_parser_help(parser) + assert parser.parse_args(["--x=1"]).x == 1 + with pytest.raises(ArgumentError) as ctx: + parser.parse_args(["--x=abc"]) + ctx.match("Expected a ") + + +def test_new_type_of_container(parser): + parser.add_argument("--x", type=Vector) + assert parser.parse_args(["--x=[1.0, 2.0]"]).x == [1.0, 2.0] + with pytest.raises(ArgumentError) as ctx: + parser.parse_args(['--x=["a"]']) + ctx.match("Expected a ") + + +def test_new_type_of_new_type(parser): + parser.add_argument("--x", type=NestedUserId) + assert "(type: NestedUserId, default: null)" in get_parser_help(parser) + assert parser.parse_args(["--x=1"]).x == 1 + with pytest.raises(ArgumentError) as ctx: + parser.parse_args(["--x=abc"]) + ctx.match("Expected a ") + + +def test_new_type_nested_in_container(parser): + parser.add_argument("--x", type=Dict[str, UserId]) + assert parser.parse_args(['--x={"a": 1}']).x == {"a": 1} + with pytest.raises(ArgumentError) as ctx: + parser.parse_args(['--x={"a": "b"}']) + ctx.match("Expected a ") + + +def test_optional_new_type(parser): + parser.add_argument("--x", type=Optional[UserId]) + assert parser.parse_args(["--x=1"]).x == 1 + assert parser.parse_args(["--x=null"]).x is None + with pytest.raises(ArgumentError) as ctx: + parser.parse_args(["--x=abc"]) + ctx.match("Expected a ") + + +def test_new_type_of_class(parser): + parser.add_argument("--x", type=CalendarType) + cfg = parser.parse_args(["--x=calendar.Calendar"]) + assert cfg.x.class_path == "calendar.Calendar" + assert isinstance(parser.instantiate(cfg).x, calendar.Calendar) + + +def test_new_type_signature_parameter(parser): + def func(x: UserId = UserId(1)): + return x # pragma: no cover + + parser.add_function_arguments(func) + assert "(type: UserId, default: 1)" in get_parser_help(parser) + assert parser.parse_args(["--x=2"]).x == 2 + with pytest.raises(ArgumentError) as ctx: + parser.parse_args(["--x=abc"]) + ctx.match("Expected a ") + + +def test_new_type_dump(parser): + parser.add_argument("--x", type=Vector) + cfg = parser.parse_args(["--x=[1.0]"]) + assert json_or_yaml_load(parser.dump(cfg)) == {"x": [1.0]} + + +def test_new_type_of_unsupported_supertype(parser): + def func(x: Optional[UnsupportedNewType] = None): + return x # pragma: no cover + + parser.add_function_arguments(func) + assert "Unvalidated" in get_parser_help(parser) + assert parser.parse_args(["--x=any value"]).x == "any value" + + +# LiteralString tests. At runtime it is a str, so it is validated as one. + + +@pytest.mark.skipif( + not LiteralString, reason="LiteralString introduced in python 3.11 or backported in typing_extensions" +) +def test_literal_string(parser): + parser.add_argument("--x", type=LiteralString) + assert "(type: LiteralString, default: null)" in get_parser_help(parser) + assert parser.parse_args(["--x=abc"]).x == "abc" + + +# subscripted generic NamedTuple tests + +# a generic NamedTuple requires python 3.11 or later +skip_if_no_generic_namedtuple = pytest.mark.skipif( + sys.version_info < (3, 11), reason="generic NamedTuple introduced in python 3.11" +) + +if sys.version_info >= (3, 11): + + class GenericCoord(NamedTuple, Generic[GenericVar]): + """Generic coordinate.""" + + item: GenericVar + n: int = 1 + + +@skip_if_no_generic_namedtuple +def test_subscripted_generic_namedtuple(parser): + parser.add_argument("--coord", type=GenericCoord[int]) + assert parser.parse_args(['--coord={"item": 5}']).coord == GenericCoord(5, 1) + with pytest.raises(ArgumentError) as ctx: + parser.parse_args(['--coord={"item": "x"}']) + ctx.match("Expected a ") + help_str = get_parse_args_stdout(parser, ["--coord.help"]) + assert "--coord.item ITEM (required, type: int)" in help_str + + +@skip_if_no_generic_namedtuple +def test_unsubscripted_generic_namedtuple(parser): + # the field annotated with an unbound TypeVar accepts any value + parser.add_argument("--coord", type=GenericCoord) + assert parser.parse_args(['--coord={"item": "anything"}']).coord == GenericCoord("anything", 1) + + +@skip_if_no_generic_namedtuple +def test_subscripted_generic_namedtuple_add_class_arguments(parser): + parser.add_class_arguments(GenericCoord[int], "coord") + assert "--coord.item ITEM (required, type: int)" in get_parser_help(parser) + assert parser.instantiate(parser.parse_args(["--coord.item=3"])).coord == GenericCoord(3, 1) + + # unsubscripted typing alias tests @@ -1540,6 +1896,73 @@ def test_typeddict_union_help_unexpected_name(parser): ctx.match('"Unexpected" is not a typed dict') +# NamedTuple --*.help tests + + +def test_namedtuple_help(parser): + parser.add_argument("--coord", type=Coord) + help_str = get_parser_help(parser) + assert "Show the help for Coord and exit" in help_str + assert "CLASS_PATH_OR_NAME" not in help_str # a named tuple is a value, not a subclass type + assert "(type: , default: null)" in help_str + help_str = get_parse_args_stdout(parser, ["--coord.help"]) + assert f"Help for --coord.help={__name__}.Coord" in help_str + assert "--coord.x X" in help_str + assert "(required, type: int)" in help_str + assert "--coord.y Y" in help_str + assert "(type: int, default: 3)" in help_str + + +@skip_if_docstring_parser_unavailable +def test_namedtuple_help_docstrings(parser): + parser.add_argument("--coord", type=Coord) + help_str = get_parse_args_stdout(parser, ["--coord.help"]) + assert "A coordinate:" in help_str + assert "the x (required, type: int)" in help_str + assert "the y (type: int, default: 3)" in help_str + + +@pytest.mark.parametrize("typehint", [Optional[Coord], List[Coord]], ids=type_to_str) +def test_namedtuple_in_container_help(parser, typehint): + parser.add_argument("--coord", type=typehint) + help_str = get_parse_args_stdout(parser, ["--coord.help"]) + assert f"Help for --coord.help={__name__}.Coord" in help_str + assert "--coord.x X" in help_str + + +@pytest.mark.parametrize( + ["typehint", "kind"], + [ + (Union[Coord, NestedCoord], "named tuple"), + (Union[Coord, HelpTypedDict], "typed dict or named tuple"), + ], + ids=["named_tuples", "typed_dict_and_named_tuple"], +) +def test_namedtuple_union_named_types_help(parser, typehint, kind): + parser.add_argument("--val", type=typehint) + help_str = get_parser_help(parser) + assert "--val.help NAME" in help_str + assert f"Show the help for the given {kind}" in help_str + help_str = get_parse_args_stdout(parser, ["--val.help=Coord"]) + assert f"Help for --val.help={__name__}.Coord" in help_str + assert "--val.x X" in help_str + with pytest.raises(ArgumentError) as ctx: + parser.parse_args(["--val.help=Unexpected"]) + ctx.match(f'"Unexpected" is not a {kind}') + + +def test_namedtuple_union_class_help(parser): + parser.add_argument("--val", type=Union[Coord, BaseC]) + help_str = get_parser_help(parser) + assert "--val.help CLASS_PATH_OR_NAME" in help_str + assert "Show the help for the given class or named tuple" in help_str + help_str = get_parse_args_stdout(parser, ["--val.help=Coord"]) + assert f"Help for --val.help={__name__}.Coord" in help_str + assert "--val.x X" in help_str + help_str = get_parse_args_stdout(parser, [f"--val.help={__name__}.SubC"]) + assert f"Help for --val.help={__name__}.SubC" in help_str + + # type[TypedDict] tests. TypedDicts don't support issubclass, so the check is structural. @@ -2431,7 +2854,7 @@ def test_callable_function_path(parser): def make_closure_callable(): def unbound_closure(): - return "closure" + return "closure" # pragma: no cover return unbound_closure