Python recursive override path function and mixin class
Here’s a recursive utility function in Python that allows overriding nested paths through objects that can be mutable mappings, mutable sequences or dataclass instances.
The function allows doing things like this:
override_paths(
{
"foo0": "bar",
"lvl1": {
"foo1": "bar1",
"list1": [1, 2, 3, 4, 5, 6],
"foobar1": Foobar(foo="hello1"),
},
},
foo0="123",
lvl1__foo1="456",
lvl1__list1__3=23,
lvl1__foobar1__foo="hello2",
)
{
"foo0": "123",
"lvl1": {
"foo1": "456",
"list1": [1, 2, 3, 23, 5, 6],
"foobar1": Foobar(foo="hello2"),
},
}
There is also a OverridableMixin
that makes it convenient to do things like
this inline:
Foobar().overridden(foo__bar__baz=123)
import dataclasses
from copy import deepcopy
from dataclasses import is_dataclass
from itertools import chain
from typing import (
TYPE_CHECKING,
Any,
ClassVar,
MutableMapping,
MutableSequence,
Protocol,
Self,
TypeVar,
)
class Dataclass(Protocol):
"""Identifies an instance of any dataclass."""
__dataclass_fields__: ClassVar[dict[str, Any]]
if TYPE_CHECKING:
Overridable = MutableMapping | MutableSequence | Dataclass # pragma: no cover
else:
# Having to add `| object` to the end of this union to keep PyCharm happy as
# the PyCharm type checker can't figure out the Dataclass Protocol above.
# Doing this inside a TYPE_CHECKING condition so that mypy still validates
# the protocol correctly.
Overridable = MutableMapping | MutableSequence | Dataclass | object
# Using TypeVar instead of the inline type parameter syntax because pydocstyle
# seems unable to cope with the newer syntax.
T = TypeVar("T", bound=Overridable)
def override_paths(
target: T,
*args: tuple[str | list[str], Any],
**kwargs: Any,
) -> T:
"""
Override keys, indexes or fields specified via a flat path syntax.
E.g.
override_paths({"foo": {"bar": 123}}, foo__bar=456)
{"foo": {"bar": 456}}
The override paths can be given as either kwargs, or as args as tuples with
the path and the value as pairs. The path can be a __ string or a list of
strings.
override_paths({"foo": {"bar": 123}}, ("foo__bar", 456))
{"foo": {"bar": 456}}
override_paths({"foo": {"bar": 123}}, (["foo, "bar"], 456))
{"foo": {"bar": 456}}
Mutable mappings, mutable sequences and dataclass instances can be
overridden.
"""
target = deepcopy(target)
for key_path, value in chain(args, kwargs.items()):
if not key_path:
return value
if isinstance(key_path, str):
key_path: list[str] = key_path.split("__") # type: ignore[no-redef]
path_entry: str = key_path[0]
if isinstance(target, MutableMapping):
if path_entry not in target:
raise KeyError(f"Key '{path_entry}' not found in target")
target[path_entry] = override_paths(
target[path_entry],
(key_path[1:], value),
)
continue
if path_entry.isdigit() and isinstance(target, MutableSequence):
override_index = int(path_entry)
if len(target) - 1 < override_index:
raise IndexError(f"index {override_index} out of range in target")
target[override_index] = override_paths(
target[int(path_entry)],
(key_path[1:], value),
)
continue
if has_dataclass_field(target, path_entry):
setattr(
target,
path_entry,
override_paths(
getattr(target, path_entry),
(key_path[1:], value),
),
)
continue
raise KeyError(f"Key / index / field '{path_entry}' not found in target")
return target
def has_dataclass_field(
target: Any,
field_name: str,
) -> bool:
"""
Check if an object is a dataclass that has a dataclass field with the given
name.
"""
return is_dataclass(target) and any(
True for f in dataclasses.fields(target) if f.name == field_name
)
@dataclasses.dataclass
class OverridableMixin:
"""
Mixin class which adds a convenience method to override paths on the
instance and get back an overridden copy.
This makes it convenient to override paths inline with a new instance
without needing a variable referencing the instance, e.g.
Foobar().overridden(foo__bar__baz=123)
Can be dropped in place to get an instance of Foobar with that path
overridden with the new value.
"""
def overridden(
self,
**kwargs: Any,
) -> Self:
"""
Get a copy of this instance with the given paths overridden.
"""
return override_paths(self, **kwargs)