Diff Coverage

Diff: origin/master...HEAD, staged and unstaged changes

Source File Diff Coverage (%) Missing Lines
hyper_parallel/trainer/config/parser.py 86.7% 51,56,90-93
hyper_parallel/trainer/config/resolver.py 65.1% 56,60,69,74,78,85,88,108-109,135,141,159,175,211,233-235,237,249,252,272-275,282-285,295,297,313,360-361,394,398,416-417,420,450,459,464,473,478,491,509,543,545,552,559,581,586,592,621,650,676,690,697,702,711,725
hyper_parallel/trainer/config/parser.py
47
48
49
50
51
52
53
54
55
56
57
58
59
                "CLI", f"expected a dotted override in '--field=value' form, got {token!r}"
            )
        option = token[2:]
        if "=" not in option:
            raise ConfigResolutionError(
                "CLI", f"expected a dotted override in '--field=value' form, got {token!r}"
            )
        path, raw_value = option.split("=", 1)
        if not path or any(not part for part in path.split(".")):
            raise ConfigResolutionError("CLI", f"invalid override path {path!r}")
        parsed.append((path, raw_value))
    return parsed


86
87
88
89
90
91
92
93
94
95
96

    config_path = Path(config_file)
    try:
        raw = yaml.safe_load(config_path.read_text(encoding="utf-8"))
    except OSError as exc:
        raise ConfigResolutionError(config_path, f"could not read config file: {exc}") from exc
    except yaml.YAMLError as exc:
        raise ConfigResolutionError(config_path, f"invalid YAML: {exc}") from exc

    config = resolve_config(raw)
    return _apply_typed_overrides(config, cli_overrides)
hyper_parallel/trainer/config/resolver.py
52
53
54
55
56
57
58
59
60
61
62
63
64
def import_target(target_path: str, *, location: str) -> object:
    """Import a dotted callable, including nested callable attributes."""

    if not isinstance(target_path, str) or not target_path.strip():
        raise ConfigResolutionError(location, "_target_ must be a non-empty dotted path")

    parts = target_path.split(".")
    if any(not part for part in parts):
        raise ConfigResolutionError(location, f"invalid target path {target_path!r}")

    for split_at in range(len(parts), 0, -1):
        module_name = ".".join(parts[:split_at])
        try:

65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
            target = importlib.import_module(module_name)
        except ModuleNotFoundError as exc:
            if exc.name == module_name or module_name.startswith(f"{exc.name}."):
                continue
            raise ConfigResolutionError(
                location,
                f"target {target_path!r} failed while importing dependency {exc.name!r}",
            ) from exc
        except ImportError as exc:
            raise ConfigResolutionError(location, f"target {target_path!r} could not be imported: {exc}") from exc

        for attribute in parts[split_at:]:
            if not hasattr(target, attribute):
                raise ConfigResolutionError(
                    location,
                    f"target {target_path!r} has no attribute {attribute!r}",
                )
            target = getattr(target, attribute)

81
82
83
84
85
86
87
88
89
90
91
92
                )
            target = getattr(target, attribute)

        if not callable(target):
            raise ConfigResolutionError(location, f"target {target_path!r} is not callable")
        return target

    raise ConfigResolutionError(location, f"target {target_path!r} could not be imported")


def _is_union(annotation: object) -> bool:
    return get_origin(annotation) in (Union, types.UnionType)

104
105
106
107
108
109
110
111
112
113
    """Resolve annotations for a target function or class constructor."""
    hint_source = target.__init__ if inspect.isclass(target) else target
    try:
        return get_type_hints(hint_source)
    except (NameError, TypeError) as exc:
        raise ConfigResolutionError(path, f"could not resolve target type annotations: {exc}") from exc


# Annotations whose payload is a container of other values.
_COLLECTION_TYPES: tuple[type, ...] = (list, tuple, dict, Mapping)

131
132
133
134
135
136
137
138
139
    ``tuple[int, ...]``).  An empty ``item_types`` means a bare ``tuple``, which
    is passed through without normalizing its items.
    """
    if not isinstance(value, (list, tuple)):
        raise ConfigResolutionError(path, f"expected {kind}, got {type(value).__name__}")
    if not item_types:
        return tuple(value)
    if uniform:
        item_types = item_types * len(value)

137
138
139
140
141
142
143
144
145
        return tuple(value)
    if uniform:
        item_types = item_types * len(value)
    if len(item_types) != len(value):
        raise ConfigResolutionError(
            path,
            f"expected {kind} of length {len(item_types)}, got {len(value)}",
        )
    normalized = [

155
156
157
158
159
160
161
162
163
    if annotation is types.NoneType or (
        _is_union(annotation) and types.NoneType in get_args(annotation)
    ):
        return None
    raise ConfigResolutionError(path, f"expected {_annotation_name(annotation)}, got None")


def _normalize_union(value: object, annotation: object, *, path: str) -> object:
    """Normalize a value against an ``Optional`` or general union."""

171
172
173
174
175
176
177
178
        try:
            return normalize_value(value, member, path=path)
        except ConfigResolutionError:
            continue
    raise ConfigResolutionError(
        path,
        f"expected {_annotation_name(annotation)}, got {type(value).__name__}",
    )

207
208
209
210
211
212
213
214
215
                return word
    if any(type(value) is type(choice) and value == choice for choice in choices):
        return value
    expected = ", ".join(repr(choice) for choice in choices)
    raise ConfigResolutionError(path, f"expected one of ({expected}), got {value!r}")


def _normalize_collection(
    value: object,

229
230
231
232
233
234
235
236
237
238
239
240
241
            kind="list",
            path=path,
        )
    if origin is tuple or annotation is tuple:
        if len(args) == 2 and args[1] is Ellipsis:
            return _normalize_sequence(value, args[:1], uniform=True, kind="tuple", path=path)
        return _normalize_sequence(value, args, uniform=False, kind="tuple", path=path)
    if not isinstance(value, Mapping):
        raise ConfigResolutionError(path, f"expected mapping, got {type(value).__name__}")
    return dict(value)


def _normalize_class(value: object, annotation: object, *, path: str) -> object:

245
246
247
248
249
250
251
252
253
254
255
    reported here as unsupported.
    """

    if not isinstance(annotation, type):
        raise ConfigResolutionError(path, f"unsupported type annotation {_annotation_name(annotation)}")
    if isinstance(value, annotation):
        return value
    raise ConfigResolutionError(
        path,
        f"expected {_annotation_name(annotation)}, got {type(value).__name__}",
    )

268
269
270
271
272
273
274
275
276
277
278
279
    if not isinstance(value, str):
        return value
    target = annotation
    if get_origin(target) in (Union, types.UnionType):
        members = [member for member in get_args(target) if member is not types.NoneType]
        if len(members) != 1:
            return value
        target = members[0]
    if target is int:
        try:
            return int(value)
        except ValueError:

278
279
280
281
282
283
284
285
286
287
288
289
            return int(value)
        except ValueError:
            return value
    if target is float:
        try:
            return float(value)
        except ValueError:
            return value
    return value


def normalize_value(value: object, annotation: object, *, path: str) -> object:

291
292
293
294
295
296
297
298
299
300
301

    if annotation in (Any, object):
        return value
    if isinstance(annotation, dataclasses.InitVar):
        return normalize_value(value, annotation.type, path=path)
    if value is None:
        return _require_none_allowed(annotation, path=path)
    if _is_union(annotation):
        return _normalize_union(value, annotation, path=path)
    if annotation in (bool, int, float, str):
        return _normalize_scalar(value, annotation, path=path)

309
310
311
312
313
314
315
316
    if isinstance(annotation, type) and dataclasses.is_dataclass(annotation):
        # Nested dataclass items resolve from mappings in the same way as
        # top-level dataclass components.
        return _resolve_dataclass(value, annotation, path=path)
    return _normalize_class(value, annotation, path=path)


# ── Dotted overrides on the resolved config tree ──

356
357
358
359
360
361
362
363
364

    if len(parts) > 1:
        try:
            child = getattr(config, name)
        except AttributeError as exc:
            raise ConfigResolutionError(
                f"CLI.{full_path}", "target argument is not configured"
            ) from exc
        return config.replace(**{name: replace_override_path(child, parts[1:], value, path=full_path)})

390
391
392
393
394
395
396
397
398
399
400
401
402
        if name not in config:
            raise ConfigResolutionError(f"CLI.{path}", f"unknown mapping key {name!r}")
        if len(parts) == 1:
            return {**config, name: value}
        return {**config, name: replace_override_path(config[name], parts[1:], value, path=full_path)}

    location = f"CLI.{path}" if path else "CLI"
    if not is_dataclass(config):
        raise ConfigResolutionError(
            location, f"value of type {type(config).__name__} has no configurable fields"
        )

    config_fields = {field.name: field for field in fields(config)}

412
413
414
415
416
417
418
419
420
421
422
423
424
        )

    full_path = f"{path}.{name}" if path else name
    if len(parts) == 1:
        annotation = get_type_hints(type(config))[name]
        normalized = normalize_value(
            _normalize_override_scalar(value, annotation), annotation, path=f"CLI.{full_path}"
        )
        return replace(config, **{name: normalized})

    child = getattr(config, name)
    if child is None:
        raise ConfigResolutionError(

446
447
448
449
450
451
452
453
        try:
            return resolve_component(node, annotation=member, path=path)
        except ConfigResolutionError as exc:
            last_error = exc
    raise ConfigResolutionError(
        path,
        f"expected {_annotation_name(annotation)}, got {type(node).__name__}",
    ) from last_error

455
456
457
458
459
460
461
462
463
464
465
466
467
468

def _resolve_dataclass(node: object, config_type: type, *, path: str) -> object:
    """Construct one pure-parameter dataclass from a YAML mapping."""
    if not isinstance(node, Mapping):
        raise ConfigResolutionError(path, "configuration section must be a YAML mapping")

    config_fields = {field.name: field for field in fields(config_type)}
    unknown = sorted(set(node) - set(config_fields))
    if unknown:
        raise ConfigResolutionError(path, f"unknown configuration fields: {unknown}")

    required = [
        field.name
        for field in config_fields.values()

469
470
471
472
473
474
475
476
477
478
479
480
481
482
        if field.default is MISSING and field.default_factory is MISSING
    ]
    missing = [name for name in required if name not in node]
    if missing:
        raise ConfigResolutionError(path, f"missing required configuration fields: {missing}")

    try:
        hints = get_type_hints(config_type)
    except (NameError, TypeError) as exc:
        raise ConfigResolutionError(path, f"could not resolve configuration type annotations: {exc}") from exc

    resolved = {
        name: resolve_component(
            value,

487
488
489
490
491
492
493
494
495
    }
    try:
        return config_type(**resolved)
    except TypeError as exc:
        raise ConfigResolutionError(path, f"could not construct {config_type.__name__}: {exc}") from exc


def _resolve_target_args(
    raw_args: Mapping[str, object],

505
506
507
508
509
510
511
512
513
    """
    try:
        signature.bind_partial(**raw_args)
    except TypeError as exc:
        raise ConfigResolutionError(path, f"target arguments are invalid: {exc}") from exc

    normalized = {}
    for name, value in raw_args.items():
        parameter = signature.parameters.get(name)

539
540
541
542
543
544
545
546
547
548
549

def _resolve_target(node: object, *, path: str) -> Target[Any]:
    """Resolve one YAML target without invoking its callable."""
    if not isinstance(node, Mapping):
        raise ConfigResolutionError(path, "target section must be a YAML mapping")
    if "_target_" not in node:
        raise ConfigResolutionError(path, "target section is missing required _target_")

    target_path = node["_target_"]
    target = import_target(target_path, location=f"{path}._target_")
    try:

548
549
550
551
552
553
554
555
556
    target = import_target(target_path, location=f"{path}._target_")
    try:
        signature = inspect.signature(target)
    except (TypeError, ValueError) as exc:
        raise ConfigResolutionError(path, f"target signature is unavailable: {exc}") from exc

    for parameter in signature.parameters.values():
        if (
            parameter.kind is inspect.Parameter.POSITIONAL_ONLY

555
556
557
558
559
560
561
562
        if (
            parameter.kind is inspect.Parameter.POSITIONAL_ONLY
            and parameter.default is inspect.Signature.empty
        ):
            raise ConfigResolutionError(
                path,
                f"target parameter {parameter.name!r} must be callable by keyword",
            )

577
578
579
580
581
582
583
584
585
586
587
588
589
590

def _resolve_dataloader_config(node: object, *, path: str) -> DataLoaderConfig:
    """Resolve a DataLoader target with nested collator and batch adapter."""
    if not isinstance(node, Mapping):
        raise ConfigResolutionError(path, "DataLoader configuration must be a YAML mapping")

    target_node = dict(node)
    collate_node = target_node.pop("collate_fn", None)
    get_batch_node = target_node.pop("get_batch", None)
    dataloader_type = normalize_value(
        target_node.pop("dataloader_type", "single"),
        Literal["single", "cyclic"],
        path=f"{path}.dataloader_type",
    )

588
589
590
591
592
593
594
595
596
        Literal["single", "cyclic"],
        path=f"{path}.dataloader_type",
    )
    data_rearrange_map = target_node.pop("data_rearrange_map", None)
    data_sharding = normalize_value(
        target_node.pop("data_sharding", False),
        bool,
        path=f"{path}.data_sharding",
    )

617
618
619
620
621
622
623
624
625

def _resolve_dataset_config(node: object, *, path: str) -> DatasetConfig:
    """Resolve a Dataset target with its assets and sample transform."""
    if not isinstance(node, Mapping):
        raise ConfigResolutionError(path, "Dataset configuration must be a YAML mapping")

    target_node = dict(node)
    model_assets_node = target_node.pop("model_assets", {})
    data_transform_node = target_node.pop("data_transform", None)

646
647
648
649
650
651
652
653
654

def _resolve_optimizer_config(node: object, *, path: str) -> OptimizerConfig:
    """Resolve an optimizer target and its fp32 main-parameter policy."""
    if not isinstance(node, Mapping):
        raise ConfigResolutionError(path, "Optimizer configuration must be a YAML mapping")

    target_node = dict(node)
    fp32_main_params = normalize_value(
        target_node.pop("fp32_main_params", False),

672
673
674
675
676
677
678
679
680
    Returns:
        The resolved configuration value.
    """
    if node is None:
        return _require_none_allowed(annotation, path=path)
    if _is_union(annotation):
        return _resolve_union(node, annotation, path=path)

    origin = get_origin(annotation)

686
687
688
689
690
691
692
693
694
        return _resolve_dataloader_config(node, path=path)
    if annotation is OptimizerConfig:
        return _resolve_optimizer_config(node, path=path)
    if isinstance(annotation, type) and dataclasses.is_dataclass(annotation):
        return _resolve_dataclass(node, annotation, path=path)
    return normalize_value(node, annotation, path=path)


def resolve_config(raw: object) -> TrainerConfig:

693
694
695
696
697
698
699
700
701
702
703
704
705
706

def resolve_config(raw: object) -> TrainerConfig:
    """Resolve YAML root fields and construct ``TrainerConfig``."""
    if not isinstance(raw, Mapping):
        raise ConfigResolutionError("$", "YAML root must be a mapping")

    root_fields = {field.name: field for field in fields(TrainerConfig)}
    unknown = sorted(set(raw) - set(root_fields))
    if unknown:
        raise ConfigResolutionError("$", f"unknown configuration fields: {unknown}")

    required = [
        field.name
        for field in root_fields.values()

707
708
709
710
711
712
713
714
715
        if field.default is MISSING and field.default_factory is MISSING
    ]
    missing = [name for name in required if name not in raw]
    if missing:
        raise ConfigResolutionError("$", f"missing required configuration fields: {missing}")

    root_hints = get_type_hints(TrainerConfig)
    resolved = {
        name: resolve_component(

721
722
723
724
725
    }
    try:
        return TrainerConfig(**resolved)
    except TypeError as exc:
        raise ConfigResolutionError("$", f"could not construct TrainerConfig: {exc}") from exc