Diff Coverage

Diff: origin/master...HEAD, staged and unstaged changes

Source File Diff Coverage (%) Missing Lines
hyper_parallel/trainer/config/parser.py 86.7% 50,55,87-90
hyper_parallel/trainer/config/resolver.py 65.1% 67,71,80,85,89,96,99,121-122,147,153,170,185,219,240-242,244,251,254,271-274,281-284,306,308,324,371-372,418,422,440-441,444,474,483,488,497,502,515,533,567,569,576,583,605,610,616,645,674,704,718,736,741,750,764
hyper_parallel/trainer/config/parser.py
46
47
48
49
50
51
52
53
54
55
56
57
58
                "CLI", f"expected a dotted override in '--field=value' form, got {token!r}"
            )
        option = token[2:]
        if "=" not in option:
            raise ConfigResolutionError(
                "CLI", f"expected a dotted override in '--field=value' form, got {token!r}"
            )
        path, raw_value = option.split("=", 1)
        if not path or any(not part for part in path.split(".")):
            raise ConfigResolutionError("CLI", f"invalid override path {path!r}")
        parsed.append((path, raw_value))
    return parsed


83
84
85
86
87
88
89
90
91
92
93
    """Load a YAML configuration and apply CLI overrides."""
    config_path = Path(config_file)
    try:
        raw = yaml.safe_load(config_path.read_text(encoding="utf-8"))
    except OSError as exc:
        raise ConfigResolutionError(config_path, f"could not read config file: {exc}") from exc
    except yaml.YAMLError as exc:
        raise ConfigResolutionError(config_path, f"invalid YAML: {exc}") from exc

    config = resolve_config(raw)
    return _apply_typed_overrides(config, cli_overrides)
hyper_parallel/trainer/config/resolver.py
63
64
65
66
67
68
69
70
71
72
73
74
75
        ConfigResolutionError: The path is invalid, cannot be imported, or
            does not identify a callable.
    """
    if not isinstance(target_path, str) or not target_path.strip():
        raise ConfigResolutionError(location, "_target_ must be a non-empty dotted path")

    parts = target_path.split(".")
    if any(not part for part in parts):
        raise ConfigResolutionError(location, f"invalid target path {target_path!r}")

    for split_at in range(len(parts), 0, -1):
        module_name = ".".join(parts[:split_at])
        try:

76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
            target = importlib.import_module(module_name)
        except ModuleNotFoundError as exc:
            if exc.name == module_name or module_name.startswith(f"{exc.name}."):
                continue
            raise ConfigResolutionError(
                location,
                f"target {target_path!r} failed while importing dependency {exc.name!r}",
            ) from exc
        except ImportError as exc:
            raise ConfigResolutionError(location, f"target {target_path!r} could not be imported: {exc}") from exc

        for attribute in parts[split_at:]:
            if not hasattr(target, attribute):
                raise ConfigResolutionError(
                    location,
                    f"target {target_path!r} has no attribute {attribute!r}",
                )
            target = getattr(target, attribute)

 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
                )
            target = getattr(target, attribute)

        if not callable(target):
            raise ConfigResolutionError(location, f"target {target_path!r} is not callable")
        return target

    raise ConfigResolutionError(location, f"target {target_path!r} could not be imported")


def _is_union(annotation: object) -> bool:
    """Return whether the annotation is a ``Union`` or a PEP 604 union."""

117
118
119
120
121
122
123
124
125
126
    """Resolve annotations for a target function or class constructor."""
    hint_source = target.__init__ if inspect.isclass(target) else target
    try:
        return get_type_hints(hint_source)
    except (NameError, TypeError) as exc:
        raise ConfigResolutionError(path, f"could not resolve target type annotations: {exc}") from exc


# Annotations whose payload is a container of other values.
_COLLECTION_TYPES: tuple[type, ...] = (list, tuple, dict, Mapping)

143
144
145
146
147
148
149
150
151
    ``item_types`` must contain one type per element. Empty ``item_types``
    produces a tuple without normalizing its items.
    """
    if not isinstance(value, (list, tuple)):
        raise ConfigResolutionError(path, f"expected {kind}, got {type(value).__name__}")
    if not item_types:
        return tuple(value)
    if uniform:
        item_types = item_types * len(value)

149
150
151
152
153
154
155
156
157
        return tuple(value)
    if uniform:
        item_types = item_types * len(value)
    if len(item_types) != len(value):
        raise ConfigResolutionError(
            path,
            f"expected {kind} of length {len(item_types)}, got {len(value)}",
        )
    normalized = [

166
167
168
169
170
171
172
173
174
    if annotation is types.NoneType or (
        _is_union(annotation) and types.NoneType in get_args(annotation)
    ):
        return None
    raise ConfigResolutionError(path, f"expected {_annotation_name(annotation)}, got None")


def _normalize_union(value: object, annotation: object, *, path: str) -> object:
    """Normalize a value against the first compatible union member."""

181
182
183
184
185
186
187
188
        try:
            return normalize_value(value, member, path=path)
        except ConfigResolutionError:
            continue
    raise ConfigResolutionError(
        path,
        f"expected {_annotation_name(annotation)}, got {type(value).__name__}",
    )

215
216
217
218
219
220
221
222
223
                return word
    if any(type(value) is type(choice) and value == choice for choice in choices):
        return value
    expected = ", ".join(repr(choice) for choice in choices)
    raise ConfigResolutionError(path, f"expected one of ({expected}), got {value!r}")


def _normalize_collection(
    value: object,

236
237
238
239
240
241
242
243
244
245
246
247
248
            kind="list",
            path=path,
        )
    if origin is tuple or annotation is tuple:
        if len(args) == 2 and args[1] is Ellipsis:
            return _normalize_sequence(value, args[:1], uniform=True, kind="tuple", path=path)
        return _normalize_sequence(value, args, uniform=False, kind="tuple", path=path)
    if not isinstance(value, Mapping):
        raise ConfigResolutionError(path, f"expected mapping, got {type(value).__name__}")
    return dict(value)


def _normalize_class(value: object, annotation: object, *, path: str) -> object:

247
248
249
250
251
252
253
254
255
256
257

def _normalize_class(value: object, annotation: object, *, path: str) -> object:
    """Validate that a value is an instance of the annotated class."""
    if not isinstance(annotation, type):
        raise ConfigResolutionError(path, f"unsupported type annotation {_annotation_name(annotation)}")
    if isinstance(value, annotation):
        return value
    raise ConfigResolutionError(
        path,
        f"expected {_annotation_name(annotation)}, got {type(value).__name__}",
    )

267
268
269
270
271
272
273
274
275
276
277
278
    if not isinstance(value, str):
        return value
    target = annotation
    if get_origin(target) in (Union, types.UnionType):
        members = [member for member in get_args(target) if member is not types.NoneType]
        if len(members) != 1:
            return value
        target = members[0]
    if target is int:
        try:
            return int(value)
        except ValueError:

277
278
279
280
281
282
283
284
285
286
287
288
            return int(value)
        except ValueError:
            return value
    if target is float:
        try:
            return float(value)
        except ValueError:
            return value
    return value


def normalize_value(value: object, annotation: object, *, path: str) -> object:

302
303
304
305
306
307
308
309
310
311
312
    """
    if annotation in (Any, object):
        return value
    if isinstance(annotation, dataclasses.InitVar):
        return normalize_value(value, annotation.type, path=path)
    if value is None:
        return _require_none_allowed(annotation, path=path)
    if _is_union(annotation):
        return _normalize_union(value, annotation, path=path)
    if annotation in (bool, int, float, str):
        return _normalize_scalar(value, annotation, path=path)

320
321
322
323
324
325
326
327
    if isinstance(annotation, type) and dataclasses.is_dataclass(annotation):
        # Nested dataclass items resolve from mappings in the same way as
        # top-level dataclass components.
        return _resolve_dataclass(value, annotation, path=path)
    return _normalize_class(value, annotation, path=path)


# ── Dotted overrides ────────────────────────────────────────────────────

367
368
369
370
371
372
373
374
375

    if len(parts) > 1:
        try:
            child = getattr(config, name)
        except AttributeError as exc:
            raise ConfigResolutionError(
                f"CLI.{full_path}", "target argument is not configured"
            ) from exc
        return config.replace(**{name: replace_override_path(child, parts[1:], value, path=full_path)})

414
415
416
417
418
419
420
421
422
423
424
425
426
        if name not in config:
            raise ConfigResolutionError(f"CLI.{path}", f"unknown mapping key {name!r}")
        if len(parts) == 1:
            return {**config, name: value}
        return {**config, name: replace_override_path(config[name], parts[1:], value, path=full_path)}

    location = f"CLI.{path}" if path else "CLI"
    if not is_dataclass(config):
        raise ConfigResolutionError(
            location, f"value of type {type(config).__name__} has no configurable fields"
        )

    config_fields = {field.name: field for field in fields(config)}

436
437
438
439
440
441
442
443
444
445
446
447
448
        )

    full_path = f"{path}.{name}" if path else name
    if len(parts) == 1:
        annotation = get_type_hints(type(config))[name]
        normalized = normalize_value(
            _normalize_override_scalar(value, annotation), annotation, path=f"CLI.{full_path}"
        )
        return replace(config, **{name: normalized})

    child = getattr(config, name)
    if child is None:
        raise ConfigResolutionError(

470
471
472
473
474
475
476
477
        try:
            return resolve_component(node, annotation=member, path=path)
        except ConfigResolutionError as exc:
            last_error = exc
    raise ConfigResolutionError(
        path,
        f"expected {_annotation_name(annotation)}, got {type(node).__name__}",
    ) from last_error

479
480
481
482
483
484
485
486
487
488
489
490
491
492

def _resolve_dataclass(node: object, config_type: type, *, path: str) -> object:
    """Resolve YAML fields and construct a configuration dataclass."""
    if not isinstance(node, Mapping):
        raise ConfigResolutionError(path, "configuration section must be a YAML mapping")

    config_fields = {field.name: field for field in fields(config_type)}
    unknown = sorted(set(node) - set(config_fields))
    if unknown:
        raise ConfigResolutionError(path, f"unknown configuration fields: {unknown}")

    required = [
        field.name
        for field in config_fields.values()

493
494
495
496
497
498
499
500
501
502
503
504
505
506
        if field.default is MISSING and field.default_factory is MISSING
    ]
    missing = [name for name in required if name not in node]
    if missing:
        raise ConfigResolutionError(path, f"missing required configuration fields: {missing}")

    try:
        hints = get_type_hints(config_type)
    except (NameError, TypeError) as exc:
        raise ConfigResolutionError(path, f"could not resolve configuration type annotations: {exc}") from exc

    resolved = {
        name: resolve_component(
            value,

511
512
513
514
515
516
517
518
519
    }
    try:
        return config_type(**resolved)
    except TypeError as exc:
        raise ConfigResolutionError(path, f"could not construct {config_type.__name__}: {exc}") from exc


def _resolve_target_args(
    raw_args: Mapping[str, object],

529
530
531
532
533
534
535
536
537
    """
    try:
        signature.bind_partial(**raw_args)
    except TypeError as exc:
        raise ConfigResolutionError(path, f"target arguments are invalid: {exc}") from exc

    normalized = {}
    for name, value in raw_args.items():
        parameter = signature.parameters.get(name)

563
564
565
566
567
568
569
570
571
572
573

def _resolve_target(node: object, *, path: str) -> Target[Any]:
    """Resolve a YAML mapping into a ``Target`` without invoking its callable."""
    if not isinstance(node, Mapping):
        raise ConfigResolutionError(path, "target section must be a YAML mapping")
    if "_target_" not in node:
        raise ConfigResolutionError(path, "target section is missing required _target_")

    target_path = node["_target_"]
    target = import_target(target_path, location=f"{path}._target_")
    try:

572
573
574
575
576
577
578
579
580
    target = import_target(target_path, location=f"{path}._target_")
    try:
        signature = inspect.signature(target)
    except (TypeError, ValueError) as exc:
        raise ConfigResolutionError(path, f"target signature is unavailable: {exc}") from exc

    for parameter in signature.parameters.values():
        if (
            parameter.kind is inspect.Parameter.POSITIONAL_ONLY

579
580
581
582
583
584
585
586
        if (
            parameter.kind is inspect.Parameter.POSITIONAL_ONLY
            and parameter.default is inspect.Signature.empty
        ):
            raise ConfigResolutionError(
                path,
                f"target parameter {parameter.name!r} must be callable by keyword",
            )

601
602
603
604
605
606
607
608
609
610
611
612
613
614

def _resolve_dataloader_config(node: object, *, path: str) -> DataLoaderConfig:
    """Resolve a ``DataLoaderConfig`` with its collator and batch adapter."""
    if not isinstance(node, Mapping):
        raise ConfigResolutionError(path, "DataLoader configuration must be a YAML mapping")

    target_node = dict(node)
    collate_node = target_node.pop("collate_fn", None)
    get_batch_node = target_node.pop("get_batch", None)
    dataloader_type = normalize_value(
        target_node.pop("dataloader_type", "single"),
        Literal["single", "cyclic"],
        path=f"{path}.dataloader_type",
    )

612
613
614
615
616
617
618
619
620
        Literal["single", "cyclic"],
        path=f"{path}.dataloader_type",
    )
    data_rearrange_map = target_node.pop("data_rearrange_map", None)
    data_sharding = normalize_value(
        target_node.pop("data_sharding", False),
        bool,
        path=f"{path}.data_sharding",
    )

641
642
643
644
645
646
647
648
649

def _resolve_dataset_config(node: object, *, path: str) -> DatasetConfig:
    """Resolve a ``DatasetConfig`` with its model assets and sample transform."""
    if not isinstance(node, Mapping):
        raise ConfigResolutionError(path, "Dataset configuration must be a YAML mapping")

    target_node = dict(node)
    model_assets_node = target_node.pop("model_assets", {})
    data_transform_node = target_node.pop("data_transform", None)

670
671
672
673
674
675
676
677
678

def _resolve_optimizer_config(node: object, *, path: str) -> OptimizerConfig:
    """Resolve an ``OptimizerConfig`` with its FP32 main-parameter policy."""
    if not isinstance(node, Mapping):
        raise ConfigResolutionError(path, "Optimizer configuration must be a YAML mapping")

    target_node = dict(node)
    fp32_main_params = normalize_value(
        target_node.pop("fp32_main_params", False),

700
701
702
703
704
705
706
707
708
        ConfigResolutionError: The value or target declaration is invalid for
            the annotation.
    """
    if node is None:
        return _require_none_allowed(annotation, path=path)
    if _is_union(annotation):
        return _resolve_union(node, annotation, path=path)

    origin = get_origin(annotation)

714
715
716
717
718
719
720
721
722
        return _resolve_dataloader_config(node, path=path)
    if annotation is OptimizerConfig:
        return _resolve_optimizer_config(node, path=path)
    if isinstance(annotation, type) and dataclasses.is_dataclass(annotation):
        return _resolve_dataclass(node, annotation, path=path)
    return normalize_value(node, annotation, path=path)


def resolve_config(raw: object) -> TrainerConfig:

732
733
734
735
736
737
738
739
740
741
742
743
744
745
        ConfigResolutionError: The root is not a mapping, required fields are
            missing, or a field or target declaration is invalid.
    """
    if not isinstance(raw, Mapping):
        raise ConfigResolutionError("$", "YAML root must be a mapping")

    root_fields = {field.name: field for field in fields(TrainerConfig)}
    unknown = sorted(set(raw) - set(root_fields))
    if unknown:
        raise ConfigResolutionError("$", f"unknown configuration fields: {unknown}")

    required = [
        field.name
        for field in root_fields.values()

746
747
748
749
750
751
752
753
754
        if field.default is MISSING and field.default_factory is MISSING
    ]
    missing = [name for name in required if name not in raw]
    if missing:
        raise ConfigResolutionError("$", f"missing required configuration fields: {missing}")

    root_hints = get_type_hints(TrainerConfig)
    resolved = {
        name: resolve_component(

760
761
762
763
764
    }
    try:
        return TrainerConfig(**resolved)
    except TypeError as exc:
        raise ConfigResolutionError("$", f"could not construct TrainerConfig: {exc}") from exc