Diff Coverage

Diff: origin/master...HEAD, staged and unstaged changes

Source File Diff Coverage (%) Missing Lines
hyper_parallel/data/omni/build_dataset.py 15.0% 39-40,42-46,48,50-55,89-91
hyper_parallel/data/omni/build_dataset.py
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59


def _get_train_valid_test_num_samples(training_config: Any) -> tuple[int, int, int]:
    """Calculate train, validation, and test sizes from the training plan."""
    if training_config is None:
        raise ValueError("Dataset sample planning requires a training configuration")

    global_batch_size = training_config.global_batch_size
    if training_config.train_iters is not None:
        train_iters = training_config.train_iters
    elif training_config.train_samples:
        train_iters = training_config.train_samples // global_batch_size
    else:
        raise ValueError("training.train_iters and training.train_samples cannot both be None")

    train_samples = training_config.train_samples or train_iters * global_batch_size
    eval_iters = training_config.eval_iters
    valid_iters = (train_iters // eval_iters + 1) * eval_iters if eval_iters else 0
    split_sizes = (train_samples, valid_iters * global_batch_size, eval_iters * global_batch_size)
    logger.debug("Dataset target sizes: train=%d, validation=%d, test=%d", *split_sizes)
    return split_sizes


def build_online_omni_mapping_dataset(
    *,

85
86
87
88
89
90
91
92
93
94
95

    dataset_config = dict(data_config)
    training_seed = getattr(training_config, "seed", None)
    dataset_config["random_seed"] = 42 if training_seed is None else int(training_seed)
    train_valid_test_num_samples = None
    if training_config is not None:
        train_valid_test_num_samples = _get_train_valid_test_num_samples(training_config)

    sample_filter = transform.is_valid_sample
    source_dataset = build_online_mapping_source(
        data_path=data_path,