Diff Coverage

Diff: origin/master...HEAD, staged and unstaged changes

Source File Diff Coverage (%) Missing Lines
hyper_parallel/trainer/callbacks/environ_meter_callback.py 57.1% 350,353-354
hyper_parallel/trainer/callbacks/tqdm_callback.py 50.0% 148
hyper_parallel/trainer/config/training.py 100%  
hyper_parallel/trainer/runtime/flops.py 14.8% 44-45,47-55,57-58,61,67-69,73-75,82,84-85
hyper_parallel/trainer/callbacks/environ_meter_callback.py
346
347
348
349
350
351
352
353
354
355
356
357
358
            and global_samples > 0
            and global_tokens > 0
            and global_step_time > 0
        ):
            flops_per_token = estimate_flops_per_token(
                model_config, global_tokens / global_samples
            )
            if flops_per_token is not None:
                env_metrics["performance/tflops_per_device"] = (
                    global_tokens * flops_per_token
                    / (global_step_time * get_world_size_safe())
                    / 1e12
                )
hyper_parallel/trainer/callbacks/tqdm_callback.py
144
145
146
147
148
149
150
151
152
            return f"{scalar:.3f}"
        if name == "performance/step_time":
            return f"{scalar:.3f}s"
        if name == "performance/tflops_per_device":
            return f"{scalar:.3f}"
        return f"{scalar:.6g}"

    def _create_progress_bar(self, total: int, initial: int) -> Any:
        """Create tqdm lazily because it is an optional dependency."""
hyper_parallel/trainer/runtime/flops.py
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
        Estimated FLOPs per token, or ``None`` when the model family is not
        supported. Communication, recomputation, normalization, dropout,
        RoPE and optimizer work are not included.
    """
    if getattr(config, "model_type", None) != "qwen3_moe":
        return None

    hidden = int(config.hidden_size)
    layers = int(config.num_hidden_layers)
    heads = int(config.num_attention_heads)
    kv_heads = int(getattr(config, "num_key_value_heads", None) or heads)
    head_dim = int(getattr(config, "head_dim", None) or hidden // heads)
    intermediate = int(config.intermediate_size)
    moe_intermediate = int(config.moe_intermediate_size)
    topk = int(config.num_experts_per_tok)
    vocab = int(config.vocab_size)

    query = heads * head_dim
    key_value = kv_heads * head_dim

    # Per token, per standard-attention layer.
    attention = 6.0 * (
        hidden * (query + key_value + key_value)
        + query * float(seq_length)
        + query * hidden
    )

63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
        + query * float(seq_length)
        + query * hidden
    )

    sigma = 3.0 if getattr(config, "hidden_act", "") in _GATED_ACTS else 2.0
    dense_mlp = 6.0 * hidden * intermediate * sigma
    moe_mlp = 6.0 * hidden * moe_intermediate * topk * sigma

    # Match Qwen3MoeDecoderLayer: MoE when the layer is not in
    # ``mlp_only_layers`` and ``(layer_idx + 1) % decoder_sparse_step == 0``.
    mlp_only = set(getattr(config, "mlp_only_layers", None) or ())
    sparse_step = max(1, int(getattr(config, "decoder_sparse_step", 1) or 1))
    num_moe = sum(
        1
        for layer_idx in range(layers)
        if layer_idx not in mlp_only
        and int(getattr(config, "num_experts", 0) or 0) > 0

78
79
80
81
82
83
84
85
86
87
88
89
        if layer_idx not in mlp_only
        and int(getattr(config, "num_experts", 0) or 0) > 0
        and (layer_idx + 1) % sparse_step == 0
    )
    num_dense = layers - num_moe

    logits = 6.0 * hidden * vocab
    return (
        attention * layers
        + dense_mlp * num_dense
        + moe_mlp * num_moe
        + logits