Coverage for  / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / data / vl_dummy.py: 16%

1# Copyright 2026 Huawei Technologies Co., Ltd 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14# ============================================================================ 

15"""``vl_dummy`` — synthetic VL dataset for native Qwen3-VL smoke tests. 

16 

17Each sample's pixel grid, image-token slice and trailing token stream are 

18seeded by ``train.seed + idx`` so 1-card vs N-card runs see identical content 

19after the distributed sampler interleaves them. 

20""" 

21import logging 

22from typing import Any 

23 

24import torch 

25from torch.utils.data import Dataset 

26 

27from hyper_parallel.data.registry import DATASET_REGISTRY 

28 

29 

30logger = logging.getLogger(__name__) 

31 

32 

33class DummyVLDataset(Dataset): 

34 """Synthetic multimodal samples with stable shapes per index.""" 

35 

36 def __init__( 

37 self, 

38 num_samples: int, 

39 seq_length: int, 

40 grid_t: int, 

41 grid_h: int, 

42 grid_w: int, 

43 row_width: int, 

44 image_tokens: int, 

45 image_token_id: int, 

46 base_seed: int, 

47 video_token_id: int = 151656, 

48 is_video: bool = False, 

49 ) -> None: 

50 self.num_samples = num_samples 

51 self.seq_length = seq_length 

52 self.grid_t = grid_t 

53 self.grid_h = grid_h 

54 self.grid_w = grid_w 

55 self.row_width = row_width 

56 self.image_tokens = image_tokens 

57 self.image_token_id = image_token_id 

58 self.base_seed = base_seed 

59 self.video_token_id = video_token_id 

60 self.is_video = is_video 

61 

62 def __len__(self) -> int: 

63 return self.num_samples 

64 

65 def __getitem__(self, idx: int): 

66 g = torch.Generator().manual_seed(self.base_seed + idx) 

67 if self.is_video: 

68 return self._video_item(idx, g) 

69 pixel_values = torch.randn( 

70 self.grid_t * self.grid_h * self.grid_w, self.row_width, 

71 generator=g, dtype=torch.float32, 

72 ) 

73 input_ids = torch.full((self.seq_length,), 100, dtype=torch.long) 

74 input_ids[0] = 151643 

75 input_ids[1: 1 + self.image_tokens] = self.image_token_id 

76 tail = torch.arange( 

77 200 + idx % 17, 

78 200 + idx % 17 + self.seq_length - 1 - self.image_tokens, 

79 dtype=torch.long, 

80 ) 

81 input_ids[1 + self.image_tokens:] = tail 

82 labels = input_ids.clone() 

83 mm_token_type_ids = torch.zeros(self.seq_length, dtype=torch.int32) 

84 mm_token_type_ids[1: 1 + self.image_tokens] = 1 

85 return { 

86 "input_ids": input_ids, 

87 "labels": labels, 

88 "attention_mask": torch.ones(self.seq_length, dtype=torch.long), 

89 "mm_token_type_ids": mm_token_type_ids, 

90 "pixel_values": pixel_values, 

91 "image_grid_thw": torch.tensor( 

92 [self.grid_t, self.grid_h, self.grid_w], dtype=torch.long, 

93 ), 

94 } 

95 

96 def _video_item(self, idx: int, g: torch.Generator): 

97 """Build one deterministic video sample.""" 

98 # A video lays each of grid_t frames as its own mm==2 run (a text 

99 # separator between frames) so get_rope_index consumes one per-frame 

100 # [1,h,w] grid per run; an image is a single contiguous run. 

101 tokens_per_frame = self.image_tokens // self.grid_t 

102 pixel_values_videos = torch.randn( 

103 self.grid_t * self.grid_h * self.grid_w, self.row_width, 

104 generator=g, dtype=torch.float32, 

105 ) 

106 input_ids = torch.full((self.seq_length,), 100, dtype=torch.long) 

107 input_ids[0] = 151643 

108 mm_token_type_ids = torch.zeros(self.seq_length, dtype=torch.int32) 

109 pos = 1 

110 frame_idx = 0 

111 while frame_idx < self.grid_t: 

112 input_ids[pos] = 150 # text separator → split per-frame runs 

113 pos += 1 

114 input_ids[pos: pos + tokens_per_frame] = self.video_token_id 

115 mm_token_type_ids[pos: pos + tokens_per_frame] = 2 

116 pos += tokens_per_frame 

117 frame_idx += 1 

118 tail = torch.arange( 

119 200 + idx % 17, 200 + idx % 17 + self.seq_length - pos, 

120 dtype=torch.long, 

121 ) 

122 input_ids[pos:] = tail 

123 labels = input_ids.clone() 

124 return { 

125 "input_ids": input_ids, 

126 "labels": labels, 

127 "attention_mask": torch.ones(self.seq_length, dtype=torch.long), 

128 "mm_token_type_ids": mm_token_type_ids, 

129 "pixel_values_videos": pixel_values_videos, 

130 "video_grid_thw": torch.tensor( 

131 [self.grid_t, self.grid_h, self.grid_w], dtype=torch.long, 

132 ), 

133 } 

134 

135 

136@DATASET_REGISTRY.register("vl_dummy") 

137def build_vl_dummy(*, base: Any, args: Any, **_: Any) -> DummyVLDataset: 

138 """Build the deterministic VL dummy dataset. 

139 

140 Pulls ``patch_size`` / ``temporal_patch_size`` / ``in_channels`` / 

141 ``spatial_merge_size`` from ``model.config_overrides.vision_config`` so 

142 the row width matches the Qwen3-VL vision-tower expectation; falls back 

143 to the published defaults when those keys are absent. 

144 """ 

145 max_steps = args.train.max_steps 

146 global_bs = args.train.global_batch_size 

147 total_samples = max_steps * global_bs 

148 

149 data_cfg = args.data 

150 model_cfg = args.model 

151 extra = model_cfg.config_overrides or {} 

152 vision_extra = extra.get("vision_config", {}) if isinstance(extra, dict) else {} 

153 patch_size = int(vision_extra.get("patch_size", 16)) 

154 temporal_patch_size = int(vision_extra.get("temporal_patch_size", 2)) 

155 in_channels = int(vision_extra.get("in_channels", 3)) 

156 spatial_merge = int(vision_extra.get("spatial_merge_size", 2)) 

157 grid_t = int(data_cfg.vl_grid_t) 

158 grid_h = int(data_cfg.vl_grid_h) 

159 grid_w = int(data_cfg.vl_grid_w) 

160 image_token_id = int(data_cfg.image_token_id) 

161 video_token_id = int(data_cfg.video_token_id) 

162 is_video = bool(data_cfg.vl_video) 

163 image_tokens = grid_t * grid_h * grid_w // (spatial_merge ** 2) 

164 tokens_per_frame = grid_h * grid_w // (spatial_merge ** 2) 

165 row_width = in_channels * temporal_patch_size * patch_size * patch_size 

166 min_len = grid_t * (tokens_per_frame + 1) + 2 if is_video else image_tokens + 4 

167 seq_len = max(int(data_cfg.max_seq_len), min_len) 

168 base_seed = int(args.train.seed) 

169 

170 ds = DummyVLDataset( 

171 num_samples=total_samples, seq_length=seq_len, 

172 grid_t=grid_t, grid_h=grid_h, grid_w=grid_w, row_width=row_width, 

173 image_tokens=image_tokens, image_token_id=image_token_id, base_seed=base_seed, 

174 video_token_id=video_token_id, is_video=is_video, 

175 ) 

176 base.state.max_steps = max_steps 

177 logger.info( 

178 "VL dummy dataset created: samples=%d seq_len=%d grid=(%d,%d,%d) image_tokens=%d", 

179 total_samples, seq_len, grid_t, grid_h, grid_w, image_tokens, 

180 ) 

181 return ds