Coverage for  / home / jenkins / .local / lib / python3.10 / site-packages / hyper_parallel / data / constants.py: 100%

1# Copyright 2025-2026 Bytedance Ltd. and/or its affiliates 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14"""Static constants used by text, Omni, Online, batching, and indexed datasets.""" 

15 

16 

17# Text and Omni token semantics. 

18IGNORE_INDEX = -100 

19ROLE_SUPPORTED = ["system", "user", "assistant", "tool"] 

20TEXT_LAZY_EXPORTS = { 

21 "AutoTokenizer": "hyper_parallel.data.text.build_tokenizer", 

22 "ChatTemplate": "hyper_parallel.data.text.chat_template", 

23 "IdentityDataTransform": "hyper_parallel.data.text.text_transform", 

24 "PlaintextTransform": "hyper_parallel.data.text.text_transform", 

25 "TextConversationTransform": "hyper_parallel.data.text.text_transform", 

26 "build_chat_template": "hyper_parallel.data.text.chat_template", 

27 "build_indexed_text_dataset": "hyper_parallel.data.text.build_dataset", 

28 "build_text_transform": "hyper_parallel.data.text.text_transform", 

29 "build_online_iterable_dataset": "hyper_parallel.data.text.build_dataset", 

30 "build_online_text_mapping_dataset": "hyper_parallel.data.text.build_dataset", 

31 "build_tokenizer": "hyper_parallel.data.text.build_tokenizer", 

32} 

33IMAGE_INPUT_INDEX = -200 

34VIDEO_INPUT_INDEX = -300 

35AUDIO_INPUT_INDEX = -400 

36IMAGE_OUTPUT_INDEX = -201 

37VIDEO_OUTPUT_INDEX = -301 

38AUDIO_OUTPUT_INDEX = -401 

39TYPE2INDEX = { 

40 "input": { 

41 "image": IMAGE_INPUT_INDEX, 

42 "video": VIDEO_INPUT_INDEX, 

43 "audio": AUDIO_INPUT_INDEX, 

44 }, 

45 "output": { 

46 "image": IMAGE_OUTPUT_INDEX, 

47 "video": VIDEO_OUTPUT_INDEX, 

48 "audio": AUDIO_OUTPUT_INDEX, 

49 }, 

50} 

51MODALITY = TYPE2INDEX["input"].keys() | TYPE2INDEX["output"].keys() 

52 

53# Online source and split policy. 

54ONLINE_SPLIT_NAMES = ("train", "valid", "test") 

55ONLINE_SPLIT_COUNT = len(ONLINE_SPLIT_NAMES) 

56ONLINE_SOURCE_PATH_KEY = "__online_source_path__" 

57ONLINE_STOPPING_STRATEGIES = ( 

58 "first_exhausted", 

59 "all_exhausted", 

60 "all_exhausted_without_replacement", 

61) 

62ONLINE_BLEND_STRATEGIES = ("local_weighted", "global_interleave") 

63ONLINE_FILE_FORMATS = { 

64 ".arrow": "arrow", 

65 ".csv": "csv", 

66 ".json": "json", 

67 ".jsonl": "json", 

68 ".parquet": "parquet", 

69} 

70 

71# Packing and Omni batch field rules. 

72DEFAULT_FIELD_PACK_DIMS = { 

73 "input_ids": -1, 

74 "labels": -1, 

75 "attention_mask": -1, 

76 "loss_mask": -1, 

77 "position_ids": -1, 

78 "text_position_ids": -1, 

79 "router_attention_mask": -1, 

80 "mm_token_type_ids": -1, 

81 "token_types": -1, 

82 "pixel_values": 0, 

83 "pixel_values_videos": 0, 

84 "image_grid_thw": 0, 

85 "video_grid_thw": 0, 

86 "audio_values": 0, 

87} 

88PACKING_METADATA_FIELDS = {"packing_length", "cu_seq_lens"} 

89OMNI_CONCAT_FIELDS = { 

90 "audio_values", 

91 "image_grid_thw", 

92 "pixel_values", 

93 "pixel_values_videos", 

94 "video_grid_thw", 

95} 

96OMNI_LOSS_INPUT_FIELDS = {"labels", "loss_mask", "stream_loss_mask"} 

97OMNI_INTERNAL_FIELDS = {"loss_mask", "stream_loss_mask", "cu_seq_lens", "packing_length"} 

98OMNI_CP_TOKEN_FIELDS = frozenset({ 

99 "input_ids", "labels", "loss_mask", "stream_loss_mask", "attention_mask", "position_ids", 

100 "text_position_ids", "router_attention_mask", "mm_token_type_ids", "token_types", 

101}) 

102OMNI_CP_PAD_VALUES = {"labels": -100, "token_types": -1, "mm_token_type_ids": -1} 

103 

104# Indexed dataset storage and object-store paths. 

105INDEX_HEADER = b"MMIDIDX\x00\x00" 

106INDEX_VERSION = 1 

107INDEX_DTYPE_CODES = { 

108 1: "uint8", 

109 2: "int8", 

110 3: "int16", 

111 4: "int32", 

112 5: "int64", 

113 6: "float64", 

114 7: "float32", 

115 8: "uint16", 

116} 

117PAD_TOKEN_ID = -1 

118S3_PREFIX = "s3://" 

119MSC_PREFIX = "msc://" 

120 

121# Dataset logger presentation. 

122DATASET_LOGGER_NAME = "hyper_parallel.data" 

123DATASET_LOG_FORMAT = ( 

124 "[%(asctime)s] [%(levelname)s] [%(filename)s:%(lineno)d] " 

125 "[rank:%(rank_id)d] \t> %(message)s" 

126) 

127DATASET_DATE_FORMAT = "%Y-%m-%d %H:%M:%S.%f"