Skip to content

vllm.transformers_utils.configs.deepseek_vl2

DeepseekV2Config

Bases: PretrainedConfig

Source code in vllm/transformers_utils/configs/deepseek_vl2.py 
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
class DeepseekV2Config(PretrainedConfig):

    model_type = "deepseek_v2"
    keys_to_ignore_at_inference = ["past_key_values"]

    def __init__(
        self,
        vocab_size=102400,
        hidden_size=4096,
        intermediate_size=11008,
        moe_intermediate_size=1407,
        num_hidden_layers=30,
        num_attention_heads=32,
        num_key_value_heads=32,
        n_shared_experts=None,
        n_routed_experts=None,
        ep_size=1,
        routed_scaling_factor=1.0,
        kv_lora_rank=512,
        q_lora_rank=1536,
        qk_rope_head_dim=64,
        v_head_dim=128,
        qk_nope_head_dim=128,
        topk_method='gready',
        n_group=None,
        topk_group=None,
        num_experts_per_tok=None,
        moe_layer_freq=1,
        first_k_dense_replace=0,
        norm_topk_prob=False,
        scoring_func='softmax',
        aux_loss_alpha=0.001,
        seq_aux=True,
        hidden_act="silu",
        max_position_embeddings=2048,
        initializer_range=0.02,
        rms_norm_eps=1e-6,
        use_cache=True,
        pad_token_id=None,
        bos_token_id=100000,
        eos_token_id=100001,
        pretraining_tp=1,
        tie_word_embeddings=False,
        rope_theta=10000.0,
        rope_scaling=None,
        attention_bias=False,
        attention_dropout=0.0,
        use_mla=True,
        **kwargs,
    ):
        self.vocab_size = vocab_size
        self.max_position_embeddings = max_position_embeddings
        self.hidden_size = hidden_size
        self.intermediate_size = intermediate_size
        self.moe_intermediate_size = moe_intermediate_size
        self.num_hidden_layers = num_hidden_layers
        self.num_attention_heads = num_attention_heads
        self.n_shared_experts = n_shared_experts
        self.n_routed_experts = n_routed_experts
        self.ep_size = ep_size
        self.routed_scaling_factor = routed_scaling_factor
        self.kv_lora_rank = kv_lora_rank
        self.q_lora_rank = q_lora_rank
        self.qk_rope_head_dim = qk_rope_head_dim
        self.v_head_dim = v_head_dim
        self.qk_nope_head_dim = qk_nope_head_dim
        self.topk_method = topk_method
        self.n_group = n_group
        self.topk_group = topk_group
        self.num_experts_per_tok = num_experts_per_tok
        self.moe_layer_freq = moe_layer_freq
        self.first_k_dense_replace = first_k_dense_replace
        self.norm_topk_prob = norm_topk_prob
        self.scoring_func = scoring_func
        self.aux_loss_alpha = aux_loss_alpha
        self.seq_aux = seq_aux
        # for backward compatibility
        if num_key_value_heads is None:
            num_key_value_heads = num_attention_heads

        self.num_key_value_heads = num_key_value_heads
        self.hidden_act = hidden_act
        self.initializer_range = initializer_range
        self.rms_norm_eps = float(rms_norm_eps)
        self.pretraining_tp = pretraining_tp
        self.use_cache = use_cache
        self.rope_theta = rope_theta
        self.rope_scaling = rope_scaling
        self.attention_bias = attention_bias
        self.attention_dropout = attention_dropout
        self.use_mla = use_mla

        super().__init__(
            pad_token_id=pad_token_id,
            bos_token_id=bos_token_id,
            eos_token_id=eos_token_id,
            tie_word_embeddings=tie_word_embeddings,
            **kwargs,
        )

attention_bias instance-attribute 

attention_bias = attention_bias

attention_dropout instance-attribute 

attention_dropout = attention_dropout

aux_loss_alpha instance-attribute 

aux_loss_alpha = aux_loss_alpha

ep_size instance-attribute 

ep_size = ep_size

first_k_dense_replace instance-attribute 

first_k_dense_replace = first_k_dense_replace

hidden_act instance-attribute 

hidden_act = hidden_act

hidden_size instance-attribute 

hidden_size = hidden_size

initializer_range instance-attribute 

initializer_range = initializer_range

intermediate_size instance-attribute 

intermediate_size = intermediate_size

keys_to_ignore_at_inference class-attribute instance-attribute 

keys_to_ignore_at_inference = ['past_key_values']

kv_lora_rank instance-attribute 

kv_lora_rank = kv_lora_rank

max_position_embeddings instance-attribute 

max_position_embeddings = max_position_embeddings

model_type class-attribute instance-attribute 

model_type = 'deepseek_v2'

moe_intermediate_size instance-attribute 

moe_intermediate_size = moe_intermediate_size

moe_layer_freq instance-attribute 

moe_layer_freq = moe_layer_freq

n_group instance-attribute 

n_group = n_group

n_routed_experts instance-attribute 

n_routed_experts = n_routed_experts

n_shared_experts instance-attribute 

n_shared_experts = n_shared_experts

norm_topk_prob instance-attribute 

norm_topk_prob = norm_topk_prob

num_attention_heads instance-attribute 

num_attention_heads = num_attention_heads

num_experts_per_tok instance-attribute 

num_experts_per_tok = num_experts_per_tok

num_hidden_layers instance-attribute 

num_hidden_layers = num_hidden_layers

num_key_value_heads instance-attribute 

num_key_value_heads = num_key_value_heads

pretraining_tp instance-attribute 

pretraining_tp = pretraining_tp

q_lora_rank instance-attribute 

q_lora_rank = q_lora_rank

qk_nope_head_dim instance-attribute 

qk_nope_head_dim = qk_nope_head_dim

qk_rope_head_dim instance-attribute 

qk_rope_head_dim = qk_rope_head_dim

rms_norm_eps instance-attribute 

rms_norm_eps = float(rms_norm_eps)

rope_scaling instance-attribute 

rope_scaling = rope_scaling

rope_theta instance-attribute 

rope_theta = rope_theta

routed_scaling_factor instance-attribute 

routed_scaling_factor = routed_scaling_factor

scoring_func instance-attribute 

scoring_func = scoring_func

seq_aux instance-attribute 

seq_aux = seq_aux

topk_group instance-attribute 

topk_group = topk_group

topk_method instance-attribute 

topk_method = topk_method

use_cache instance-attribute 

use_cache = use_cache

use_mla instance-attribute 

use_mla = use_mla

v_head_dim instance-attribute 

v_head_dim = v_head_dim

vocab_size instance-attribute 

vocab_size = vocab_size

__init__ 

__init__(
    vocab_size=102400,
    hidden_size=4096,
    intermediate_size=11008,
    moe_intermediate_size=1407,
    num_hidden_layers=30,
    num_attention_heads=32,
    num_key_value_heads=32,
    n_shared_experts=None,
    n_routed_experts=None,
    ep_size=1,
    routed_scaling_factor=1.0,
    kv_lora_rank=512,
    q_lora_rank=1536,
    qk_rope_head_dim=64,
    v_head_dim=128,
    qk_nope_head_dim=128,
    topk_method="gready",
    n_group=None,
    topk_group=None,
    num_experts_per_tok=None,
    moe_layer_freq=1,
    first_k_dense_replace=0,
    norm_topk_prob=False,
    scoring_func="softmax",
    aux_loss_alpha=0.001,
    seq_aux=True,
    hidden_act="silu",
    max_position_embeddings=2048,
    initializer_range=0.02,
    rms_norm_eps=1e-06,
    use_cache=True,
    pad_token_id=None,
    bos_token_id=100000,
    eos_token_id=100001,
    pretraining_tp=1,
    tie_word_embeddings=False,
    rope_theta=10000.0,
    rope_scaling=None,
    attention_bias=False,
    attention_dropout=0.0,
    use_mla=True,
    **kwargs,
)
Source code in vllm/transformers_utils/configs/deepseek_vl2.py 
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
def __init__(
    self,
    vocab_size=102400,
    hidden_size=4096,
    intermediate_size=11008,
    moe_intermediate_size=1407,
    num_hidden_layers=30,
    num_attention_heads=32,
    num_key_value_heads=32,
    n_shared_experts=None,
    n_routed_experts=None,
    ep_size=1,
    routed_scaling_factor=1.0,
    kv_lora_rank=512,
    q_lora_rank=1536,
    qk_rope_head_dim=64,
    v_head_dim=128,
    qk_nope_head_dim=128,
    topk_method='gready',
    n_group=None,
    topk_group=None,
    num_experts_per_tok=None,
    moe_layer_freq=1,
    first_k_dense_replace=0,
    norm_topk_prob=False,
    scoring_func='softmax',
    aux_loss_alpha=0.001,
    seq_aux=True,
    hidden_act="silu",
    max_position_embeddings=2048,
    initializer_range=0.02,
    rms_norm_eps=1e-6,
    use_cache=True,
    pad_token_id=None,
    bos_token_id=100000,
    eos_token_id=100001,
    pretraining_tp=1,
    tie_word_embeddings=False,
    rope_theta=10000.0,
    rope_scaling=None,
    attention_bias=False,
    attention_dropout=0.0,
    use_mla=True,
    **kwargs,
):
    self.vocab_size = vocab_size
    self.max_position_embeddings = max_position_embeddings
    self.hidden_size = hidden_size
    self.intermediate_size = intermediate_size
    self.moe_intermediate_size = moe_intermediate_size
    self.num_hidden_layers = num_hidden_layers
    self.num_attention_heads = num_attention_heads
    self.n_shared_experts = n_shared_experts
    self.n_routed_experts = n_routed_experts
    self.ep_size = ep_size
    self.routed_scaling_factor = routed_scaling_factor
    self.kv_lora_rank = kv_lora_rank
    self.q_lora_rank = q_lora_rank
    self.qk_rope_head_dim = qk_rope_head_dim
    self.v_head_dim = v_head_dim
    self.qk_nope_head_dim = qk_nope_head_dim
    self.topk_method = topk_method
    self.n_group = n_group
    self.topk_group = topk_group
    self.num_experts_per_tok = num_experts_per_tok
    self.moe_layer_freq = moe_layer_freq
    self.first_k_dense_replace = first_k_dense_replace
    self.norm_topk_prob = norm_topk_prob
    self.scoring_func = scoring_func
    self.aux_loss_alpha = aux_loss_alpha
    self.seq_aux = seq_aux
    # for backward compatibility
    if num_key_value_heads is None:
        num_key_value_heads = num_attention_heads

    self.num_key_value_heads = num_key_value_heads
    self.hidden_act = hidden_act
    self.initializer_range = initializer_range
    self.rms_norm_eps = float(rms_norm_eps)
    self.pretraining_tp = pretraining_tp
    self.use_cache = use_cache
    self.rope_theta = rope_theta
    self.rope_scaling = rope_scaling
    self.attention_bias = attention_bias
    self.attention_dropout = attention_dropout
    self.use_mla = use_mla

    super().__init__(
        pad_token_id=pad_token_id,
        bos_token_id=bos_token_id,
        eos_token_id=eos_token_id,
        tie_word_embeddings=tie_word_embeddings,
        **kwargs,
    )

DeepseekVLV2Config

Bases: PretrainedConfig

Source code in vllm/transformers_utils/configs/deepseek_vl2.py 
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
class DeepseekVLV2Config(PretrainedConfig):
    model_type = "deepseek_vl_v2"
    vision_config: VisionEncoderConfig
    projector_config: MlpProjectorConfig

    tile_tag: str = "2D"
    global_view_pos: str = "head"
    candidate_resolutions: tuple[tuple[int, int]] = ((384, 384), )

    def __init__(self,
                 tile_tag: str = "tile_tag",
                 global_view_pos: str = "head",
                 candidate_resolutions: tuple[tuple[int,
                                                    int]] = ((384, 384), ),
                 **kwargs):
        super().__init__(**kwargs)

        vision_config = kwargs.get("vision_config", {})
        self.vision_config = VisionEncoderConfig(**vision_config)

        projector_config = kwargs.get("projector_config", {})
        self.projector_config = MlpProjectorConfig(**projector_config)

        language_config = kwargs.get("language_config", {})
        self.text_config = DeepseekV2Config(**language_config)

        self.tile_tag = tile_tag
        self.global_view_pos = global_view_pos
        self.candidate_resolutions = candidate_resolutions
        self.vocab_size = self.text_config.vocab_size

candidate_resolutions class-attribute instance-attribute 

candidate_resolutions: tuple[tuple[int, int]] = (
    candidate_resolutions
)

global_view_pos class-attribute instance-attribute 

global_view_pos: str = global_view_pos

model_type class-attribute instance-attribute 

model_type = 'deepseek_vl_v2'

projector_config instance-attribute 

projector_config: MlpProjectorConfig = MlpProjectorConfig(
    **projector_config
)

text_config instance-attribute 

text_config = DeepseekV2Config(**language_config)

tile_tag class-attribute instance-attribute 

tile_tag: str = tile_tag

vision_config instance-attribute 

vision_config: VisionEncoderConfig = VisionEncoderConfig(
    **vision_config
)

vocab_size instance-attribute 

vocab_size = vocab_size

__init__ 

__init__(
    tile_tag: str = "tile_tag",
    global_view_pos: str = "head",
    candidate_resolutions: tuple[tuple[int, int]] = (
        (384, 384),
    ),
    **kwargs,
)
Source code in vllm/transformers_utils/configs/deepseek_vl2.py 
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
def __init__(self,
             tile_tag: str = "tile_tag",
             global_view_pos: str = "head",
             candidate_resolutions: tuple[tuple[int,
                                                int]] = ((384, 384), ),
             **kwargs):
    super().__init__(**kwargs)

    vision_config = kwargs.get("vision_config", {})
    self.vision_config = VisionEncoderConfig(**vision_config)

    projector_config = kwargs.get("projector_config", {})
    self.projector_config = MlpProjectorConfig(**projector_config)

    language_config = kwargs.get("language_config", {})
    self.text_config = DeepseekV2Config(**language_config)

    self.tile_tag = tile_tag
    self.global_view_pos = global_view_pos
    self.candidate_resolutions = candidate_resolutions
    self.vocab_size = self.text_config.vocab_size

MlpProjectorConfig

Bases: PretrainedConfig

Source code in vllm/transformers_utils/configs/deepseek_vl2.py 
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
class MlpProjectorConfig(PretrainedConfig):
    model_type = "mlp_projector"
    projector_type: str = "downsample_mlp_gelu"
    input_dim: int = 1152
    n_embed: int = 2048
    depth: int = 2
    mlp_ratio: int = 1
    downsample_ratio: int = 2
    token_pooling: bool = False

    def __init__(self,
                 projector_type: str = "downsample_mlp_gelu",
                 input_dim: int = 1152,
                 n_embed: int = 2048,
                 depth: int = 2,
                 mlp_ratio: int = 1,
                 downsample_ratio: int = 2,
                 **kwargs):
        self.projector_type = projector_type
        self.input_dim = input_dim
        self.n_embed = n_embed
        self.depth = depth
        self.mlp_ratio = mlp_ratio
        self.downsample_ratio = downsample_ratio

        super().__init__(**kwargs)

depth class-attribute instance-attribute 

depth: int = depth

downsample_ratio class-attribute instance-attribute 

downsample_ratio: int = downsample_ratio

input_dim class-attribute instance-attribute 

input_dim: int = input_dim

mlp_ratio class-attribute instance-attribute 

mlp_ratio: int = mlp_ratio

model_type class-attribute instance-attribute 

model_type = 'mlp_projector'

n_embed class-attribute instance-attribute 

n_embed: int = n_embed

projector_type class-attribute instance-attribute 

projector_type: str = projector_type

token_pooling class-attribute instance-attribute 

token_pooling: bool = False

__init__ 

__init__(
    projector_type: str = "downsample_mlp_gelu",
    input_dim: int = 1152,
    n_embed: int = 2048,
    depth: int = 2,
    mlp_ratio: int = 1,
    downsample_ratio: int = 2,
    **kwargs,
)
Source code in vllm/transformers_utils/configs/deepseek_vl2.py 
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
def __init__(self,
             projector_type: str = "downsample_mlp_gelu",
             input_dim: int = 1152,
             n_embed: int = 2048,
             depth: int = 2,
             mlp_ratio: int = 1,
             downsample_ratio: int = 2,
             **kwargs):
    self.projector_type = projector_type
    self.input_dim = input_dim
    self.n_embed = n_embed
    self.depth = depth
    self.mlp_ratio = mlp_ratio
    self.downsample_ratio = downsample_ratio

    super().__init__(**kwargs)

VisionEncoderConfig

Bases: PretrainedConfig

Source code in vllm/transformers_utils/configs/deepseek_vl2.py 
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
class VisionEncoderConfig(PretrainedConfig):
    model_type: str = "vision"

    model_name: str = "vit_so400m_patch14_siglip_384.webli"
    image_size: int = 384
    patch_size: int = 16
    width: int = 1024
    layers: int = 24
    heads: int = 16
    mlp_ratio: int = 4
    global_pool: str = "map"
    ignore_head: bool = True
    class_token: bool = False
    num_classes: int = 0
    use_checkpoint: bool = False
    weight_init: str = "skip"
    deterministic: bool = False
    num_recomputing_layers: int = 0

    def __init__(self,
                 model_name: str = "vit_so400m_patch14_siglip_384.webli",
                 image_size: int = 384,
                 patch_size: int = 16,
                 width: int = 1024,
                 layers: int = 24,
                 heads: int = 16,
                 mlp_ratio: int = 4,
                 global_pool: str = "map",
                 ignore_head: bool = True,
                 class_token: bool = False,
                 num_classes: int = 0,
                 use_checkpoint: bool = False,
                 **kwargs):
        self.model_name = model_name
        self.image_size = image_size
        self.patch_size = patch_size
        self.width = width
        self.layers = layers
        self.heads = heads
        self.mlp_ratio = mlp_ratio
        self.global_pool = global_pool
        self.ignore_head = ignore_head
        self.class_token = class_token
        self.num_classes = num_classes
        self.use_checkpoint = use_checkpoint

        super().__init__(**kwargs)

class_token class-attribute instance-attribute 

class_token: bool = class_token

deterministic class-attribute instance-attribute 

deterministic: bool = False

global_pool class-attribute instance-attribute 

global_pool: str = global_pool

heads class-attribute instance-attribute 

heads: int = heads

ignore_head class-attribute instance-attribute 

ignore_head: bool = ignore_head

image_size class-attribute instance-attribute 

image_size: int = image_size

layers class-attribute instance-attribute 

layers: int = layers

mlp_ratio class-attribute instance-attribute 

mlp_ratio: int = mlp_ratio

model_name class-attribute instance-attribute 

model_name: str = model_name

model_type class-attribute instance-attribute 

model_type: str = 'vision'

num_classes class-attribute instance-attribute 

num_classes: int = num_classes

num_recomputing_layers class-attribute instance-attribute 

num_recomputing_layers: int = 0

patch_size class-attribute instance-attribute 

patch_size: int = patch_size

use_checkpoint class-attribute instance-attribute 

use_checkpoint: bool = use_checkpoint

weight_init class-attribute instance-attribute 

weight_init: str = 'skip'

width class-attribute instance-attribute 

width: int = width

__init__ 

__init__(
    model_name: str = "vit_so400m_patch14_siglip_384.webli",
    image_size: int = 384,
    patch_size: int = 16,
    width: int = 1024,
    layers: int = 24,
    heads: int = 16,
    mlp_ratio: int = 4,
    global_pool: str = "map",
    ignore_head: bool = True,
    class_token: bool = False,
    num_classes: int = 0,
    use_checkpoint: bool = False,
    **kwargs,
)
Source code in vllm/transformers_utils/configs/deepseek_vl2.py 
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
def __init__(self,
             model_name: str = "vit_so400m_patch14_siglip_384.webli",
             image_size: int = 384,
             patch_size: int = 16,
             width: int = 1024,
             layers: int = 24,
             heads: int = 16,
             mlp_ratio: int = 4,
             global_pool: str = "map",
             ignore_head: bool = True,
             class_token: bool = False,
             num_classes: int = 0,
             use_checkpoint: bool = False,
             **kwargs):
    self.model_name = model_name
    self.image_size = image_size
    self.patch_size = patch_size
    self.width = width
    self.layers = layers
    self.heads = heads
    self.mlp_ratio = mlp_ratio
    self.global_pool = global_pool
    self.ignore_head = ignore_head
    self.class_token = class_token
    self.num_classes = num_classes
    self.use_checkpoint = use_checkpoint

    super().__init__(**kwargs)