Skip to content

vllm.model_executor.models.intern_vit

NORM2FN module-attribute 

NORM2FN = {'rms_norm': RMSNorm, 'layer_norm': LayerNorm}

InternMLP

Bases: Module

Source code in vllm/model_executor/models/intern_vit.py 
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
class InternMLP(nn.Module):

    def __init__(
        self,
        config: PretrainedConfig,
        quant_config: Optional[QuantizationConfig] = None,
        prefix: str = "",
    ) -> None:
        super().__init__()

        self.config = config
        self.activation_fn = get_act_fn(config.hidden_act)
        self.fc1 = ColumnParallelLinear(config.hidden_size,
                                        config.intermediate_size,
                                        bias=True,
                                        quant_config=quant_config,
                                        prefix=f"{prefix}.fc1")
        self.fc2 = RowParallelLinear(config.intermediate_size,
                                     config.hidden_size,
                                     bias=True,
                                     quant_config=quant_config,
                                     prefix=f"{prefix}.fc2")

    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
        hidden_states, _ = self.fc1(hidden_states)
        hidden_states = self.activation_fn(hidden_states)
        hidden_states, _ = self.fc2(hidden_states)

        return hidden_states

activation_fn instance-attribute 

activation_fn = get_act_fn(hidden_act)

config instance-attribute 

config = config

fc1 instance-attribute 

fc1 = ColumnParallelLinear(
    hidden_size,
    intermediate_size,
    bias=True,
    quant_config=quant_config,
    prefix=f"{prefix}.fc1",
)

fc2 instance-attribute 

fc2 = RowParallelLinear(
    intermediate_size,
    hidden_size,
    bias=True,
    quant_config=quant_config,
    prefix=f"{prefix}.fc2",
)

__init__ 

__init__(
    config: PretrainedConfig,
    quant_config: Optional[QuantizationConfig] = None,
    prefix: str = "",
) -> None
Source code in vllm/model_executor/models/intern_vit.py 
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
def __init__(
    self,
    config: PretrainedConfig,
    quant_config: Optional[QuantizationConfig] = None,
    prefix: str = "",
) -> None:
    super().__init__()

    self.config = config
    self.activation_fn = get_act_fn(config.hidden_act)
    self.fc1 = ColumnParallelLinear(config.hidden_size,
                                    config.intermediate_size,
                                    bias=True,
                                    quant_config=quant_config,
                                    prefix=f"{prefix}.fc1")
    self.fc2 = RowParallelLinear(config.intermediate_size,
                                 config.hidden_size,
                                 bias=True,
                                 quant_config=quant_config,
                                 prefix=f"{prefix}.fc2")

forward 

forward(hidden_states: Tensor) -> Tensor
Source code in vllm/model_executor/models/intern_vit.py 
305
306
307
308
309
310
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
    hidden_states, _ = self.fc1(hidden_states)
    hidden_states = self.activation_fn(hidden_states)
    hidden_states, _ = self.fc2(hidden_states)

    return hidden_states

InternParallelAttention

Bases: Module

Multi-headed attention from 'Attention Is All You Need' paper

Source code in vllm/model_executor/models/intern_vit.py 
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
class InternParallelAttention(nn.Module):
    """Multi-headed attention from 'Attention Is All You Need' paper"""

    def __init__(
        self,
        config: PretrainedConfig,
        quant_config: Optional[QuantizationConfig] = None,
        *,
        num_dummy_heads: int = 0,
        prefix: str = "",
    ) -> None:
        super().__init__()

        self.config = config
        self.embed_dim = config.hidden_size
        self.num_heads = config.num_attention_heads
        self.head_dim = self.embed_dim // self.num_heads
        if self.head_dim * self.num_heads != self.embed_dim:
            raise ValueError(
                f'embed_dim must be divisible by num_heads '
                f'(got `embed_dim`: {self.embed_dim} and `num_heads`:'
                f' {self.num_heads}).')

        self.tp_size = get_tensor_model_parallel_world_size()
        self.tp_rank = get_tensor_model_parallel_rank()

        # Additional dummy heads are used to enable TP for common GPU counts.
        self.dummy_dim = (num_dummy_heads + self.num_heads) * self.head_dim
        self.num_heads_per_partition = divide(num_dummy_heads + self.num_heads,
                                              self.tp_size)

        self.scale = self.head_dim**-0.5
        self.qkv = QKVParallelLinear(
            self.embed_dim,
            self.head_dim,
            num_dummy_heads + self.num_heads,
            bias=config.qkv_bias,
            quant_config=quant_config,
            prefix=f"{prefix}.qkv",
        )

        self.qk_normalization = config.qk_normalization

        if self.qk_normalization:
            self.q_norm = RMSNorm(self.dummy_dim,
                                  eps=config.layer_norm_eps,
                                  var_hidden_size=self.embed_dim)
            self.k_norm = RMSNorm(self.dummy_dim,
                                  eps=config.layer_norm_eps,
                                  var_hidden_size=self.embed_dim)

        self.proj = RowParallelLinear(
            self.dummy_dim,
            self.embed_dim,
            quant_config=quant_config,
            prefix=f"{prefix}.proj",
        )

        self.attn = MultiHeadAttention(self.num_heads_per_partition,
                                       self.head_dim, self.scale)

    def _apply_qk_norm(self, q: torch.Tensor, k: torch.Tensor):
        if self.tp_size > 1:
            q = tensor_model_parallel_all_gather(q.contiguous())
            k = tensor_model_parallel_all_gather(k.contiguous())
        q = self.q_norm(q)
        k = self.k_norm(k)
        if self.tp_size > 1:
            splitter = partial(split_tensor_along_last_dim,
                               num_partitions=self.tp_size)
            q = splitter(q)[self.tp_rank]
            k = splitter(k)[self.tp_rank]
        return q, k

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        B, N, _ = x.shape
        qkv, _ = self.qkv(x)
        q, k, v = qkv.chunk(3, dim=-1)

        if self.qk_normalization:
            q, k = self._apply_qk_norm(q, k)

        out = self.attn(q, k, v)
        out, _ = self.proj(out)
        return out

attn instance-attribute 

attn = MultiHeadAttention(
    num_heads_per_partition, head_dim, scale
)

config instance-attribute 

config = config

dummy_dim instance-attribute 

dummy_dim = num_dummy_heads + num_heads * head_dim

embed_dim instance-attribute 

embed_dim = hidden_size

head_dim instance-attribute 

head_dim = embed_dim // num_heads

k_norm instance-attribute 

k_norm = RMSNorm(
    dummy_dim, eps=layer_norm_eps, var_hidden_size=embed_dim
)

num_heads instance-attribute 

num_heads = num_attention_heads

num_heads_per_partition instance-attribute 

num_heads_per_partition = divide(
    num_dummy_heads + num_heads, tp_size
)

proj instance-attribute 

proj = RowParallelLinear(
    dummy_dim,
    embed_dim,
    quant_config=quant_config,
    prefix=f"{prefix}.proj",
)

q_norm instance-attribute 

q_norm = RMSNorm(
    dummy_dim, eps=layer_norm_eps, var_hidden_size=embed_dim
)

qk_normalization instance-attribute 

qk_normalization = qk_normalization

qkv instance-attribute 

qkv = QKVParallelLinear(
    embed_dim,
    head_dim,
    num_dummy_heads + num_heads,
    bias=qkv_bias,
    quant_config=quant_config,
    prefix=f"{prefix}.qkv",
)

scale instance-attribute 

scale = head_dim ** -0.5

tp_rank instance-attribute 

tp_rank = get_tensor_model_parallel_rank()

tp_size instance-attribute 

tp_size = get_tensor_model_parallel_world_size()

__init__ 

__init__(
    config: PretrainedConfig,
    quant_config: Optional[QuantizationConfig] = None,
    *,
    num_dummy_heads: int = 0,
    prefix: str = "",
) -> None
Source code in vllm/model_executor/models/intern_vit.py 
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
def __init__(
    self,
    config: PretrainedConfig,
    quant_config: Optional[QuantizationConfig] = None,
    *,
    num_dummy_heads: int = 0,
    prefix: str = "",
) -> None:
    super().__init__()

    self.config = config
    self.embed_dim = config.hidden_size
    self.num_heads = config.num_attention_heads
    self.head_dim = self.embed_dim // self.num_heads
    if self.head_dim * self.num_heads != self.embed_dim:
        raise ValueError(
            f'embed_dim must be divisible by num_heads '
            f'(got `embed_dim`: {self.embed_dim} and `num_heads`:'
            f' {self.num_heads}).')

    self.tp_size = get_tensor_model_parallel_world_size()
    self.tp_rank = get_tensor_model_parallel_rank()

    # Additional dummy heads are used to enable TP for common GPU counts.
    self.dummy_dim = (num_dummy_heads + self.num_heads) * self.head_dim
    self.num_heads_per_partition = divide(num_dummy_heads + self.num_heads,
                                          self.tp_size)

    self.scale = self.head_dim**-0.5
    self.qkv = QKVParallelLinear(
        self.embed_dim,
        self.head_dim,
        num_dummy_heads + self.num_heads,
        bias=config.qkv_bias,
        quant_config=quant_config,
        prefix=f"{prefix}.qkv",
    )

    self.qk_normalization = config.qk_normalization

    if self.qk_normalization:
        self.q_norm = RMSNorm(self.dummy_dim,
                              eps=config.layer_norm_eps,
                              var_hidden_size=self.embed_dim)
        self.k_norm = RMSNorm(self.dummy_dim,
                              eps=config.layer_norm_eps,
                              var_hidden_size=self.embed_dim)

    self.proj = RowParallelLinear(
        self.dummy_dim,
        self.embed_dim,
        quant_config=quant_config,
        prefix=f"{prefix}.proj",
    )

    self.attn = MultiHeadAttention(self.num_heads_per_partition,
                                   self.head_dim, self.scale)

_apply_qk_norm 

_apply_qk_norm(q: Tensor, k: Tensor)
Source code in vllm/model_executor/models/intern_vit.py 
191
192
193
194
195
196
197
198
199
200
201
202
def _apply_qk_norm(self, q: torch.Tensor, k: torch.Tensor):
    if self.tp_size > 1:
        q = tensor_model_parallel_all_gather(q.contiguous())
        k = tensor_model_parallel_all_gather(k.contiguous())
    q = self.q_norm(q)
    k = self.k_norm(k)
    if self.tp_size > 1:
        splitter = partial(split_tensor_along_last_dim,
                           num_partitions=self.tp_size)
        q = splitter(q)[self.tp_rank]
        k = splitter(k)[self.tp_rank]
    return q, k

forward 

forward(x: Tensor) -> Tensor
Source code in vllm/model_executor/models/intern_vit.py 
204
205
206
207
208
209
210
211
212
213
214
def forward(self, x: torch.Tensor) -> torch.Tensor:
    B, N, _ = x.shape
    qkv, _ = self.qkv(x)
    q, k, v = qkv.chunk(3, dim=-1)

    if self.qk_normalization:
        q, k = self._apply_qk_norm(q, k)

    out = self.attn(q, k, v)
    out, _ = self.proj(out)
    return out

InternSdpaAttention

Bases: Module

Multi-headed attention from 'Attention Is All You Need' paper

Source code in vllm/model_executor/models/intern_vit.py 
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
class InternSdpaAttention(nn.Module):
    """Multi-headed attention from 'Attention Is All You Need' paper"""

    def __init__(
        self,
        config: PretrainedConfig,
        *,
        num_dummy_heads: int = 0,
    ) -> None:
        super().__init__()

        self.config = config
        self.embed_dim = config.hidden_size
        self.num_heads = config.num_attention_heads
        self.head_dim = self.embed_dim // self.num_heads
        if self.head_dim * self.num_heads != self.embed_dim:
            raise ValueError(
                f'embed_dim must be divisible by num_heads '
                f'(got `embed_dim`: {self.embed_dim} and `num_heads`:'
                f' {self.num_heads}).')

        # Additional dummy heads are used to enable TP for common GPU counts.
        self.dummy_dim = (num_dummy_heads + self.num_heads) * self.head_dim

        self.scale = self.head_dim**-0.5
        self.qkv = nn.Linear(self.embed_dim,
                             3 * self.dummy_dim,
                             bias=config.qkv_bias)

        self.qk_normalization = config.qk_normalization

        if self.qk_normalization:
            self.q_norm = RMSNorm(self.dummy_dim,
                                  eps=config.layer_norm_eps,
                                  var_hidden_size=self.embed_dim)
            self.k_norm = RMSNorm(self.dummy_dim,
                                  eps=config.layer_norm_eps,
                                  var_hidden_size=self.embed_dim)

        self.proj = nn.Linear(self.dummy_dim, self.embed_dim)

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        B, N, C = x.shape
        qkv = self.qkv(x)
        q, k, v = qkv.chunk(3, dim=-1)

        q = q.view(B, N, self.num_heads, self.head_dim)
        k = k.view(B, N, self.num_heads, self.head_dim)
        v = v.view(B, N, self.num_heads, self.head_dim)

        if self.qk_normalization:
            B_, N_, H_, D_ = q.shape
            q = self.q_norm(q.flatten(-2, -1)).view(B_, N_, H_, D_)
            k = self.k_norm(k.flatten(-2, -1)).view(B_, N_, H_, D_)
        q = q.transpose(1, 2)
        k = k.transpose(1, 2)
        v = v.transpose(1, 2)

        x = F.scaled_dot_product_attention(q, k, v, scale=self.scale)
        x = x.transpose(1, 2).reshape(B, N, -1)

        x = self.proj(x)
        return x

config instance-attribute 

config = config

dummy_dim instance-attribute 

dummy_dim = num_dummy_heads + num_heads * head_dim

embed_dim instance-attribute 

embed_dim = hidden_size

head_dim instance-attribute 

head_dim = embed_dim // num_heads

k_norm instance-attribute 

k_norm = RMSNorm(
    dummy_dim, eps=layer_norm_eps, var_hidden_size=embed_dim
)

num_heads instance-attribute 

num_heads = num_attention_heads

proj instance-attribute 

proj = Linear(dummy_dim, embed_dim)

q_norm instance-attribute 

q_norm = RMSNorm(
    dummy_dim, eps=layer_norm_eps, var_hidden_size=embed_dim
)

qk_normalization instance-attribute 

qk_normalization = qk_normalization

qkv instance-attribute 

qkv = Linear(embed_dim, 3 * dummy_dim, bias=qkv_bias)

scale instance-attribute 

scale = head_dim ** -0.5

__init__ 

__init__(
    config: PretrainedConfig, *, num_dummy_heads: int = 0
) -> None
Source code in vllm/model_executor/models/intern_vit.py 
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
def __init__(
    self,
    config: PretrainedConfig,
    *,
    num_dummy_heads: int = 0,
) -> None:
    super().__init__()

    self.config = config
    self.embed_dim = config.hidden_size
    self.num_heads = config.num_attention_heads
    self.head_dim = self.embed_dim // self.num_heads
    if self.head_dim * self.num_heads != self.embed_dim:
        raise ValueError(
            f'embed_dim must be divisible by num_heads '
            f'(got `embed_dim`: {self.embed_dim} and `num_heads`:'
            f' {self.num_heads}).')

    # Additional dummy heads are used to enable TP for common GPU counts.
    self.dummy_dim = (num_dummy_heads + self.num_heads) * self.head_dim

    self.scale = self.head_dim**-0.5
    self.qkv = nn.Linear(self.embed_dim,
                         3 * self.dummy_dim,
                         bias=config.qkv_bias)

    self.qk_normalization = config.qk_normalization

    if self.qk_normalization:
        self.q_norm = RMSNorm(self.dummy_dim,
                              eps=config.layer_norm_eps,
                              var_hidden_size=self.embed_dim)
        self.k_norm = RMSNorm(self.dummy_dim,
                              eps=config.layer_norm_eps,
                              var_hidden_size=self.embed_dim)

    self.proj = nn.Linear(self.dummy_dim, self.embed_dim)

forward 

forward(x: Tensor) -> Tensor
Source code in vllm/model_executor/models/intern_vit.py 
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
def forward(self, x: torch.Tensor) -> torch.Tensor:
    B, N, C = x.shape
    qkv = self.qkv(x)
    q, k, v = qkv.chunk(3, dim=-1)

    q = q.view(B, N, self.num_heads, self.head_dim)
    k = k.view(B, N, self.num_heads, self.head_dim)
    v = v.view(B, N, self.num_heads, self.head_dim)

    if self.qk_normalization:
        B_, N_, H_, D_ = q.shape
        q = self.q_norm(q.flatten(-2, -1)).view(B_, N_, H_, D_)
        k = self.k_norm(k.flatten(-2, -1)).view(B_, N_, H_, D_)
    q = q.transpose(1, 2)
    k = k.transpose(1, 2)
    v = v.transpose(1, 2)

    x = F.scaled_dot_product_attention(q, k, v, scale=self.scale)
    x = x.transpose(1, 2).reshape(B, N, -1)

    x = self.proj(x)
    return x

InternVisionEmbeddings

Bases: Module

Source code in vllm/model_executor/models/intern_vit.py 
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
class InternVisionEmbeddings(nn.Module):

    def __init__(self, config: PretrainedConfig):
        super().__init__()
        self.config = config
        self.embed_dim = config.hidden_size
        self.image_size = config.image_size
        self.patch_size = config.patch_size

        self.class_embedding = nn.Parameter(torch.randn(1, 1, self.embed_dim))

        self.patch_embedding = nn.Conv2d(in_channels=3,
                                         out_channels=self.embed_dim,
                                         kernel_size=self.patch_size,
                                         stride=self.patch_size)

        self.num_patches = (self.image_size // self.patch_size)**2
        self.num_positions = self.num_patches + 1

        self.position_embedding = nn.Parameter(
            torch.randn(1, self.num_positions, self.embed_dim))

    def _get_pos_embed(self, pos_embed: torch.Tensor, H: int, W: int):
        target_dtype = pos_embed.dtype
        pos_embed = pos_embed.float().reshape(
            1, self.image_size // self.patch_size,
            self.image_size // self.patch_size, -1).permute(0, 3, 1, 2)
        pos_embed = F.interpolate(pos_embed,
                                  size=(H, W),
                                  mode='bicubic',
                                  align_corners=False)
        return pos_embed.reshape(1, -1, H * W).permute(0, 2,
                                                       1).to(target_dtype)

    def _get_position_embedding(self, H: int, W: int) -> torch.Tensor:
        position_embedding = self.position_embedding
        if self.num_patches == H * W:
            return position_embedding

        return torch.cat(
            [
                position_embedding[:, :1, :],
                self._get_pos_embed(position_embedding[:, 1:, :], H, W),
            ],
            dim=1,
        )

    def forward(self, pixel_values: torch.FloatTensor) -> torch.Tensor:
        target_dtype = self.patch_embedding.weight.dtype
        patch_embeds = self.patch_embedding(pixel_values.to(
            target_dtype))  # shape = [*, channel, width, height]
        batch_size, _, height, width = patch_embeds.shape
        patch_embeds = patch_embeds.flatten(2).transpose(1, 2)
        class_embeds = self.class_embedding.expand(batch_size, 1,
                                                   -1).to(target_dtype)
        embeddings = torch.cat([class_embeds, patch_embeds], dim=1)
        position_embedding = self._get_position_embedding(height, width)
        embeddings = embeddings + position_embedding.to(target_dtype)
        return embeddings

class_embedding instance-attribute 

class_embedding = Parameter(randn(1, 1, embed_dim))

config instance-attribute 

config = config

embed_dim instance-attribute 

embed_dim = hidden_size

image_size instance-attribute 

image_size = image_size

num_patches instance-attribute 

num_patches = image_size // patch_size ** 2

num_positions instance-attribute 

num_positions = num_patches + 1

patch_embedding instance-attribute 

patch_embedding = Conv2d(
    in_channels=3,
    out_channels=embed_dim,
    kernel_size=patch_size,
    stride=patch_size,
)

patch_size instance-attribute 

patch_size = patch_size

position_embedding instance-attribute 

position_embedding = Parameter(
    randn(1, num_positions, embed_dim)
)

__init__ 

__init__(config: PretrainedConfig)
Source code in vllm/model_executor/models/intern_vit.py 
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
def __init__(self, config: PretrainedConfig):
    super().__init__()
    self.config = config
    self.embed_dim = config.hidden_size
    self.image_size = config.image_size
    self.patch_size = config.patch_size

    self.class_embedding = nn.Parameter(torch.randn(1, 1, self.embed_dim))

    self.patch_embedding = nn.Conv2d(in_channels=3,
                                     out_channels=self.embed_dim,
                                     kernel_size=self.patch_size,
                                     stride=self.patch_size)

    self.num_patches = (self.image_size // self.patch_size)**2
    self.num_positions = self.num_patches + 1

    self.position_embedding = nn.Parameter(
        torch.randn(1, self.num_positions, self.embed_dim))

_get_pos_embed 

_get_pos_embed(pos_embed: Tensor, H: int, W: int)
Source code in vllm/model_executor/models/intern_vit.py 
60
61
62
63
64
65
66
67
68
69
70
def _get_pos_embed(self, pos_embed: torch.Tensor, H: int, W: int):
    target_dtype = pos_embed.dtype
    pos_embed = pos_embed.float().reshape(
        1, self.image_size // self.patch_size,
        self.image_size // self.patch_size, -1).permute(0, 3, 1, 2)
    pos_embed = F.interpolate(pos_embed,
                              size=(H, W),
                              mode='bicubic',
                              align_corners=False)
    return pos_embed.reshape(1, -1, H * W).permute(0, 2,
                                                   1).to(target_dtype)

_get_position_embedding 

_get_position_embedding(H: int, W: int) -> Tensor
Source code in vllm/model_executor/models/intern_vit.py 
72
73
74
75
76
77
78
79
80
81
82
83
def _get_position_embedding(self, H: int, W: int) -> torch.Tensor:
    position_embedding = self.position_embedding
    if self.num_patches == H * W:
        return position_embedding

    return torch.cat(
        [
            position_embedding[:, :1, :],
            self._get_pos_embed(position_embedding[:, 1:, :], H, W),
        ],
        dim=1,
    )

forward 

forward(pixel_values: FloatTensor) -> Tensor
Source code in vllm/model_executor/models/intern_vit.py 
85
86
87
88
89
90
91
92
93
94
95
96
def forward(self, pixel_values: torch.FloatTensor) -> torch.Tensor:
    target_dtype = self.patch_embedding.weight.dtype
    patch_embeds = self.patch_embedding(pixel_values.to(
        target_dtype))  # shape = [*, channel, width, height]
    batch_size, _, height, width = patch_embeds.shape
    patch_embeds = patch_embeds.flatten(2).transpose(1, 2)
    class_embeds = self.class_embedding.expand(batch_size, 1,
                                               -1).to(target_dtype)
    embeddings = torch.cat([class_embeds, patch_embeds], dim=1)
    position_embedding = self._get_position_embedding(height, width)
    embeddings = embeddings + position_embedding.to(target_dtype)
    return embeddings

InternVisionEncoder

Bases: Module

Source code in vllm/model_executor/models/intern_vit.py 
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
class InternVisionEncoder(nn.Module):

    def __init__(
        self,
        config: PretrainedConfig,
        quant_config: Optional[QuantizationConfig] = None,
        *,
        num_hidden_layers_override: Optional[int] = None,
        num_dummy_heads: int = 0,
        prefix: str = "",
    ):
        super().__init__()

        self.config = config

        if num_hidden_layers_override is None:
            num_hidden_layers = config.num_hidden_layers
        else:
            num_hidden_layers = num_hidden_layers_override

        self.layers = nn.ModuleList([
            InternVisionEncoderLayer(config,
                                     quant_config,
                                     num_dummy_heads=num_dummy_heads,
                                     prefix=f"{prefix}.layers.{layer_idx}")
            for layer_idx in range(num_hidden_layers)
        ])

    def forward(self, inputs_embeds: torch.Tensor):

        hidden_states = inputs_embeds
        for encoder_layer in self.layers:
            hidden_states = encoder_layer(hidden_states)

        return hidden_states

config instance-attribute 

config = config

layers instance-attribute 

layers = ModuleList(
    [
        InternVisionEncoderLayer(
            config,
            quant_config,
            num_dummy_heads=num_dummy_heads,
            prefix=f"{prefix}.layers.{layer_idx}",
        )
        for layer_idx in range(num_hidden_layers)
    ]
)

__init__ 

__init__(
    config: PretrainedConfig,
    quant_config: Optional[QuantizationConfig] = None,
    *,
    num_hidden_layers_override: Optional[int] = None,
    num_dummy_heads: int = 0,
    prefix: str = "",
)
Source code in vllm/model_executor/models/intern_vit.py 
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
def __init__(
    self,
    config: PretrainedConfig,
    quant_config: Optional[QuantizationConfig] = None,
    *,
    num_hidden_layers_override: Optional[int] = None,
    num_dummy_heads: int = 0,
    prefix: str = "",
):
    super().__init__()

    self.config = config

    if num_hidden_layers_override is None:
        num_hidden_layers = config.num_hidden_layers
    else:
        num_hidden_layers = num_hidden_layers_override

    self.layers = nn.ModuleList([
        InternVisionEncoderLayer(config,
                                 quant_config,
                                 num_dummy_heads=num_dummy_heads,
                                 prefix=f"{prefix}.layers.{layer_idx}")
        for layer_idx in range(num_hidden_layers)
    ])

forward 

forward(inputs_embeds: Tensor)
Source code in vllm/model_executor/models/intern_vit.py 
408
409
410
411
412
413
414
def forward(self, inputs_embeds: torch.Tensor):

    hidden_states = inputs_embeds
    for encoder_layer in self.layers:
        hidden_states = encoder_layer(hidden_states)

    return hidden_states

InternVisionEncoderLayer

Bases: Module

Source code in vllm/model_executor/models/intern_vit.py 
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
class InternVisionEncoderLayer(nn.Module):

    def __init__(
        self,
        config: PretrainedConfig,
        quant_config: Optional[QuantizationConfig] = None,
        *,
        num_dummy_heads: int = 0,
        prefix: str = "",
    ) -> None:
        super().__init__()

        self.embed_dim = config.hidden_size
        self.intermediate_size = config.intermediate_size
        self.norm_type = config.norm_type

        self.attn = self._init_attn(config,
                                    quant_config,
                                    num_dummy_heads=num_dummy_heads,
                                    prefix=f"{prefix}.attn")

        self.mlp = InternMLP(config,
                             quant_config=quant_config,
                             prefix=f"{prefix}.mlp")
        self.norm1 = NORM2FN[self.norm_type](self.embed_dim,
                                             eps=config.layer_norm_eps)
        self.norm2 = NORM2FN[self.norm_type](self.embed_dim,
                                             eps=config.layer_norm_eps)

        self.ls1 = nn.Parameter(config.initializer_factor *
                                torch.ones(self.embed_dim))
        self.ls2 = nn.Parameter(config.initializer_factor *
                                torch.ones(self.embed_dim))

    def _init_attn(
        self,
        config: PretrainedConfig,
        quant_config: Optional[QuantizationConfig],
        *,
        num_dummy_heads: int,
        prefix: str = "",
    ):
        # fallback to sdpa attention if tp unavailable
        tp_size = get_tensor_model_parallel_world_size()
        num_heads = config.num_attention_heads

        if (num_heads + num_dummy_heads) % tp_size == 0:
            return InternParallelAttention(config,
                                           quant_config=quant_config,
                                           num_dummy_heads=num_dummy_heads,
                                           prefix=prefix)

        return InternSdpaAttention(config, num_dummy_heads=num_dummy_heads)

    def forward(
        self,
        hidden_states: torch.Tensor,
    ):
        hidden_states = hidden_states + self.attn(
            self.norm1(hidden_states)) * self.ls1

        hidden_states = hidden_states + self.mlp(
            self.norm2(hidden_states)) * self.ls2

        return hidden_states

attn instance-attribute 

attn = _init_attn(
    config,
    quant_config,
    num_dummy_heads=num_dummy_heads,
    prefix=f"{prefix}.attn",
)

embed_dim instance-attribute 

embed_dim = hidden_size

intermediate_size instance-attribute 

intermediate_size = intermediate_size

ls1 instance-attribute 

ls1 = Parameter(initializer_factor * ones(embed_dim))

ls2 instance-attribute 

ls2 = Parameter(initializer_factor * ones(embed_dim))

mlp instance-attribute 

mlp = InternMLP(
    config,
    quant_config=quant_config,
    prefix=f"{prefix}.mlp",
)

norm1 instance-attribute 

norm1 = NORM2FN[norm_type](embed_dim, eps=layer_norm_eps)

norm2 instance-attribute 

norm2 = NORM2FN[norm_type](embed_dim, eps=layer_norm_eps)

norm_type instance-attribute 

norm_type = norm_type

__init__ 

__init__(
    config: PretrainedConfig,
    quant_config: Optional[QuantizationConfig] = None,
    *,
    num_dummy_heads: int = 0,
    prefix: str = "",
) -> None
Source code in vllm/model_executor/models/intern_vit.py 
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
def __init__(
    self,
    config: PretrainedConfig,
    quant_config: Optional[QuantizationConfig] = None,
    *,
    num_dummy_heads: int = 0,
    prefix: str = "",
) -> None:
    super().__init__()

    self.embed_dim = config.hidden_size
    self.intermediate_size = config.intermediate_size
    self.norm_type = config.norm_type

    self.attn = self._init_attn(config,
                                quant_config,
                                num_dummy_heads=num_dummy_heads,
                                prefix=f"{prefix}.attn")

    self.mlp = InternMLP(config,
                         quant_config=quant_config,
                         prefix=f"{prefix}.mlp")
    self.norm1 = NORM2FN[self.norm_type](self.embed_dim,
                                         eps=config.layer_norm_eps)
    self.norm2 = NORM2FN[self.norm_type](self.embed_dim,
                                         eps=config.layer_norm_eps)

    self.ls1 = nn.Parameter(config.initializer_factor *
                            torch.ones(self.embed_dim))
    self.ls2 = nn.Parameter(config.initializer_factor *
                            torch.ones(self.embed_dim))

_init_attn 

_init_attn(
    config: PretrainedConfig,
    quant_config: Optional[QuantizationConfig],
    *,
    num_dummy_heads: int,
    prefix: str = "",
)
Source code in vllm/model_executor/models/intern_vit.py 
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
def _init_attn(
    self,
    config: PretrainedConfig,
    quant_config: Optional[QuantizationConfig],
    *,
    num_dummy_heads: int,
    prefix: str = "",
):
    # fallback to sdpa attention if tp unavailable
    tp_size = get_tensor_model_parallel_world_size()
    num_heads = config.num_attention_heads

    if (num_heads + num_dummy_heads) % tp_size == 0:
        return InternParallelAttention(config,
                                       quant_config=quant_config,
                                       num_dummy_heads=num_dummy_heads,
                                       prefix=prefix)

    return InternSdpaAttention(config, num_dummy_heads=num_dummy_heads)

forward 

forward(hidden_states: Tensor)
Source code in vllm/model_executor/models/intern_vit.py 
367
368
369
370
371
372
373
374
375
376
377
def forward(
    self,
    hidden_states: torch.Tensor,
):
    hidden_states = hidden_states + self.attn(
        self.norm1(hidden_states)) * self.ls1

    hidden_states = hidden_states + self.mlp(
        self.norm2(hidden_states)) * self.ls2

    return hidden_states

InternVisionModel

Bases: Module

Source code in vllm/model_executor/models/intern_vit.py 
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
class InternVisionModel(nn.Module):

    packed_modules_mapping = {
        "qkv": ["qkv"],
    }

    def __init__(
        self,
        config: PretrainedConfig,
        quant_config: Optional[QuantizationConfig] = None,
        *,
        num_hidden_layers_override: Optional[int] = None,
        num_dummy_heads: int = 0,
        prefix: str = "",
    ) -> None:
        super().__init__()

        self.config = config

        self.embeddings = InternVisionEmbeddings(config)
        self.encoder = InternVisionEncoder(
            config=config,
            quant_config=quant_config,
            num_hidden_layers_override=num_hidden_layers_override,
            num_dummy_heads=num_dummy_heads,
            prefix=f"{prefix}.encoder",
        )

    def get_input_embeddings(self):
        return self.embeddings

    def forward(
        self,
        pixel_values: Optional[torch.Tensor] = None,
        pixel_embeds: Optional[torch.Tensor] = None,
    ) -> torch.FloatTensor:
        if pixel_values is None and pixel_embeds is None:
            raise ValueError(
                'You have to specify pixel_values or pixel_embeds')

        if pixel_embeds is not None:
            hidden_states = pixel_embeds
        elif pixel_values is not None:
            if pixel_values.ndim == 4:
                hidden_states = self.embeddings(pixel_values)
            else:
                raise ValueError(
                    f'wrong pixel_values size: {pixel_values.shape}')

        encoder_outputs = self.encoder(inputs_embeds=hidden_states)

        return encoder_outputs

    def load_weights(self, weights: Iterable[tuple[str,
                                                   torch.Tensor]]) -> set[str]:
        params_dict = dict(self.named_parameters())
        loaded_params: set[str] = set()
        for name, loaded_weight in weights:
            param = params_dict[name]
            weight_loader = getattr(param, "weight_loader",
                                    default_weight_loader)
            weight_loader(param, loaded_weight)
            loaded_params.add(name)
        return loaded_params

config instance-attribute 

config = config

embeddings instance-attribute 

embeddings = InternVisionEmbeddings(config)

encoder instance-attribute 

encoder = InternVisionEncoder(
    config=config,
    quant_config=quant_config,
    num_hidden_layers_override=num_hidden_layers_override,
    num_dummy_heads=num_dummy_heads,
    prefix=f"{prefix}.encoder",
)

packed_modules_mapping class-attribute instance-attribute 

packed_modules_mapping = {'qkv': ['qkv']}

__init__ 

__init__(
    config: PretrainedConfig,
    quant_config: Optional[QuantizationConfig] = None,
    *,
    num_hidden_layers_override: Optional[int] = None,
    num_dummy_heads: int = 0,
    prefix: str = "",
) -> None
Source code in vllm/model_executor/models/intern_vit.py 
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
def __init__(
    self,
    config: PretrainedConfig,
    quant_config: Optional[QuantizationConfig] = None,
    *,
    num_hidden_layers_override: Optional[int] = None,
    num_dummy_heads: int = 0,
    prefix: str = "",
) -> None:
    super().__init__()

    self.config = config

    self.embeddings = InternVisionEmbeddings(config)
    self.encoder = InternVisionEncoder(
        config=config,
        quant_config=quant_config,
        num_hidden_layers_override=num_hidden_layers_override,
        num_dummy_heads=num_dummy_heads,
        prefix=f"{prefix}.encoder",
    )

forward 

forward(
    pixel_values: Optional[Tensor] = None,
    pixel_embeds: Optional[Tensor] = None,
) -> FloatTensor
Source code in vllm/model_executor/models/intern_vit.py 
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
def forward(
    self,
    pixel_values: Optional[torch.Tensor] = None,
    pixel_embeds: Optional[torch.Tensor] = None,
) -> torch.FloatTensor:
    if pixel_values is None and pixel_embeds is None:
        raise ValueError(
            'You have to specify pixel_values or pixel_embeds')

    if pixel_embeds is not None:
        hidden_states = pixel_embeds
    elif pixel_values is not None:
        if pixel_values.ndim == 4:
            hidden_states = self.embeddings(pixel_values)
        else:
            raise ValueError(
                f'wrong pixel_values size: {pixel_values.shape}')

    encoder_outputs = self.encoder(inputs_embeds=hidden_states)

    return encoder_outputs

get_input_embeddings 

get_input_embeddings()
Source code in vllm/model_executor/models/intern_vit.py 
445
446
def get_input_embeddings(self):
    return self.embeddings

load_weights 

load_weights(
    weights: Iterable[tuple[str, Tensor]],
) -> set[str]
Source code in vllm/model_executor/models/intern_vit.py 
470
471
472
473
474
475
476
477
478
479
480
def load_weights(self, weights: Iterable[tuple[str,
                                               torch.Tensor]]) -> set[str]:
    params_dict = dict(self.named_parameters())
    loaded_params: set[str] = set()
    for name, loaded_weight in weights:
        param = params_dict[name]
        weight_loader = getattr(param, "weight_loader",
                                default_weight_loader)
        weight_loader(param, loaded_weight)
        loaded_params.add(name)
    return loaded_params

InternVisionPatchModel

Bases: Module

Source code in vllm/model_executor/models/intern_vit.py 
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
class InternVisionPatchModel(nn.Module):

    def __init__(self, config: PretrainedConfig):
        super().__init__()
        self.config = config
        self.embeddings = InternVisionEmbeddings(config)

    def get_input_embeddings(self):
        return self.embeddings

    def forward(
        self,
        pixel_values: Optional[torch.Tensor] = None,
        pixel_embeds: Optional[torch.Tensor] = None,
    ) -> torch.FloatTensor:
        if pixel_values is None and pixel_embeds is None:
            raise ValueError(
                'You have to specify pixel_values or pixel_embeds')

        if pixel_embeds is not None:
            hidden_states = pixel_embeds
        elif pixel_values is not None:
            if pixel_values.ndim == 4:
                hidden_states = self.embeddings(pixel_values)
            else:
                raise ValueError(
                    f'wrong pixel_values size: {pixel_values.shape}')

        return hidden_states

config instance-attribute 

config = config

embeddings instance-attribute 

embeddings = InternVisionEmbeddings(config)

__init__ 

__init__(config: PretrainedConfig)
Source code in vllm/model_executor/models/intern_vit.py 
101
102
103
104
def __init__(self, config: PretrainedConfig):
    super().__init__()
    self.config = config
    self.embeddings = InternVisionEmbeddings(config)

forward 

forward(
    pixel_values: Optional[Tensor] = None,
    pixel_embeds: Optional[Tensor] = None,
) -> FloatTensor
Source code in vllm/model_executor/models/intern_vit.py 
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
def forward(
    self,
    pixel_values: Optional[torch.Tensor] = None,
    pixel_embeds: Optional[torch.Tensor] = None,
) -> torch.FloatTensor:
    if pixel_values is None and pixel_embeds is None:
        raise ValueError(
            'You have to specify pixel_values or pixel_embeds')

    if pixel_embeds is not None:
        hidden_states = pixel_embeds
    elif pixel_values is not None:
        if pixel_values.ndim == 4:
            hidden_states = self.embeddings(pixel_values)
        else:
            raise ValueError(
                f'wrong pixel_values size: {pixel_values.shape}')

    return hidden_states

get_input_embeddings 

get_input_embeddings()
Source code in vllm/model_executor/models/intern_vit.py 
106
107
def get_input_embeddings(self):
    return self.embeddings