Skip to content

vllm.transformers_utils.processors.deepseek_vl2

DeepseekVLV2Processor

Bases: ProcessorMixin

Source code in vllm/transformers_utils/processors/deepseek_vl2.py 
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
class DeepseekVLV2Processor(ProcessorMixin):
    tokenizer_class = ("LlamaTokenizer", "LlamaTokenizerFast")
    attributes = ["tokenizer"]

    def __init__(
        self,
        tokenizer: LlamaTokenizerFast,
        candidate_resolutions: tuple[tuple[int, int]],
        patch_size: int,
        downsample_ratio: int,
        image_mean: tuple[float, float, float] = (0.5, 0.5, 0.5),
        image_std: tuple[float, float, float] = (0.5, 0.5, 0.5),
        normalize: bool = True,
        image_token: str = "<image>",
        pad_token: str = "<｜▁pad▁｜>",
        add_special_token: bool = False,
        sft_format: str = "deepseek",
        mask_prompt: bool = True,
        ignore_id: int = -100,
        **kwargs,
    ):

        self.candidate_resolutions = candidate_resolutions
        self.image_size = candidate_resolutions[0][0]
        self.patch_size = patch_size
        self.image_mean = image_mean
        self.image_std = image_std
        self.normalize = normalize
        self.downsample_ratio = downsample_ratio

        self.image_transform = ImageTransform(mean=image_mean, std=image_std, normalize=normalize)
        self.tokenizer = tokenizer
        self.tokenizer.padding_side = 'left'  # must set this，padding side with make a difference in batch inference

        # add the pad_token as special token to use 'tokenizer.pad_token' and 'tokenizer.pad_token_id'
        if tokenizer.pad_token is None:
            self.tokenizer.add_special_tokens({'pad_token': pad_token})

        # add image token
        image_token_id = self.tokenizer.vocab.get(image_token)
        if image_token_id is None:
            special_tokens = [image_token]
            special_tokens_dict = {"additional_special_tokens": special_tokens}
            self.tokenizer.add_special_tokens(special_tokens_dict)
        self.image_token_id = self.tokenizer.vocab.get(image_token)

        # add five special tokens for grounding-related tasks
        # <|ref|>, <|/ref|>, <|det|>, <|/det|>, <|grounding|>
        special_tokens = ['<|ref|>', '<|/ref|>', '<|det|>', '<|/det|>', '<|grounding|>']
        special_tokens_dict = {"additional_special_tokens": special_tokens}
        self.tokenizer.add_special_tokens(special_tokens_dict)

        # add special tokens for SFT data
        special_tokens = ["<|User|>", "<|Assistant|>"]
        special_tokens_dict = {"additional_special_tokens": special_tokens}
        self.tokenizer.add_special_tokens(special_tokens_dict)

        self.image_token = image_token
        self.pad_token = pad_token
        self.add_special_token = add_special_token
        self.sft_format = sft_format
        self.mask_prompt = mask_prompt
        self.ignore_id = ignore_id

        super().__init__(
            tokenizer,
            **kwargs,
        )

    def select_best_resolution(self, image_size):
        # used for cropping
        original_width, original_height = image_size
        best_fit = None
        max_effective_resolution = 0
        min_wasted_resolution = float("inf")

        for width, height in self.candidate_resolutions:
            scale = min(width / original_width, height / original_height)
            downscaled_width, downscaled_height = int(
                original_width * scale), int(original_height * scale)
            effective_resolution = min(downscaled_width * downscaled_height,
                                       original_width * original_height)
            wasted_resolution = (width * height) - effective_resolution

            if effective_resolution > max_effective_resolution or (
                    effective_resolution == max_effective_resolution
                    and wasted_resolution < min_wasted_resolution):
                max_effective_resolution = effective_resolution
                min_wasted_resolution = wasted_resolution
                best_fit = (width, height)

        return best_fit

    @property
    def bos_id(self):
        return self.tokenizer.bos_token_id

    @property
    def eos_id(self):
        return self.tokenizer.eos_token_id

    @property
    def pad_id(self):
        return self.tokenizer.pad_token_id

    def encode(self, text: str, bos: bool = True, eos: bool = False):
        t = self.tokenizer.encode(text, add_special_tokens=False)

        if bos:
            t = [self.bos_id] + t
        if eos:
            t = t + [self.eos_id]

        return t

    def decode(self, t: list[int], **kwargs) -> str:
        return self.tokenizer.decode(t, **kwargs)

    def process_one(
        self,
        prompt: str,
        images: list[Image.Image],
        inference_mode: bool = True,
        **kwargs,
    ):
        """

        Args:
            prompt (str): the formatted prompt;
            conversations (list[dict]): conversations with a list of messages;
            images (list[ImageType]): the list of images;
            inference_mode (bool): if True, then remove the last eos token;
            system_prompt (str): the system prompt;
            **kwargs:

        Returns:
            outputs (BaseProcessorOutput): the output of the processor,
                - input_ids (torch.LongTensor): [N + image tokens]
                - target_ids (torch.LongTensor): [N + image tokens]
                - pixel_values (torch.FloatTensor): [n_patches, 3, H, W]
                - image_id (int): the id of the image token
                - num_image_tokens (list[int]): the number of image tokens
        """

        assert (prompt is not None and images is not None
                ), "prompt and images must be used at the same time."

        sft_format = prompt
        tokenized_str, images_list, images_seq_mask, images_spatial_crop, num_image_tokens = self.tokenize_with_images(
            sft_format, images, bos=True, eos=True, cropping=len(images) <= 2)
        masked_tokenized_str = []
        for token_index in tokenized_str:
            if token_index != self.image_token_id:
                masked_tokenized_str.append(token_index)
            else:
                masked_tokenized_str.append(self.ignore_id)

        assert len(tokenized_str) == len(images_seq_mask) == len(masked_tokenized_str), \
            (f"tokenized_str's length {len(tokenized_str)}, input_ids' length {len(masked_tokenized_str)}, "
             f"imags_seq_mask's length {len(images_seq_mask)}, are not equal")

        input_ids = torch.LongTensor(tokenized_str)
        target_ids = torch.LongTensor(masked_tokenized_str)
        images_seq_mask = torch.tensor(images_seq_mask, dtype=torch.bool)

        # set input_ids < 0 | input_ids == self.image_token_id as ignore_id
        target_ids[(input_ids < 0) |
                   (input_ids == self.image_token_id)] = self.ignore_id
        input_ids[input_ids < 0] = self.pad_id

        if inference_mode:
            # Remove the ending eos token
            assert input_ids[-1] == self.eos_id
            input_ids = input_ids[:-1]
            target_ids = target_ids[:-1]
            images_seq_mask = images_seq_mask[:-1]

        if len(images_list) == 0:
            pixel_values = torch.zeros((1, 3, self.image_size, self.image_size))
            images_spatial_crop = torch.zeros((1, 2), dtype=torch.long)
        else:
            pixel_values = torch.stack(images_list, dim=0)
            images_spatial_crop = torch.tensor(images_spatial_crop, dtype=torch.long)

        input_ids = input_ids.unsqueeze(0)

        prepare = BatchFeature(
            data=dict(
                input_ids=input_ids,
                pixel_values=pixel_values,
                images_seq_mask=images_seq_mask,
                images_spatial_crop=images_spatial_crop,
                num_image_tokens=num_image_tokens,
            ),
            tensor_type="pt",
        )
        return prepare

    def __call__(
        self,
        *,
        prompt: str,
        images: list[Image.Image],
        inference_mode: bool = True,
        **kwargs,
    ):
        """

        Args:
            prompt (str): the formatted prompt;
            images (list[ImageType]): the list of images;
            inference_mode (bool): if True, then remove the last eos token;
            **kwargs:

        Returns:
            outputs (BaseProcessorOutput): the output of the processor,
                - input_ids (torch.LongTensor): [N + image tokens]
                - images (torch.FloatTensor): [n_images, 3, H, W]
                - image_id (int): the id of the image token
                - num_image_tokens (list[int]): the number of image tokens
        """

        prepare = self.process_one(
            prompt=prompt,
            images=images,
            inference_mode=inference_mode,
        )

        return prepare

    def tokenize_with_images(
        self,
        conversation: str,
        images: list[Image.Image],
        bos: bool = True,
        eos: bool = True,
        cropping: bool = True,
    ):
        """Tokenize text with <image> tags."""
        assert conversation.count(self.image_token) == len(images)
        text_splits = conversation.split(self.image_token)
        images_list, images_seq_mask, images_spatial_crop = [], [], []
        num_image_tokens = []
        tokenized_str = []
        for text_sep, image in zip(text_splits, images):
            """encode text_sep"""
            tokenized_sep = self.encode(text_sep, bos=False, eos=False)
            tokenized_str += tokenized_sep
            images_seq_mask += [False] * len(tokenized_sep)

            """select best resolution for anyres"""
            if cropping:
                best_width, best_height = self.select_best_resolution(image.size)
            else:
                best_width, best_height = self.image_size, self.image_size

            """process the global view"""
            global_view = ImageOps.pad(image, (self.image_size, self.image_size),
                                       color=tuple(int(x * 255) for x in self.image_transform.mean))
            images_list.append(self.image_transform(global_view))

            """process the local views"""
            local_view = ImageOps.pad(image, (best_width, best_height),
                                      color=tuple(int(x * 255) for x in self.image_transform.mean))
            for i in range(0, best_height, self.image_size):
                for j in range(0, best_width, self.image_size):
                    images_list.append(
                        self.image_transform(local_view.crop((j, i, j + self.image_size, i + self.image_size))))

            """record height / width crop num"""
            num_width_tiles, num_height_tiles = best_width // self.image_size, best_height // self.image_size
            images_spatial_crop.append([num_width_tiles, num_height_tiles])

            """add image tokens"""
            h = w = math.ceil((self.image_size // self.patch_size) / self.downsample_ratio)
            # global views tokens h * (w + 1), 1 is for line separator
            tokenized_image = [self.image_token_id] * h * (w + 1)
            # add a separator between global and local views
            tokenized_image += [self.image_token_id]
            # local views tokens, (num_height_tiles * h) * (num_width_tiles * w + 1)
            tokenized_image += [self.image_token_id] * (num_height_tiles * h) * (num_width_tiles * w + 1)

            tokenized_str += tokenized_image
            images_seq_mask += [True] * len(tokenized_image)
            num_image_tokens.append(len(tokenized_image))

        """process the last text split"""
        tokenized_sep = self.encode(text_splits[-1], bos=False, eos=False)
        tokenized_str += tokenized_sep
        images_seq_mask += [False] * len(tokenized_sep)

        """add the bos and eos tokens"""
        if bos:
            tokenized_str = [self.bos_id] + tokenized_str
            images_seq_mask = [False] + images_seq_mask
        if eos:
            tokenized_str = tokenized_str + [self.eos_id]
            images_seq_mask = images_seq_mask + [False]

        assert len(tokenized_str) == len(
            images_seq_mask), f"tokenize_with_images func: tokenized_str's length {len(tokenized_str)} is not equal to imags_seq_mask's length {len(images_seq_mask)}"

        return tokenized_str, images_list, images_seq_mask, images_spatial_crop, num_image_tokens

add_special_token instance-attribute 

add_special_token = add_special_token

attributes class-attribute instance-attribute 

attributes = ['tokenizer']

bos_id property 

bos_id

candidate_resolutions instance-attribute 

candidate_resolutions = candidate_resolutions

downsample_ratio instance-attribute 

downsample_ratio = downsample_ratio

eos_id property 

eos_id

ignore_id instance-attribute 

ignore_id = ignore_id

image_mean instance-attribute 

image_mean = image_mean

image_size instance-attribute 

image_size = candidate_resolutions[0][0]

image_std instance-attribute 

image_std = image_std

image_token instance-attribute 

image_token = image_token

image_token_id instance-attribute 

image_token_id = get(image_token)

image_transform instance-attribute 

image_transform = ImageTransform(
    mean=image_mean, std=image_std, normalize=normalize
)

mask_prompt instance-attribute 

mask_prompt = mask_prompt

normalize instance-attribute 

normalize = normalize

pad_id property 

pad_id

pad_token instance-attribute 

pad_token = pad_token

patch_size instance-attribute 

patch_size = patch_size

sft_format instance-attribute 

sft_format = sft_format

tokenizer instance-attribute 

tokenizer = tokenizer

tokenizer_class class-attribute instance-attribute 

tokenizer_class = ('LlamaTokenizer', 'LlamaTokenizerFast')

__call__ 

__call__(
    *,
    prompt: str,
    images: list[Image],
    inference_mode: bool = True,
    **kwargs,
)

Parameters:

Name Type Description Default
prompt str

the formatted prompt;

 required
images list[ImageType]

the list of images;

 required
inference_mode bool

if True, then remove the last eos token;

 True
**kwargs
{}

Returns:

Name Type Description
outputs BaseProcessorOutput

the output of the processor, - input_ids (torch.LongTensor): [N + image tokens] - images (torch.FloatTensor): [n_images, 3, H, W] - image_id (int): the id of the image token - num_image_tokens (list[int]): the number of image tokens
Source code in vllm/transformers_utils/processors/deepseek_vl2.py 
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
def __call__(
    self,
    *,
    prompt: str,
    images: list[Image.Image],
    inference_mode: bool = True,
    **kwargs,
):
    """

    Args:
        prompt (str): the formatted prompt;
        images (list[ImageType]): the list of images;
        inference_mode (bool): if True, then remove the last eos token;
        **kwargs:

    Returns:
        outputs (BaseProcessorOutput): the output of the processor,
            - input_ids (torch.LongTensor): [N + image tokens]
            - images (torch.FloatTensor): [n_images, 3, H, W]
            - image_id (int): the id of the image token
            - num_image_tokens (list[int]): the number of image tokens
    """

    prepare = self.process_one(
        prompt=prompt,
        images=images,
        inference_mode=inference_mode,
    )

    return prepare

__init__ 

__init__(
    tokenizer: LlamaTokenizerFast,
    candidate_resolutions: tuple[tuple[int, int]],
    patch_size: int,
    downsample_ratio: int,
    image_mean: tuple[float, float, float] = (
        0.5,
        0.5,
        0.5,
    ),
    image_std: tuple[float, float, float] = (0.5, 0.5, 0.5),
    normalize: bool = True,
    image_token: str = "<image>",
    pad_token: str = "<｜▁pad▁｜>",
    add_special_token: bool = False,
    sft_format: str = "deepseek",
    mask_prompt: bool = True,
    ignore_id: int = -100,
    **kwargs,
)
Source code in vllm/transformers_utils/processors/deepseek_vl2.py 
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
def __init__(
    self,
    tokenizer: LlamaTokenizerFast,
    candidate_resolutions: tuple[tuple[int, int]],
    patch_size: int,
    downsample_ratio: int,
    image_mean: tuple[float, float, float] = (0.5, 0.5, 0.5),
    image_std: tuple[float, float, float] = (0.5, 0.5, 0.5),
    normalize: bool = True,
    image_token: str = "<image>",
    pad_token: str = "<｜▁pad▁｜>",
    add_special_token: bool = False,
    sft_format: str = "deepseek",
    mask_prompt: bool = True,
    ignore_id: int = -100,
    **kwargs,
):

    self.candidate_resolutions = candidate_resolutions
    self.image_size = candidate_resolutions[0][0]
    self.patch_size = patch_size
    self.image_mean = image_mean
    self.image_std = image_std
    self.normalize = normalize
    self.downsample_ratio = downsample_ratio

    self.image_transform = ImageTransform(mean=image_mean, std=image_std, normalize=normalize)
    self.tokenizer = tokenizer
    self.tokenizer.padding_side = 'left'  # must set this，padding side with make a difference in batch inference

    # add the pad_token as special token to use 'tokenizer.pad_token' and 'tokenizer.pad_token_id'
    if tokenizer.pad_token is None:
        self.tokenizer.add_special_tokens({'pad_token': pad_token})

    # add image token
    image_token_id = self.tokenizer.vocab.get(image_token)
    if image_token_id is None:
        special_tokens = [image_token]
        special_tokens_dict = {"additional_special_tokens": special_tokens}
        self.tokenizer.add_special_tokens(special_tokens_dict)
    self.image_token_id = self.tokenizer.vocab.get(image_token)

    # add five special tokens for grounding-related tasks
    # <|ref|>, <|/ref|>, <|det|>, <|/det|>, <|grounding|>
    special_tokens = ['<|ref|>', '<|/ref|>', '<|det|>', '<|/det|>', '<|grounding|>']
    special_tokens_dict = {"additional_special_tokens": special_tokens}
    self.tokenizer.add_special_tokens(special_tokens_dict)

    # add special tokens for SFT data
    special_tokens = ["<|User|>", "<|Assistant|>"]
    special_tokens_dict = {"additional_special_tokens": special_tokens}
    self.tokenizer.add_special_tokens(special_tokens_dict)

    self.image_token = image_token
    self.pad_token = pad_token
    self.add_special_token = add_special_token
    self.sft_format = sft_format
    self.mask_prompt = mask_prompt
    self.ignore_id = ignore_id

    super().__init__(
        tokenizer,
        **kwargs,
    )

decode 

decode(t: list[int], **kwargs) -> str
Source code in vllm/transformers_utils/processors/deepseek_vl2.py 
173
174
def decode(self, t: list[int], **kwargs) -> str:
    return self.tokenizer.decode(t, **kwargs)

encode 

encode(text: str, bos: bool = True, eos: bool = False)
Source code in vllm/transformers_utils/processors/deepseek_vl2.py 
163
164
165
166
167
168
169
170
171
def encode(self, text: str, bos: bool = True, eos: bool = False):
    t = self.tokenizer.encode(text, add_special_tokens=False)

    if bos:
        t = [self.bos_id] + t
    if eos:
        t = t + [self.eos_id]

    return t

process_one 

process_one(
    prompt: str,
    images: list[Image],
    inference_mode: bool = True,
    **kwargs,
)

Parameters:

Name Type Description Default
prompt str

the formatted prompt;

 required
conversations list[dict]

conversations with a list of messages;

 required
images list[ImageType]

the list of images;

 required
inference_mode bool

if True, then remove the last eos token;

 True
system_prompt str

the system prompt;

 required
**kwargs
{}

Returns:

Name Type Description
outputs BaseProcessorOutput

the output of the processor, - input_ids (torch.LongTensor): [N + image tokens] - target_ids (torch.LongTensor): [N + image tokens] - pixel_values (torch.FloatTensor): [n_patches, 3, H, W] - image_id (int): the id of the image token - num_image_tokens (list[int]): the number of image tokens
Source code in vllm/transformers_utils/processors/deepseek_vl2.py 
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
def process_one(
    self,
    prompt: str,
    images: list[Image.Image],
    inference_mode: bool = True,
    **kwargs,
):
    """

    Args:
        prompt (str): the formatted prompt;
        conversations (list[dict]): conversations with a list of messages;
        images (list[ImageType]): the list of images;
        inference_mode (bool): if True, then remove the last eos token;
        system_prompt (str): the system prompt;
        **kwargs:

    Returns:
        outputs (BaseProcessorOutput): the output of the processor,
            - input_ids (torch.LongTensor): [N + image tokens]
            - target_ids (torch.LongTensor): [N + image tokens]
            - pixel_values (torch.FloatTensor): [n_patches, 3, H, W]
            - image_id (int): the id of the image token
            - num_image_tokens (list[int]): the number of image tokens
    """

    assert (prompt is not None and images is not None
            ), "prompt and images must be used at the same time."

    sft_format = prompt
    tokenized_str, images_list, images_seq_mask, images_spatial_crop, num_image_tokens = self.tokenize_with_images(
        sft_format, images, bos=True, eos=True, cropping=len(images) <= 2)
    masked_tokenized_str = []
    for token_index in tokenized_str:
        if token_index != self.image_token_id:
            masked_tokenized_str.append(token_index)
        else:
            masked_tokenized_str.append(self.ignore_id)

    assert len(tokenized_str) == len(images_seq_mask) == len(masked_tokenized_str), \
        (f"tokenized_str's length {len(tokenized_str)}, input_ids' length {len(masked_tokenized_str)}, "
         f"imags_seq_mask's length {len(images_seq_mask)}, are not equal")

    input_ids = torch.LongTensor(tokenized_str)
    target_ids = torch.LongTensor(masked_tokenized_str)
    images_seq_mask = torch.tensor(images_seq_mask, dtype=torch.bool)

    # set input_ids < 0 | input_ids == self.image_token_id as ignore_id
    target_ids[(input_ids < 0) |
               (input_ids == self.image_token_id)] = self.ignore_id
    input_ids[input_ids < 0] = self.pad_id

    if inference_mode:
        # Remove the ending eos token
        assert input_ids[-1] == self.eos_id
        input_ids = input_ids[:-1]
        target_ids = target_ids[:-1]
        images_seq_mask = images_seq_mask[:-1]

    if len(images_list) == 0:
        pixel_values = torch.zeros((1, 3, self.image_size, self.image_size))
        images_spatial_crop = torch.zeros((1, 2), dtype=torch.long)
    else:
        pixel_values = torch.stack(images_list, dim=0)
        images_spatial_crop = torch.tensor(images_spatial_crop, dtype=torch.long)

    input_ids = input_ids.unsqueeze(0)

    prepare = BatchFeature(
        data=dict(
            input_ids=input_ids,
            pixel_values=pixel_values,
            images_seq_mask=images_seq_mask,
            images_spatial_crop=images_spatial_crop,
            num_image_tokens=num_image_tokens,
        ),
        tensor_type="pt",
    )
    return prepare

select_best_resolution 

select_best_resolution(image_size)
Source code in vllm/transformers_utils/processors/deepseek_vl2.py 
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
def select_best_resolution(self, image_size):
    # used for cropping
    original_width, original_height = image_size
    best_fit = None
    max_effective_resolution = 0
    min_wasted_resolution = float("inf")

    for width, height in self.candidate_resolutions:
        scale = min(width / original_width, height / original_height)
        downscaled_width, downscaled_height = int(
            original_width * scale), int(original_height * scale)
        effective_resolution = min(downscaled_width * downscaled_height,
                                   original_width * original_height)
        wasted_resolution = (width * height) - effective_resolution

        if effective_resolution > max_effective_resolution or (
                effective_resolution == max_effective_resolution
                and wasted_resolution < min_wasted_resolution):
            max_effective_resolution = effective_resolution
            min_wasted_resolution = wasted_resolution
            best_fit = (width, height)

    return best_fit

tokenize_with_images 

tokenize_with_images(
    conversation: str,
    images: list[Image],
    bos: bool = True,
    eos: bool = True,
    cropping: bool = True,
)

Tokenize text with tags.

Source code in vllm/transformers_utils/processors/deepseek_vl2.py 
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
def tokenize_with_images(
    self,
    conversation: str,
    images: list[Image.Image],
    bos: bool = True,
    eos: bool = True,
    cropping: bool = True,
):
    """Tokenize text with <image> tags."""
    assert conversation.count(self.image_token) == len(images)
    text_splits = conversation.split(self.image_token)
    images_list, images_seq_mask, images_spatial_crop = [], [], []
    num_image_tokens = []
    tokenized_str = []
    for text_sep, image in zip(text_splits, images):
        """encode text_sep"""
        tokenized_sep = self.encode(text_sep, bos=False, eos=False)
        tokenized_str += tokenized_sep
        images_seq_mask += [False] * len(tokenized_sep)

        """select best resolution for anyres"""
        if cropping:
            best_width, best_height = self.select_best_resolution(image.size)
        else:
            best_width, best_height = self.image_size, self.image_size

        """process the global view"""
        global_view = ImageOps.pad(image, (self.image_size, self.image_size),
                                   color=tuple(int(x * 255) for x in self.image_transform.mean))
        images_list.append(self.image_transform(global_view))

        """process the local views"""
        local_view = ImageOps.pad(image, (best_width, best_height),
                                  color=tuple(int(x * 255) for x in self.image_transform.mean))
        for i in range(0, best_height, self.image_size):
            for j in range(0, best_width, self.image_size):
                images_list.append(
                    self.image_transform(local_view.crop((j, i, j + self.image_size, i + self.image_size))))

        """record height / width crop num"""
        num_width_tiles, num_height_tiles = best_width // self.image_size, best_height // self.image_size
        images_spatial_crop.append([num_width_tiles, num_height_tiles])

        """add image tokens"""
        h = w = math.ceil((self.image_size // self.patch_size) / self.downsample_ratio)
        # global views tokens h * (w + 1), 1 is for line separator
        tokenized_image = [self.image_token_id] * h * (w + 1)
        # add a separator between global and local views
        tokenized_image += [self.image_token_id]
        # local views tokens, (num_height_tiles * h) * (num_width_tiles * w + 1)
        tokenized_image += [self.image_token_id] * (num_height_tiles * h) * (num_width_tiles * w + 1)

        tokenized_str += tokenized_image
        images_seq_mask += [True] * len(tokenized_image)
        num_image_tokens.append(len(tokenized_image))

    """process the last text split"""
    tokenized_sep = self.encode(text_splits[-1], bos=False, eos=False)
    tokenized_str += tokenized_sep
    images_seq_mask += [False] * len(tokenized_sep)

    """add the bos and eos tokens"""
    if bos:
        tokenized_str = [self.bos_id] + tokenized_str
        images_seq_mask = [False] + images_seq_mask
    if eos:
        tokenized_str = tokenized_str + [self.eos_id]
        images_seq_mask = images_seq_mask + [False]

    assert len(tokenized_str) == len(
        images_seq_mask), f"tokenize_with_images func: tokenized_str's length {len(tokenized_str)} is not equal to imags_seq_mask's length {len(images_seq_mask)}"

    return tokenized_str, images_list, images_seq_mask, images_spatial_crop, num_image_tokens

ImageTransform

Source code in vllm/transformers_utils/processors/deepseek_vl2.py 
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
class ImageTransform:

    def __init__(self,
                 mean: tuple[float, float, float] = (0.5, 0.5, 0.5),
                 std: tuple[float, float, float] = (0.5, 0.5, 0.5),
                 normalize: bool = True):
        self.mean = mean
        self.std = std
        self.normalize = normalize

        transform_pipelines = [T.ToTensor()]

        if normalize:
            transform_pipelines.append(T.Normalize(mean, std))

        self.transform = T.Compose(transform_pipelines)

    def __call__(self, pil_img: Image.Image):
        x = self.transform(pil_img)
        return x

mean instance-attribute 

mean = mean

normalize instance-attribute 

normalize = normalize

std instance-attribute 

std = std

transform instance-attribute 

transform = Compose(transform_pipelines)

__call__ 

__call__(pil_img: Image)
Source code in vllm/transformers_utils/processors/deepseek_vl2.py 
53
54
55
def __call__(self, pil_img: Image.Image):
    x = self.transform(pil_img)
    return x

__init__ 

__init__(
    mean: tuple[float, float, float] = (0.5, 0.5, 0.5),
    std: tuple[float, float, float] = (0.5, 0.5, 0.5),
    normalize: bool = True,
)
Source code in vllm/transformers_utils/processors/deepseek_vl2.py 
38
39
40
41
42
43
44
45
46
47
48
49
50
51
def __init__(self,
             mean: tuple[float, float, float] = (0.5, 0.5, 0.5),
             std: tuple[float, float, float] = (0.5, 0.5, 0.5),
             normalize: bool = True):
    self.mean = mean
    self.std = std
    self.normalize = normalize

    transform_pipelines = [T.ToTensor()]

    if normalize:
        transform_pipelines.append(T.Normalize(mean, std))

    self.transform = T.Compose(transform_pipelines)