Skip to content

vllm.core.block.interfaces

BlockId module-attribute 

BlockId = int

Block

Bases: ABC

Source code in vllm/core/block/interfaces.py 
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
class Block(ABC):

    @abstractmethod
    def append_token_ids(self, token_ids: List[int]) -> None:
        pass

    @property
    @abstractmethod
    def block_id(self) -> Optional[int]:
        pass

    @block_id.setter
    @abstractmethod
    def block_id(self, value: Optional[int]) -> None:
        """NOTE: Do not use this API outside Block."""
        self._block_id = value

    @property
    @abstractmethod
    def token_ids(self) -> List[int]:
        pass

    @property
    @abstractmethod
    def num_tokens_total(self) -> int:
        """The number of tokens till the current block (inclusive)
        """
        pass

    @property
    @abstractmethod
    def num_empty_slots(self) -> int:
        pass

    @property
    @abstractmethod
    def is_full(self) -> bool:
        pass

    @property
    @abstractmethod
    def prev_block(self) -> Optional["Block"]:
        pass

    @property
    @abstractmethod
    def extra_hash(self) -> Optional[int]:
        return None

    @property
    @abstractmethod
    def computed(self) -> bool:
        raise NotImplementedError

    @computed.setter
    @abstractmethod
    def computed(self, value) -> bool:
        """Should be only used by PrefixCacingAllocator"""
        raise NotImplementedError

    @property
    @abstractmethod
    def last_accessed(self) -> float:
        raise NotImplementedError

    @last_accessed.setter
    @abstractmethod
    def last_accessed(self, last_accessed_ts: float):
        raise NotImplementedError

    class Factory(Protocol):

        @abstractmethod
        def __call__(
            self,
            prev_block: Optional["Block"],
            token_ids: List[int],
            block_size: int,
            allocator: "BlockAllocator",
            block_id: Optional[int] = None,
            computed: bool = False,
            extra_hash: Optional[int] = None,
        ) -> "Block":
            pass

    @property
    @abstractmethod
    def content_hash(self) -> Optional[int]:
        """Return the content-based hash of the current block, or None if it is
        not yet defined or not supported.

        For the content-based hash to be defined, the current block must be
        full.
        """
        return None

block_id abstractmethod property writable 

block_id: Optional[int]

computed abstractmethod property writable 

computed: bool

content_hash abstractmethod property 

content_hash: Optional[int]

Return the content-based hash of the current block, or None if it is not yet defined or not supported.

For the content-based hash to be defined, the current block must be full.

extra_hash abstractmethod property 

extra_hash: Optional[int]

is_full abstractmethod property 

is_full: bool

last_accessed abstractmethod property writable 

last_accessed: float

num_empty_slots abstractmethod property 

num_empty_slots: int

num_tokens_total abstractmethod property 

num_tokens_total: int

The number of tokens till the current block (inclusive)

prev_block abstractmethod property 

prev_block: Optional[Block]

token_ids abstractmethod property 

token_ids: List[int]

Factory

Bases: Protocol

Source code in vllm/core/block/interfaces.py 
82
83
84
85
86
87
88
89
90
91
92
93
94
95
class Factory(Protocol):

    @abstractmethod
    def __call__(
        self,
        prev_block: Optional["Block"],
        token_ids: List[int],
        block_size: int,
        allocator: "BlockAllocator",
        block_id: Optional[int] = None,
        computed: bool = False,
        extra_hash: Optional[int] = None,
    ) -> "Block":
        pass

__call__ abstractmethod 

__call__(
    prev_block: Optional[Block],
    token_ids: List[int],
    block_size: int,
    allocator: BlockAllocator,
    block_id: Optional[int] = None,
    computed: bool = False,
    extra_hash: Optional[int] = None,
) -> Block
Source code in vllm/core/block/interfaces.py 
84
85
86
87
88
89
90
91
92
93
94
95
@abstractmethod
def __call__(
    self,
    prev_block: Optional["Block"],
    token_ids: List[int],
    block_size: int,
    allocator: "BlockAllocator",
    block_id: Optional[int] = None,
    computed: bool = False,
    extra_hash: Optional[int] = None,
) -> "Block":
    pass

append_token_ids abstractmethod 

append_token_ids(token_ids: List[int]) -> None
Source code in vllm/core/block/interfaces.py 
14
15
16
@abstractmethod
def append_token_ids(self, token_ids: List[int]) -> None:
    pass

BlockAllocator

Bases: ABC

Source code in vllm/core/block/interfaces.py 
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
class BlockAllocator(ABC):

    @abstractmethod
    def allocate_mutable_block(self, prev_block: Optional[Block],
                               extra_hash: Optional[int]) -> Block:
        pass

    @abstractmethod
    def allocate_immutable_block(self, prev_block: Optional[Block],
                                 token_ids: List[int],
                                 extra_hash: Optional[int]) -> Block:
        pass

    @abstractmethod
    def allocate_immutable_blocks(self, prev_block: Optional[Block],
                                  block_token_ids: List[List[int]],
                                  extra_hash: Optional[int]) -> List[Block]:
        pass

    @abstractmethod
    def free(self, block: Block) -> None:
        pass

    @abstractmethod
    def fork(self, last_block: Block) -> List[Block]:
        pass

    @abstractmethod
    def get_num_total_blocks(self) -> int:
        pass

    @abstractmethod
    def get_num_free_blocks(self) -> int:
        pass

    @abstractmethod
    def get_physical_block_id(self, absolute_id: int) -> int:
        pass

    @abstractmethod
    def swap_out(self, blocks: List[Block]) -> None:
        pass

    @abstractmethod
    def swap_in(self, blocks: List[Block]) -> None:
        pass

    @property
    @abstractmethod
    def all_block_ids(self) -> FrozenSet[int]:
        pass

    @abstractmethod
    def clear_copy_on_writes(self) -> List[Tuple[int, int]]:
        pass

    @abstractmethod
    def mark_blocks_as_accessed(self, block_ids: List[int],
                                now: float) -> None:
        pass

    @abstractmethod
    def mark_blocks_as_computed(self, block_ids: List[int]) -> None:
        pass

    @abstractmethod
    def get_common_computed_block_ids(
            self, computed_seq_block_ids: List[List[int]]) -> List[int]:
        pass

    @abstractmethod
    def cow_block_if_not_appendable(self, block: Block) -> BlockId:
        """NOTE: This should not be used besides Block"""
        pass

    @abstractmethod
    def promote_to_immutable_block(self, block: Block) -> BlockId:
        """NOTE: This should not be used besides Block"""
        pass

    @abstractmethod
    def get_num_full_blocks_touched(self, blocks: List[Block]) -> int:
        pass

    @abstractmethod
    def get_prefix_cache_hit_rate(self) -> float:
        """Prefix cache hit rate. -1 means not supported or disabled."""
        pass

    @abstractmethod
    def reset_prefix_cache(self) -> bool:
        """Reset prefix cache."""
        pass

    class NoFreeBlocksError(ValueError):
        pass

    @abstractmethod
    def find_cached_blocks_prefix(
        self,
        block_hashes: List[int],
    ) -> List[int]:
        pass

all_block_ids abstractmethod property 

all_block_ids: FrozenSet[int]

NoFreeBlocksError

Bases: ValueError

Source code in vllm/core/block/interfaces.py 
203
204
class NoFreeBlocksError(ValueError):
    pass

allocate_immutable_block abstractmethod 

allocate_immutable_block(
    prev_block: Optional[Block],
    token_ids: List[int],
    extra_hash: Optional[int],
) -> Block
Source code in vllm/core/block/interfaces.py 
116
117
118
119
120
@abstractmethod
def allocate_immutable_block(self, prev_block: Optional[Block],
                             token_ids: List[int],
                             extra_hash: Optional[int]) -> Block:
    pass

allocate_immutable_blocks abstractmethod 

allocate_immutable_blocks(
    prev_block: Optional[Block],
    block_token_ids: List[List[int]],
    extra_hash: Optional[int],
) -> List[Block]
Source code in vllm/core/block/interfaces.py 
122
123
124
125
126
@abstractmethod
def allocate_immutable_blocks(self, prev_block: Optional[Block],
                              block_token_ids: List[List[int]],
                              extra_hash: Optional[int]) -> List[Block]:
    pass

allocate_mutable_block abstractmethod 

allocate_mutable_block(
    prev_block: Optional[Block], extra_hash: Optional[int]
) -> Block
Source code in vllm/core/block/interfaces.py 
111
112
113
114
@abstractmethod
def allocate_mutable_block(self, prev_block: Optional[Block],
                           extra_hash: Optional[int]) -> Block:
    pass

clear_copy_on_writes abstractmethod 

clear_copy_on_writes() -> List[Tuple[int, int]]
Source code in vllm/core/block/interfaces.py 
161
162
163
@abstractmethod
def clear_copy_on_writes(self) -> List[Tuple[int, int]]:
    pass

cow_block_if_not_appendable abstractmethod 

cow_block_if_not_appendable(block: Block) -> BlockId

NOTE: This should not be used besides Block

Source code in vllm/core/block/interfaces.py 
179
180
181
182
@abstractmethod
def cow_block_if_not_appendable(self, block: Block) -> BlockId:
    """NOTE: This should not be used besides Block"""
    pass

find_cached_blocks_prefix abstractmethod 

find_cached_blocks_prefix(
    block_hashes: List[int],
) -> List[int]
Source code in vllm/core/block/interfaces.py 
206
207
208
209
210
211
@abstractmethod
def find_cached_blocks_prefix(
    self,
    block_hashes: List[int],
) -> List[int]:
    pass

fork abstractmethod 

fork(last_block: Block) -> List[Block]
Source code in vllm/core/block/interfaces.py 
132
133
134
@abstractmethod
def fork(self, last_block: Block) -> List[Block]:
    pass

free abstractmethod 

free(block: Block) -> None
Source code in vllm/core/block/interfaces.py 
128
129
130
@abstractmethod
def free(self, block: Block) -> None:
    pass

get_common_computed_block_ids abstractmethod 

get_common_computed_block_ids(
    computed_seq_block_ids: List[List[int]],
) -> List[int]
Source code in vllm/core/block/interfaces.py 
174
175
176
177
@abstractmethod
def get_common_computed_block_ids(
        self, computed_seq_block_ids: List[List[int]]) -> List[int]:
    pass

get_num_free_blocks abstractmethod 

get_num_free_blocks() -> int
Source code in vllm/core/block/interfaces.py 
140
141
142
@abstractmethod
def get_num_free_blocks(self) -> int:
    pass

get_num_full_blocks_touched abstractmethod 

get_num_full_blocks_touched(blocks: List[Block]) -> int
Source code in vllm/core/block/interfaces.py 
189
190
191
@abstractmethod
def get_num_full_blocks_touched(self, blocks: List[Block]) -> int:
    pass

get_num_total_blocks abstractmethod 

get_num_total_blocks() -> int
Source code in vllm/core/block/interfaces.py 
136
137
138
@abstractmethod
def get_num_total_blocks(self) -> int:
    pass

get_physical_block_id abstractmethod 

get_physical_block_id(absolute_id: int) -> int
Source code in vllm/core/block/interfaces.py 
144
145
146
@abstractmethod
def get_physical_block_id(self, absolute_id: int) -> int:
    pass

get_prefix_cache_hit_rate abstractmethod 

get_prefix_cache_hit_rate() -> float

Prefix cache hit rate. -1 means not supported or disabled.

Source code in vllm/core/block/interfaces.py 
193
194
195
196
@abstractmethod
def get_prefix_cache_hit_rate(self) -> float:
    """Prefix cache hit rate. -1 means not supported or disabled."""
    pass

mark_blocks_as_accessed abstractmethod 

mark_blocks_as_accessed(
    block_ids: List[int], now: float
) -> None
Source code in vllm/core/block/interfaces.py 
165
166
167
168
@abstractmethod
def mark_blocks_as_accessed(self, block_ids: List[int],
                            now: float) -> None:
    pass

mark_blocks_as_computed abstractmethod 

mark_blocks_as_computed(block_ids: List[int]) -> None
Source code in vllm/core/block/interfaces.py 
170
171
172
@abstractmethod
def mark_blocks_as_computed(self, block_ids: List[int]) -> None:
    pass

promote_to_immutable_block abstractmethod 

promote_to_immutable_block(block: Block) -> BlockId

NOTE: This should not be used besides Block

Source code in vllm/core/block/interfaces.py 
184
185
186
187
@abstractmethod
def promote_to_immutable_block(self, block: Block) -> BlockId:
    """NOTE: This should not be used besides Block"""
    pass

reset_prefix_cache abstractmethod 

reset_prefix_cache() -> bool

Reset prefix cache.

Source code in vllm/core/block/interfaces.py 
198
199
200
201
@abstractmethod
def reset_prefix_cache(self) -> bool:
    """Reset prefix cache."""
    pass

swap_in abstractmethod 

swap_in(blocks: List[Block]) -> None
Source code in vllm/core/block/interfaces.py 
152
153
154
@abstractmethod
def swap_in(self, blocks: List[Block]) -> None:
    pass

swap_out abstractmethod 

swap_out(blocks: List[Block]) -> None
Source code in vllm/core/block/interfaces.py 
148
149
150
@abstractmethod
def swap_out(self, blocks: List[Block]) -> None:
    pass

DeviceAwareBlockAllocator

Bases: ABC

Source code in vllm/core/block/interfaces.py 
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
class DeviceAwareBlockAllocator(ABC):

    @abstractmethod
    def allocate_mutable_block(self,
                               prev_block: Optional[Block],
                               device: Device,
                               extra_hash: Optional[int] = None) -> Block:
        pass

    @abstractmethod
    def allocate_immutable_block(self,
                                 prev_block: Optional[Block],
                                 token_ids: List[int],
                                 device: Device,
                                 extra_hash: Optional[int] = None) -> Block:
        pass

    @abstractmethod
    def allocate_immutable_blocks(
        self,
        prev_block: Optional[Block],
        block_token_ids: List[List[int]],
        device: Device,
        extra_hash: Optional[int] = None,
    ) -> List[Block]:
        pass

    @abstractmethod
    def get_num_free_blocks(self, device: Device) -> int:
        pass

    @abstractmethod
    def get_num_total_blocks(self, device: Device) -> int:
        pass

    @abstractmethod
    def free(self, block: Block) -> None:
        pass

    @abstractmethod
    def fork(self, last_block: Block) -> List[Block]:
        pass

    @property
    @abstractmethod
    def all_block_ids(self) -> FrozenSet[int]:
        pass

    @abstractmethod
    def clear_copy_on_writes(self) -> List[Tuple[int, int]]:
        pass

    @abstractmethod
    def mark_blocks_as_accessed(self, block_ids: List[int],
                                now: float) -> None:
        pass

    @abstractmethod
    def mark_blocks_as_computed(self, block_ids: List[int]) -> None:
        pass

    @abstractmethod
    def get_common_computed_block_ids(
            self, computed_seq_block_ids: List[List[int]]) -> List[int]:
        pass

    @abstractmethod
    def get_num_full_blocks_touched(self, blocks: List[Block],
                                    device: Device) -> int:
        pass

    @abstractmethod
    def swap(self, blocks: List[Block], src_device: Device,
             dst_device: Device) -> Dict[int, int]:
        pass

    @abstractmethod
    def get_physical_block_id(self, device: Device, absolute_id: int) -> int:
        pass

    @abstractmethod
    def allocate_or_get_null_block(self) -> Block:
        """
        Null blocks are used as a placeholders for KV cache blocks that have
        been dropped due to sliding window.
        There is at most one null block per allocator.
        """
        pass

    @abstractmethod
    def get_prefix_cache_hit_rate(self, device: Device) -> float:
        """Prefix cache hit rate. -1 means not supported or disabled."""
        pass

    @abstractmethod
    def reset_prefix_cache(self, device: Optional[Device] = None) -> bool:
        """Reset prefix cache."""
        pass

    @abstractmethod
    def find_cached_blocks_prefix(
        self,
        block_hashes: List[int],
        device: Device = Device.GPU,
    ) -> List[int]:
        pass

all_block_ids abstractmethod property 

all_block_ids: FrozenSet[int]

allocate_immutable_block abstractmethod 

allocate_immutable_block(
    prev_block: Optional[Block],
    token_ids: List[int],
    device: Device,
    extra_hash: Optional[int] = None,
) -> Block
Source code in vllm/core/block/interfaces.py 
223
224
225
226
227
228
229
@abstractmethod
def allocate_immutable_block(self,
                             prev_block: Optional[Block],
                             token_ids: List[int],
                             device: Device,
                             extra_hash: Optional[int] = None) -> Block:
    pass

allocate_immutable_blocks abstractmethod 

allocate_immutable_blocks(
    prev_block: Optional[Block],
    block_token_ids: List[List[int]],
    device: Device,
    extra_hash: Optional[int] = None,
) -> List[Block]
Source code in vllm/core/block/interfaces.py 
231
232
233
234
235
236
237
238
239
@abstractmethod
def allocate_immutable_blocks(
    self,
    prev_block: Optional[Block],
    block_token_ids: List[List[int]],
    device: Device,
    extra_hash: Optional[int] = None,
) -> List[Block]:
    pass

allocate_mutable_block abstractmethod 

allocate_mutable_block(
    prev_block: Optional[Block],
    device: Device,
    extra_hash: Optional[int] = None,
) -> Block
Source code in vllm/core/block/interfaces.py 
216
217
218
219
220
221
@abstractmethod
def allocate_mutable_block(self,
                           prev_block: Optional[Block],
                           device: Device,
                           extra_hash: Optional[int] = None) -> Block:
    pass

allocate_or_get_null_block abstractmethod 

allocate_or_get_null_block() -> Block

Null blocks are used as a placeholders for KV cache blocks that have been dropped due to sliding window. There is at most one null block per allocator.

Source code in vllm/core/block/interfaces.py 
294
295
296
297
298
299
300
301
@abstractmethod
def allocate_or_get_null_block(self) -> Block:
    """
    Null blocks are used as a placeholders for KV cache blocks that have
    been dropped due to sliding window.
    There is at most one null block per allocator.
    """
    pass

clear_copy_on_writes abstractmethod 

clear_copy_on_writes() -> List[Tuple[int, int]]
Source code in vllm/core/block/interfaces.py 
262
263
264
@abstractmethod
def clear_copy_on_writes(self) -> List[Tuple[int, int]]:
    pass

find_cached_blocks_prefix abstractmethod 

find_cached_blocks_prefix(
    block_hashes: List[int], device: Device = GPU
) -> List[int]
Source code in vllm/core/block/interfaces.py 
313
314
315
316
317
318
319
@abstractmethod
def find_cached_blocks_prefix(
    self,
    block_hashes: List[int],
    device: Device = Device.GPU,
) -> List[int]:
    pass

fork abstractmethod 

fork(last_block: Block) -> List[Block]
Source code in vllm/core/block/interfaces.py 
253
254
255
@abstractmethod
def fork(self, last_block: Block) -> List[Block]:
    pass

free abstractmethod 

free(block: Block) -> None
Source code in vllm/core/block/interfaces.py 
249
250
251
@abstractmethod
def free(self, block: Block) -> None:
    pass

get_common_computed_block_ids abstractmethod 

get_common_computed_block_ids(
    computed_seq_block_ids: List[List[int]],
) -> List[int]
Source code in vllm/core/block/interfaces.py 
275
276
277
278
@abstractmethod
def get_common_computed_block_ids(
        self, computed_seq_block_ids: List[List[int]]) -> List[int]:
    pass

get_num_free_blocks abstractmethod 

get_num_free_blocks(device: Device) -> int
Source code in vllm/core/block/interfaces.py 
241
242
243
@abstractmethod
def get_num_free_blocks(self, device: Device) -> int:
    pass

get_num_full_blocks_touched abstractmethod 

get_num_full_blocks_touched(
    blocks: List[Block], device: Device
) -> int
Source code in vllm/core/block/interfaces.py 
280
281
282
283
@abstractmethod
def get_num_full_blocks_touched(self, blocks: List[Block],
                                device: Device) -> int:
    pass

get_num_total_blocks abstractmethod 

get_num_total_blocks(device: Device) -> int
Source code in vllm/core/block/interfaces.py 
245
246
247
@abstractmethod
def get_num_total_blocks(self, device: Device) -> int:
    pass

get_physical_block_id abstractmethod 

get_physical_block_id(
    device: Device, absolute_id: int
) -> int
Source code in vllm/core/block/interfaces.py 
290
291
292
@abstractmethod
def get_physical_block_id(self, device: Device, absolute_id: int) -> int:
    pass

get_prefix_cache_hit_rate abstractmethod 

get_prefix_cache_hit_rate(device: Device) -> float

Prefix cache hit rate. -1 means not supported or disabled.

Source code in vllm/core/block/interfaces.py 
303
304
305
306
@abstractmethod
def get_prefix_cache_hit_rate(self, device: Device) -> float:
    """Prefix cache hit rate. -1 means not supported or disabled."""
    pass

mark_blocks_as_accessed abstractmethod 

mark_blocks_as_accessed(
    block_ids: List[int], now: float
) -> None
Source code in vllm/core/block/interfaces.py 
266
267
268
269
@abstractmethod
def mark_blocks_as_accessed(self, block_ids: List[int],
                            now: float) -> None:
    pass

mark_blocks_as_computed abstractmethod 

mark_blocks_as_computed(block_ids: List[int]) -> None
Source code in vllm/core/block/interfaces.py 
271
272
273
@abstractmethod
def mark_blocks_as_computed(self, block_ids: List[int]) -> None:
    pass

reset_prefix_cache abstractmethod 

reset_prefix_cache(device: Optional[Device] = None) -> bool

Reset prefix cache.

Source code in vllm/core/block/interfaces.py 
308
309
310
311
@abstractmethod
def reset_prefix_cache(self, device: Optional[Device] = None) -> bool:
    """Reset prefix cache."""
    pass

swap abstractmethod 

swap(
    blocks: List[Block],
    src_device: Device,
    dst_device: Device,
) -> Dict[int, int]
Source code in vllm/core/block/interfaces.py 
285
286
287
288
@abstractmethod
def swap(self, blocks: List[Block], src_device: Device,
         dst_device: Device) -> Dict[int, int]:
    pass