Skip to content

vllm.core.interfaces

AllocStatus

Bases: Enum

Result for BlockSpaceManager.can_allocate

  1. Ok: seq_group can be allocated now.
  2. Later: seq_group cannot be allocated. The capacity of allocator is larger than seq_group required.
  3. Never: seq_group can never be allocated. The seq_group is too large to allocated in GPU.
Source code in vllm/core/interfaces.py 
14
15
16
17
18
19
20
21
22
23
24
25
class AllocStatus(enum.Enum):
    """Result for BlockSpaceManager.can_allocate

    1. Ok: seq_group can be allocated now.
    2. Later: seq_group cannot be allocated.
      The capacity of allocator is larger than seq_group required.
    3. Never: seq_group can never be allocated.
      The seq_group is too large to allocated in GPU.
    """
    OK = enum.auto()
    LATER = enum.auto()
    NEVER = enum.auto()

LATER class-attribute instance-attribute 

LATER = auto()

NEVER class-attribute instance-attribute 

NEVER = auto()

OK class-attribute instance-attribute 

OK = auto()

BlockSpaceManager

Bases: ABC

Source code in vllm/core/interfaces.py 
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
class BlockSpaceManager(ABC):

    @staticmethod
    def get_block_space_manager_class(version: str):
        version = version.lower()

        if version == "selfattn":
            from vllm.core.block_manager import SelfAttnBlockSpaceManager
            return SelfAttnBlockSpaceManager

        if version == "placeholder":
            from vllm.core.placeholder_block_space_manager import (
                PlaceholderBlockSpaceManager)
            return PlaceholderBlockSpaceManager

        raise ValueError(f"Unknown version {version=}")

    @abstractmethod
    def can_allocate(self,
                     seq_group: SequenceGroup,
                     num_lookahead_slots: int = 0) -> AllocStatus:
        pass

    @abstractmethod
    def allocate(self, seq_group: SequenceGroup) -> None:
        pass

    @abstractmethod
    def can_append_slots(self, seq_group: SequenceGroup,
                         num_lookahead_slots: int) -> bool:
        pass

    @abstractmethod
    def append_slots(
        self,
        seq: Sequence,
        num_lookahead_slots: int,
    ) -> List[Tuple[int, int]]:
        pass

    @abstractmethod
    def fork(self, parent_seq: Sequence, child_seq: Sequence) -> None:
        pass

    @abstractmethod
    def can_swap_in(self, seq_group: SequenceGroup,
                    num_lookahead_slots: int) -> AllocStatus:
        pass

    @abstractmethod
    def swap_in(self, seq_group: SequenceGroup) -> List[Tuple[int, int]]:
        pass

    @abstractmethod
    def can_swap_out(self, seq_group: SequenceGroup) -> bool:
        pass

    @abstractmethod
    def swap_out(self, seq_group: SequenceGroup) -> List[Tuple[int, int]]:
        pass

    @abstractmethod
    def free(self, seq: Sequence) -> None:
        pass

    @abstractmethod
    def get_block_table(self, seq: Sequence) -> List[int]:
        pass

    @abstractmethod
    def get_num_free_gpu_blocks(self) -> int:
        pass

    @abstractmethod
    def get_num_free_cpu_blocks(self) -> int:
        pass

    @abstractmethod
    def access_all_blocks_in_seq(
        self,
        seq: Sequence,
        access_time: float,
    ) -> None:
        pass

    @abstractmethod
    def get_common_computed_block_ids(
            self, seqs: List[Sequence]) -> GenericSequence[int]:
        pass

    @abstractmethod
    def mark_blocks_as_computed(self, seq_group: SequenceGroup,
                                token_chunk_size: int):
        pass

    @abstractmethod
    def get_prefix_cache_hit_rate(self, device: Device) -> float:
        """Prefix cache hit rate. -1 means not supported or disabled."""
        pass

    @abstractmethod
    def reset_prefix_cache(self, device: Optional[Device] = None) -> bool:
        """Reset prefix cache for specified or all devices."""
        pass

    @abstractmethod
    def get_num_cached_tokens(self, seq: Sequence) -> int:
        pass

    @abstractmethod
    def remove_seq_from_computed_blocks_tracker(self, seq: Sequence) -> None:
        pass

access_all_blocks_in_seq abstractmethod 

access_all_blocks_in_seq(
    seq: Sequence, access_time: float
) -> None
Source code in vllm/core/interfaces.py 
105
106
107
108
109
110
111
@abstractmethod
def access_all_blocks_in_seq(
    self,
    seq: Sequence,
    access_time: float,
) -> None:
    pass

allocate abstractmethod 

allocate(seq_group: SequenceGroup) -> None
Source code in vllm/core/interfaces.py 
51
52
53
@abstractmethod
def allocate(self, seq_group: SequenceGroup) -> None:
    pass

append_slots abstractmethod 

append_slots(
    seq: Sequence, num_lookahead_slots: int
) -> List[Tuple[int, int]]
Source code in vllm/core/interfaces.py 
60
61
62
63
64
65
66
@abstractmethod
def append_slots(
    self,
    seq: Sequence,
    num_lookahead_slots: int,
) -> List[Tuple[int, int]]:
    pass

can_allocate abstractmethod 

can_allocate(
    seq_group: SequenceGroup, num_lookahead_slots: int = 0
) -> AllocStatus
Source code in vllm/core/interfaces.py 
45
46
47
48
49
@abstractmethod
def can_allocate(self,
                 seq_group: SequenceGroup,
                 num_lookahead_slots: int = 0) -> AllocStatus:
    pass

can_append_slots abstractmethod 

can_append_slots(
    seq_group: SequenceGroup, num_lookahead_slots: int
) -> bool
Source code in vllm/core/interfaces.py 
55
56
57
58
@abstractmethod
def can_append_slots(self, seq_group: SequenceGroup,
                     num_lookahead_slots: int) -> bool:
    pass

can_swap_in abstractmethod 

can_swap_in(
    seq_group: SequenceGroup, num_lookahead_slots: int
) -> AllocStatus
Source code in vllm/core/interfaces.py 
72
73
74
75
@abstractmethod
def can_swap_in(self, seq_group: SequenceGroup,
                num_lookahead_slots: int) -> AllocStatus:
    pass

can_swap_out abstractmethod 

can_swap_out(seq_group: SequenceGroup) -> bool
Source code in vllm/core/interfaces.py 
81
82
83
@abstractmethod
def can_swap_out(self, seq_group: SequenceGroup) -> bool:
    pass

fork abstractmethod 

fork(parent_seq: Sequence, child_seq: Sequence) -> None
Source code in vllm/core/interfaces.py 
68
69
70
@abstractmethod
def fork(self, parent_seq: Sequence, child_seq: Sequence) -> None:
    pass

free abstractmethod 

free(seq: Sequence) -> None
Source code in vllm/core/interfaces.py 
89
90
91
@abstractmethod
def free(self, seq: Sequence) -> None:
    pass

get_block_space_manager_class staticmethod 

get_block_space_manager_class(version: str)
Source code in vllm/core/interfaces.py 
30
31
32
33
34
35
36
37
38
39
40
41
42
43
@staticmethod
def get_block_space_manager_class(version: str):
    version = version.lower()

    if version == "selfattn":
        from vllm.core.block_manager import SelfAttnBlockSpaceManager
        return SelfAttnBlockSpaceManager

    if version == "placeholder":
        from vllm.core.placeholder_block_space_manager import (
            PlaceholderBlockSpaceManager)
        return PlaceholderBlockSpaceManager

    raise ValueError(f"Unknown version {version=}")

get_block_table abstractmethod 

get_block_table(seq: Sequence) -> List[int]
Source code in vllm/core/interfaces.py 
93
94
95
@abstractmethod
def get_block_table(self, seq: Sequence) -> List[int]:
    pass

get_common_computed_block_ids abstractmethod 

get_common_computed_block_ids(
    seqs: List[Sequence],
) -> Sequence[int]
Source code in vllm/core/interfaces.py 
113
114
115
116
@abstractmethod
def get_common_computed_block_ids(
        self, seqs: List[Sequence]) -> GenericSequence[int]:
    pass

get_num_cached_tokens abstractmethod 

get_num_cached_tokens(seq: Sequence) -> int
Source code in vllm/core/interfaces.py 
133
134
135
@abstractmethod
def get_num_cached_tokens(self, seq: Sequence) -> int:
    pass

get_num_free_cpu_blocks abstractmethod 

get_num_free_cpu_blocks() -> int
Source code in vllm/core/interfaces.py 
101
102
103
@abstractmethod
def get_num_free_cpu_blocks(self) -> int:
    pass

get_num_free_gpu_blocks abstractmethod 

get_num_free_gpu_blocks() -> int
Source code in vllm/core/interfaces.py 
97
98
99
@abstractmethod
def get_num_free_gpu_blocks(self) -> int:
    pass

get_prefix_cache_hit_rate abstractmethod 

get_prefix_cache_hit_rate(device: Device) -> float

Prefix cache hit rate. -1 means not supported or disabled.

Source code in vllm/core/interfaces.py 
123
124
125
126
@abstractmethod
def get_prefix_cache_hit_rate(self, device: Device) -> float:
    """Prefix cache hit rate. -1 means not supported or disabled."""
    pass

mark_blocks_as_computed abstractmethod 

mark_blocks_as_computed(
    seq_group: SequenceGroup, token_chunk_size: int
)
Source code in vllm/core/interfaces.py 
118
119
120
121
@abstractmethod
def mark_blocks_as_computed(self, seq_group: SequenceGroup,
                            token_chunk_size: int):
    pass

remove_seq_from_computed_blocks_tracker abstractmethod 

remove_seq_from_computed_blocks_tracker(
    seq: Sequence,
) -> None
Source code in vllm/core/interfaces.py 
137
138
139
@abstractmethod
def remove_seq_from_computed_blocks_tracker(self, seq: Sequence) -> None:
    pass

reset_prefix_cache abstractmethod 

reset_prefix_cache(device: Optional[Device] = None) -> bool

Reset prefix cache for specified or all devices.

Source code in vllm/core/interfaces.py 
128
129
130
131
@abstractmethod
def reset_prefix_cache(self, device: Optional[Device] = None) -> bool:
    """Reset prefix cache for specified or all devices."""
    pass

swap_in abstractmethod 

swap_in(seq_group: SequenceGroup) -> List[Tuple[int, int]]
Source code in vllm/core/interfaces.py 
77
78
79
@abstractmethod
def swap_in(self, seq_group: SequenceGroup) -> List[Tuple[int, int]]:
    pass

swap_out abstractmethod 

swap_out(seq_group: SequenceGroup) -> List[Tuple[int, int]]
Source code in vllm/core/interfaces.py 
85
86
87
@abstractmethod
def swap_out(self, seq_group: SequenceGroup) -> List[Tuple[int, int]]:
    pass