Skip to content

vllm.v1.core.sched.request_queue

FCFSRequestQueue

Bases: deque[Request], RequestQueue

A first-come-first-served queue that supports deque operations.

Source code in vllm/v1/core/sched/request_queue.py 
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
class FCFSRequestQueue(deque[Request], RequestQueue):
    """A first-come-first-served queue that supports deque operations."""

    def add_request(self, request: Request) -> None:
        """Add a request to the queue according to FCFS policy."""
        self.append(request)

    def pop_request(self) -> Request:
        """Pop a request from the queue according to FCFS policy."""
        return self.popleft()

    def peek_request(self) -> Request:
        """Peek at the next request in the queue without removing it."""
        if not self:
            raise IndexError("peek from an empty queue")
        return self[0]

    def prepend_request(self, request: Request) -> None:
        """Prepend a request to the front of the queue."""
        self.appendleft(request)

    def prepend_requests(self, requests: RequestQueue) -> None:
        """Prepend all requests from another queue to the front of this
        queue."""
        self.extendleft(reversed(requests))

    def remove_request(self, request: Request) -> None:
        """Remove a specific request from the queue."""
        self.remove(request)

    def remove_requests(self, requests: Iterable[Request]) -> None:
        """Remove multiple specific requests from the queue."""
        requests_to_remove = set(requests)
        filtered_requests = [
            req for req in self if req not in requests_to_remove
        ]
        # deque does not support in-place filtering, so we need to clear
        # and extend
        self.clear()
        self.extend(filtered_requests)

    def __bool__(self) -> bool:
        """Check if queue has any requests."""
        return len(self) > 0

    def __len__(self) -> int:
        """Get number of requests in queue."""
        return super().__len__()

    def __iter__(self) -> Iterator[Request]:
        """Iterate over the queue according to FCFS policy."""
        return super().__iter__()

    def __reversed__(self) -> Iterator[Request]:
        """Iterate over the queue in reverse order."""
        return super().__reversed__()

__bool__ 

__bool__() -> bool

Check if queue has any requests.

Source code in vllm/v1/core/sched/request_queue.py 
122
123
124
def __bool__(self) -> bool:
    """Check if queue has any requests."""
    return len(self) > 0

__iter__ 

__iter__() -> Iterator[Request]

Iterate over the queue according to FCFS policy.

Source code in vllm/v1/core/sched/request_queue.py 
130
131
132
def __iter__(self) -> Iterator[Request]:
    """Iterate over the queue according to FCFS policy."""
    return super().__iter__()

__len__ 

__len__() -> int

Get number of requests in queue.

Source code in vllm/v1/core/sched/request_queue.py 
126
127
128
def __len__(self) -> int:
    """Get number of requests in queue."""
    return super().__len__()

__reversed__ 

__reversed__() -> Iterator[Request]

Iterate over the queue in reverse order.

Source code in vllm/v1/core/sched/request_queue.py 
134
135
136
def __reversed__(self) -> Iterator[Request]:
    """Iterate over the queue in reverse order."""
    return super().__reversed__()

add_request 

add_request(request: Request) -> None

Add a request to the queue according to FCFS policy.

Source code in vllm/v1/core/sched/request_queue.py 
84
85
86
def add_request(self, request: Request) -> None:
    """Add a request to the queue according to FCFS policy."""
    self.append(request)

peek_request 

peek_request() -> Request

Peek at the next request in the queue without removing it.

Source code in vllm/v1/core/sched/request_queue.py 
92
93
94
95
96
def peek_request(self) -> Request:
    """Peek at the next request in the queue without removing it."""
    if not self:
        raise IndexError("peek from an empty queue")
    return self[0]

pop_request 

pop_request() -> Request

Pop a request from the queue according to FCFS policy.

Source code in vllm/v1/core/sched/request_queue.py 
88
89
90
def pop_request(self) -> Request:
    """Pop a request from the queue according to FCFS policy."""
    return self.popleft()

prepend_request 

prepend_request(request: Request) -> None

Prepend a request to the front of the queue.

Source code in vllm/v1/core/sched/request_queue.py 
 98
 99
100
def prepend_request(self, request: Request) -> None:
    """Prepend a request to the front of the queue."""
    self.appendleft(request)

prepend_requests 

prepend_requests(requests: RequestQueue) -> None

Prepend all requests from another queue to the front of this queue.

Source code in vllm/v1/core/sched/request_queue.py 
102
103
104
105
def prepend_requests(self, requests: RequestQueue) -> None:
    """Prepend all requests from another queue to the front of this
    queue."""
    self.extendleft(reversed(requests))

remove_request 

remove_request(request: Request) -> None

Remove a specific request from the queue.

Source code in vllm/v1/core/sched/request_queue.py 
107
108
109
def remove_request(self, request: Request) -> None:
    """Remove a specific request from the queue."""
    self.remove(request)

remove_requests 

remove_requests(requests: Iterable[Request]) -> None

Remove multiple specific requests from the queue.

Source code in vllm/v1/core/sched/request_queue.py 
111
112
113
114
115
116
117
118
119
120
def remove_requests(self, requests: Iterable[Request]) -> None:
    """Remove multiple specific requests from the queue."""
    requests_to_remove = set(requests)
    filtered_requests = [
        req for req in self if req not in requests_to_remove
    ]
    # deque does not support in-place filtering, so we need to clear
    # and extend
    self.clear()
    self.extend(filtered_requests)

PriorityRequestQueue

Bases: RequestQueue

A priority queue that supports heap operations.

Requests with a smaller value of priority are processed first. If multiple requests have the same priority, the one with the earlier arrival_time is processed first.

Source code in vllm/v1/core/sched/request_queue.py 
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
class PriorityRequestQueue(RequestQueue):
    """
    A priority queue that supports heap operations.

    Requests with a smaller value of `priority` are processed first.
    If multiple requests have the same priority, the one with the earlier
    `arrival_time` is processed first.
    """

    def __init__(self) -> None:
        self._heap: list[tuple[int, float, Request]] = []

    def add_request(self, request: Request) -> None:
        """Add a request to the queue according to priority policy."""
        heapq.heappush(self._heap,
                       (request.priority, request.arrival_time, request))

    def pop_request(self) -> Request:
        """Pop a request from the queue according to priority policy."""
        if not self._heap:
            raise IndexError("pop from empty heap")
        _, _, request = heapq.heappop(self._heap)
        return request

    def peek_request(self) -> Request:
        """Peek at the next request in the queue without removing it."""
        if not self._heap:
            raise IndexError("peek from empty heap")
        _, _, request = self._heap[0]
        return request

    def prepend_request(self, request: Request) -> None:
        """Add a request to the queue according to priority policy.

        Note: In a priority queue, there is no concept of prepending to the 
        front. Requests are ordered by (priority, arrival_time)."""
        self.add_request(request)

    def prepend_requests(self, requests: RequestQueue) -> None:
        """Add all requests from another queue according to priority policy.

        Note: In a priority queue, there is no concept of prepending to the 
        front. Requests are ordered by (priority, arrival_time)."""
        for request in requests:
            self.add_request(request)

    def remove_request(self, request: Request) -> None:
        """Remove a specific request from the queue."""
        self._heap = [(p, t, r) for p, t, r in self._heap if r != request]
        heapq.heapify(self._heap)

    def remove_requests(self, requests: Iterable[Request]) -> None:
        """Remove multiple specific requests from the queue."""
        requests_to_remove = set(requests)
        self._heap = [(p, t, r) for p, t, r in self._heap
                      if r not in requests_to_remove]
        heapq.heapify(self._heap)

    def __bool__(self) -> bool:
        """Check if queue has any requests."""
        return bool(self._heap)

    def __len__(self) -> int:
        """Get number of requests in queue."""
        return len(self._heap)

    def __iter__(self) -> Iterator[Request]:
        """Iterate over the queue according to priority policy."""
        heap_copy = self._heap[:]
        while heap_copy:
            _, _, request = heapq.heappop(heap_copy)
            yield request

    def __reversed__(self) -> Iterator[Request]:
        """Iterate over the queue in reverse priority order."""
        return reversed(list(self))

_heap instance-attribute 

_heap: list[tuple[int, float, Request]] = []

__bool__ 

__bool__() -> bool

Check if queue has any requests.

Source code in vllm/v1/core/sched/request_queue.py 
197
198
199
def __bool__(self) -> bool:
    """Check if queue has any requests."""
    return bool(self._heap)

__init__ 

__init__() -> None
Source code in vllm/v1/core/sched/request_queue.py 
148
149
def __init__(self) -> None:
    self._heap: list[tuple[int, float, Request]] = []

__iter__ 

__iter__() -> Iterator[Request]

Iterate over the queue according to priority policy.

Source code in vllm/v1/core/sched/request_queue.py 
205
206
207
208
209
210
def __iter__(self) -> Iterator[Request]:
    """Iterate over the queue according to priority policy."""
    heap_copy = self._heap[:]
    while heap_copy:
        _, _, request = heapq.heappop(heap_copy)
        yield request

__len__ 

__len__() -> int

Get number of requests in queue.

Source code in vllm/v1/core/sched/request_queue.py 
201
202
203
def __len__(self) -> int:
    """Get number of requests in queue."""
    return len(self._heap)

__reversed__ 

__reversed__() -> Iterator[Request]

Iterate over the queue in reverse priority order.

Source code in vllm/v1/core/sched/request_queue.py 
212
213
214
def __reversed__(self) -> Iterator[Request]:
    """Iterate over the queue in reverse priority order."""
    return reversed(list(self))

add_request 

add_request(request: Request) -> None

Add a request to the queue according to priority policy.

Source code in vllm/v1/core/sched/request_queue.py 
151
152
153
154
def add_request(self, request: Request) -> None:
    """Add a request to the queue according to priority policy."""
    heapq.heappush(self._heap,
                   (request.priority, request.arrival_time, request))

peek_request 

peek_request() -> Request

Peek at the next request in the queue without removing it.

Source code in vllm/v1/core/sched/request_queue.py 
163
164
165
166
167
168
def peek_request(self) -> Request:
    """Peek at the next request in the queue without removing it."""
    if not self._heap:
        raise IndexError("peek from empty heap")
    _, _, request = self._heap[0]
    return request

pop_request 

pop_request() -> Request

Pop a request from the queue according to priority policy.

Source code in vllm/v1/core/sched/request_queue.py 
156
157
158
159
160
161
def pop_request(self) -> Request:
    """Pop a request from the queue according to priority policy."""
    if not self._heap:
        raise IndexError("pop from empty heap")
    _, _, request = heapq.heappop(self._heap)
    return request

prepend_request 

prepend_request(request: Request) -> None

Add a request to the queue according to priority policy.

Note: In a priority queue, there is no concept of prepending to the front. Requests are ordered by (priority, arrival_time).

Source code in vllm/v1/core/sched/request_queue.py 
170
171
172
173
174
175
def prepend_request(self, request: Request) -> None:
    """Add a request to the queue according to priority policy.

    Note: In a priority queue, there is no concept of prepending to the 
    front. Requests are ordered by (priority, arrival_time)."""
    self.add_request(request)

prepend_requests 

prepend_requests(requests: RequestQueue) -> None

Add all requests from another queue according to priority policy.

Note: In a priority queue, there is no concept of prepending to the front. Requests are ordered by (priority, arrival_time).

Source code in vllm/v1/core/sched/request_queue.py 
177
178
179
180
181
182
183
def prepend_requests(self, requests: RequestQueue) -> None:
    """Add all requests from another queue according to priority policy.

    Note: In a priority queue, there is no concept of prepending to the 
    front. Requests are ordered by (priority, arrival_time)."""
    for request in requests:
        self.add_request(request)

remove_request 

remove_request(request: Request) -> None

Remove a specific request from the queue.

Source code in vllm/v1/core/sched/request_queue.py 
185
186
187
188
def remove_request(self, request: Request) -> None:
    """Remove a specific request from the queue."""
    self._heap = [(p, t, r) for p, t, r in self._heap if r != request]
    heapq.heapify(self._heap)

remove_requests 

remove_requests(requests: Iterable[Request]) -> None

Remove multiple specific requests from the queue.

Source code in vllm/v1/core/sched/request_queue.py 
190
191
192
193
194
195
def remove_requests(self, requests: Iterable[Request]) -> None:
    """Remove multiple specific requests from the queue."""
    requests_to_remove = set(requests)
    self._heap = [(p, t, r) for p, t, r in self._heap
                  if r not in requests_to_remove]
    heapq.heapify(self._heap)

RequestQueue

Bases: ABC

Abstract base class for request queues.

Source code in vllm/v1/core/sched/request_queue.py 
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
class RequestQueue(ABC):
    """Abstract base class for request queues."""

    @abstractmethod
    def add_request(self, request: Request) -> None:
        """Add a request to the queue according to the policy."""
        pass

    @abstractmethod
    def pop_request(self) -> Request:
        """Pop a request from the queue according to the policy."""
        pass

    @abstractmethod
    def peek_request(self) -> Request:
        """Peek at the request at the front of the queue without removing it."""
        pass

    @abstractmethod
    def prepend_request(self, request: Request) -> None:
        """Prepend a request to the front of the queue."""
        pass

    @abstractmethod
    def prepend_requests(self, requests: RequestQueue) -> None:
        """Prepend all requests from another queue to the front of this
        queue."""
        pass

    @abstractmethod
    def remove_request(self, request: Request) -> None:
        """Remove a specific request from the queue."""
        pass

    @abstractmethod
    def remove_requests(self, requests: Iterable[Request]) -> None:
        """Remove multiple specific requests from the queue."""
        pass

    @abstractmethod
    def __bool__(self) -> bool:
        """Check if queue has any requests."""
        pass

    @abstractmethod
    def __len__(self) -> int:
        """Get number of requests in queue."""
        pass

    @abstractmethod
    def __iter__(self) -> Iterator[Request]:
        """Iterate over the queue according to the policy."""
        pass

    @abstractmethod
    def __reversed__(self) -> Iterator[Request]:
        """Iterate over the queue in reverse order."""
        pass

__bool__ abstractmethod 

__bool__() -> bool

Check if queue has any requests.

Source code in vllm/v1/core/sched/request_queue.py 
60
61
62
63
@abstractmethod
def __bool__(self) -> bool:
    """Check if queue has any requests."""
    pass

__iter__ abstractmethod 

__iter__() -> Iterator[Request]

Iterate over the queue according to the policy.

Source code in vllm/v1/core/sched/request_queue.py 
70
71
72
73
@abstractmethod
def __iter__(self) -> Iterator[Request]:
    """Iterate over the queue according to the policy."""
    pass

__len__ abstractmethod 

__len__() -> int

Get number of requests in queue.

Source code in vllm/v1/core/sched/request_queue.py 
65
66
67
68
@abstractmethod
def __len__(self) -> int:
    """Get number of requests in queue."""
    pass

__reversed__ abstractmethod 

__reversed__() -> Iterator[Request]

Iterate over the queue in reverse order.

Source code in vllm/v1/core/sched/request_queue.py 
75
76
77
78
@abstractmethod
def __reversed__(self) -> Iterator[Request]:
    """Iterate over the queue in reverse order."""
    pass

add_request abstractmethod 

add_request(request: Request) -> None

Add a request to the queue according to the policy.

Source code in vllm/v1/core/sched/request_queue.py 
24
25
26
27
@abstractmethod
def add_request(self, request: Request) -> None:
    """Add a request to the queue according to the policy."""
    pass

peek_request abstractmethod 

peek_request() -> Request

Peek at the request at the front of the queue without removing it.

Source code in vllm/v1/core/sched/request_queue.py 
34
35
36
37
@abstractmethod
def peek_request(self) -> Request:
    """Peek at the request at the front of the queue without removing it."""
    pass

pop_request abstractmethod 

pop_request() -> Request

Pop a request from the queue according to the policy.

Source code in vllm/v1/core/sched/request_queue.py 
29
30
31
32
@abstractmethod
def pop_request(self) -> Request:
    """Pop a request from the queue according to the policy."""
    pass

prepend_request abstractmethod 

prepend_request(request: Request) -> None

Prepend a request to the front of the queue.

Source code in vllm/v1/core/sched/request_queue.py 
39
40
41
42
@abstractmethod
def prepend_request(self, request: Request) -> None:
    """Prepend a request to the front of the queue."""
    pass

prepend_requests abstractmethod 

prepend_requests(requests: RequestQueue) -> None

Prepend all requests from another queue to the front of this queue.

Source code in vllm/v1/core/sched/request_queue.py 
44
45
46
47
48
@abstractmethod
def prepend_requests(self, requests: RequestQueue) -> None:
    """Prepend all requests from another queue to the front of this
    queue."""
    pass

remove_request abstractmethod 

remove_request(request: Request) -> None

Remove a specific request from the queue.

Source code in vllm/v1/core/sched/request_queue.py 
50
51
52
53
@abstractmethod
def remove_request(self, request: Request) -> None:
    """Remove a specific request from the queue."""
    pass

remove_requests abstractmethod 

remove_requests(requests: Iterable[Request]) -> None

Remove multiple specific requests from the queue.

Source code in vllm/v1/core/sched/request_queue.py 
55
56
57
58
@abstractmethod
def remove_requests(self, requests: Iterable[Request]) -> None:
    """Remove multiple specific requests from the queue."""
    pass

SchedulingPolicy

Bases: Enum

Enum for scheduling policies.

Source code in vllm/v1/core/sched/request_queue.py 
15
16
17
18
class SchedulingPolicy(Enum):
    """Enum for scheduling policies."""
    FCFS = "fcfs"
    PRIORITY = "priority"

FCFS class-attribute instance-attribute 

FCFS = 'fcfs'

PRIORITY class-attribute instance-attribute 

PRIORITY = 'priority'

create_request_queue 

create_request_queue(
    policy: SchedulingPolicy,
) -> RequestQueue

Create request queue based on scheduling policy.

Source code in vllm/v1/core/sched/request_queue.py 
217
218
219
220
221
222
223
224
def create_request_queue(policy: SchedulingPolicy) -> RequestQueue:
    """Create request queue based on scheduling policy."""
    if policy == SchedulingPolicy.PRIORITY:
        return PriorityRequestQueue()
    elif policy == SchedulingPolicy.FCFS:
        return FCFSRequestQueue()
    else:
        raise ValueError(f"Unknown scheduling policy: {policy}")