Skip to content

vllm.utils.gc_utils

logger module-attribute 

logger = init_logger(__name__)

GCDebugConfig

Config for GC Debugger. - 0: disable GC debugger - 1: enable GC debugger with gc.collect elpased times - '{"top_objects":5}': enable GC debugger with top 5 collected objects

Source code in vllm/utils/gc_utils.py 
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
class GCDebugConfig:
    """
    Config for GC Debugger.
    - 0: disable GC debugger
    - 1: enable GC debugger with gc.collect elpased times
    - '{"top_objects":5}': enable GC debugger with top 5 collected objects
    """

    def __init__(self, gc_debug_conf: str | None = None) -> None:
        self.enabled: bool = False
        self.top_objects: int = -1

        if not gc_debug_conf or gc_debug_conf == "0":
            pass
        elif gc_debug_conf == "1":
            self.enabled = True
        else:
            try:
                json_conf = json.loads(gc_debug_conf)
                self.enabled = True
                self.top_objects = json_conf.get("top_objects", -1)
            except Exception:
                self.enabled = False
                logger.error("Failed to parse VLLM_GC_DEBUG(%s)", VLLM_GC_DEBUG)
        logger.info("GC Debug Config. %s", str(self))

    def __repr__(self) -> str:
        return f"enabled:{self.enabled},top_objects:{self.top_objects}"

enabled instance-attribute 

enabled: bool = True

top_objects instance-attribute 

top_objects: int = get('top_objects', -1)

__init__ 

__init__(gc_debug_conf: str | None = None) -> None
Source code in vllm/utils/gc_utils.py 
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
def __init__(self, gc_debug_conf: str | None = None) -> None:
    self.enabled: bool = False
    self.top_objects: int = -1

    if not gc_debug_conf or gc_debug_conf == "0":
        pass
    elif gc_debug_conf == "1":
        self.enabled = True
    else:
        try:
            json_conf = json.loads(gc_debug_conf)
            self.enabled = True
            self.top_objects = json_conf.get("top_objects", -1)
        except Exception:
            self.enabled = False
            logger.error("Failed to parse VLLM_GC_DEBUG(%s)", VLLM_GC_DEBUG)
    logger.info("GC Debug Config. %s", str(self))

__repr__ 

__repr__() -> str
Source code in vllm/utils/gc_utils.py 
42
43
def __repr__(self) -> str:
    return f"enabled:{self.enabled},top_objects:{self.top_objects}"

GCDebugger

Debugger for GC which logs helpful information for GC understanding. To enable, you should call maybe_attach_gc_debug_callback in the process.

Source code in vllm/utils/gc_utils.py 
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
class GCDebugger:
    """
    Debugger for GC which logs helpful information for GC understanding.
    To enable, you should call maybe_attach_gc_debug_callback in the process.
    """

    def __init__(self, config: GCDebugConfig) -> None:
        self.config = config
        # Start time in micro second of this GC cycle
        self.start_time_ns: int = time.monotonic_ns()
        # If config.top_objects is positive,
        # compute top collected objects by object types
        self.gc_top_collected_objects: str = ""

    def handle(self, phase: str, info: dict[str, int]) -> None:
        """
        Handles a GC event (e.g. GC start or GC finish)
        """
        generation = info.get("generation")
        if generation is None:
            return
        if phase == "start":
            # Before GC started, record GC start time
            # and top collected objects
            self.start_time_ns = time.monotonic_ns()
            self.gc_top_collected_objects = _compute_top_gc_collected_objects(
                gc.get_objects(generation), self.config.top_objects
            )
        elif phase == "stop":
            # After GC finished, Record GC elapsed time and
            # optionally top collected objects
            elpased_ms = (time.monotonic_ns() - self.start_time_ns) / 1e6
            logger.info(
                "GC took %.3fms to complete. "
                "Collected %s objects in GC generation %d.%s",
                elpased_ms,
                str(info.get("collected", "?")),
                generation,
                (
                    f" Top collected objects: \n{self.gc_top_collected_objects}"
                    if self.gc_top_collected_objects
                    else ""
                ),
            )

config instance-attribute 

config = config

gc_top_collected_objects instance-attribute 

gc_top_collected_objects: str = ''

start_time_ns instance-attribute 

start_time_ns: int = monotonic_ns()

__init__ 

__init__(config: GCDebugConfig) -> None
Source code in vllm/utils/gc_utils.py 
52
53
54
55
56
57
58
def __init__(self, config: GCDebugConfig) -> None:
    self.config = config
    # Start time in micro second of this GC cycle
    self.start_time_ns: int = time.monotonic_ns()
    # If config.top_objects is positive,
    # compute top collected objects by object types
    self.gc_top_collected_objects: str = ""

handle 

handle(phase: str, info: dict[str, int]) -> None

Handles a GC event (e.g. GC start or GC finish)

Source code in vllm/utils/gc_utils.py 
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
def handle(self, phase: str, info: dict[str, int]) -> None:
    """
    Handles a GC event (e.g. GC start or GC finish)
    """
    generation = info.get("generation")
    if generation is None:
        return
    if phase == "start":
        # Before GC started, record GC start time
        # and top collected objects
        self.start_time_ns = time.monotonic_ns()
        self.gc_top_collected_objects = _compute_top_gc_collected_objects(
            gc.get_objects(generation), self.config.top_objects
        )
    elif phase == "stop":
        # After GC finished, Record GC elapsed time and
        # optionally top collected objects
        elpased_ms = (time.monotonic_ns() - self.start_time_ns) / 1e6
        logger.info(
            "GC took %.3fms to complete. "
            "Collected %s objects in GC generation %d.%s",
            elpased_ms,
            str(info.get("collected", "?")),
            generation,
            (
                f" Top collected objects: \n{self.gc_top_collected_objects}"
                if self.gc_top_collected_objects
                else ""
            ),
        )

_compute_detailed_type 

_compute_detailed_type(o: Any) -> str

Detailed object type.

TODO(Jialin): Further enhance the detailed type with element types for easier debugging. We tried but occasionally it would run into signals which kills the engine.

Source code in vllm/utils/gc_utils.py 
106
107
108
109
110
111
112
113
114
115
116
117
118
119
def _compute_detailed_type(o: Any) -> str:
    """
    Detailed object type.

    TODO(Jialin): Further enhance the detailed type with element types for
    easier debugging. We tried but occasionally it would run into signals
    which kills the engine.
    """
    size_str: str = ""
    # Object doesn't support len() - this can happen with type objects
    # or other objects that don't implement __len__ properly
    with suppress(Exception):
        size_str = f"(size:{len(o)})"
    return f"{str(type(o))}{size_str}"

_compute_top_gc_collected_objects 

_compute_top_gc_collected_objects(
    objects: list[Any], top: int
) -> str

Group collected objects by types.

Source code in vllm/utils/gc_utils.py 
122
123
124
125
126
127
128
129
130
131
132
def _compute_top_gc_collected_objects(objects: list[Any], top: int) -> str:
    """
    Group collected objects by types.
    """
    if top <= 0:
        return ""
    object_types = [_compute_detailed_type(o) for o in objects]
    return "\n".join(
        f"{count:>5}:{object_type}"
        for object_type, count in Counter(object_types).most_common(top)
    )

maybe_attach_gc_debug_callback 

maybe_attach_gc_debug_callback() -> None

Attached a callback for GC debug when VLLM_GC_DEBUG is enabled.

Source code in vllm/utils/gc_utils.py 
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
def maybe_attach_gc_debug_callback() -> None:
    """
    Attached a callback for GC debug when VLLM_GC_DEBUG is enabled.
    """
    config = GCDebugConfig(VLLM_GC_DEBUG)
    if config.enabled:
        debugger: GCDebugger = GCDebugger(config)

        def gc_callback(phase: str, info: dict[str, int]) -> None:
            debugger.handle(phase, info)

        gc.callbacks.append(gc_callback)