Skip to content

vllm.transformers_utils.configs.mpt

A HuggingFace-style model configuration.

attn_config_defaults module-attribute 

attn_config_defaults: dict = {
    "attn_type": "multihead_attention",
    "attn_pdrop": 0.0,
    "attn_impl": "triton",
    "qk_ln": False,
    "clip_qkv": None,
    "softmax_scale": None,
    "prefix_lm": False,
    "attn_uses_sequence_id": False,
    "alibi": False,
    "alibi_bias_max": 8,
}

ffn_config_defaults module-attribute 

ffn_config_defaults: dict = {'ffn_type': 'mptmlp'}

init_config_defaults module-attribute 

init_config_defaults: dict = {
    "name": "kaiming_normal_",
    "fan_mode": "fan_in",
    "init_nonlinearity": "relu",
    "init_div_is_residual": True,
    "emb_init_std": None,
    "emb_init_uniform_lim": None,
    "init_std": None,
    "init_gain": 0.0,
}

MPTConfig

Bases: PretrainedConfig

Source code in vllm/transformers_utils/configs/mpt.py 
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
class MPTConfig(PretrainedConfig):
    model_type = 'mpt'
    attribute_map = {
        'num_attention_heads': 'n_heads',
        'hidden_size': 'd_model',
        'num_hidden_layers': 'n_layers',
    }

    # pylint: disable=dangerous-default-value
    def __init__(self,
                 d_model: int = 2048,
                 n_heads: int = 16,
                 n_layers: int = 24,
                 expansion_ratio: int = 4,
                 max_seq_len: int = 2048,
                 vocab_size: int = 50368,
                 resid_pdrop: float = 0.0,
                 emb_pdrop: float = 0.0,
                 learned_pos_emb: bool = True,
                 attn_config: dict = attn_config_defaults,
                 ffn_config: dict = ffn_config_defaults,
                 init_device: str = 'cpu',
                 logit_scale: Optional[Union[float, str]] = None,
                 no_bias: bool = False,
                 embedding_fraction: float = 1.0,
                 norm_type: str = 'low_precision_layernorm',
                 use_cache: bool = False,
                 init_config: dict = init_config_defaults,
                 fc_type: str = 'torch',
                 verbose: Optional[int] = None,
                 **kwargs: Any):
        self.d_model = d_model
        self.n_heads = n_heads
        self.n_layers = n_layers
        self.expansion_ratio = expansion_ratio
        self.max_seq_len = max_seq_len
        self.vocab_size = vocab_size
        self.resid_pdrop = resid_pdrop
        self.emb_pdrop = emb_pdrop
        self.learned_pos_emb = learned_pos_emb
        self.attn_config = attn_config
        self.ffn_config = ffn_config
        self.init_device = init_device
        self.logit_scale = logit_scale
        self.no_bias = no_bias
        self.embedding_fraction = embedding_fraction
        self.norm_type = norm_type
        self.use_cache = use_cache
        self.init_config = init_config
        self.fc_type = fc_type
        if verbose is not None:
            warnings.warn(DeprecationWarning(
                'verbose argument for MPTConfig is now ignored and '
                'will be removed. Use python_log_level instead.'),
                          stacklevel=2)
        if 'name' in kwargs:
            del kwargs['name']
        if 'loss_fn' in kwargs:
            del kwargs['loss_fn']
        if self.attn_config.get('alibi', False):
            self.learned_pos_emb = False
            warnings.warn(
                f'alibi is turned on, setting `learned_pos_emb` '
                f'to {self.learned_pos_emb}`',
                stacklevel=2)
        super().__init__(**kwargs)
        self._validate_config()

    def _set_config_defaults(
            self, config: dict[str, Any],
            config_defaults: dict[str, Any]) -> dict[str, Any]:
        for (k, v) in config_defaults.items():
            if k not in config:
                config[k] = v
        return config

    def _validate_config(self) -> None:
        self.attn_config = self._set_config_defaults(self.attn_config,
                                                     attn_config_defaults)
        self.ffn_config = self._set_config_defaults(self.ffn_config,
                                                    ffn_config_defaults)
        self.init_config = self._set_config_defaults(self.init_config,
                                                     init_config_defaults)
        if self.d_model % self.n_heads != 0:
            raise ValueError('d_model must be divisible by n_heads')
        if any(
                prob < 0 or prob > 1 for prob in
            [self.attn_config['attn_pdrop'], self.resid_pdrop, self.emb_pdrop
             ]):
            raise ValueError(
                "self.attn_config['attn_pdrop'], resid_pdrop, emb_pdrop are "
                "probabilities and must be between 0 and 1")
        if self.attn_config['attn_impl'] not in ['torch', 'flash', 'triton']:
            raise ValueError(
                f"Unknown attn_impl={self.attn_config['attn_impl']}")
        if self.attn_config['prefix_lm'] and self.attn_config[
                'attn_impl'] not in ['torch', 'triton']:
            raise NotImplementedError(
                'prefix_lm only implemented with torch and triton attention.')
        if self.attn_config['alibi'] and self.attn_config['attn_impl'] not in [
                'torch', 'triton'
        ]:
            raise NotImplementedError(
                'alibi only implemented with torch and triton attention.')
        if self.attn_config['attn_uses_sequence_id'] and self.attn_config[
                'attn_impl'] not in ['torch', 'triton']:
            raise NotImplementedError(
                'attn_uses_sequence_id only implemented with torch '
                'and triton attention.')
        if self.embedding_fraction > 1 or self.embedding_fraction <= 0:
            raise ValueError(
                'model.embedding_fraction must be between 0 (exclusive) '
                'and 1 (inclusive)!')
        if isinstance(self.logit_scale,
                      str) and self.logit_scale != 'inv_sqrt_d_model':
            raise ValueError(
                f"self.logit_scale={self.logit_scale!r} is not recognized as "
                "an option; use numeric value or 'inv_sqrt_d_model'.")
        if self.init_config.get('name', None) is None:
            raise ValueError(
                f"self.init_config={self.init_config!r} 'name' needs to be set."
            )
        if not self.learned_pos_emb and (not self.attn_config['alibi']):
            warnings.warn(
                'Positional information not being provided to the model.',
                stacklevel=2)
        if self.fc_type == 'te' or self.ffn_config['ffn_type'] == 'te_ln_mlp':
            try:
                # pylint: disable=import-outside-toplevel
                import transformer_engine.pytorch as te
                del te
            except Exception as exc:
                raise ImportError(
                    'TransformerEngine import fail. `fc_type: te` requires '
                    'TransformerEngine be installed. '
                    'The required version of transformer_engine also requires '
                    'FlashAttention v1.0.6 is installed:\n'
                    'pip install flash-attn==1.0.6 --no-build-isolation \n'
                    'pip install git+https://github.com/NVIDIA/TransformerEngine.git@144e4888b2cdd60bd52e706d5b7a79cb9c1a7156'
                ) from exc
        if self.ffn_config['ffn_type'] == 'mptmlp':
            self.ffn_config['fc_type'] = self.fc_type
        elif self.ffn_config['ffn_type'] == 'te_ln_mlp':
            self.ffn_config['bias'] = not self.no_bias

attn_config instance-attribute 

attn_config = attn_config

attribute_map class-attribute instance-attribute 

attribute_map = {
    "num_attention_heads": "n_heads",
    "hidden_size": "d_model",
    "num_hidden_layers": "n_layers",
}

d_model instance-attribute 

d_model = d_model

emb_pdrop instance-attribute 

emb_pdrop = emb_pdrop

embedding_fraction instance-attribute 

embedding_fraction = embedding_fraction

expansion_ratio instance-attribute 

expansion_ratio = expansion_ratio

fc_type instance-attribute 

fc_type = fc_type

ffn_config instance-attribute 

ffn_config = ffn_config

init_config instance-attribute 

init_config = init_config

init_device instance-attribute 

init_device = init_device

learned_pos_emb instance-attribute 

learned_pos_emb = learned_pos_emb

logit_scale instance-attribute 

logit_scale = logit_scale

max_seq_len instance-attribute 

max_seq_len = max_seq_len

model_type class-attribute instance-attribute 

model_type = 'mpt'

n_heads instance-attribute 

n_heads = n_heads

n_layers instance-attribute 

n_layers = n_layers

no_bias instance-attribute 

no_bias = no_bias

norm_type instance-attribute 

norm_type = norm_type

resid_pdrop instance-attribute 

resid_pdrop = resid_pdrop

use_cache instance-attribute 

use_cache = use_cache

vocab_size instance-attribute 

vocab_size = vocab_size

__init__ 

__init__(
    d_model: int = 2048,
    n_heads: int = 16,
    n_layers: int = 24,
    expansion_ratio: int = 4,
    max_seq_len: int = 2048,
    vocab_size: int = 50368,
    resid_pdrop: float = 0.0,
    emb_pdrop: float = 0.0,
    learned_pos_emb: bool = True,
    attn_config: dict = attn_config_defaults,
    ffn_config: dict = ffn_config_defaults,
    init_device: str = "cpu",
    logit_scale: Optional[Union[float, str]] = None,
    no_bias: bool = False,
    embedding_fraction: float = 1.0,
    norm_type: str = "low_precision_layernorm",
    use_cache: bool = False,
    init_config: dict = init_config_defaults,
    fc_type: str = "torch",
    verbose: Optional[int] = None,
    **kwargs: Any,
)
Source code in vllm/transformers_utils/configs/mpt.py 
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
def __init__(self,
             d_model: int = 2048,
             n_heads: int = 16,
             n_layers: int = 24,
             expansion_ratio: int = 4,
             max_seq_len: int = 2048,
             vocab_size: int = 50368,
             resid_pdrop: float = 0.0,
             emb_pdrop: float = 0.0,
             learned_pos_emb: bool = True,
             attn_config: dict = attn_config_defaults,
             ffn_config: dict = ffn_config_defaults,
             init_device: str = 'cpu',
             logit_scale: Optional[Union[float, str]] = None,
             no_bias: bool = False,
             embedding_fraction: float = 1.0,
             norm_type: str = 'low_precision_layernorm',
             use_cache: bool = False,
             init_config: dict = init_config_defaults,
             fc_type: str = 'torch',
             verbose: Optional[int] = None,
             **kwargs: Any):
    self.d_model = d_model
    self.n_heads = n_heads
    self.n_layers = n_layers
    self.expansion_ratio = expansion_ratio
    self.max_seq_len = max_seq_len
    self.vocab_size = vocab_size
    self.resid_pdrop = resid_pdrop
    self.emb_pdrop = emb_pdrop
    self.learned_pos_emb = learned_pos_emb
    self.attn_config = attn_config
    self.ffn_config = ffn_config
    self.init_device = init_device
    self.logit_scale = logit_scale
    self.no_bias = no_bias
    self.embedding_fraction = embedding_fraction
    self.norm_type = norm_type
    self.use_cache = use_cache
    self.init_config = init_config
    self.fc_type = fc_type
    if verbose is not None:
        warnings.warn(DeprecationWarning(
            'verbose argument for MPTConfig is now ignored and '
            'will be removed. Use python_log_level instead.'),
                      stacklevel=2)
    if 'name' in kwargs:
        del kwargs['name']
    if 'loss_fn' in kwargs:
        del kwargs['loss_fn']
    if self.attn_config.get('alibi', False):
        self.learned_pos_emb = False
        warnings.warn(
            f'alibi is turned on, setting `learned_pos_emb` '
            f'to {self.learned_pos_emb}`',
            stacklevel=2)
    super().__init__(**kwargs)
    self._validate_config()

_set_config_defaults 

_set_config_defaults(
    config: dict[str, Any], config_defaults: dict[str, Any]
) -> dict[str, Any]
Source code in vllm/transformers_utils/configs/mpt.py 
105
106
107
108
109
110
111
def _set_config_defaults(
        self, config: dict[str, Any],
        config_defaults: dict[str, Any]) -> dict[str, Any]:
    for (k, v) in config_defaults.items():
        if k not in config:
            config[k] = v
    return config

_validate_config 

_validate_config() -> None
Source code in vllm/transformers_utils/configs/mpt.py 
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
def _validate_config(self) -> None:
    self.attn_config = self._set_config_defaults(self.attn_config,
                                                 attn_config_defaults)
    self.ffn_config = self._set_config_defaults(self.ffn_config,
                                                ffn_config_defaults)
    self.init_config = self._set_config_defaults(self.init_config,
                                                 init_config_defaults)
    if self.d_model % self.n_heads != 0:
        raise ValueError('d_model must be divisible by n_heads')
    if any(
            prob < 0 or prob > 1 for prob in
        [self.attn_config['attn_pdrop'], self.resid_pdrop, self.emb_pdrop
         ]):
        raise ValueError(
            "self.attn_config['attn_pdrop'], resid_pdrop, emb_pdrop are "
            "probabilities and must be between 0 and 1")
    if self.attn_config['attn_impl'] not in ['torch', 'flash', 'triton']:
        raise ValueError(
            f"Unknown attn_impl={self.attn_config['attn_impl']}")
    if self.attn_config['prefix_lm'] and self.attn_config[
            'attn_impl'] not in ['torch', 'triton']:
        raise NotImplementedError(
            'prefix_lm only implemented with torch and triton attention.')
    if self.attn_config['alibi'] and self.attn_config['attn_impl'] not in [
            'torch', 'triton'
    ]:
        raise NotImplementedError(
            'alibi only implemented with torch and triton attention.')
    if self.attn_config['attn_uses_sequence_id'] and self.attn_config[
            'attn_impl'] not in ['torch', 'triton']:
        raise NotImplementedError(
            'attn_uses_sequence_id only implemented with torch '
            'and triton attention.')
    if self.embedding_fraction > 1 or self.embedding_fraction <= 0:
        raise ValueError(
            'model.embedding_fraction must be between 0 (exclusive) '
            'and 1 (inclusive)!')
    if isinstance(self.logit_scale,
                  str) and self.logit_scale != 'inv_sqrt_d_model':
        raise ValueError(
            f"self.logit_scale={self.logit_scale!r} is not recognized as "
            "an option; use numeric value or 'inv_sqrt_d_model'.")
    if self.init_config.get('name', None) is None:
        raise ValueError(
            f"self.init_config={self.init_config!r} 'name' needs to be set."
        )
    if not self.learned_pos_emb and (not self.attn_config['alibi']):
        warnings.warn(
            'Positional information not being provided to the model.',
            stacklevel=2)
    if self.fc_type == 'te' or self.ffn_config['ffn_type'] == 'te_ln_mlp':
        try:
            # pylint: disable=import-outside-toplevel
            import transformer_engine.pytorch as te
            del te
        except Exception as exc:
            raise ImportError(
                'TransformerEngine import fail. `fc_type: te` requires '
                'TransformerEngine be installed. '
                'The required version of transformer_engine also requires '
                'FlashAttention v1.0.6 is installed:\n'
                'pip install flash-attn==1.0.6 --no-build-isolation \n'
                'pip install git+https://github.com/NVIDIA/TransformerEngine.git@144e4888b2cdd60bd52e706d5b7a79cb9c1a7156'
            ) from exc
    if self.ffn_config['ffn_type'] == 'mptmlp':
        self.ffn_config['fc_type'] = self.fc_type
    elif self.ffn_config['ffn_type'] == 'te_ln_mlp':
        self.ffn_config['bias'] = not self.no_bias