Skip to content

vllm.model_executor.layers.quantization.ipex_quant

MIN_IPEX_VERSION module-attribute 

MIN_IPEX_VERSION = '2.6.0'

IPEXAWQLinearMethod

Bases: AWQLinearMethod

AWQ linear method using IPEX for the CPU/XPU backend.

Source code in vllm/model_executor/layers/quantization/ipex_quant.py 
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
class IPEXAWQLinearMethod(AWQLinearMethod):
    """AWQ linear method using IPEX for the CPU/XPU backend.
    """

    def __init__(self, quant_config: IPEXConfig):
        self.quant_config = quant_config  # type: ignore

    def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
        super().process_weights_after_loading(layer=layer)

        bias = layer.bias if not layer.skip_bias_add else None

        try:
            import intel_extension_for_pytorch as ipex
            if ipex.__version__ < MIN_IPEX_VERSION:
                raise ImportError(
                    "intel_extension_for_pytorch version is "
                    "wrong. Please install "
                    f"intel_extension_for_pytorch>={MIN_IPEX_VERSION}.")
        except ImportError as err:
            raise ImportError(
                "Please install "
                f"intel_extension_for_pytorch>={MIN_IPEX_VERSION} via "
                f"`pip install intel_extension_for_pytorch>={MIN_IPEX_VERSION}`"
                " to use IPEX-AWQ linear method.") from err

        # Using the compute dtype (lowp_mode) as INT8 to leverage instructions
        # with better performance.
        lowp_mode = ipex.quantization.WoqLowpMode.INT8
        # The weight will be de-packed from INT4 to INT8.
        weight_dtype = ipex.quantization.WoqWeightDtype.INT4
        # The float activation will be quantized (dynamic, per-token) to INT8.
        act_quant_mode = ipex.quantization.WoqActQuantMode.PER_BATCH

        qconfig = ipex.quantization.get_weight_only_quant_qconfig_mapping(
            weight_dtype=weight_dtype,
            lowp_mode=lowp_mode,
            act_quant_mode=act_quant_mode,
            group_size=self.quant_config.group_size,
        )

        layer.ipex_output_size = layer.qweight.size(
            1) * self.quant_config.pack_factor
        layer.ipex_qlinear = ipex.llm.quantization.woq_linear. \
            IPEXWeightOnlyQuantizedLinear.from_weight(
            layer.qweight,
            layer.scales,
            layer.qzeros,
            layer.qweight.size(0),
            layer.ipex_output_size,
            qconfig=qconfig,
            bias=bias,
            group_size=self.quant_config.group_size,
            quant_method=IPEXConfig.IPEX_QUANT_METHOD_MAP["awq"]  # type: ignore
        )

    def apply(self,
              layer: torch.nn.Module,
              x: torch.Tensor,
              bias: Optional[torch.Tensor] = None) -> torch.Tensor:
        reshaped_x = x.reshape(-1, x.shape[-1])
        out = layer.ipex_qlinear(reshaped_x)
        return out.reshape(x.shape[:-1] + (layer.ipex_output_size, ))

quant_config instance-attribute 

quant_config = quant_config

__init__ 

__init__(quant_config: IPEXConfig)
Source code in vllm/model_executor/layers/quantization/ipex_quant.py 
192
193
def __init__(self, quant_config: IPEXConfig):
    self.quant_config = quant_config  # type: ignore

apply 

apply(
    layer: Module, x: Tensor, bias: Optional[Tensor] = None
) -> Tensor
Source code in vllm/model_executor/layers/quantization/ipex_quant.py 
244
245
246
247
248
249
250
def apply(self,
          layer: torch.nn.Module,
          x: torch.Tensor,
          bias: Optional[torch.Tensor] = None) -> torch.Tensor:
    reshaped_x = x.reshape(-1, x.shape[-1])
    out = layer.ipex_qlinear(reshaped_x)
    return out.reshape(x.shape[:-1] + (layer.ipex_output_size, ))

process_weights_after_loading 

process_weights_after_loading(layer: Module) -> None
Source code in vllm/model_executor/layers/quantization/ipex_quant.py 
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
    super().process_weights_after_loading(layer=layer)

    bias = layer.bias if not layer.skip_bias_add else None

    try:
        import intel_extension_for_pytorch as ipex
        if ipex.__version__ < MIN_IPEX_VERSION:
            raise ImportError(
                "intel_extension_for_pytorch version is "
                "wrong. Please install "
                f"intel_extension_for_pytorch>={MIN_IPEX_VERSION}.")
    except ImportError as err:
        raise ImportError(
            "Please install "
            f"intel_extension_for_pytorch>={MIN_IPEX_VERSION} via "
            f"`pip install intel_extension_for_pytorch>={MIN_IPEX_VERSION}`"
            " to use IPEX-AWQ linear method.") from err

    # Using the compute dtype (lowp_mode) as INT8 to leverage instructions
    # with better performance.
    lowp_mode = ipex.quantization.WoqLowpMode.INT8
    # The weight will be de-packed from INT4 to INT8.
    weight_dtype = ipex.quantization.WoqWeightDtype.INT4
    # The float activation will be quantized (dynamic, per-token) to INT8.
    act_quant_mode = ipex.quantization.WoqActQuantMode.PER_BATCH

    qconfig = ipex.quantization.get_weight_only_quant_qconfig_mapping(
        weight_dtype=weight_dtype,
        lowp_mode=lowp_mode,
        act_quant_mode=act_quant_mode,
        group_size=self.quant_config.group_size,
    )

    layer.ipex_output_size = layer.qweight.size(
        1) * self.quant_config.pack_factor
    layer.ipex_qlinear = ipex.llm.quantization.woq_linear. \
        IPEXWeightOnlyQuantizedLinear.from_weight(
        layer.qweight,
        layer.scales,
        layer.qzeros,
        layer.qweight.size(0),
        layer.ipex_output_size,
        qconfig=qconfig,
        bias=bias,
        group_size=self.quant_config.group_size,
        quant_method=IPEXConfig.IPEX_QUANT_METHOD_MAP["awq"]  # type: ignore
    )

IPEXConfig

Bases: QuantizationConfig

INT8 quantization config class using IPEX for the CPU/XPU backend, including AWQ, GPTQ.

Source code in vllm/model_executor/layers/quantization/ipex_quant.py 
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
class IPEXConfig(QuantizationConfig):
    """INT8 quantization config class using IPEX for the CPU/XPU backend,
    including AWQ, GPTQ.
    """

    IPEX_QUANT_METHOD_MAP = {
        "awq": 1,
        "gptq": 0,
    }

    def __init__(
        self,
        method: str,
        weight_bits: int,
        group_size: int,
        modules_to_not_convert: Optional[list[str]] = None,
        desc_act: Optional[bool] = None,
        lm_head_quantized: Optional[bool] = None,
    ) -> None:
        super().__init__()
        self.method = method
        self.weight_bits = weight_bits
        self.group_size = group_size
        self.modules_to_not_convert = modules_to_not_convert or []
        self.desc_act = desc_act
        self.lm_head_quantized = lm_head_quantized
        self.pack_factor = 32 // self.weight_bits

        if self.weight_bits not in [4]:
            raise ValueError(f"IPEX quantization supports weight bits [4], "
                             f"but got {self.weight_bits}.")

        if self.method not in ["awq", "gptq"]:
            raise ValueError(f"IPEX quantization supports [awq, gptq], "
                             f"but got {self.method}.")

    def __repr__(self) -> str:
        return (f"IPEXConfig(method={self.method},"
                f"weight_bits={self.weight_bits}, "
                f"group_size={self.group_size})")

    @classmethod
    def get_name(cls) -> QuantizationMethods:
        return "ipex"

    @classmethod
    def get_supported_act_dtypes(cls) -> list[torch.dtype]:
        return [torch.bfloat16, torch.float16]

    @classmethod
    def get_min_capability(cls) -> int:
        return -1

    @staticmethod
    def get_config_filenames() -> list[str]:
        return [
            "quant_config.json",
            "quantize_config.json",
        ]

    @classmethod
    def from_config(cls, config: dict[str, Any]) -> "IPEXConfig":
        method = cls.get_from_keys(config, ["quant_method"]).lower()
        if method == "awq":
            weight_bits = cls.get_from_keys(config, ["w_bit", "bits"])
            group_size = cls.get_from_keys(config,
                                           ["q_group_size", "group_size"])
            modules_to_not_convert = cls.get_from_keys_or(
                config, ["modules_to_not_convert"], None)
            return cls(method, weight_bits, group_size, modules_to_not_convert,
                       False, False)
        # otherwise for gptq
        weight_bits = cls.get_from_keys(config, ["bits"])
        group_size = cls.get_from_keys(config, ["group_size"])
        lm_head_quantized = cls.get_from_keys_or(config, ["lm_head"],
                                                 default=False)
        desc_act = cls.get_from_keys_or(config, ["desc_act"], default=False)
        return cls(method, weight_bits, group_size, [], desc_act,
                   lm_head_quantized)

    @classmethod
    def override_quantization_method(
            cls, hf_quant_cfg, user_quant) -> Optional[QuantizationMethods]:
        if not current_platform.is_cpu() and not current_platform.is_xpu():
            return None

        quant_method = hf_quant_cfg.get("quant_method", "").lower()

        if quant_method in ["awq", "gptq"]:
            return cls.get_name()

        return None

    def get_quant_method(self, layer: torch.nn.Module,
                         prefix: str) -> Optional["LinearMethodBase"]:
        if isinstance(layer, LinearBase):
            if self.method == "awq":
                if is_layer_skipped_awq(prefix, self.modules_to_not_convert):
                    return UnquantizedLinearMethod()
                return IPEXAWQLinearMethod(self)
            if self.method == "gptq":
                return IPEXGPTQLinearMethod(self)
        return None

IPEX_QUANT_METHOD_MAP class-attribute instance-attribute 

IPEX_QUANT_METHOD_MAP = {'awq': 1, 'gptq': 0}

desc_act instance-attribute 

desc_act = desc_act

group_size instance-attribute 

group_size = group_size

lm_head_quantized instance-attribute 

lm_head_quantized = lm_head_quantized

method instance-attribute 

method = method

modules_to_not_convert instance-attribute 

modules_to_not_convert = modules_to_not_convert or []

pack_factor instance-attribute 

pack_factor = 32 // weight_bits

weight_bits instance-attribute 

weight_bits = weight_bits

__init__ 

__init__(
    method: str,
    weight_bits: int,
    group_size: int,
    modules_to_not_convert: Optional[list[str]] = None,
    desc_act: Optional[bool] = None,
    lm_head_quantized: Optional[bool] = None,
) -> None
Source code in vllm/model_executor/layers/quantization/ipex_quant.py 
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
def __init__(
    self,
    method: str,
    weight_bits: int,
    group_size: int,
    modules_to_not_convert: Optional[list[str]] = None,
    desc_act: Optional[bool] = None,
    lm_head_quantized: Optional[bool] = None,
) -> None:
    super().__init__()
    self.method = method
    self.weight_bits = weight_bits
    self.group_size = group_size
    self.modules_to_not_convert = modules_to_not_convert or []
    self.desc_act = desc_act
    self.lm_head_quantized = lm_head_quantized
    self.pack_factor = 32 // self.weight_bits

    if self.weight_bits not in [4]:
        raise ValueError(f"IPEX quantization supports weight bits [4], "
                         f"but got {self.weight_bits}.")

    if self.method not in ["awq", "gptq"]:
        raise ValueError(f"IPEX quantization supports [awq, gptq], "
                         f"but got {self.method}.")

__repr__ 

__repr__() -> str
Source code in vllm/model_executor/layers/quantization/ipex_quant.py 
57
58
59
60
def __repr__(self) -> str:
    return (f"IPEXConfig(method={self.method},"
            f"weight_bits={self.weight_bits}, "
            f"group_size={self.group_size})")

from_config classmethod 

from_config(config: dict[str, Any]) -> IPEXConfig
Source code in vllm/model_executor/layers/quantization/ipex_quant.py 
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
@classmethod
def from_config(cls, config: dict[str, Any]) -> "IPEXConfig":
    method = cls.get_from_keys(config, ["quant_method"]).lower()
    if method == "awq":
        weight_bits = cls.get_from_keys(config, ["w_bit", "bits"])
        group_size = cls.get_from_keys(config,
                                       ["q_group_size", "group_size"])
        modules_to_not_convert = cls.get_from_keys_or(
            config, ["modules_to_not_convert"], None)
        return cls(method, weight_bits, group_size, modules_to_not_convert,
                   False, False)
    # otherwise for gptq
    weight_bits = cls.get_from_keys(config, ["bits"])
    group_size = cls.get_from_keys(config, ["group_size"])
    lm_head_quantized = cls.get_from_keys_or(config, ["lm_head"],
                                             default=False)
    desc_act = cls.get_from_keys_or(config, ["desc_act"], default=False)
    return cls(method, weight_bits, group_size, [], desc_act,
               lm_head_quantized)

get_config_filenames staticmethod 

get_config_filenames() -> list[str]
Source code in vllm/model_executor/layers/quantization/ipex_quant.py 
74
75
76
77
78
79
@staticmethod
def get_config_filenames() -> list[str]:
    return [
        "quant_config.json",
        "quantize_config.json",
    ]

get_min_capability classmethod 

get_min_capability() -> int
Source code in vllm/model_executor/layers/quantization/ipex_quant.py 
70
71
72
@classmethod
def get_min_capability(cls) -> int:
    return -1

get_name classmethod 

get_name() -> QuantizationMethods
Source code in vllm/model_executor/layers/quantization/ipex_quant.py 
62
63
64
@classmethod
def get_name(cls) -> QuantizationMethods:
    return "ipex"

get_quant_method 

get_quant_method(
    layer: Module, prefix: str
) -> Optional[LinearMethodBase]
Source code in vllm/model_executor/layers/quantization/ipex_quant.py 
114
115
116
117
118
119
120
121
122
123
def get_quant_method(self, layer: torch.nn.Module,
                     prefix: str) -> Optional["LinearMethodBase"]:
    if isinstance(layer, LinearBase):
        if self.method == "awq":
            if is_layer_skipped_awq(prefix, self.modules_to_not_convert):
                return UnquantizedLinearMethod()
            return IPEXAWQLinearMethod(self)
        if self.method == "gptq":
            return IPEXGPTQLinearMethod(self)
    return None

get_supported_act_dtypes classmethod 

get_supported_act_dtypes() -> list[dtype]
Source code in vllm/model_executor/layers/quantization/ipex_quant.py 
66
67
68
@classmethod
def get_supported_act_dtypes(cls) -> list[torch.dtype]:
    return [torch.bfloat16, torch.float16]

override_quantization_method classmethod 

override_quantization_method(
    hf_quant_cfg, user_quant
) -> Optional[QuantizationMethods]
Source code in vllm/model_executor/layers/quantization/ipex_quant.py 
101
102
103
104
105
106
107
108
109
110
111
112
@classmethod
def override_quantization_method(
        cls, hf_quant_cfg, user_quant) -> Optional[QuantizationMethods]:
    if not current_platform.is_cpu() and not current_platform.is_xpu():
        return None

    quant_method = hf_quant_cfg.get("quant_method", "").lower()

    if quant_method in ["awq", "gptq"]:
        return cls.get_name()

    return None

IPEXGPTQLinearMethod

Bases: GPTQLinearMethod

GPTQ linear method using IPEX for the CPU/XPU backend.

Source code in vllm/model_executor/layers/quantization/ipex_quant.py 
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
class IPEXGPTQLinearMethod(GPTQLinearMethod):
    """GPTQ linear method using IPEX for the CPU/XPU backend.
    """

    def __init__(self, quant_config: IPEXConfig):
        self.quant_config = quant_config  # type: ignore

    def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
        bias = layer.bias if not layer.skip_bias_add else None

        try:
            import intel_extension_for_pytorch as ipex
            if ipex.__version__ < MIN_IPEX_VERSION:
                raise ImportError(
                    "intel_extension_for_pytorch version is "
                    "wrong. Please install "
                    f"intel_extension_for_pytorch>={MIN_IPEX_VERSION}.")
        except ImportError as err:
            raise ImportError(
                "Please install "
                f"intel_extension_for_pytorch>={MIN_IPEX_VERSION} via "
                f"`pip install intel_extension_for_pytorch>={MIN_IPEX_VERSION}`"
                " to use IPEX-AWQ linear method.") from err
        # Using the compute dtype (lowp_mode) as INT8 to leverage instructions
        # with better performance.
        lowp_mode = ipex.quantization.WoqLowpMode.INT8
        # The weight will be de-packed from INT4 to INT8.
        weight_dtype = ipex.quantization.WoqWeightDtype.INT4
        # The float activation will be quantized (dynamic, per-token) to INT8.
        act_quant_mode = ipex.quantization.WoqActQuantMode.PER_BATCH_IC_BLOCK

        qconfig = ipex.quantization.get_weight_only_quant_qconfig_mapping(
            weight_dtype=weight_dtype,
            lowp_mode=lowp_mode,
            act_quant_mode=act_quant_mode,
            group_size=self.quant_config.group_size,
        )
        layer.ipex_output_size = layer.qweight.shape[-1]
        g_idx = layer.g_idx if self.quant_config.desc_act else None
        layer.ipex_qlinear = ipex.llm.quantization.woq_linear. \
            IPEXWeightOnlyQuantizedLinear.from_weight(
            layer.qweight,
            layer.scales,
            layer.qzeros,
            layer.qweight.size(0),
            layer.ipex_output_size,
            qconfig=qconfig,
            g_idx=g_idx,
            bias=bias,
            group_size=self.quant_config.group_size,
            quant_method=IPEXConfig.IPEX_QUANT_METHOD_MAP["gptq"]
        )

    def apply(self,
              layer: torch.nn.Module,
              x: torch.Tensor,
              bias: Optional[torch.Tensor] = None) -> torch.Tensor:
        reshaped_x = x.reshape(-1, x.shape[-1])
        out = layer.ipex_qlinear(reshaped_x)
        return out.reshape(x.shape[:-1] + (layer.ipex_output_size, ))

quant_config instance-attribute 

quant_config = quant_config

__init__ 

__init__(quant_config: IPEXConfig)
Source code in vllm/model_executor/layers/quantization/ipex_quant.py 
130
131
def __init__(self, quant_config: IPEXConfig):
    self.quant_config = quant_config  # type: ignore

apply 

apply(
    layer: Module, x: Tensor, bias: Optional[Tensor] = None
) -> Tensor
Source code in vllm/model_executor/layers/quantization/ipex_quant.py 
179
180
181
182
183
184
185
def apply(self,
          layer: torch.nn.Module,
          x: torch.Tensor,
          bias: Optional[torch.Tensor] = None) -> torch.Tensor:
    reshaped_x = x.reshape(-1, x.shape[-1])
    out = layer.ipex_qlinear(reshaped_x)
    return out.reshape(x.shape[:-1] + (layer.ipex_output_size, ))

process_weights_after_loading 

process_weights_after_loading(layer: Module) -> None
Source code in vllm/model_executor/layers/quantization/ipex_quant.py 
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
    bias = layer.bias if not layer.skip_bias_add else None

    try:
        import intel_extension_for_pytorch as ipex
        if ipex.__version__ < MIN_IPEX_VERSION:
            raise ImportError(
                "intel_extension_for_pytorch version is "
                "wrong. Please install "
                f"intel_extension_for_pytorch>={MIN_IPEX_VERSION}.")
    except ImportError as err:
        raise ImportError(
            "Please install "
            f"intel_extension_for_pytorch>={MIN_IPEX_VERSION} via "
            f"`pip install intel_extension_for_pytorch>={MIN_IPEX_VERSION}`"
            " to use IPEX-AWQ linear method.") from err
    # Using the compute dtype (lowp_mode) as INT8 to leverage instructions
    # with better performance.
    lowp_mode = ipex.quantization.WoqLowpMode.INT8
    # The weight will be de-packed from INT4 to INT8.
    weight_dtype = ipex.quantization.WoqWeightDtype.INT4
    # The float activation will be quantized (dynamic, per-token) to INT8.
    act_quant_mode = ipex.quantization.WoqActQuantMode.PER_BATCH_IC_BLOCK

    qconfig = ipex.quantization.get_weight_only_quant_qconfig_mapping(
        weight_dtype=weight_dtype,
        lowp_mode=lowp_mode,
        act_quant_mode=act_quant_mode,
        group_size=self.quant_config.group_size,
    )
    layer.ipex_output_size = layer.qweight.shape[-1]
    g_idx = layer.g_idx if self.quant_config.desc_act else None
    layer.ipex_qlinear = ipex.llm.quantization.woq_linear. \
        IPEXWeightOnlyQuantizedLinear.from_weight(
        layer.qweight,
        layer.scales,
        layer.qzeros,
        layer.qweight.size(0),
        layer.ipex_output_size,
        qconfig=qconfig,
        g_idx=g_idx,
        bias=bias,
        group_size=self.quant_config.group_size,
        quant_method=IPEXConfig.IPEX_QUANT_METHOD_MAP["gptq"]
    )