Support for compressed-tensors

tests/models/test_load_compressed_tensors_model.py

@@ -20,7 +20,7 @@
 
 
 @pytest.mark.parametrize("model_pair", MODELS)
-@pytest.mark.parametrize("dtype", ["float16", "bfloat16"])
+@pytest.mark.parametrize("dtype", ["float16"])
 @pytest.mark.parametrize("max_tokens", [32])
 @pytest.mark.parametrize("num_logprobs", [3])
 def test_models(

vllm/config.py

@@ -205,7 +205,8 @@ def _verify_sparsity(self) -> None:
                             "inferred from the config: "
                             f"{sparsity_structure} with: {self.sparsity}")
             self.sparsity = self.sparsity or sparsity_structure
-            if self.sparsity not in supported_sparsity and self.sparsity is not None:  # noqa E501
+            if (self.sparsity not in supported_sparsity) and \
+               (self.sparsity is not None):
                 raise ValueError(
                     f"Unknown sparsity_structure: {self.sparsity}. Must "
                     f"be one of {supported_sparsity}. Running the models "
@@ -238,21 +239,21 @@ def _sparsity_structure_from_config(
         # check for valid dtype
         if dtype not in supported_sparsity_dtypes:
             logger.warning(
-                "Sparsity is only supported for float16 and bfloat16 "
+                f"Sparsity is only supported for {supported_sparsity_dtypes}"
                 "dtypes. Running the models without sparse kernels.")
             return None
 
         # choose the sparsity structure based on the sparsity config
         if sparsity_config["sparsity_structure"] in {"unstructured", "0:0"}:
-            return SparsityStructure.sparse_w16a16
+            return SparsityStructure.sparse_w16a16.value
 
         elif sparsity_config["sparsity_structure"] == "2:4":
-            return SparsityStructure.semi_structured_sparse_w16a16
+            return SparsityStructure.semi_structured_sparse_w16a16.value
 
         # if the sparsity config is not recognized, return None
         logger.warning("The valid sparsity structure cannot be inferred from "
-                       "the valid sparsity config. Running the models without "
-                       "sparse kernels.")
+                       "the valid sparsity config:\n{sparsity_config}"
+                       "\n Running the models without sparse kernels.")
         return None
 
     def _verify_quantization(self) -> None: