[Move] Common Evaluation Modules to Sparsezoo`

src/deepsparse/evaluation/cli.py

@@ -59,7 +59,7 @@
 ##########
 Example command for evaluating a quantized MPT model from SparseZoo using the Deepsparse Engine.
 The evaluation will be run using `lm-evaluation-harness` on `hellaswag` and `gsm8k` datasets:
-deepsparse.eval zoo:mpt-7b-mpt_pretrain-base_quantized \
+deepsparse.eval --target zoo:mpt-7b-mpt_pretrain-base_quantized \
                 --dataset hellaswag \
                 --dataset gsm8k \
                 --integration lm-evaluation-harness \
@@ -72,13 +72,13 @@
 import click
 
 from deepsparse.evaluation.evaluator import evaluate
-from deepsparse.evaluation.results import Result, save_result
 from deepsparse.evaluation.utils import args_to_dict, get_save_path
 from deepsparse.operators.engine_operator import (
     DEEPSPARSE_ENGINE,
     ORT_ENGINE,
     TORCHSCRIPT_ENGINE,
 )
+from sparsezoo.evaluation.results import Result, save_result
 
 
 _LOGGER = logging.getLogger(__name__)

src/deepsparse/evaluation/evaluator.py

@@ -11,23 +11,20 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import logging
 from typing import Any, List, Optional, Union
 
-from deepsparse.evaluation.registry import EvaluationRegistry
-from deepsparse.evaluation.results import Result
+from deepsparse.evaluation.registry import DeepSparseEvaluationRegistry
 from deepsparse.evaluation.utils import create_model_from_target
 from deepsparse.operators.engine_operator import (
     DEEPSPARSE_ENGINE,
     ORT_ENGINE,
     TORCHSCRIPT_ENGINE,
 )
+from sparsezoo.evaluation.results import Result
 
 
 __all__ = ["evaluate"]
 
-_LOGGER = logging.getLogger(__name__)
-
 
 def evaluate(
     target: Any,
@@ -50,7 +47,9 @@ def evaluate(
         else target
     )
 
-    eval_integration = EvaluationRegistry.resolve(model, datasets, integration)
+    eval_integration = DeepSparseEvaluationRegistry.resolve(
+        model=model, datasets=datasets, integration=integration
+    )
 
     return eval_integration(
         model=model,

src/deepsparse/evaluation/integrations/lm_evaluation_harness.py

@@ -27,17 +27,17 @@
 
 import torch
 from deepsparse import Pipeline
-from deepsparse.evaluation.registry import EvaluationRegistry
-from deepsparse.evaluation.results import Dataset, Evaluation, Metric, Result
+from deepsparse.evaluation.registry import DeepSparseEvaluationRegistry
 from lm_eval import base, evaluator, tasks, utils
+from sparsezoo.evaluation.results import Dataset, Evaluation, Metric, Result
 
 
 _LOGGER = logging.getLogger(__name__)
 
 __all__ = ["integration_eval"]
 
 
-@EvaluationRegistry.register(name="lm-evaluation-harness")
+@DeepSparseEvaluationRegistry.register(name="lm-evaluation-harness")
 def integration_eval(
     model: Any,
     datasets: Union[List[str], str],

src/deepsparse/evaluation/registry.py

@@ -17,23 +17,23 @@
 import logging
 from typing import Any, Callable, List, Optional, Union
 
-from sparsezoo.utils.registry import RegistryMixin
+from sparsezoo.evaluation import EvaluationRegistry
 
 
-__all__ = ["EvaluationRegistry"]
+__all__ = ["DeepSparseEvaluationRegistry"]
 
 _LOGGER = logging.getLogger(__name__)
 
 
-class EvaluationRegistry(RegistryMixin):
+class DeepSparseEvaluationRegistry(EvaluationRegistry):
     """
     Extends the RegistryMixin to enable registering
-    and loading of evaluation functions.
-    """
+    and loading of evaluation functions for DeepSparse.
 
-    @classmethod
-    def load_from_registry(cls, name: str) -> Callable[..., "Result"]:  # noqa: F821
-        return cls.get_value_from_registry(name=name)
+    Adds a resolve method to automatically infer the integration
+    from the model and datasets if not specified, and returns
+    the appropriate evaluation function as a callable.
+    """
 
     @classmethod
     def resolve(
@@ -43,7 +43,7 @@ def resolve(
         integration: Optional[str] = None,
     ) -> Callable[..., "Result"]:  # noqa: F821
         """
-        Chooses an evaluation function from the registry based on the target,
+        Chooses an evaluation function from the registry based on the model,
         datasets and integration.
 
         If integration is specified, attempts to load the evaluation function
@@ -70,7 +70,4 @@ def resolve(
 
         potentially_check_dependency_import(integration)
 
-        try:
-            return cls.load_from_registry(name=integration)
-        except KeyError as err:
-            raise KeyError(err)
+        return cls.get_value_from_registry(name=integration)

tests/deepsparse/evaluation/test_evaluator.py

@@ -21,7 +21,7 @@
 import pytest
 from deepsparse.evaluation.evaluator import evaluate
 from deepsparse.evaluation.integrations import try_import_lm_evaluation_harness
-from deepsparse.evaluation.registry import EvaluationRegistry
+from deepsparse.evaluation.registry import DeepSparseEvaluationRegistry
 from deepsparse.evaluation.results import (
     Dataset,
     EvalSample,
@@ -31,7 +31,7 @@
 )
 
 
-@EvaluationRegistry.register()
+@DeepSparseEvaluationRegistry.register()
 def dummy_integration(*args, **kwargs):
     result_formatted = [
         Evaluation(