Tests success in evaluator

evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/SuiteBuilder.kt

@@ -35,7 +35,7 @@ data class SuiteSpec(
   val model: CreateChatCompletionRequestModel
 ) {
 
-  suspend inline fun <reified E> evaluate(): SuiteResults<E> where
+  suspend inline fun <reified E> evaluate(success: List<E>): SuiteResults<E> where
   E : AI.PromptClassifier,
   E : Enum<E> {
     val items =
@@ -46,7 +46,13 @@ data class SuiteSpec(
             val classification =
               AI.classify<E>(item.input, item.context, output.value, model = model)
             println(" |_ ${output.description.value} = classification $classification")
-            OutputResult(output.description.value, item.context, output.value, classification)
+            OutputResult(
+              output.description.value,
+              item.context,
+              output.value,
+              classification,
+              success.contains(classification)
+            )
           }
         ItemResult(item.input, outputResults)
       }

evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/metrics/ContextualRelevancy.kt

@@ -0,0 +1,26 @@
+package com.xebia.functional.xef.evaluator.metrics
+
+import com.xebia.functional.xef.AI
+
+enum class ContextualRelevancy : AI.PromptClassifier {
+  high,
+  mid,
+  low;
+
+  override fun template(input: String, output: String, context: String): String {
+    return """|
+      |You are an expert en evaluating whether the `output` is consistent with the given `context`.
+      | <output>
+      | $output
+      | </output>
+      | <context>
+      | $context
+      | </context>
+      |Return one of the following:
+      | - if the answer is high consistent: `high`
+      | - if the answer is middle consistent: `mid`
+      | - if the answer is low consistent: `low`
+    """
+      .trimMargin()
+  }
+}

evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/ItemResult.kt

@@ -21,5 +21,6 @@ data class OutputResult<E>(
   val description: String,
   val contextDescription: String,
   val output: String,
-  val result: E
+  val result: E,
+  val success: Boolean
 ) where E : AI.PromptClassifier, E : Enum<E>

evaluator/src/main/resources/web/script.js

@@ -53,6 +53,7 @@ document.addEventListener('DOMContentLoaded', function() {
             blockDiv.appendChild(outputDiv);
 
             const result = document.createElement('div');
+            result.classList.add('score', test.success ? 'score-passed' : 'score-failed');
             result.textContent = 'Result: ' + test.result;
             blockDiv.appendChild(result);
 

examples/src/main/kotlin/com/xebia/functional/xef/evaluator/TestExample.kt

@@ -46,6 +46,6 @@ object TestExample {
           +OutputResponse(description = fakeOutputs, value = "The movie is Jurassic Park")
         }
       }
-    spec.evaluate<AnswerAccuracy>()
+    spec.evaluate<AnswerAccuracy>(success = listOf(AnswerAccuracy.yes))
   }
 }