Removing var in TestSuite and adding messsages in console (#699)

xebia-functional · Mar 25, 2024 · 3a30855 · 3a30855
1 parent 807adf9
commit 3a30855
Show file tree

Hide file tree

Showing 4 changed files with 13 additions and 18 deletions.
diff --git a/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/SuiteBuilder.kt b/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/SuiteBuilder.kt
@@ -40,10 +40,12 @@ data class SuiteSpec(
   E : Enum<E> {
     val items =
       items.map { item ->
+        println("Evaluating item: ${item.input}")
         val outputResults =
           item.outputs.map { output ->
             val classification =
               AI.classify<E>(item.input, item.context, output.value, model = model)
+            println(" |_ ${output.description.value} = classification $classification")
             OutputResult(output.description.value, item.context, output.value, classification)
           }
         ItemResult(item.input, outputResults)
@@ -107,7 +109,8 @@ data class ItemSpec(
     @JvmSynthetic
     suspend operator fun invoke(
       input: String,
+      context: String,
       block: suspend TestItemBuilder.() -> Unit
-    ): ItemSpec = TestItemBuilder(input).apply { block() }.build()
+    ): ItemSpec = TestItemBuilder(input, context).apply { block() }.build()
   }
 }
diff --git a/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/TestItemBuilder.kt b/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/TestItemBuilder.kt
@@ -1,18 +1,11 @@
 package com.xebia.functional.xef.evaluator
 
-import com.xebia.functional.xef.evaluator.models.ContextDescription
 import com.xebia.functional.xef.evaluator.models.OutputResponse
 
-class TestItemBuilder(val input: String) {
-
-  private lateinit var context: String
+class TestItemBuilder(val input: String, val context: String) {
 
   private val outputs = mutableListOf<OutputResponse>()
 
-  operator fun ContextDescription.unaryPlus() {
-    context = value
-  }
-
   operator fun OutputResponse.unaryPlus() {
     outputs.add(this)
   }

diff --git a/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/TestModels.kt b/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/TestModels.kt
@@ -15,5 +15,3 @@ data class OutputResponse(val description: OutputDescription, val value: String)
     ): OutputResponse = OutputResponse(description, block())
   }
 }
-
-@Serializable data class ContextDescription(val value: String)
diff --git a/examples/src/main/kotlin/com/xebia/functional/xef/evaluator/TestExample.kt b/examples/src/main/kotlin/com/xebia/functional/xef/evaluator/TestExample.kt
@@ -5,7 +5,6 @@ import arrow.continuations.SuspendApp
 import com.xebia.functional.openai.models.CreateChatCompletionRequestModel
 import com.xebia.functional.xef.conversation.Conversation
 import com.xebia.functional.xef.evaluator.metrics.AnswerAccuracy
-import com.xebia.functional.xef.evaluator.models.ContextDescription
 import com.xebia.functional.xef.evaluator.models.OutputDescription
 import com.xebia.functional.xef.evaluator.models.OutputResponse
 import com.xebia.functional.xef.prompt.Prompt
@@ -25,19 +24,21 @@ object TestExample {
         val gpt35Description = OutputDescription("Using GPT3.5")
         val fakeOutputs = OutputDescription("Fake outputs with errors")
 
-        +ItemSpec("Please provide a movie title, genre and director") {
-          +ContextDescription("Contains information about a movie")
-
+        +ItemSpec(
+          input = "Please provide a movie title, genre and director",
+          context = "Contains information about a movie"
+        ) {
           +OutputResponse(gpt35Description) {
             Conversation { promptMessage(Prompt(model) { +user(input) }) }
           }
 
           +OutputResponse(description = fakeOutputs, value = "I don't know")
         }
 
-        +ItemSpec("Recipe for a chocolate cake") {
-          +ContextDescription("Contains instructions for making a cake")
-
+        +ItemSpec(
+          input = "Recipe for a chocolate cake",
+          context = "Contains instructions for making a cake"
+        ) {
           +OutputResponse(gpt35Description) {
             Conversation { promptMessage(Prompt(model) { +user(input) }) }
           }