disallow unversioned gpt-4 model

Author: davidsbailey

API.md

@@ -13,7 +13,7 @@
 
 `POST /assessment`: Issue a rubric assessment to the AI agent and wait for a response.
 
-* `model`: The model to use. Default: `gpt-4`
+* `model`: The model to use. Default: see DEFAULT_MODEL
 * `api-key`: The API key associated with the model. Default: the configured key
 * `code`: The code to assess. Required.
 * `prompt`: The system prompt. Required.
@@ -60,7 +60,7 @@
 
 `(GET|POST) /test/assessment`: Issue a test rubric assessment to the AI agent and wait for a response.
 
-* `model`: The model to use. Default: `gpt-4`
+* `model`: The model to use. Default: see DEFAULT_MODEL
 * `api-key`: The API key associated with the model. Default: the configured key
 * `remove-comments`: When `1`, attempts to strip comments out of the code before assessment. Default: 0
 * `num-responses`: The number of times it should ask the AI model. It votes on the final answer. Default: 1

TESTING.md

@@ -7,13 +7,13 @@
 import logging
 
 # Import our support classes
-from lib.assessment.config import SUPPORTED_MODELS, VALID_LABELS
+from lib.assessment.config import SUPPORTED_MODELS, DEFAULT_MODEL, VALID_LABELS
 from lib.assessment.label import Label
 
 class KeyConceptError(Exception):
   pass
 
-def label(code, prompt, rubric, examples=[], api_key='', llm_model='gpt-4', num_responses=1, temperature=0.2, remove_comments=False):
+def label(code, prompt, rubric, examples=[], api_key='', llm_model=DEFAULT_MODEL, num_responses=1, temperature=0.2, remove_comments=False):
   OPENAI_API_KEY = api_key
 
   # Set the key

lib/assessment/assess.py

@@ -1,5 +1,7 @@
 VALID_LABELS = ["Extensive Evidence", "Convincing Evidence", "Limited Evidence", "No Evidence"]
-SUPPORTED_MODELS = ['gpt-4', 'gpt-4-0314', 'gpt-4-32k', 'gpt-4-32k-0314']
+# do not include gpt-4, so that we always know what version of the model we are using.
+SUPPORTED_MODELS = ['gpt-4-0314', 'gpt-4-32k', 'gpt-4-32k-0314', 'gpt-4-0613', 'gpt-4-32k-0613']
+DEFAULT_MODEL = 'gpt-4-0613'
 LESSONS = {
     "U3-2022-L10" : "1ROCbvHb3yWGVoQqzKAjwdaF0dSRPUjy_",
     "U3-2022-L13" : "1kGHeY5LRpFJ9xVRoBEWbyOJyKm4wClqw",

lib/assessment/config.py

@@ -15,7 +15,7 @@
 from sklearn.metrics import accuracy_score, confusion_matrix
 from collections import defaultdict
 
-from lib.assessment.config import SUPPORTED_MODELS, VALID_LABELS, LESSONS
+from lib.assessment.config import SUPPORTED_MODELS, DEFAULT_MODEL, VALID_LABELS, LESSONS
 from lib.assessment.label import Label
 from lib.assessment.report import Report
 
@@ -37,8 +37,8 @@ def command_line_options():
                         help='Output filename within output directory')
     parser.add_argument('-c', '--use-cached', action='store_true',
                         help='Use cached responses from the API.')
-    parser.add_argument('-l', '--llm-model', type=str, default='gpt-4',
-                        help=f"Which LLM model to use. Supported models: {', '.join(SUPPORTED_MODELS)}. Default: gpt-4")
+    parser.add_argument('-l', '--llm-model', type=str, default=DEFAULT_MODEL,
+                        help=f"Which LLM model to use. Supported models: {', '.join(SUPPORTED_MODELS)}. Default: {DEFAULT_MODEL}")
     parser.add_argument('-n', '--num-responses', type=int, default=1,
                         help='Number of responses to generate for each student. Defaults to 1.')
     parser.add_argument('-p', '--num-passing-labels', type=int,

lib/assessment/rubric_tester.py

@@ -7,6 +7,8 @@
 import openai
 import json
 
+from lib.assessment.config import DEFAULT_MODEL
+
 # Our assessment code
 from lib.assessment import assess
 from lib.assessment.assess import KeyConceptError
@@ -37,7 +39,7 @@ def post_assessment():
             rubric=request.values.get("rubric", ""),
             examples=examples,
             api_key=request.values.get("api-key", openai.api_key),
-            llm_model=request.values.get("model", "gpt-4"),
+            llm_model=request.values.get("model", DEFAULT_MODEL),
             remove_comments=(request.values.get("remove-comments", "0") != "0"),
             num_responses=int(request.values.get("num-responses", "1")),
             temperature=float(request.values.get("temperature", "0.2")),
@@ -76,7 +78,7 @@ def test_assessment():
             prompt=prompt,
             rubric=rubric,
             api_key=request.values.get("api-key", openai.api_key),
-            llm_model=request.values.get("model", "gpt-4"),
+            llm_model=request.values.get("model", DEFAULT_MODEL),
             remove_comments=(request.values.get("remove-comments", "0") != "0"),
             num_responses=int(request.values.get("num-responses", "1")),
             temperature=float(request.values.get("temperature", "0.2")),
@@ -110,7 +112,7 @@ def test_assessment_blank():
             prompt=prompt,
             rubric=rubric,
             api_key=request.values.get("api-key", openai.api_key),
-            llm_model=request.values.get("model", "gpt-4"),
+            llm_model=request.values.get("model", DEFAULT_MODEL),
             remove_comments=(request.values.get("remove-comments", "0") != "0"),
             num_responses=int(request.values.get("num-responses", "1")),
             temperature=float(request.values.get("temperature", "0.2")),
@@ -152,7 +154,7 @@ def test_assessment_examples():
             rubric=rubric,
             examples=[examples],
             api_key=request.values.get("api-key", openai.api_key),
-            llm_model=request.values.get("model", "gpt-4"),
+            llm_model=request.values.get("model", DEFAULT_MODEL),
             remove_comments=(request.values.get("remove-comments", "0") != "0"),
             num_responses=int(request.values.get("num-responses", "1")),
             temperature=float(request.values.get("temperature", "0.2")),