Skip to content

Commit

Permalink
fix: add heuristicFnLanguage to evaluator (#223)
Browse files Browse the repository at this point in the history
  • Loading branch information
@xke
xke committed Apr 29, 2024
1 parent 925c780 commit 1109895
Show file tree
Hide file tree
Showing 3 changed files with 25 additions and 4 deletions.
1 change: 1 addition & 0 deletions package/docs/models/EvaluatorV2.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ Key | Input Type | Accessed Type | Description | Notes
**organizationId** | str, uuid.UUID, | str, | The ID of the organization that the evaluator belongs to | [optional] value must be a uuid
**templateDescription** | str, | str, | For evaluator templates, the description of the template | [optional]
**heuristicFn** | None, str, | NoneClass, str, | For heuristic evaluators, the heuristic function to use | [optional]
**heuristicFnLanguage** | str, | str, | For heuristic evaluators, the coding language of the heuristic function (such as \"JAVASCRIPT\", \"PYTHON\") | [optional]
**aiPromptFormat** | None, str, | NoneClass, str, | For AI evaluators, the prompt template that should be sent to the AI model | [optional]
**[aiImageUrls](#aiImageUrls)** | list, tuple, | tuple, | For AI image evaluators, the paths to the image URLs | [optional]
**humanPrompt** | None, str, | NoneClass, str, | For human evaluators, the instructions for the human to follow | [optional]
Expand Down
14 changes: 12 additions & 2 deletions package/gentrace/model/evaluator_v2.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,7 @@ def __new__(
*_args,
_configuration=_configuration,
)
heuristicFnLanguage = schemas.StrSchema


class aiPromptFormat(
Expand Down Expand Up @@ -337,6 +338,7 @@ def __new__(
"organizationId": organizationId,
"templateDescription": templateDescription,
"heuristicFn": heuristicFn,
"heuristicFnLanguage": heuristicFnLanguage,
"aiPromptFormat": aiPromptFormat,
"aiImageUrls": aiImageUrls,
"humanPrompt": humanPrompt,
Expand Down Expand Up @@ -403,6 +405,9 @@ def __getitem__(self, name: typing_extensions.Literal["templateDescription"]) ->
@typing.overload
def __getitem__(self, name: typing_extensions.Literal["heuristicFn"]) -> MetaOapg.properties.heuristicFn: ...

@typing.overload
def __getitem__(self, name: typing_extensions.Literal["heuristicFnLanguage"]) -> MetaOapg.properties.heuristicFnLanguage: ...

@typing.overload
def __getitem__(self, name: typing_extensions.Literal["aiPromptFormat"]) -> MetaOapg.properties.aiPromptFormat: ...

Expand Down Expand Up @@ -430,7 +435,7 @@ def __getitem__(self, name: typing_extensions.Literal["samplingProbability"]) ->
@typing.overload
def __getitem__(self, name: str) -> schemas.UnsetAnyTypeSchema: ...

def __getitem__(self, name: typing.Union[typing_extensions.Literal["id", "createdAt", "updatedAt", "name", "who", "valueType", "runCondition", "archivedAt", "icon", "options", "aiModel", "pipelineId", "processorId", "organizationId", "templateDescription", "heuristicFn", "aiPromptFormat", "aiImageUrls", "humanPrompt", "classifierValuePath", "classifierExpectedValuePath", "multiClassOptions", "prodEvalActive", "samplingProbability", ], str]):
def __getitem__(self, name: typing.Union[typing_extensions.Literal["id", "createdAt", "updatedAt", "name", "who", "valueType", "runCondition", "archivedAt", "icon", "options", "aiModel", "pipelineId", "processorId", "organizationId", "templateDescription", "heuristicFn", "heuristicFnLanguage", "aiPromptFormat", "aiImageUrls", "humanPrompt", "classifierValuePath", "classifierExpectedValuePath", "multiClassOptions", "prodEvalActive", "samplingProbability", ], str]):
# dict_instance[name] accessor
return super().__getitem__(name)

Expand Down Expand Up @@ -483,6 +488,9 @@ def get_item_oapg(self, name: typing_extensions.Literal["templateDescription"])
@typing.overload
def get_item_oapg(self, name: typing_extensions.Literal["heuristicFn"]) -> typing.Union[MetaOapg.properties.heuristicFn, schemas.Unset]: ...

@typing.overload
def get_item_oapg(self, name: typing_extensions.Literal["heuristicFnLanguage"]) -> typing.Union[MetaOapg.properties.heuristicFnLanguage, schemas.Unset]: ...

@typing.overload
def get_item_oapg(self, name: typing_extensions.Literal["aiPromptFormat"]) -> typing.Union[MetaOapg.properties.aiPromptFormat, schemas.Unset]: ...

Expand Down Expand Up @@ -510,7 +518,7 @@ def get_item_oapg(self, name: typing_extensions.Literal["samplingProbability"])
@typing.overload
def get_item_oapg(self, name: str) -> typing.Union[schemas.UnsetAnyTypeSchema, schemas.Unset]: ...

def get_item_oapg(self, name: typing.Union[typing_extensions.Literal["id", "createdAt", "updatedAt", "name", "who", "valueType", "runCondition", "archivedAt", "icon", "options", "aiModel", "pipelineId", "processorId", "organizationId", "templateDescription", "heuristicFn", "aiPromptFormat", "aiImageUrls", "humanPrompt", "classifierValuePath", "classifierExpectedValuePath", "multiClassOptions", "prodEvalActive", "samplingProbability", ], str]):
def get_item_oapg(self, name: typing.Union[typing_extensions.Literal["id", "createdAt", "updatedAt", "name", "who", "valueType", "runCondition", "archivedAt", "icon", "options", "aiModel", "pipelineId", "processorId", "organizationId", "templateDescription", "heuristicFn", "heuristicFnLanguage", "aiPromptFormat", "aiImageUrls", "humanPrompt", "classifierValuePath", "classifierExpectedValuePath", "multiClassOptions", "prodEvalActive", "samplingProbability", ], str]):
return super().get_item_oapg(name)


Expand All @@ -533,6 +541,7 @@ def __new__(
organizationId: typing.Union[MetaOapg.properties.organizationId, str, uuid.UUID, schemas.Unset] = schemas.unset,
templateDescription: typing.Union[MetaOapg.properties.templateDescription, str, schemas.Unset] = schemas.unset,
heuristicFn: typing.Union[MetaOapg.properties.heuristicFn, None, str, schemas.Unset] = schemas.unset,
heuristicFnLanguage: typing.Union[MetaOapg.properties.heuristicFnLanguage, str, schemas.Unset] = schemas.unset,
aiPromptFormat: typing.Union[MetaOapg.properties.aiPromptFormat, None, str, schemas.Unset] = schemas.unset,
aiImageUrls: typing.Union[MetaOapg.properties.aiImageUrls, list, tuple, schemas.Unset] = schemas.unset,
humanPrompt: typing.Union[MetaOapg.properties.humanPrompt, None, str, schemas.Unset] = schemas.unset,
Expand Down Expand Up @@ -563,6 +572,7 @@ def __new__(
organizationId=organizationId,
templateDescription=templateDescription,
heuristicFn=heuristicFn,
heuristicFnLanguage=heuristicFnLanguage,
aiPromptFormat=aiPromptFormat,
aiImageUrls=aiImageUrls,
humanPrompt=humanPrompt,
Expand Down
14 changes: 12 additions & 2 deletions package/gentrace/model/evaluator_v2.pyi
View file
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,7 @@ class EvaluatorV2(
*_args,
_configuration=_configuration,
)
heuristicFnLanguage = schemas.StrSchema


class aiPromptFormat(
Expand Down Expand Up @@ -337,6 +338,7 @@ class EvaluatorV2(
"organizationId": organizationId,
"templateDescription": templateDescription,
"heuristicFn": heuristicFn,
"heuristicFnLanguage": heuristicFnLanguage,
"aiPromptFormat": aiPromptFormat,
"aiImageUrls": aiImageUrls,
"humanPrompt": humanPrompt,
Expand Down Expand Up @@ -403,6 +405,9 @@ class EvaluatorV2(
@typing.overload
def __getitem__(self, name: typing_extensions.Literal["heuristicFn"]) -> MetaOapg.properties.heuristicFn: ...

@typing.overload
def __getitem__(self, name: typing_extensions.Literal["heuristicFnLanguage"]) -> MetaOapg.properties.heuristicFnLanguage: ...

@typing.overload
def __getitem__(self, name: typing_extensions.Literal["aiPromptFormat"]) -> MetaOapg.properties.aiPromptFormat: ...

Expand Down Expand Up @@ -430,7 +435,7 @@ class EvaluatorV2(
@typing.overload
def __getitem__(self, name: str) -> schemas.UnsetAnyTypeSchema: ...

def __getitem__(self, name: typing.Union[typing_extensions.Literal["id", "createdAt", "updatedAt", "name", "who", "valueType", "runCondition", "archivedAt", "icon", "options", "aiModel", "pipelineId", "processorId", "organizationId", "templateDescription", "heuristicFn", "aiPromptFormat", "aiImageUrls", "humanPrompt", "classifierValuePath", "classifierExpectedValuePath", "multiClassOptions", "prodEvalActive", "samplingProbability", ], str]):
def __getitem__(self, name: typing.Union[typing_extensions.Literal["id", "createdAt", "updatedAt", "name", "who", "valueType", "runCondition", "archivedAt", "icon", "options", "aiModel", "pipelineId", "processorId", "organizationId", "templateDescription", "heuristicFn", "heuristicFnLanguage", "aiPromptFormat", "aiImageUrls", "humanPrompt", "classifierValuePath", "classifierExpectedValuePath", "multiClassOptions", "prodEvalActive", "samplingProbability", ], str]):
# dict_instance[name] accessor
return super().__getitem__(name)

Expand Down Expand Up @@ -483,6 +488,9 @@ class EvaluatorV2(
@typing.overload
def get_item_oapg(self, name: typing_extensions.Literal["heuristicFn"]) -> typing.Union[MetaOapg.properties.heuristicFn, schemas.Unset]: ...

@typing.overload
def get_item_oapg(self, name: typing_extensions.Literal["heuristicFnLanguage"]) -> typing.Union[MetaOapg.properties.heuristicFnLanguage, schemas.Unset]: ...

@typing.overload
def get_item_oapg(self, name: typing_extensions.Literal["aiPromptFormat"]) -> typing.Union[MetaOapg.properties.aiPromptFormat, schemas.Unset]: ...

Expand Down Expand Up @@ -510,7 +518,7 @@ class EvaluatorV2(
@typing.overload
def get_item_oapg(self, name: str) -> typing.Union[schemas.UnsetAnyTypeSchema, schemas.Unset]: ...

def get_item_oapg(self, name: typing.Union[typing_extensions.Literal["id", "createdAt", "updatedAt", "name", "who", "valueType", "runCondition", "archivedAt", "icon", "options", "aiModel", "pipelineId", "processorId", "organizationId", "templateDescription", "heuristicFn", "aiPromptFormat", "aiImageUrls", "humanPrompt", "classifierValuePath", "classifierExpectedValuePath", "multiClassOptions", "prodEvalActive", "samplingProbability", ], str]):
def get_item_oapg(self, name: typing.Union[typing_extensions.Literal["id", "createdAt", "updatedAt", "name", "who", "valueType", "runCondition", "archivedAt", "icon", "options", "aiModel", "pipelineId", "processorId", "organizationId", "templateDescription", "heuristicFn", "heuristicFnLanguage", "aiPromptFormat", "aiImageUrls", "humanPrompt", "classifierValuePath", "classifierExpectedValuePath", "multiClassOptions", "prodEvalActive", "samplingProbability", ], str]):
return super().get_item_oapg(name)


Expand All @@ -533,6 +541,7 @@ class EvaluatorV2(
organizationId: typing.Union[MetaOapg.properties.organizationId, str, uuid.UUID, schemas.Unset] = schemas.unset,
templateDescription: typing.Union[MetaOapg.properties.templateDescription, str, schemas.Unset] = schemas.unset,
heuristicFn: typing.Union[MetaOapg.properties.heuristicFn, None, str, schemas.Unset] = schemas.unset,
heuristicFnLanguage: typing.Union[MetaOapg.properties.heuristicFnLanguage, str, schemas.Unset] = schemas.unset,
aiPromptFormat: typing.Union[MetaOapg.properties.aiPromptFormat, None, str, schemas.Unset] = schemas.unset,
aiImageUrls: typing.Union[MetaOapg.properties.aiImageUrls, list, tuple, schemas.Unset] = schemas.unset,
humanPrompt: typing.Union[MetaOapg.properties.humanPrompt, None, str, schemas.Unset] = schemas.unset,
Expand Down Expand Up @@ -563,6 +572,7 @@ class EvaluatorV2(
organizationId=organizationId,
templateDescription=templateDescription,
heuristicFn=heuristicFn,
heuristicFnLanguage=heuristicFnLanguage,
aiPromptFormat=aiPromptFormat,
aiImageUrls=aiImageUrls,
humanPrompt=humanPrompt,
Expand Down

0 comments on commit 1109895

Please sign in to comment.