-
Notifications
You must be signed in to change notification settings - Fork 107
TM030 Check Results API
Alex St Laurent edited this page Apr 1, 2017
·
6 revisions
See the #checker-api branch for progress.
Goal: define and implement a (JavaScript) interface that can be used from both pyret-lang and code.pyret.org that easily transforms the monolithic, black-boxed "run result" into a relatively-flat plain object/record which gives a detailed account of either errors or all check results.
-
Automated grading, e.g. for Brown's CS019 and CS173, tends to be an iterated exercise in complexity which requires some TA have understanding of Pyret internals. An API like this, supported from
pyret-lang, would make that much easier. NOTE: We aren't discussing being able to override themy-gdriveand/orshared-gdriveimports in this document, but that would also be nice. -
code.pyret.org could use this to save the headache of maintaining separate code that must do runtime/stack management in order to dive into Pyret objects to extract relevant data
These examples were generated right before the pull request of the #checker-api branch was opened.
For each example, I ran
node build/phaseA/pyret.jarr --run-full-report <file.arr> | /home/alex/bin/formatjson.js
and copied the resulting (formatted) JSON data into this document. (Without the formatjson.js script, the JSON blob has no white-space, because the blob is meant to be parsed by other programs.)
print("Invocation of print\n")
check "first block":
empty.first raises "not-found"
empty.first raises "foo"
[list: 1,2,3,4,5].first is 1
[list: 2,4,6,8].first is 1
end
print-error("Invocation of print-error\n")
check "second block":
[list: 1].first raises "not-found"
end
print("Another invocation of print\n")
fun add-one(x :: Number) -> Number:
x + 1
where:
add-one(0) is 1
add-one(1) is 1
end
check "third block":
add-one(1) is 2
end
print-error("Another invocation of print-error\n")and get data that looks something like this:
{
"timestamp": 1491068390993,
"is-error": false,
"message": "",
"error": null,
"report": {
"result": {
"blocks": [
{
"is-error": false,
"reason": "",
"tests": [
{
"passed": true,
"reason": "",
"fancy-reason": "",
"loc": "line 4, column 2"
},
{
"passed": false,
"reason": "Got unexpected exception Evaluating the field lookup expression at file:///home/alex/pyret-examples/example.arr:5:2-5:13 errored. The left side was an object that did not have a field named `first`: [list: ] when expecting \"foo\"",
"fancy-reason": "Got unexpected exception Evaluating the field lookup expression at file:///home/alex/pyret-examples/example.arr:5:2-5:13 errored. The left side was an object that did not have a field named `first`: [list: ] when expecting \"foo\"",
"loc": "line 5, column 2"
},
{
"passed": true,
"reason": "",
"fancy-reason": "",
"loc": "line 6, column 2"
},
{
"passed": false,
"reason": "Values not equal 2 1",
"fancy-reason": "Values not equal 2 1",
"loc": "line 7, column 2"
}
],
"failed": 2,
"total": 4,
"passed": 2
},
{
"is-error": false,
"reason": "",
"tests": [
{
"passed": false,
"reason": "No exception raised, expected \"not-found\"",
"fancy-reason": "No exception raised, expected \"not-found\"",
"loc": "line 13, column 2"
}
],
"failed": 1,
"total": 1,
"passed": 0
},
{
"is-error": false,
"reason": "",
"tests": [
{
"passed": true,
"reason": "",
"fancy-reason": "",
"loc": "line 21, column 2"
},
{
"passed": false,
"reason": "Values not equal 2 1",
"fancy-reason": "Values not equal 2 1",
"loc": "line 22, column 2"
}
],
"failed": 1,
"total": 2,
"passed": 1
},
{
"is-error": false,
"reason": "",
"tests": [
{
"passed": true,
"reason": "",
"fancy-reason": "",
"loc": "line 26, column 2"
}
],
"passed": 1,
"total": 1,
"failed": 0
}
],
"message": "Passed: 4; Failed: 4; Ended in Error: 0; Total: 8\n",
"errored": 0,
"passed": 4,
"failed": 4,
"total": 8
},
"stats": {
"bounces": 16,
"tos": 16,
"time": [
0,
15138850
]
},
"stdout": "Invocation of print\nAnother invocation of print\n",
"stderr": "Invocation of print-error\nAnother invocation of print-error\n"
}
}"{
"error": [
"There were 0 potential parses.\nParse failed, next token is ('UNTERMINATED-STRING \"\\\"\") at ../../pyret-examples/parse-err.arr, 1:0-1:1",
"Pyret thinks your program has an incomplete string literal around ../../pyret-examples/parse-err.arr:1:0-1:1; you may be missing closing punctuation."
],
"is-error": true,
"message": "There were parse errors",
"report": {
"result": null,
"stats": null
},
"timestamp": 1491068644269
}provide _ end
data Node: node(a, a, a) end
fun f({k;v;}, {a;k;c;}): a + c end
fun f(): block: 5 3 end end{
"error": [
"The declaration of the identifier named `a` at file:///home/alex/pyret-examples/comp-err.arr:3:19-3:20 is preceeded in the same scope by a declaration of an identifier also named `a` at file:///home/alex/pyret-examples/comp-err.arr:3:16-3:17",
"The declaration of the identifier named `a` at file:///home/alex/pyret-examples/comp-err.arr:3:22-3:23 is preceeded in the same scope by a declaration of an identifier also named `a` at file:///home/alex/pyret-examples/comp-err.arr:3:16-3:17",
"The declaration of the identifier named `k` at file:///home/alex/pyret-examples/comp-err.arr:5:17-5:18 is preceeded in the same scope by a declaration of an identifier also named `k` at file:///home/alex/pyret-examples/comp-err.arr:5:7-5:8",
"Pyret expects each expression within a block to have its own line, but the expression at file:///home/alex/pyret-examples/comp-err.arr:7:16-7:17 is on the same line as the expression at file:///home/alex/pyret-examples/comp-err.arr:7:18-7:19.",
"Couldn't read the program because the provide statement must contain an object literal at file:///home/alex/pyret-examples/comp-err.arr:1:0-1:13"
],
"is-error": true,
"message": "There were compilation errors",
"report": {
"result": null,
"stats": null
},
"timestamp": 1491068760442
}print("print statement before error\n")
print-error("print-error statement before error\n")
empty.first()
print("print statement after error\n")
print-error("print-error statement after error\n"){
"timestamp": 1491068878212,
"is-error": true,
"message": "The run ended in error",
"error": "Evaluating the field lookup expression at file:///home/alex/pyret-examples/run-err.arr:4:0-4:13 errored. The left side was an object that did not have a field named `first`: [list: ]\nStack trace:\n file:///home/alex/pyret-examples/run-err.arr: line 4, column 0",
"report": {
"result": null,
"stats": {
"bounces": 1,
"tos": 1,
"time": [
0,
7661589
]
},
"stdout": "print statement before error\n",
"stderr": "print-error statement before error\n"
}
}import system as SYS
SYS.exit(0){
"timestamp": 1491070778719,
"is-error": true,
"message": "The run ended in error",
"error": "Exited with code 0\nStack trace:\n builtin://system: line 9, column 2\n file:///home/alex/pyret-examples/exit-0.arr: line 2, column 0",
"report": {
"result": null,
"stats": {
"bounces": 1,
"tos": 1,
"time": [
0,
7030927
]
},
"stdout": "",
"stderr": ""
}
}check "this block runs four of its four tests":
"x" is "x"
"x" is-not "x"
"x" is "xx"
"x" is-not "xx"
end
check "this block runs zero of its four tests":
raise("err")
"y" is "y"
"y" is-not "y"
"y" is "yy"
"y" is-not "yy"
end
check "this block runs two of its four tests":
"z" is "z"
"z" is-not "z"
raise("err")
"z" is "zz"
"z" is-not "zz"
end{
"timestamp": 1491068950704,
"is-error": false,
"message": "",
"error": null,
"report": {
"result": {
"blocks": [
{
"is-error": false,
"reason": "",
"tests": [
{
"passed": true,
"reason": "",
"fancy-reason": "",
"loc": "line 2, column 4"
},
{
"passed": false,
"reason": "Values not different \"x\" \"x\"",
"fancy-reason": "Values not different \"x\" \"x\"",
"loc": "line 3, column 4"
},
{
"passed": false,
"reason": "Values not equal \"x\" \"xx\"",
"fancy-reason": "Values not equal \"x\" \"xx\"",
"loc": "line 4, column 4"
},
{
"passed": true,
"reason": "",
"fancy-reason": "",
"loc": "line 5, column 4"
}
],
"passed": 2,
"total": 4,
"failed": 2
},
{
"is-error": true,
"reason": "\n Block ended in the following error (all tests may not have ran): \n\n \"err\"file:///home/alex/pyret-examples/check-err.arr:9:4-9:16\n\n",
"tests": [],
"passed": 0,
"failed": 0,
"total": 0
},
{
"is-error": true,
"reason": "\n Block ended in the following error (all tests may not have ran): \n\n \"err\"file:///home/alex/pyret-examples/check-err.arr:19:4-19:16\n\n",
"tests": [
{
"passed": true,
"reason": "",
"fancy-reason": "",
"loc": "line 17, column 4"
},
{
"passed": false,
"reason": "Values not different \"z\" \"z\"",
"fancy-reason": "Values not different \"z\" \"z\"",
"loc": "line 18, column 4"
}
],
"failed": 1,
"total": 2,
"passed": 1
}
],
"message": "Passed: 3; Failed: 3; Ended in Error: 2; Total: 6\n",
"errored": 2,
"passed": 3,
"failed": 3,
"total": 6
},
"stats": {
"bounces": 14,
"tos": 14,
"time": [
0,
14721872
]
},
"stdout": "",
"stderr": ""
}
}