Implement pre-compilation of exercises and graders

.github/workflows/build-and-test.yml

@@ -21,12 +21,14 @@ jobs:
         uses: actions/checkout@v3
       - name: Build Docker images
         run: "make docker-images"
+      - name: Fix permissions
+        run: "chmod -R a+wX demo-repository"
       - name: Run learn-ocaml build on demo-repository
         run: "docker run --rm -v $(pwd)/demo-repository:/repository learn-ocaml -- build"
       - name: Clone learn-ocaml-corpus inside tests/corpuses
-        run: "mkdir tests/corpuses && cd tests/corpuses && git clone --depth=1 https://github.com/ocaml-sf/learn-ocaml-corpus.git && cd ../.."
+        run: "git clone --depth=1 https://github.com/ocaml-sf/learn-ocaml-corpus.git tests/corpuses/learn-ocaml-corpus"
       - name: Run tests
-        run: "cd tests && bash -c ./runtests.sh"
+        run: "tests/runtests.sh"
 
   client_using_other_server:
     name: Build learn-ocaml-client and run quick tests
@@ -36,8 +38,8 @@ jobs:
       fail-fast: false
       matrix:
         server_image:
-          - 'ocamlsf/learn-ocaml:0.12'
-          - 'ocamlsf/learn-ocaml:0.13.0'
+          # - 'ocamlsf/learn-ocaml:0.12'
+          # - 'ocamlsf/learn-ocaml:0.13.0'
           - 'learn-ocaml'  # use learn-ocaml image built from master
     env:
       USE_CLIENT_IMAGE: 'true'

.gitignore

@@ -35,3 +35,8 @@ tests/corpuses/*
 detect-libs.*
 
 docs/odoc.html
+
+demo-repository/exercises/**/*.cmo
+demo-repository/exercises/**/*.cmi
+demo-repository/exercises/**/*.cma
+demo-repository/exercises/**/*.js

Dockerfile

@@ -66,6 +66,18 @@ ARG opam_switch="/home/opam/.opam/4.12"
 COPY --from=compilation /home/opam/install-prefix /usr
 COPY --from=compilation "$opam_switch/bin"/ocaml* "$opam_switch/bin/"
 COPY --from=compilation "$opam_switch/lib/ocaml" "$opam_switch/lib/ocaml/"
+COPY --from=compilation "$opam_switch/bin/js_of_ocaml" "$opam_switch/bin/"
+COPY --from=compilation "$opam_switch/lib/js_of_ocaml" "$opam_switch/lib/js_of_ocaml"
+COPY --from=compilation "$opam_switch/lib/vg" "$opam_switch/lib/vg"
+COPY --from=compilation "$opam_switch/lib/gg" "$opam_switch/lib/gg"
+
+# Fixes for ocamlfind
+COPY --from=compilation "$opam_switch/lib/findlib.conf" "$opam_switch/lib/"
+COPY --from=compilation "$opam_switch/lib/stdlib" "$opam_switch/lib/stdlib"
+ENV PATH="${opam_switch}/bin:${PATH}"
+ENV OCAMLPATH="/usr/lib"
+RUN ln -sf "$opam_switch/lib/vg" "/usr/lib"
+RUN ln -sf "$opam_switch/lib/gg" "/usr/lib"
 
 ENTRYPOINT ["dumb-init","/usr/bin/learn-ocaml","--sync-dir=/sync","--repo=/repository"]
 CMD ["build","serve"]

META.learn-ocaml.template

@@ -0,0 +1,7 @@
+package "test_lib" (
+  directory = "test_lib"
+  version = "0.13.2"
+  description = "Learn-ocaml dependencies for automatic graders"
+  requires = "compiler-libs"
+)
+# DUNE_GEN

Makefile

@@ -70,7 +70,7 @@ REPO ?= demo-repository
 
 testrun: build install
 	rm -rf www/css
-	learn-ocaml build --repo $(REPO) -j1
+	learn-ocaml build --repo $(REPO)
 	rm -rf www/css
 	ln -s ../static/css www
 	LEARNOCAML_SERVER_NOCACHE=1 learn-ocaml serve
@@ -82,7 +82,7 @@ docker-images: Dockerfile learn-ocaml.opam
 	@docker build -t learn-ocaml-compilation --target compilation docker
 	@docker build -t learn-ocaml --target program docker
 	@docker build -t learn-ocaml-client --target client docker
-	@echo "Use with 'docker run --rm -v \$$PWD/sync:/sync -v \$$PWD:/repository -p PORT:8080 learn-ocaml -- ARGS'"
+	@echo "Use with 'docker run --rm -v learn-ocaml-sync:/sync -v \$$PWD:/repository -p PORT:8080 learn-ocaml -- ARGS'"
 
 VERSION = $(shell opam show ./learn-ocaml.opam -f version)
 

README.md

@@ -53,7 +53,7 @@ The Inconsolata font is released under the Open Font License.
 See [http://www.levien.com/type/myfonts/inconsolata.html](http://www.levien.com/type/myfonts/inconsolata.html).
 
 The Biolinum font is licensed under the GNU General Public License with
-a the 'Font-Exception'.
+a 'Font-Exception'.
 See [http://www.linuxlibertine.org](http://www.linuxlibertine.org).
 
 The public instance of Learn OCaml uses the Fontin font instead of
@@ -78,9 +78,9 @@ It was written by OCamlPro from 2015 to 2018.
 
 The current main contributors are Érik Martin-Dorel, Yann Régis-Gianas, and Louis Gesbert.
 
-The initial authors were Benjamin Canou, Çağdaş Bozman, and Grégoire Henry.
+The initial authors were Benjamin Canou, Çağdaş Bozman, Grégoire Henry, and Louis Gesbert.
 
-It builds on the previous experience of Try OCaml, by Çağdaş Bozman, and Fabrice Le Fessant.
+It builds on the previous experience of Try OCaml, by Çağdaş Bozman and Fabrice Le Fessant.
 
 We heavily use js_of_ocaml, so thanks to the Ocsigen team.
 

demo-repository/exercises/demo/test_libs.txt

@@ -0,0 +1 @@
+learn-ocaml.mutation_testing

docs/exercises_format.md

@@ -54,6 +54,30 @@ An exercise is described by a directory containing at most the following files:
 - solution.ml
 - test.ml
 - max_score.txt
+- test_libs.txt
+
+> Note: as of learn-ocaml 1.0, the `.ml` files get compiled into the exercise.
+> It is therefore not possible to use directives like `#install_printer`.
+> However, you can still define your own printers in a way similar to defining
+> custom `sample_<type>` functions:
+>
+> ```ocaml
+> (* Custom printer for a pre-defined type *)
+> let print_float ppf x = Format.fprintf ppf "%.2f" x
+> 
+> (* Name the alias to define a printer for a specific instanciation of a
+>    generic type *)
+> type int_list = int list
+> let print_int_list ppf l = ...
+>
+> (* Define a generic printer for a generic type *)
+> let print_result ppok pperr ppf = function
+>   | Ok ok -> Format.fprintf ppf "OK(%a)" ppok ok
+>   | Error err -> Format.fprintf ppf "ERR(%a)" pperr err
+> ```
+>
+> Printers defined in `prelude.ml` or `prepare.ml` affect the toplevel and the
+> grader. Printers defined in `test.ml`, obviously, affect only the grader.
 
 ### meta.json
 
@@ -130,6 +154,12 @@ code, which will be described and detailed in another section.
 Maximum score that is possible to get for this exercise, even if the grader
 grades more. Overridden by the field `max_score`, if present in `meta.json`.
 
+### test_libs.txt
+
+List of additional libraries (one per line) to be used by the grader. The
+libraries will be looked up using `ocamlfind`, available to `test.ml` during its
+compilation, and bundled in the exercise grader.
+
 # Metadata
 
 When building the corpus and extracting the metadatas of all exercises, the

docs/exercises_tests.md

@@ -20,23 +20,23 @@ A classic `test.ml` file is as follows:
 open Test_lib
 open Report
 
-let exercise_1 = ..
+let exercise_1 () = ..
 
-let exercise_2 = ..
+let exercise_2 () = ..
 
-let exercise_3 = ..
+let exercise_3 () = ..
 
 
 let () =
   set_result @@
   ast_sanity_check code_ast @@ fun () ->
-  [ exercise_1 ; exercise_2 ; exercise_3 ]
+  [ exercise_1 (); exercise_2 (); exercise_3 () ]
 
 ```
 
-The values `exercise_x` are values of type `Learnocaml_report.report`, which is
+The return values of `exercise_x` are of type `Learnocaml_report.report`, which is
 a representation of the report given by the grader. In this example, each of
-these values are referring to a specific question from the exercise. Their
+these values is referring to a specific question from the exercise. Their
 content is detailed in the next section. These reports are then given to the
 function `ast_sanity_check`, which ensures that some modules are never used
 (`Obj`, `Marshall`, all the modules from `compiler-libs` or the library that
@@ -46,7 +46,7 @@ allows introspection), and also excludes some syntactic features of the language
 
 # Writing tests and reports
 
-The format of reports can be found in `src/state/learnocaml_report.ml`. A report
+The format of reports can be found in `src/grader/learnocaml_report.ml`. A report
 describes the result of what should be outputted and interpreted by the
 grader. It can be classified into sections for lisibility, and return many kind
 of messages:
@@ -252,3 +252,23 @@ forbidden or required. The two functions `ast_check_expr` and
 pattern-matching on some specific patterns into the code. The function
 `find_binding` look for a toplevel value and apply a given function on its
 syntax tree.
+
+### Using helper libraries for testing
+
+Using a `test_libs.txt` file, it is possible to include libraries that define
+helpers for grading.
+
+The file should contain the ocamlfind names of the libraries, one per line.
+
+Example of such libs include
+[mutation_testing](https://github.com/ocaml-sf/learn-ocaml/blob/master/src/grader-plugins/mutation_test.ml)
+(from McGill University, included in this repository), or
+[easy-check](https://github.com/lsylvestre/easy-check) from University Paris 6.
+
+See `src/grader-plugins/dune` to get how to build such libraries. Like
+`test.ml`, they can access the `Introspection` and `Test_lib` interfaces. They
+cannot, at the time of writing, define new samplers or printers, but if you need
+that feature and are ready to contribute, all that is missing is the inclusion
+of their `cmi` files in the grading-toplevel environment (these features rely
+on dynamic typing, and the `cma` library doesn't include the required typing
+information).

docs/howto-setup-exercise-development-environment.md

@@ -12,7 +12,6 @@ GNU/Linux and MacOS X are supported.
 > use:
 >
 >     docker version # If this fails, find out how to run Docker, first
->     docker login
 >     docker run --rm \
 >       -v $REPOSITORY:/repository:ro \
 >       -v learn-ocaml-sync:/sync \
@@ -63,7 +62,6 @@ ready:
 
 ```
 opam switch create . --deps-only --locked
-opam install opam-installer
 eval $(opam env)
 ```
 
@@ -74,7 +72,7 @@ your current opam switch, without creating a dedicated one.)
 Second, compile and install the platform:
 
 ```
-make && make opaminstall
+make && make install
 ```
 
 At this point, you should get a working `learn-ocaml` program in

docs/howto-write-exercises.md

@@ -65,9 +65,3 @@ get the files for the second step, and so on and so forth.
 [Step 6 : Grading functions for variables](tutorials/step-6.md)
 
 [Step 7 : Modifying the comparison functions (testers) with the optional arguments [~test], [~test_stdout], [~test_stderr]](tutorials/step-7.md)
-
-[Step 8 : Reusing the grader code](tutorials/step-8.md)
-
-- Separating the grader code
-
-[Step 9 : Introspection of students code](tutorials/step-9.md)    

docs/index.md

@@ -53,7 +53,7 @@ The Inconsolata font is released under the Open Font License.
 See [http://www.levien.com/type/myfonts/inconsolata.html](http://www.levien.com/type/myfonts/inconsolata.html).
 
 The Biolinum font is licensed under the GNU General Public License with
-a the 'Font-Exception'.
+a 'Font-Exception'.
 See [http://www.linuxlibertine.org](http://www.linuxlibertine.org).
 
 The public instance of Learn OCaml uses the Fontin font instead of
@@ -78,9 +78,9 @@ It was written by OCamlPro from 2015 to 2018.
 
 The current main contributors are Érik Martin-Dorel, Yann Régis-Gianas, and Louis Gesbert.
 
-The initial authors were Benjamin Canou, Çağdaş Bozman, and Grégoire Henry.
+The initial authors were Benjamin Canou, Çağdaş Bozman, Grégoire Henry, and Louis Gesbert.
 
-It builds on the previous experience of Try OCaml, by Çağdaş Bozman, and Fabrice Le Fessant.
+It builds on the previous experience of Try OCaml, by Çağdaş Bozman and Fabrice Le Fessant.
 
 We heavily use js_of_ocaml, so thanks to the Ocsigen team.
 

docs/tutorials/step-0.md

@@ -13,7 +13,8 @@ specific shape, illustrated by the following ascii art:
 │   │   ├── prepare.ml
 │   │   ├── solution.ml
 │   │   ├── template.ml
-│   │   └── test.ml
+│   │   ├── test.ml
+│   │   └── test_libs.txt
 │   ├── exercise2
 │   │   ├── ...
 │   ├── index.json
@@ -68,6 +69,8 @@ The complete format specification for exercise description is given in
 
   - `test.ml` is the grader code.
 
+  - `test_libs.txt` optionally lists grader-helper libraries used by `test.ml`
+
 - `lessons` and `tutorials` are ignored in this tutorial.
 
 ## Do it yourself!