{"payload":{"feedbackUrl":"https://github.com/orgs/community/discussions/53140","repo":{"id":386992479,"defaultBranch":"main","name":"ScandEval","ownerLogin":"ScandEval","currentUserCanPush":false,"isFork":false,"isEmpty":false,"createdAt":"2021-07-17T16:50:37.000Z","ownerAvatar":"https://avatars.githubusercontent.com/u/91734221?v=4","public":true,"private":false,"isOrgOwned":true},"refInfo":{"name":"","listCacheKey":"v0:1717622877.0","currentOid":""},"activityList":{"items":[{"before":"1edbf34d6335e3311a09de9540847ed5fcd7f8c4","after":null,"ref":"refs/heads/fix/mixtral-oom","pushedAt":"2024-06-05T21:27:57.000Z","pushType":"branch_deletion","commitsCount":0,"pusher":{"login":"saattrupdan","name":"Dan Saattrup Nielsen","path":"/saattrupdan","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/47701536?s=80&v=4"}},{"before":"c5cb2b1ac7f61434e0a26a03c13cce0db78d4746","after":"1edbf34d6335e3311a09de9540847ed5fcd7f8c4","ref":"refs/heads/fix/mixtral-oom","pushedAt":"2024-06-05T21:19:34.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"saattrupdan","name":"Dan Saattrup Nielsen","path":"/saattrupdan","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/47701536?s=80&v=4"},"commit":{"message":"debug: Reduce gpu_memory_utilization to 0.6","shortMessageHtmlLink":"debug: Reduce gpu_memory_utilization to 0.6"}},{"before":"4822f256e7fe827fe2780f48fd3cebe551b9e17c","after":"c5cb2b1ac7f61434e0a26a03c13cce0db78d4746","ref":"refs/heads/fix/mixtral-oom","pushedAt":"2024-06-05T21:16:43.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"saattrupdan","name":"Dan Saattrup Nielsen","path":"/saattrupdan","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/47701536?s=80&v=4"},"commit":{"message":"debug: Set gpu_memory_utilization to 0.7","shortMessageHtmlLink":"debug: Set gpu_memory_utilization to 0.7"}},{"before":null,"after":"4822f256e7fe827fe2780f48fd3cebe551b9e17c","ref":"refs/heads/fix/mixtral-oom","pushedAt":"2024-06-05T21:12:56.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"saattrupdan","name":"Dan Saattrup Nielsen","path":"/saattrupdan","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/47701536?s=80&v=4"},"commit":{"message":"feat: v12.10.4","shortMessageHtmlLink":"feat: v12.10.4"}},{"before":"26e8ed0457406a76cc43c239e30ef74808502166","after":"651a4f721aab3ac3afa52170c7ec12965af46892","ref":"refs/heads/feat/add-instruction-evaluation","pushedAt":"2024-06-03T13:08:05.000Z","pushType":"push","commitsCount":3,"pusher":{"login":"saattrupdan","name":"Dan Saattrup Nielsen","path":"/saattrupdan","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/47701536?s=80&v=4"},"commit":{"message":"Merge branch 'main' of github.com:ScandEval/ScandEval into feat/add-instruction-evaluation","shortMessageHtmlLink":"Merge branch 'main' of github.com:ScandEval/ScandEval into feat/add-i…"}},{"before":"549ab1098817315f6de408f5a942ae443ffd0ed2","after":"4822f256e7fe827fe2780f48fd3cebe551b9e17c","ref":"refs/heads/main","pushedAt":"2024-06-03T13:02:13.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"saattrupdan","name":"Dan Saattrup Nielsen","path":"/saattrupdan","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/47701536?s=80&v=4"},"commit":{"message":"feat: v12.10.4","shortMessageHtmlLink":"feat: v12.10.4"}},{"before":"069a572b0ffe8c6cbb458acd6311fa5c4807ba10","after":null,"ref":"refs/heads/fix/dataset-access","pushedAt":"2024-06-03T13:01:19.000Z","pushType":"branch_deletion","commitsCount":0,"pusher":{"login":"saattrupdan","name":"Dan Saattrup Nielsen","path":"/saattrupdan","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/47701536?s=80&v=4"}},{"before":"b5cf7d6f1dfab317c101dfca0d87f6befba6d736","after":"549ab1098817315f6de408f5a942ae443ffd0ed2","ref":"refs/heads/main","pushedAt":"2024-06-03T13:01:19.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"saattrupdan","name":"Dan Saattrup Nielsen","path":"/saattrupdan","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/47701536?s=80&v=4"},"commit":{"message":"Fix/dataset access (#445)\n\n* fix: Scramble text\r\n\r\n* fix: Scrambling","shortMessageHtmlLink":"Fix/dataset access (<a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2330994751\" data-permission-text=\"Title is private\" data-url=\"https://github.com/ScandEval/ScandEval/issues/445\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/ScandEval/ScandEval/pull/445/hovercard\" href=\"https://github.com/ScandEval/ScandEval/pull/445\">#445</a>)"}},{"before":"311a8cb615f30898feeebd75c44acebf5c41e779","after":"069a572b0ffe8c6cbb458acd6311fa5c4807ba10","ref":"refs/heads/fix/dataset-access","pushedAt":"2024-06-03T12:14:27.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"saattrupdan","name":"Dan Saattrup Nielsen","path":"/saattrupdan","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/47701536?s=80&v=4"},"commit":{"message":"fix: Scrambling","shortMessageHtmlLink":"fix: Scrambling"}},{"before":"b5cf7d6f1dfab317c101dfca0d87f6befba6d736","after":"311a8cb615f30898feeebd75c44acebf5c41e779","ref":"refs/heads/fix/dataset-access","pushedAt":"2024-06-03T12:07:01.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"saattrupdan","name":"Dan Saattrup Nielsen","path":"/saattrupdan","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/47701536?s=80&v=4"},"commit":{"message":"fix: Scramble text","shortMessageHtmlLink":"fix: Scramble text"}},{"before":null,"after":"b5cf7d6f1dfab317c101dfca0d87f6befba6d736","ref":"refs/heads/fix/dataset-access","pushedAt":"2024-06-03T12:06:18.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"saattrupdan","name":"Dan Saattrup Nielsen","path":"/saattrupdan","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/47701536?s=80&v=4"},"commit":{"message":"feat: v12.10.3","shortMessageHtmlLink":"feat: v12.10.3"}},{"before":"ddd39ce55f5678dc99aedce889480bba8a78d075","after":"26e8ed0457406a76cc43c239e30ef74808502166","ref":"refs/heads/feat/add-instruction-evaluation","pushedAt":"2024-06-03T11:43:47.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"saattrupdan","name":"Dan Saattrup Nielsen","path":"/saattrupdan","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/47701536?s=80&v=4"},"commit":{"message":"docs: Update changelog","shortMessageHtmlLink":"docs: Update changelog"}},{"before":"ab7ed1a796407d6111ba007ecd34b7cf386533df","after":"ddd39ce55f5678dc99aedce889480bba8a78d075","ref":"refs/heads/feat/add-instruction-evaluation","pushedAt":"2024-06-03T11:41:56.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"saattrupdan","name":"Dan Saattrup Nielsen","path":"/saattrupdan","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/47701536?s=80&v=4"},"commit":{"message":"chore: Update vllm and tiktoken","shortMessageHtmlLink":"chore: Update vllm and tiktoken"}},{"before":"b716f3a7a98d5800952f31b0bd4519f834eeff6a","after":"ab7ed1a796407d6111ba007ecd34b7cf386533df","ref":"refs/heads/feat/add-instruction-evaluation","pushedAt":"2024-06-03T11:38:09.000Z","pushType":"push","commitsCount":4,"pusher":{"login":"saattrupdan","name":"Dan Saattrup Nielsen","path":"/saattrupdan","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/47701536?s=80&v=4"},"commit":{"message":"Merge branch 'main' of github.com:ScandEval/ScandEval into feat/add-instruction-evaluation","shortMessageHtmlLink":"Merge branch 'main' of github.com:ScandEval/ScandEval into feat/add-i…"}},{"before":"b0d96baaf4f14bd671028546de853c6518d90ccb","after":"b5cf7d6f1dfab317c101dfca0d87f6befba6d736","ref":"refs/heads/main","pushedAt":"2024-06-03T11:33:59.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"saattrupdan","name":"Dan Saattrup Nielsen","path":"/saattrupdan","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/47701536?s=80&v=4"},"commit":{"message":"feat: v12.10.3","shortMessageHtmlLink":"feat: v12.10.3"}},{"before":"dc55605dd4df6493376c7ca28a5945b73738e728","after":"b0d96baaf4f14bd671028546de853c6518d90ccb","ref":"refs/heads/main","pushedAt":"2024-06-03T11:32:49.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"saattrupdan","name":"Dan Saattrup Nielsen","path":"/saattrupdan","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/47701536?s=80&v=4"},"commit":{"message":"docs: Update changelog","shortMessageHtmlLink":"docs: Update changelog"}},{"before":"cafac95774b3d05207dffc27497a019b9dcba024","after":"dc55605dd4df6493376c7ca28a5945b73738e728","ref":"refs/heads/main","pushedAt":"2024-06-03T11:29:09.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"saattrupdan","name":"Dan Saattrup Nielsen","path":"/saattrupdan","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/47701536?s=80&v=4"},"commit":{"message":"fix: Update dataset token","shortMessageHtmlLink":"fix: Update dataset token"}},{"before":"6020abbeb62b418d19a392e969ebe50056d0ea91","after":"b716f3a7a98d5800952f31b0bd4519f834eeff6a","ref":"refs/heads/feat/add-instruction-evaluation","pushedAt":"2024-06-03T11:11:43.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"saattrupdan","name":"Dan Saattrup Nielsen","path":"/saattrupdan","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/47701536?s=80&v=4"},"commit":{"message":"chore: Revert","shortMessageHtmlLink":"chore: Revert"}},{"before":"97bac800e685fc3e2894e091494661c9f4e15e2b","after":"6020abbeb62b418d19a392e969ebe50056d0ea91","ref":"refs/heads/feat/add-instruction-evaluation","pushedAt":"2024-06-03T11:09:03.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"saattrupdan","name":"Dan Saattrup Nielsen","path":"/saattrupdan","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/47701536?s=80&v=4"},"commit":{"message":"debug: Try NER without label_prefix","shortMessageHtmlLink":"debug: Try NER without label_prefix"}},{"before":"79d90c2eb973fb4e6aeba71a56542706235dca7c","after":"97bac800e685fc3e2894e091494661c9f4e15e2b","ref":"refs/heads/feat/add-instruction-evaluation","pushedAt":"2024-06-03T11:01:39.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"saattrupdan","name":"Dan Saattrup Nielsen","path":"/saattrupdan","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/47701536?s=80&v=4"},"commit":{"message":"debug: Try not using chat template","shortMessageHtmlLink":"debug: Try not using chat template"}},{"before":"a195240edd2a8ce89f134dbd72f17a73b1b38a44","after":"79d90c2eb973fb4e6aeba71a56542706235dca7c","ref":"refs/heads/feat/add-instruction-evaluation","pushedAt":"2024-06-03T10:55:15.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"saattrupdan","name":"Dan Saattrup Nielsen","path":"/saattrupdan","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/47701536?s=80&v=4"},"commit":{"message":"chore: Include label_prefix back in","shortMessageHtmlLink":"chore: Include label_prefix back in"}},{"before":"9f575fa350c92212919805f4813fdfa7b28bbc91","after":"a195240edd2a8ce89f134dbd72f17a73b1b38a44","ref":"refs/heads/feat/add-instruction-evaluation","pushedAt":"2024-06-03T10:53:35.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"saattrupdan","name":"Dan Saattrup Nielsen","path":"/saattrupdan","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/47701536?s=80&v=4"},"commit":{"message":"debug: Try without the label prefix","shortMessageHtmlLink":"debug: Try without the label prefix"}},{"before":"a9e2ecc1578c97506db42258f51933d433d1a51b","after":"9f575fa350c92212919805f4813fdfa7b28bbc91","ref":"refs/heads/feat/add-instruction-evaluation","pushedAt":"2024-06-03T10:52:12.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"saattrupdan","name":"Dan Saattrup Nielsen","path":"/saattrupdan","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/47701536?s=80&v=4"},"commit":{"message":"debug: Try not using the instruction prompt","shortMessageHtmlLink":"debug: Try not using the instruction prompt"}},{"before":"3383735f6ec4ef809c1f9e56759998ce8bc55c6c","after":"a9e2ecc1578c97506db42258f51933d433d1a51b","ref":"refs/heads/feat/add-instruction-evaluation","pushedAt":"2024-06-03T10:50:37.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"saattrupdan","name":"Dan Saattrup Nielsen","path":"/saattrupdan","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/47701536?s=80&v=4"},"commit":{"message":"fix: Remove breakpoint","shortMessageHtmlLink":"fix: Remove breakpoint"}},{"before":"9dfbbeae934d605daa3236a29a836e6e87ce1222","after":"3383735f6ec4ef809c1f9e56759998ce8bc55c6c","ref":"refs/heads/feat/add-instruction-evaluation","pushedAt":"2024-06-03T10:49:46.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"saattrupdan","name":"Dan Saattrup Nielsen","path":"/saattrupdan","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/47701536?s=80&v=4"},"commit":{"message":"fix: Instruction prompt formatting","shortMessageHtmlLink":"fix: Instruction prompt formatting"}},{"before":"7a9d0fee3ecdf21709577b1ffcf0e973bc7b066c","after":"9dfbbeae934d605daa3236a29a836e6e87ce1222","ref":"refs/heads/feat/add-instruction-evaluation","pushedAt":"2024-06-03T10:48:27.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"saattrupdan","name":"Dan Saattrup Nielsen","path":"/saattrupdan","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/47701536?s=80&v=4"},"commit":{"message":"fix: Instruction prompt formatting","shortMessageHtmlLink":"fix: Instruction prompt formatting"}},{"before":"041ae6cd2ab5475912324f30bead877a04c4987a","after":"7a9d0fee3ecdf21709577b1ffcf0e973bc7b066c","ref":"refs/heads/feat/add-instruction-evaluation","pushedAt":"2024-06-03T10:46:13.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"saattrupdan","name":"Dan Saattrup Nielsen","path":"/saattrupdan","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/47701536?s=80&v=4"},"commit":{"message":"debug","shortMessageHtmlLink":"debug"}},{"before":"04bd6cf5cdd6661c88dc349963f90b3a6cf6c95e","after":"041ae6cd2ab5475912324f30bead877a04c4987a","ref":"refs/heads/feat/add-instruction-evaluation","pushedAt":"2024-06-03T10:44:53.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"saattrupdan","name":"Dan Saattrup Nielsen","path":"/saattrupdan","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/47701536?s=80&v=4"},"commit":{"message":"debug","shortMessageHtmlLink":"debug"}},{"before":"c1cb518bc89bee07ceda2650b1376eb0879007aa","after":"04bd6cf5cdd6661c88dc349963f90b3a6cf6c95e","ref":"refs/heads/feat/add-instruction-evaluation","pushedAt":"2024-06-03T10:36:36.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"saattrupdan","name":"Dan Saattrup Nielsen","path":"/saattrupdan","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/47701536?s=80&v=4"},"commit":{"message":"chore: Remove temporary block","shortMessageHtmlLink":"chore: Remove temporary block"}},{"before":"cafac95774b3d05207dffc27497a019b9dcba024","after":"c1cb518bc89bee07ceda2650b1376eb0879007aa","ref":"refs/heads/feat/add-instruction-evaluation","pushedAt":"2024-06-03T10:34:01.000Z","pushType":"push","commitsCount":3,"pusher":{"login":"saattrupdan","name":"Dan Saattrup Nielsen","path":"/saattrupdan","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/47701536?s=80&v=4"},"commit":{"message":"feat: Add BenchmarkDataset._add_instruction_prompt","shortMessageHtmlLink":"feat: Add BenchmarkDataset._add_instruction_prompt"}}],"hasNextPage":true,"hasPreviousPage":false,"activityType":"all","actor":null,"timePeriod":"all","sort":"DESC","perPage":30,"cursor":"djE6ks8AAAAEXZLbpQA","startCursor":null,"endCursor":null}},"title":"Activity · ScandEval/ScandEval"}