Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Oasst data v1.1 eda #3728

Draft
wants to merge 10 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
185 changes: 185 additions & 0 deletions notebooks/openassistant-oasst1.1/bad_message_id_list.py
@@ -0,0 +1,185 @@
bad_message_id_list = [
"7e93d535-dd90-48a2-beeb-b71917912e90",
"ede2cfed-22d2-421d-b218-adc7402f8a5a",
"64bfc00b-9c53-4a5d-8187-91cd2c581e78",
"80904428-577f-4cc1-95f7-c8383b093e49",
"89a37680-a7b6-4437-bd24-ecfe8395eb29",
"549a7cf2-f6bc-4b13-855f-601c714bb033",
"36be40c8-2451-4b92-99b9-97f425f6955b",
"e6933f01-4183-45bf-892c-31bdf778eee0",
"449a995f-29a4-4d04-aa56-2dab1747a417",
"4b6c83d4-b6c1-452e-b57a-a09bec46a887",
"af60f432-7fb2-4f63-b73a-6092a35bf21b",
"5ab3a8bd-bf74-4331-9564-c7e9a663de8e",
"1c001f37-ae16-49fd-9877-48ffcc8091c4",
"4fedd950-75b6-44f7-a908-3eac7d6b6056",
"15b74473-55d4-41de-bdba-6aef17b08cc2",
"e921db26-b14e-4a98-b854-ee8c07fef2d7",
"65a641e2-bfd2-4481-b5fa-5ea4bffb7867",
"ea20c005-fd42-4b1f-9089-f0545d4f379c",
"10dd0c91-f372-423b-96f5-045ca304166f",
"be6bb6d1-60ee-4418-ba29-5d61ba979935",
"29bc510e-bba5-45a5-9005-d091f95351fb",
"b60c32ae-2cc3-4896-9f92-347b5e65bce7",
"18b6e6e2-f392-4bd4-9d12-ccd706dccbb6",
"89141477-b899-4357-ae7f-50c2d33a119b",
"ba17e6e7-bde1-41b3-99c8-e5b537f5abbe",
"e5823ee5-ffd3-4604-8973-325aa1f3b2f1",
"6b665310-180c-4a4b-95e0-b4cc4a0ff501",
"12bdba1b-e7be-432e-8619-4428e98dc144",
"4b0a0f5e-8e75-4917-b9de-5b7f0f651b2f",
"5719d9f1-1cb7-4e83-bd36-d9728cefe2a3",
"b38fbad4-b43f-42f2-8561-1523165939a9",
"7dac84b7-407b-4b8a-8b77-ef384b2c117e",
"3de67512-a8bd-4c0b-8b3a-c868fb868ee4",
"7effedf7-4857-4d60-b690-34814d9c9086",
"496ee685-a186-4498-ab9f-71b7602b9594",
"0a4d14a7-6831-4fa2-b04e-ff1186651200",
"0ed9a702-199c-4fc9-9dcc-1343e4140f88",
"803b00c5-cd83-41fd-a3f5-8c88220bfdfe",
"61e0411e-27d4-4778-80f0-2501b1a36786",
"cc193de3-0ccc-4844-b2fc-8b67d0cbf89c",
"994141fc-e7b6-462f-9b64-305379038be1",
"f3949958-ced8-4ed8-9de5-2c50e4296c1e",
"bf12f013-a961-49eb-99a4-9f520f52bd52",
"59f41793-7f3a-480f-819f-b581faae1e7a",
"a1f43d47-f0f2-42d3-804c-605714cc56f0",
"2c0b821c-2525-4c69-8480-44aaea186d14",
"7695f9be-4196-4c74-918f-984ae75f0f94",
"f70cd3b7-6df9-40fe-a73b-13528f3cf49b",
"52747b24-82dc-42f0-b764-f689e885b50b",
"419354d2-2f8e-4d14-bd87-1c27b9253fea",
"22dfa8a2-2776-47b0-972d-259b63597865",
"2840a94b-ecfe-4281-8210-a866e63ee14b",
"1c9fe3c7-19a6-4e7a-ba31-a7cacd8db4d0",
"cb551444-a1ab-4eb7-b0b2-d68c15da6d4d",
"eb1d86db-4f14-4157-90d1-3d2f2e9940df",
"66ddd46c-299f-4394-b0d5-51ab66ca06bd",
"7fdb9f89-96a4-4ee6-9d01-64ff51f8fa8f",
"c812f202-fff3-4c91-8d69-11c8b2658c0f",
"6c49c918-9a73-41ca-9dfb-74c0685e861a",
"cf659a23-c5aa-494c-bb72-c08ff3cd9f93",
"f5482168-3fbc-4a15-a6e0-8660cea70f37",
"9b48bb3f-94d4-4fa8-91ed-64683e63206c",
"68c80cb8-9998-4ca3-a4f1-3e244a7aac2a",
"833b1b12-574a-4662-b1c6-33d0202a4a00",
"02702885-6c2e-4e49-8f92-f766c2a3b940",
"6a0823fe-4302-44e0-8a19-8bdc91a8ac7e",
"f4dd77d4-24dd-4ab8-a568-9d0edca5fe17",
"8ad4358b-9dd4-4f35-b3e2-17af38abc0d4",
"68a72d7a-e42e-4e5f-b22f-9964f0463cc4",
"ec357197-0ee8-4acd-8072-29aa8e76292e",
"edbecc63-47e4-4815-8744-5ad69f93bb33",
"5fb75daf-1926-47a2-85e8-f80c00ba7d03",
"776f420b-6020-4f77-8b72-d48b580a0755",
"4e97e7f6-cd85-4d8d-b73c-438faa23dd95",
"fc010f62-4ebf-46cd-893f-180cde59f0f5",
"118f9a1c-f976-4120-8ff9-934b22545b0d",
"8f70327e-1239-4564-a938-b9649465f14a",
"b0ad34ae-4080-44bf-939f-7ece554fe9bb",
"7800454d-b340-49d8-8d52-bd26f7c550e6",
"56de418b-c125-4d26-9d77-7db05d548faa",
"8b7c85bf-da3c-40e1-b3f8-b7fcd10424b6",
"8d0d0f03-2fd0-4236-a872-3826cc9d36d6",
"976893a1-416c-448d-84e4-7ee0311b6809",
"34e3d882-9be9-44a7-859e-78a828efa0f8",
"f0d3b169-1e2a-43d7-83ce-a792304e1de0",
"2528f101-fbe9-4907-a44a-783e74e47aa6",
"84418163-5721-4653-9730-c483dd7b563e",
"7953dff8-09ec-4372-a1dc-f7c8a2ae6053",
"cfa6b4b7-406a-4990-81b0-a2169cbed8d3",
"bfbf6135-3818-4d28-ac8e-cd9946bc72f3",
"ce81ca69-5a48-43eb-8df7-580a3d68578d",
"647add48-fbc9-4ac6-9930-1901da34520a",
"24545bf7-949d-446b-9b68-2553b2392357",
"b15d9839-8e4a-4d7b-8b91-634eb1b37376",
"f4e9c4d0-8686-451a-bc90-8dd7a5a51fbe",
"bea22011-334a-4e34-8c11-b1c1566de59c",
"756cdbe4-e04e-46bf-a049-e375e5100653",
"b1aee184-1ec4-45dc-9a9c-515de51f636f",
"4f5227cc-1d36-4e19-b78f-d08a60dc2141",
"14d1d5e8-61a2-4e0b-898c-f4b2c38236c2",
"a275451b-6674-467b-b7cf-539cfec31f64",
"58adf822-cd49-4c18-b72e-526d9473bf6b",
"ae9d3f55-9f6f-41ea-a0bc-0f0eea162cb9",
"1aed98d8-de97-449d-8f10-5ee506bd3ae4",
"d71ad38d-3bdd-4eed-8d19-526156f545a2",
"ae59b5aa-39cb-4ba0-8f05-8ae516ec9b3f",
"7b0b2599-294f-4d4b-be67-724f6e17280b",
"6fc61c3c-af7a-4c7f-9556-3e2317795421",
"307d9761-8dd0-4e7f-99fe-88fe9156a989",
"4b509f26-ebb4-4de3-b665-2f506d9019ac",
"7cbd1761-35d7-482d-8c6e-2cfde4677681",
"2302958f-8b4b-467f-8955-8a991ddf7836",
"377b1d72-cb10-4959-a20a-0a3846d34fbe",
"6f272372-540c-47be-ac90-1f0b0f24b944",
"ce4c4015-b5fd-464a-ab1b-df8d994e55ea",
"c51c25d5-f632-436a-840c-e46ff07e3e79",
"598cc071-e847-48a6-a064-b1f6447654fb",
"743d0067-999c-4987-996b-8cf746a84195",
"f372c08d-6054-491b-b503-b45f5996b854",
"6df241aa-f5f8-4649-8313-1f8128f9bdcd",
"f817015f-0524-4ea4-a691-c8b6137858b4",
"3eb60a82-2a3e-4a2b-b49f-7ea21002bd3d",
"b2763984-9f09-41b4-a1b3-18ecebf8eaaf",
"01216bb7-2999-411d-9224-d9ad12aeb7ae",
"4e722d24-f373-48b9-b8df-afb5f564fd18",
"28df10e4-62a6-4b4e-84da-a9806f743b40",
"2134fb02-ac5b-4c80-bfc9-0b4f7811ac22",
"c67f523f-27a9-4648-a6ca-47856067b878",
"7a875918-1bca-4cbe-9098-0cfdcf5c6a06",
"4cf9f4c3-ab6d-4610-9fa4-6e2f6d7187c6",
"5d41aea3-e16f-4453-8623-4d1e1fa46189",
"8ab59449-3c58-4282-b491-9256056a0b06",
"287eba5e-29c9-4e67-b2ef-ebb3083b3003",
"ee378b30-e1db-4356-a3f1-57e6356fced4",
"ac7a787a-b73c-46b4-87cd-5b5674b72898",
"aa96b30b-f1fe-4887-812a-b207240838be",
"4b1f967e-15a5-4286-838f-f74a7542e365",
"bc1503ac-bf8d-417a-9200-92fb1e18089c",
"5ca77f89-9460-4939-a971-940959fe8dff",
"a55cf83d-3980-4529-af25-7c16d81825f7",
"c752db2d-6bf9-4ac7-b46b-defed71c0252",
"adf3edbf-d1c4-40db-9ec6-be096f7d7353",
"192a2529-525e-47f2-841b-b82c4b2feacf",
"f59c1667-1fd2-4120-a924-e02a1c69ac73",
"754296b6-dc97-4340-8e21-e42f22ec538b",
"0d4e9ee5-54f0-4cd0-b026-5952700a5bb4",
"4a7094e2-25f4-4589-9547-c52827002cf3",
"59973454-8b7f-44dc-b0e9-f6b9d2331e13",
"973ad122-16af-4e3f-92eb-1228a4ab04ec",
"cfccf4f0-346f-4690-94b0-2676d697aeab",
"f567161f-09c6-42de-b2b1-5e577eda5b46",
"1ae6629a-c6cf-4df2-9bde-d5d437a7c412",
"a96bdb41-f57a-46bf-9806-fc692d4a485a",
"c4d2323f-4cff-4816-8d6a-6e0fb39b4685",
"95d50a89-368d-46e9-872d-685b31a0ec6a",
"d469a34a-f234-405c-bb3a-d5d0328c2c3d",
"d3ae4235-90a4-4fad-9106-3075ee9c43ba",
"5b56e959-7ce3-4d52-82dc-c0dcbb439a91",
"753de123-8213-4f98-90ce-f6137f083db2",
"3311a64f-3d64-40a0-8647-edb12616225e",
"05c3893d-9c74-4618-b690-360317677d3f",
"15f4aeab-de6c-43a0-9530-33c99e8386ff",
"29e4ae6c-bbb6-4e07-8fc9-c400f61cbc12",
"ca34580a-e5df-4064-aac2-9eac4099e7ce",
"4fba85a6-17b9-4f7a-b18c-f6e52f772700",
"2428fc50-d942-41e9-ac43-5252f7519485",
"f3b49fe6-1ce5-44ae-b78f-95b4572a161d",
"eb2b4c9b-c040-41a8-a67c-dac68c020b0a",
"ed80741a-1e3b-4a86-879e-51369e18e796",
"1b36018f-7f42-49fe-8e83-b0af4b29a04e",
"36b09cbd-d369-47a0-813a-c5f124e18e38",
"b0f0b044-ecec-4222-ac17-d5c9cd954e11",
"29458fc2-5359-48c9-b542-cce9cb92da93",
"39a7a2e4-ca84-4c41-b229-891117eb54fb",
"d757fdd4-5748-4a02-8924-6dfb5583596d",
"94da7786-44b5-4ca9-9414-fd4f85ae6ca5",
"f52c527e-b8d9-4948-87c7-12f40e5f4c18",
"3bd4b565-e359-49ea-ae35-9ef4918a3454",
"f01e097d-774f-43ab-b9cf-ad1c817cbb71",
"ccf8373b-a8cf-49be-b612-7336abf1394c",
"3c419703-2097-4946-8901-c66d4e2c0ef7",
"31a5583c-874e-402b-970c-ba2d9a4dcaae",
"d2fd731c-23cd-4b2f-b509-cf669f221756",
]