-getting-started.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Getting Started\n",
    "\n",
    "First, install the library with extras to train models:\n",
    "```\n",
    "pip install -e git+https://github.com/bennokr/minimel.git#egg=minimel[train]\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "ename": "UnboundLocalError",
     "evalue": "local variable 'child' referenced before assignment",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "File \u001b[0;32m~/scratch/anaconda3/envs/minimel/lib/python3.9/site-packages/IPython/utils/_process_posix.py:148\u001b[0m, in \u001b[0;36mProcessHandler.system\u001b[0;34m(self, cmd)\u001b[0m\n\u001b[1;32m    147\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 148\u001b[0m     child \u001b[38;5;241m=\u001b[39m \u001b[43mpexpect\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mspawn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msh\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43margs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m-c\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcmd\u001b[49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m  \u001b[38;5;66;03m# Vanilla Pexpect\u001b[39;00m\n\u001b[1;32m    149\u001b[0m flush \u001b[38;5;241m=\u001b[39m sys\u001b[38;5;241m.\u001b[39mstdout\u001b[38;5;241m.\u001b[39mflush\n",
      "File \u001b[0;32m~/scratch/anaconda3/envs/minimel/lib/python3.9/site-packages/pexpect/pty_spawn.py:205\u001b[0m, in \u001b[0;36mspawn.__init__\u001b[0;34m(self, command, args, timeout, maxread, searchwindowsize, logfile, cwd, env, ignore_sighup, echo, preexec_fn, encoding, codec_errors, dimensions, use_poll)\u001b[0m\n\u001b[1;32m    204\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 205\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_spawn\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcommand\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpreexec_fn\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdimensions\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    206\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39muse_poll \u001b[38;5;241m=\u001b[39m use_poll\n",
      "File \u001b[0;32m~/scratch/anaconda3/envs/minimel/lib/python3.9/site-packages/pexpect/pty_spawn.py:303\u001b[0m, in \u001b[0;36mspawn._spawn\u001b[0;34m(self, command, args, preexec_fn, dimensions)\u001b[0m\n\u001b[1;32m    300\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39margs \u001b[38;5;241m=\u001b[39m [a \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(a, \u001b[38;5;28mbytes\u001b[39m) \u001b[38;5;28;01melse\u001b[39;00m a\u001b[38;5;241m.\u001b[39mencode(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mencoding)\n\u001b[1;32m    301\u001b[0m                  \u001b[38;5;28;01mfor\u001b[39;00m a \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39margs]\n\u001b[0;32m--> 303\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mptyproc \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_spawnpty\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43menv\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43menv\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    304\u001b[0m \u001b[43m                             \u001b[49m\u001b[43mcwd\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcwd\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    306\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpid \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mptyproc\u001b[38;5;241m.\u001b[39mpid\n",
      "File \u001b[0;32m~/scratch/anaconda3/envs/minimel/lib/python3.9/site-packages/pexpect/pty_spawn.py:315\u001b[0m, in \u001b[0;36mspawn._spawnpty\u001b[0;34m(self, args, **kwargs)\u001b[0m\n\u001b[1;32m    314\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m'''Spawn a pty and return an instance of PtyProcess.'''\u001b[39;00m\n\u001b[0;32m--> 315\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mptyprocess\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mPtyProcess\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mspawn\u001b[49m\u001b[43m(\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/scratch/anaconda3/envs/minimel/lib/python3.9/site-packages/ptyprocess/ptyprocess.py:315\u001b[0m, in \u001b[0;36mPtyProcess.spawn\u001b[0;34m(cls, argv, cwd, env, echo, preexec_fn, dimensions, pass_fds)\u001b[0m\n\u001b[1;32m    314\u001b[0m os\u001b[38;5;241m.\u001b[39mclose(exec_err_pipe_write)\n\u001b[0;32m--> 315\u001b[0m exec_err_data \u001b[38;5;241m=\u001b[39m \u001b[43mos\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[43mexec_err_pipe_read\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m4096\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m    316\u001b[0m os\u001b[38;5;241m.\u001b[39mclose(exec_err_pipe_read)\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: ",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mUnboundLocalError\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[12], line 4\u001b[0m\n\u001b[1;32m      2\u001b[0m root \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mwiki/\u001b[39m\u001b[38;5;124m'\u001b[39m \u001b[38;5;241m+\u001b[39m wiki\n\u001b[1;32m      3\u001b[0m get_ipython()\u001b[38;5;241m.\u001b[39msystem(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mmkdir -p $root\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m----> 4\u001b[0m \u001b[43mget_ipython\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msystem\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mwikimapper download $wiki --dir $root\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m      5\u001b[0m outdb \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mroot\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m/index_\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mwiki\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m.db\u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[1;32m      6\u001b[0m get_ipython()\u001b[38;5;241m.\u001b[39msystem(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mwikimapper create $wiki --dumpdir $root --target $outdb\u001b[39m\u001b[38;5;124m'\u001b[39m)\n",
      "File \u001b[0;32m~/scratch/anaconda3/envs/minimel/lib/python3.9/site-packages/ipykernel/zmqshell.py:655\u001b[0m, in \u001b[0;36mZMQInteractiveShell.system_piped\u001b[0;34m(self, cmd)\u001b[0m\n\u001b[1;32m    653\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39muser_ns[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m_exit_code\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m system(cmd)\n\u001b[1;32m    654\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 655\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39muser_ns[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m_exit_code\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[43msystem\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mvar_expand\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcmd\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdepth\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/scratch/anaconda3/envs/minimel/lib/python3.9/site-packages/IPython/utils/_process_posix.py:164\u001b[0m, in \u001b[0;36mProcessHandler.system\u001b[0;34m(self, cmd)\u001b[0m\n\u001b[1;32m    159\u001b[0m         out_size \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mlen\u001b[39m(child\u001b[38;5;241m.\u001b[39mbefore)\n\u001b[1;32m    160\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mKeyboardInterrupt\u001b[39;00m:\n\u001b[1;32m    161\u001b[0m     \u001b[38;5;66;03m# We need to send ^C to the process.  The ascii code for '^C' is 3\u001b[39;00m\n\u001b[1;32m    162\u001b[0m     \u001b[38;5;66;03m# (the character is known as ETX for 'End of Text', see\u001b[39;00m\n\u001b[1;32m    163\u001b[0m     \u001b[38;5;66;03m# curses.ascii.ETX).\u001b[39;00m\n\u001b[0;32m--> 164\u001b[0m     \u001b[43mchild\u001b[49m\u001b[38;5;241m.\u001b[39msendline(\u001b[38;5;28mchr\u001b[39m(\u001b[38;5;241m3\u001b[39m))\n\u001b[1;32m    165\u001b[0m     \u001b[38;5;66;03m# Read and print any more output the program might produce on its\u001b[39;00m\n\u001b[1;32m    166\u001b[0m     \u001b[38;5;66;03m# way out.\u001b[39;00m\n\u001b[1;32m    167\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n",
      "\u001b[0;31mUnboundLocalError\u001b[0m: local variable 'child' referenced before assignment"
     ]
    }
   ],
   "source": [
    "wiki = 'iawiki-latest' # use Interlingua language Wikipedia version to test\n",
    "root = 'wiki/' + wiki\n",
    "!mkdir -p $root\n",
    "!wikimapper download $wiki --dir $root\n",
    "outdb = f'{root}/index_{wiki}.db'\n",
    "!wikimapper create $wiki --dumpdir $root --target $outdb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading mapping...: 100%|█████████████| 34465/34465 [00:00<00:00, 424881.81it/s]\n",
      "INFO:root:Building IntDAWG trie...\n",
      "INFO:root:Saving to wiki/iawiki-latest/index_iawiki-latest.dawg...\n"
     ]
    }
   ],
   "source": [
    "!minimel -v index $outdb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2024-03-19 15:00:50--  https://dumps.wikimedia.org/iawiki/latest/iawiki-latest-pages-articles.xml.bz2\n",
      "Resolving dumps.wikimedia.org (dumps.wikimedia.org)... 208.80.154.71, 2620:0:861:3:208:80:154:71\n",
      "Connecting to dumps.wikimedia.org (dumps.wikimedia.org)|208.80.154.71|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 11275975 (11M) [application/octet-stream]\n",
      "Saving to: ‘wiki/iawiki-latest/iawiki-latest-pages-articles.xml.bz2.1’\n",
      "\n",
      "100%[======================================>] 11,275,975  4.19MB/s   in 2.6s   \n",
      "\n",
      "2024-03-19 15:00:53 (4.19 MB/s) - ‘wiki/iawiki-latest/iawiki-latest-pages-articles.xml.bz2.1’ saved [11275975/11275975]\n",
      "\n",
      "bunzip2: Output file wiki/iawiki-latest/iawiki-latest-pages-articles.xml already exists.\n"
     ]
    }
   ],
   "source": [
    "!wget -P $root https://dumps.wikimedia.org/iawiki/latest/iawiki-latest-pages-articles.xml.bz2\n",
    "!bunzip2 $root/iawiki-latest-pages-articles.xml.bz2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:root:Finished in 14s################] | 100% Completed | 14.5s\u001b[2K\n",
      "INFO:root:Wrote 100 partitions\n"
     ]
    }
   ],
   "source": [
    "dump = f'{root}/{wiki}-pages-articles.xml'\n",
    "dawg = f'{root}/index_{wiki}.dawg'\n",
    "!minimel -v get-paragraphs -n 100 $dump $dawg "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:root:Writing to wiki/iawiki-latest/disambig_ents.txt\n"
     ]
    }
   ],
   "source": [
    "lang = wiki.split('wiki')[0]\n",
    "disambigpages = f'{root}/disambig_ents.txt'\n",
    "!minimel -v query-pages $lang -o $disambigpages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:root:Extracting disambiguation links...\n",
      "[#####                                   ] | 13% Completed | 19.1s2024-03-19 15:04:44,782 - distributed.worker - WARNING - Compute Failed\n",
      "Key:       ('fileparts-lambda-89b4b29143df67ab14ee7650adaaff1e', 10)\n",
      "Function:  execute_task\n",
      "args:      ((subgraph_callable-070f3b3c-7bb7-4e25-bf31-b5a9bc47f39b, (subgraph_callable-7044ef03-a0ab-4656-a51a-11d636979d6e, [10], PosixPath('wiki/iawiki-latest/disambig_ents.txt'), 100, '<page>', '</page>')))\n",
      "kwargs:    {}\n",
      "Exception: \"OSError('Invalid data format')\"\n",
      "\n",
      "2024-03-19 15:04:44,783 - distributed.worker - WARNING - Compute Failed\n",
      "Key:       ('fileparts-lambda-89b4b29143df67ab14ee7650adaaff1e', 18)\n",
      "Function:  execute_task\n",
      "args:      ((subgraph_callable-070f3b3c-7bb7-4e25-bf31-b5a9bc47f39b, (subgraph_callable-7044ef03-a0ab-4656-a51a-11d636979d6e, [18], PosixPath('wiki/iawiki-latest/disambig_ents.txt'), 100, '<page>', '</page>')))\n",
      "kwargs:    {}\n",
      "Exception: \"OSError('Invalid data format')\"\n",
      "\n",
      "2024-03-19 15:04:44,783 - distributed.worker - WARNING - Compute Failed\n",
      "Key:       ('fileparts-lambda-89b4b29143df67ab14ee7650adaaff1e', 32)\n",
      "Function:  execute_task\n",
      "args:      ((subgraph_callable-070f3b3c-7bb7-4e25-bf31-b5a9bc47f39b, (subgraph_callable-7044ef03-a0ab-4656-a51a-11d636979d6e, [32], PosixPath('wiki/iawiki-latest/disambig_ents.txt'), 100, '<page>', '</page>')))\n",
      "kwargs:    {}\n",
      "Exception: \"OSError('Invalid data format')\"\n",
      "\n",
      "2024-03-19 15:04:44,784 - distributed.worker - WARNING - Compute Failed\n",
      "Key:       ('fileparts-lambda-89b4b29143df67ab14ee7650adaaff1e', 25)\n",
      "Function:  execute_task\n",
      "args:      ((subgraph_callable-070f3b3c-7bb7-4e25-bf31-b5a9bc47f39b, (subgraph_callable-7044ef03-a0ab-4656-a51a-11d636979d6e, [25], PosixPath('wiki/iawiki-latest/disambig_ents.txt'), 100, '<page>', '</page>')))\n",
      "kwargs:    {}\n",
      "Exception: \"OSError('Invalid data format')\"\n",
      "\n",
      "INFO:root:Finished in 19s                ] | 13% Completed | 19.3s\u001b[2K\n",
      "2024-03-19 15:04:46,674 - distributed.worker - WARNING - Compute Failed\n",
      "Key:       ('fileparts-lambda-89b4b29143df67ab14ee7650adaaff1e', 35)\n",
      "Function:  execute_task\n",
      "args:      ((subgraph_callable-070f3b3c-7bb7-4e25-bf31-b5a9bc47f39b, (subgraph_callable-7044ef03-a0ab-4656-a51a-11d636979d6e, [35], PosixPath('wiki/iawiki-latest/disambig_ents.txt'), 100, '<page>', '</page>')))\n",
      "kwargs:    {}\n",
      "Exception: \"OSError('Invalid data format')\"\n",
      "\n",
      "2024-03-19 15:04:46,675 - distributed.worker - WARNING - Compute Failed\n",
      "Key:       ('fileparts-lambda-89b4b29143df67ab14ee7650adaaff1e', 28)\n",
      "Function:  execute_task\n",
      "args:      ((subgraph_callable-070f3b3c-7bb7-4e25-bf31-b5a9bc47f39b, (subgraph_callable-7044ef03-a0ab-4656-a51a-11d636979d6e, [28], PosixPath('wiki/iawiki-latest/disambig_ents.txt'), 100, '<page>', '</page>')))\n",
      "kwargs:    {}\n",
      "Exception: \"OSError('Invalid data format')\"\n",
      "\n",
      "2024-03-19 15:04:46,677 - distributed.worker - WARNING - Compute Failed\n",
      "Key:       ('fileparts-lambda-89b4b29143df67ab14ee7650adaaff1e', 20)\n",
      "Function:  execute_task\n",
      "args:      ((subgraph_callable-070f3b3c-7bb7-4e25-bf31-b5a9bc47f39b, (subgraph_callable-7044ef03-a0ab-4656-a51a-11d636979d6e, [20], PosixPath('wiki/iawiki-latest/disambig_ents.txt'), 100, '<page>', '</page>')))\n",
      "kwargs:    {}\n",
      "Exception: \"OSError('Invalid data format')\"\n",
      "\n",
      "2024-03-19 15:04:46,677 - distributed.worker - WARNING - Compute Failed\n",
      "Key:       ('fileparts-lambda-89b4b29143df67ab14ee7650adaaff1e', 13)\n",
      "Function:  execute_task\n",
      "args:      ((subgraph_callable-070f3b3c-7bb7-4e25-bf31-b5a9bc47f39b, (subgraph_callable-7044ef03-a0ab-4656-a51a-11d636979d6e, [13], PosixPath('wiki/iawiki-latest/disambig_ents.txt'), 100, '<page>', '</page>')))\n",
      "kwargs:    {}\n",
      "Exception: \"OSError('Invalid data format')\"\n",
      "\n",
      "2024-03-19 15:04:46,701 - distributed.worker - WARNING - Compute Failed\n",
      "Key:       ('fileparts-lambda-89b4b29143df67ab14ee7650adaaff1e', 23)\n",
      "Function:  execute_task\n",
      "args:      ((subgraph_callable-070f3b3c-7bb7-4e25-bf31-b5a9bc47f39b, (subgraph_callable-7044ef03-a0ab-4656-a51a-11d636979d6e, [23], PosixPath('wiki/iawiki-latest/disambig_ents.txt'), 100, '<page>', '</page>')))\n",
      "kwargs:    {}\n",
      "Exception: \"OSError('Invalid data format')\"\n",
      "\n",
      "2024-03-19 15:04:46,702 - distributed.worker - WARNING - Compute Failed\n",
      "Key:       ('fileparts-lambda-89b4b29143df67ab14ee7650adaaff1e', 30)\n",
      "Function:  execute_task\n",
      "args:      ((subgraph_callable-070f3b3c-7bb7-4e25-bf31-b5a9bc47f39b, (subgraph_callable-7044ef03-a0ab-4656-a51a-11d636979d6e, [30], PosixPath('wiki/iawiki-latest/disambig_ents.txt'), 100, '<page>', '</page>')))\n",
      "kwargs:    {}\n",
      "Exception: \"OSError('Invalid data format')\"\n",
      "\n",
      "2024-03-19 15:04:46,703 - distributed.worker - WARNING - Compute Failed\n",
      "Key:       ('fileparts-lambda-89b4b29143df67ab14ee7650adaaff1e', 0)\n",
      "Function:  execute_task\n",
      "args:      ((subgraph_callable-070f3b3c-7bb7-4e25-bf31-b5a9bc47f39b, (subgraph_callable-7044ef03-a0ab-4656-a51a-11d636979d6e, [0], PosixPath('wiki/iawiki-latest/disambig_ents.txt'), 100, '<page>', '</page>')))\n",
      "kwargs:    {}\n",
      "Exception: \"OSError('Invalid data format')\"\n",
      "\n",
      "2024-03-19 15:04:46,705 - distributed.worker - WARNING - Compute Failed\n",
      "Key:       ('fileparts-lambda-89b4b29143df67ab14ee7650adaaff1e', 16)\n",
      "Function:  execute_task\n",
      "args:      ((subgraph_callable-070f3b3c-7bb7-4e25-bf31-b5a9bc47f39b, (subgraph_callable-7044ef03-a0ab-4656-a51a-11d636979d6e, [16], PosixPath('wiki/iawiki-latest/disambig_ents.txt'), 100, '<page>', '</page>')))\n",
      "kwargs:    {}\n",
      "Exception: \"OSError('Invalid data format')\"\n",
      "\n",
      "2024-03-19 15:04:46,791 - distributed.worker - WARNING - Compute Failed\n",
      "Key:       ('fileparts-lambda-89b4b29143df67ab14ee7650adaaff1e', 31)\n",
      "Function:  execute_task\n",
      "args:      ((subgraph_callable-070f3b3c-7bb7-4e25-bf31-b5a9bc47f39b, (subgraph_callable-7044ef03-a0ab-4656-a51a-11d636979d6e, [31], PosixPath('wiki/iawiki-latest/disambig_ents.txt'), 100, '<page>', '</page>')))\n",
      "kwargs:    {}\n",
      "Exception: \"OSError('Invalid data format')\"\n",
      "\n",
      "2024-03-19 15:04:46,792 - distributed.worker - WARNING - Compute Failed\n",
      "Key:       ('fileparts-lambda-89b4b29143df67ab14ee7650adaaff1e', 1)\n",
      "Function:  execute_task\n",
      "args:      ((subgraph_callable-070f3b3c-7bb7-4e25-bf31-b5a9bc47f39b, (subgraph_callable-7044ef03-a0ab-4656-a51a-11d636979d6e, [1], PosixPath('wiki/iawiki-latest/disambig_ents.txt'), 100, '<page>', '</page>')))\n",
      "kwargs:    {}\n",
      "Exception: \"OSError('Invalid data format')\"\n",
      "\n",
      "2024-03-19 15:04:46,792 - distributed.worker - WARNING - Compute Failed\n",
      "Key:       ('fileparts-lambda-89b4b29143df67ab14ee7650adaaff1e', 24)\n",
      "Function:  execute_task\n",
      "args:      ((subgraph_callable-070f3b3c-7bb7-4e25-bf31-b5a9bc47f39b, (subgraph_callable-7044ef03-a0ab-4656-a51a-11d636979d6e, [24], PosixPath('wiki/iawiki-latest/disambig_ents.txt'), 100, '<page>', '</page>')))\n",
      "kwargs:    {}\n",
      "Exception: \"OSError('Invalid data format')\"\n",
      "\n",
      "2024-03-19 15:04:46,792 - distributed.worker - WARNING - Compute Failed\n",
      "Key:       ('fileparts-lambda-89b4b29143df67ab14ee7650adaaff1e', 17)\n",
      "Function:  execute_task\n",
      "args:      ((subgraph_callable-070f3b3c-7bb7-4e25-bf31-b5a9bc47f39b, (subgraph_callable-7044ef03-a0ab-4656-a51a-11d636979d6e, [17], PosixPath('wiki/iawiki-latest/disambig_ents.txt'), 100, '<page>', '</page>')))\n",
      "kwargs:    {}\n",
      "Exception: \"OSError('Invalid data format')\"\n",
      "\n",
      "2024-03-19 15:04:46,853 - distributed.worker - WARNING - Compute Failed\n",
      "Key:       ('fileparts-lambda-89b4b29143df67ab14ee7650adaaff1e', 33)\n",
      "Function:  execute_task\n",
      "args:      ((subgraph_callable-070f3b3c-7bb7-4e25-bf31-b5a9bc47f39b, (subgraph_callable-7044ef03-a0ab-4656-a51a-11d636979d6e, [33], PosixPath('wiki/iawiki-latest/disambig_ents.txt'), 100, '<page>', '</page>')))\n",
      "kwargs:    {}\n",
      "Exception: \"OSError('Invalid data format')\"\n",
      "\n",
      "2024-03-19 15:04:46,854 - distributed.worker - WARNING - Compute Failed\n",
      "Key:       ('fileparts-lambda-89b4b29143df67ab14ee7650adaaff1e', 19)\n",
      "Function:  execute_task\n",
      "args:      ((subgraph_callable-070f3b3c-7bb7-4e25-bf31-b5a9bc47f39b, (subgraph_callable-7044ef03-a0ab-4656-a51a-11d636979d6e, [19], PosixPath('wiki/iawiki-latest/disambig_ents.txt'), 100, '<page>', '</page>')))\n",
      "kwargs:    {}\n",
      "Exception: \"OSError('Invalid data format')\"\n",
      "\n",
      "2024-03-19 15:04:46,855 - distributed.worker - WARNING - Compute Failed\n",
      "Key:       ('fileparts-lambda-89b4b29143df67ab14ee7650adaaff1e', 11)\n",
      "Function:  execute_task\n",
      "args:      ((subgraph_callable-070f3b3c-7bb7-4e25-bf31-b5a9bc47f39b, (subgraph_callable-7044ef03-a0ab-4656-a51a-11d636979d6e, [11], PosixPath('wiki/iawiki-latest/disambig_ents.txt'), 100, '<page>', '</page>')))\n",
      "kwargs:    {}\n",
      "Exception: \"OSError('Invalid data format')\"\n",
      "\n",
      "2024-03-19 15:04:46,855 - distributed.worker - WARNING - Compute Failed\n",
      "Key:       ('fileparts-lambda-89b4b29143df67ab14ee7650adaaff1e', 26)\n",
      "Function:  execute_task\n",
      "args:      ((subgraph_callable-070f3b3c-7bb7-4e25-bf31-b5a9bc47f39b, (subgraph_callable-7044ef03-a0ab-4656-a51a-11d636979d6e, [26], PosixPath('wiki/iawiki-latest/disambig_ents.txt'), 100, '<page>', '</page>')))\n",
      "kwargs:    {}\n",
      "Exception: \"OSError('Invalid data format')\"\n",
      "\n",
      "2024-03-19 15:04:46,970 - distributed.worker - WARNING - Compute Failed\n",
      "Key:       ('fileparts-lambda-89b4b29143df67ab14ee7650adaaff1e', 2)\n",
      "Function:  execute_task\n",
      "args:      ((subgraph_callable-070f3b3c-7bb7-4e25-bf31-b5a9bc47f39b, (subgraph_callable-7044ef03-a0ab-4656-a51a-11d636979d6e, [2], PosixPath('wiki/iawiki-latest/disambig_ents.txt'), 100, '<page>', '</page>')))\n",
      "kwargs:    {}\n",
      "Exception: \"OSError('Invalid data format')\"\n",
      "\n",
      "2024-03-19 15:04:46,970 - distributed.worker - WARNING - Compute Failed\n",
      "Key:       ('fileparts-lambda-89b4b29143df67ab14ee7650adaaff1e', 34)\n",
      "Function:  execute_task\n",
      "args:      ((subgraph_callable-070f3b3c-7bb7-4e25-bf31-b5a9bc47f39b, (subgraph_callable-7044ef03-a0ab-4656-a51a-11d636979d6e, [34], PosixPath('wiki/iawiki-latest/disambig_ents.txt'), 100, '<page>', '</page>')))\n",
      "kwargs:    {}\n",
      "Exception: \"OSError('Invalid data format')\"\n",
      "\n",
      "2024-03-19 15:04:46,971 - distributed.worker - WARNING - Compute Failed\n",
      "Key:       ('fileparts-lambda-89b4b29143df67ab14ee7650adaaff1e', 27)\n",
      "Function:  execute_task\n",
      "args:      ((subgraph_callable-070f3b3c-7bb7-4e25-bf31-b5a9bc47f39b, (subgraph_callable-7044ef03-a0ab-4656-a51a-11d636979d6e, [27], PosixPath('wiki/iawiki-latest/disambig_ents.txt'), 100, '<page>', '</page>')))\n",
      "kwargs:    {}\n",
      "Exception: \"OSError('Invalid data format')\"\n",
      "\n",
      "2024-03-19 15:04:46,971 - distributed.worker - WARNING - Compute Failed\n",
      "Key:       ('fileparts-lambda-89b4b29143df67ab14ee7650adaaff1e', 12)\n",
      "Function:  execute_task\n",
      "args:      ((subgraph_callable-070f3b3c-7bb7-4e25-bf31-b5a9bc47f39b, (subgraph_callable-7044ef03-a0ab-4656-a51a-11d636979d6e, [12], PosixPath('wiki/iawiki-latest/disambig_ents.txt'), 100, '<page>', '</page>')))\n",
      "kwargs:    {}\n",
      "Exception: \"OSError('Invalid data format')\"\n",
      "\n",
      "2024-03-19 15:04:47,072 - distributed.worker - WARNING - Compute Failed\n",
      "Key:       ('fileparts-lambda-89b4b29143df67ab14ee7650adaaff1e', 37)\n",
      "Function:  execute_task\n",
      "args:      ((subgraph_callable-070f3b3c-7bb7-4e25-bf31-b5a9bc47f39b, (subgraph_callable-7044ef03-a0ab-4656-a51a-11d636979d6e, [37], PosixPath('wiki/iawiki-latest/disambig_ents.txt'), 100, '<page>', '</page>')))\n",
      "kwargs:    {}\n",
      "Exception: \"OSError('Invalid data format')\"\n",
      "\n",
      "2024-03-19 15:04:47,073 - distributed.worker - WARNING - Compute Failed\n",
      "Key:       ('fileparts-lambda-89b4b29143df67ab14ee7650adaaff1e', 15)\n",
      "Function:  execute_task\n",
      "args:      ((subgraph_callable-070f3b3c-7bb7-4e25-bf31-b5a9bc47f39b, (subgraph_callable-7044ef03-a0ab-4656-a51a-11d636979d6e, [15], PosixPath('wiki/iawiki-latest/disambig_ents.txt'), 100, '<page>', '</page>')))\n",
      "kwargs:    {}\n",
      "Exception: \"OSError('Invalid data format')\"\n",
      "\n",
      "2024-03-19 15:04:47,074 - distributed.worker - WARNING - Compute Failed\n",
      "Key:       ('fileparts-lambda-89b4b29143df67ab14ee7650adaaff1e', 22)\n",
      "Function:  execute_task\n",
      "args:      ((subgraph_callable-070f3b3c-7bb7-4e25-bf31-b5a9bc47f39b, (subgraph_callable-7044ef03-a0ab-4656-a51a-11d636979d6e, [22], PosixPath('wiki/iawiki-latest/disambig_ents.txt'), 100, '<page>', '</page>')))\n",
      "kwargs:    {}\n",
      "Exception: \"OSError('Invalid data format')\"\n",
      "\n",
      "2024-03-19 15:04:47,075 - distributed.worker - WARNING - Compute Failed\n",
      "Key:       ('fileparts-lambda-89b4b29143df67ab14ee7650adaaff1e', 3)\n",
      "Function:  execute_task\n",
      "args:      ((subgraph_callable-070f3b3c-7bb7-4e25-bf31-b5a9bc47f39b, (subgraph_callable-7044ef03-a0ab-4656-a51a-11d636979d6e, [3], PosixPath('wiki/iawiki-latest/disambig_ents.txt'), 100, '<page>', '</page>')))\n",
      "kwargs:    {}\n",
      "Exception: \"OSError('Invalid data format')\"\n",
      "\n",
      "2024-03-19 15:04:47,131 - distributed.worker - WARNING - Compute Failed\n",
      "Key:       ('fileparts-lambda-89b4b29143df67ab14ee7650adaaff1e', 21)\n",
      "Function:  execute_task\n",
      "args:      ((subgraph_callable-070f3b3c-7bb7-4e25-bf31-b5a9bc47f39b, (subgraph_callable-7044ef03-a0ab-4656-a51a-11d636979d6e, [21], PosixPath('wiki/iawiki-latest/disambig_ents.txt'), 100, '<page>', '</page>')))\n",
      "kwargs:    {}\n",
      "Exception: \"OSError('Invalid data format')\"\n",
      "\n",
      "2024-03-19 15:04:47,132 - distributed.worker - WARNING - Compute Failed\n",
      "Key:       ('fileparts-lambda-89b4b29143df67ab14ee7650adaaff1e', 36)\n",
      "Function:  execute_task\n",
      "args:      ((subgraph_callable-070f3b3c-7bb7-4e25-bf31-b5a9bc47f39b, (subgraph_callable-7044ef03-a0ab-4656-a51a-11d636979d6e, [36], PosixPath('wiki/iawiki-latest/disambig_ents.txt'), 100, '<page>', '</page>')))\n",
      "kwargs:    {}\n",
      "Exception: \"OSError('Invalid data format')\"\n",
      "\n",
      "2024-03-19 15:04:47,132 - distributed.worker - WARNING - Compute Failed\n",
      "Key:       ('fileparts-lambda-89b4b29143df67ab14ee7650adaaff1e', 14)\n",
      "Function:  execute_task\n",
      "args:      ((subgraph_callable-070f3b3c-7bb7-4e25-bf31-b5a9bc47f39b, (subgraph_callable-7044ef03-a0ab-4656-a51a-11d636979d6e, [14], PosixPath('wiki/iawiki-latest/disambig_ents.txt'), 100, '<page>', '</page>')))\n",
      "kwargs:    {}\n",
      "Exception: \"OSError('Invalid data format')\"\n",
      "\n",
      "2024-03-19 15:04:47,132 - distributed.worker - WARNING - Compute Failed\n",
      "Key:       ('fileparts-lambda-89b4b29143df67ab14ee7650adaaff1e', 29)\n",
      "Function:  execute_task\n",
      "args:      ((subgraph_callable-070f3b3c-7bb7-4e25-bf31-b5a9bc47f39b, (subgraph_callable-7044ef03-a0ab-4656-a51a-11d636979d6e, [29], PosixPath('wiki/iawiki-latest/disambig_ents.txt'), 100, '<page>', '</page>')))\n",
      "kwargs:    {}\n",
      "Exception: \"OSError('Invalid data format')\"\n",
      "\n",
      "2024-03-19 15:04:49,701 - distributed.worker - WARNING - Compute Failed\n",
      "Key:       ('fileparts-lambda-89b4b29143df67ab14ee7650adaaff1e', 4)\n",
      "Function:  execute_task\n",
      "args:      ((subgraph_callable-070f3b3c-7bb7-4e25-bf31-b5a9bc47f39b, (subgraph_callable-7044ef03-a0ab-4656-a51a-11d636979d6e, [4], PosixPath('wiki/iawiki-latest/disambig_ents.txt'), 100, '<page>', '</page>')))\n",
      "kwargs:    {}\n",
      "Exception: \"OSError('Invalid data format')\"\n",
      "\n",
      "2024-03-19 15:04:51,670 - distributed.worker - WARNING - Compute Failed\n",
      "Key:       ('fileparts-lambda-89b4b29143df67ab14ee7650adaaff1e', 11)\n",
      "Function:  execute_task\n",
      "args:      ((subgraph_callable-070f3b3c-7bb7-4e25-bf31-b5a9bc47f39b, (subgraph_callable-7044ef03-a0ab-4656-a51a-11d636979d6e, [11], PosixPath('wiki/iawiki-latest/disambig_ents.txt'), 100, '<page>', '</page>')))\n",
      "kwargs:    {}\n",
      "Exception: \"OSError('Invalid data format')\"\n",
      "\n",
      "^C\n",
      "Traceback (most recent call last):\n",
      "  File \"/var/scratch/bbkruit/minimal-EL/minimel/get_disambig.py\", line 170, in get_disambig\n",
      "    ents, titles, links = zip(*links.compute())\n",
      "  File \"/home/bbkruit/scratch/anaconda3/envs/minimel/lib/python3.9/site-packages/dask/base.py\", line 381, in compute\n",
      "    (result,) = compute(self, traverse=False, **kwargs)\n",
      "  File \"/home/bbkruit/scratch/anaconda3/envs/minimel/lib/python3.9/site-packages/dask/base.py\", line 666, in compute\n",
      "    results = schedule(dsk, keys, **kwargs)\n",
      "  File \"/home/bbkruit/scratch/anaconda3/envs/minimel/lib/python3.9/site-packages/distributed/client.py\", line 3279, in get\n",
      "    results = self.gather(packed, asynchronous=asynchronous, direct=direct)\n",
      "  File \"/home/bbkruit/scratch/anaconda3/envs/minimel/lib/python3.9/site-packages/distributed/client.py\", line 2404, in gather\n",
      "    return self.sync(\n",
      "  File \"/home/bbkruit/scratch/anaconda3/envs/minimel/lib/python3.9/site-packages/distributed/utils.py\", line 359, in sync\n",
      "    return sync(\n",
      "  File \"/home/bbkruit/scratch/anaconda3/envs/minimel/lib/python3.9/site-packages/distributed/utils.py\", line 426, in sync\n",
      "    raise exc.with_traceback(tb)\n",
      "  File \"/home/bbkruit/scratch/anaconda3/envs/minimel/lib/python3.9/site-packages/distributed/utils.py\", line 399, in f\n",
      "    result = yield future\n",
      "  File \"/home/bbkruit/scratch/anaconda3/envs/minimel/lib/python3.9/site-packages/tornado/gen.py\", line 767, in run\n",
      "    value = future.result()\n",
      "  File \"/home/bbkruit/scratch/anaconda3/envs/minimel/lib/python3.9/site-packages/distributed/client.py\", line 2267, in _gather\n",
      "    raise exception.with_traceback(traceback)\n",
      "  File \"/home/bbkruit/scratch/anaconda3/envs/minimel/lib/python3.9/site-packages/dask/optimization.py\", line 992, in __call__\n",
      "    return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))\n",
      "  File \"/home/bbkruit/scratch/anaconda3/envs/minimel/lib/python3.9/site-packages/dask/core.py\", line 151, in get\n",
      "    result = _execute_task(task, cache)\n",
      "  File \"/home/bbkruit/scratch/anaconda3/envs/minimel/lib/python3.9/site-packages/dask/core.py\", line 121, in _execute_task\n",
      "    return func(*(_execute_task(a, cache) for a in args))\n",
      "  File \"/var/scratch/bbkruit/minimal-EL/minimel/get_disambig.py\", line 157, in <lambda>\n",
      "    lambda b: get_disambig_links(\n",
      "  File \"/var/scratch/bbkruit/minimal-EL/minimel/get_disambig.py\", line 91, in get_disambig_links\n",
      "    index.load(dawgfile)\n",
      "  File \"src/dawg.pyx\", line 156, in dawg.DAWG.load\n",
      "OSError: Invalid data format\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/bbkruit/scratch/anaconda3/envs/minimel/bin/minimel\", line 33, in <module>\n",
      "    sys.exit(load_entry_point('minimel', 'console_scripts', 'minimel')())\n",
      "  File \"/var/scratch/bbkruit/minimal-EL/minimel/__main__.py\", line 74, in main\n",
      "    defopt.run(subcommands)\n",
      "  File \"/home/bbkruit/scratch/anaconda3/envs/minimel/lib/python3.9/site-packages/defopt.py\", line 356, in run\n",
      "    return call()\n",
      "  File \"/var/scratch/bbkruit/minimal-EL/minimel/get_disambig.py\", line 180, in get_disambig\n",
      "    json.dump(links, fw)\n",
      "  File \"/home/bbkruit/scratch/anaconda3/envs/minimel/lib/python3.9/site-packages/distributed/deploy/spec.py\", line 293, in close\n",
      "    aw = super().close(timeout)\n",
      "  File \"/home/bbkruit/scratch/anaconda3/envs/minimel/lib/python3.9/site-packages/distributed/deploy/cluster.py\", line 226, in close\n",
      "    return self.sync(self._close, callback_timeout=timeout)\n",
      "  File \"/home/bbkruit/scratch/anaconda3/envs/minimel/lib/python3.9/threading.py\", line 581, in wait\n",
      "    signaled = self._cond.wait(timeout)\n",
      "  File \"/home/bbkruit/scratch/anaconda3/envs/minimel/lib/python3.9/threading.py\", line 316, in wait\n",
      "    gotit = waiter.acquire(True, timeout)\n",
      "KeyboardInterrupt\n"
     ]
    }
   ],
   "source": [
    "!minimel -v get-disambig -n 100 $dump $dawg $disambigpages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:root:Counting links...\n",
      "INFO:root:Finished in 2s#################] | 100% Completed |  2.5s\u001b[2K\n",
      "INFO:root:Got 32490 counts.\n",
      "INFO:root:Aggregating...\n",
      "INFO:root:Finished in 2s#################] | 100% Completed |  2.2s\u001b[2K\n",
      "INFO:root:Writing to wiki/iawiki-latest/count.min2.json\n"
     ]
    }
   ],
   "source": [
    "paragraphlinks = f'{root}/{wiki}-paragraph-links/'\n",
    "!minimel -v count $paragraphlinks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2024-03-06 10:10:14--  https://query.wikidata.org/sparql?format=json&query=%0ASELECT%20DISTINCT%20%3Fs%20WHERE%20%7B%0A%20%20%7B%3Fs%20wdt%3AP31%20wd%3AQ4167410%20.%7D%20%23%20disambig%0A%20%20UNION%0A%20%20%7B%3Fs%20wdt%3AP31%20wd%3AQ13406463%20.%7D%20%23%20list%0A%20%20UNION%0A%20%20%7B%3Fs%20wdt%3AP360%20%3Fl%20.%20%7D%20%23%20list%20of%0A%0A%20%20%3Fpage%20schema%3Aabout%20%3Fs%20.%0A%20%20%3Fpage%20schema%3AinLanguage%20%22ia%22%20.%0A%7D%0A\n",
      "Resolving query.wikidata.org (query.wikidata.org)... 185.15.59.224, 2a02:ec80:300:ed1a::1\n",
      "Connecting to query.wikidata.org (query.wikidata.org)|185.15.59.224|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: unspecified [application/sparql-results+json]\n",
      "Saving to: ‘wiki/iawiki-latest/badent.json’\n",
      "\n",
      "    [ <=>                                   ] 55,974      --.-K/s   in 0.001s  \n",
      "\n",
      "2024-03-06 10:10:14 (52.4 MB/s) - ‘wiki/iawiki-latest/badent.json’ saved [55974]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import shlex, urllib, json\n",
    "\n",
    "# Get Wikidata IDs for disambiguation and list articles\n",
    "lang = wiki.split('wiki')[0]\n",
    "query = urllib.parse.quote(\"\"\"\n",
    "SELECT DISTINCT ?s WHERE {\n",
    "  {?s wdt:P31 wd:Q4167410 .} # disambig\n",
    "  UNION\n",
    "  {?s wdt:P31 wd:Q13406463 .} # list\n",
    "  UNION\n",
    "  {?s wdt:P360 ?l . } # list of\n",
    "\n",
    "  ?page schema:about ?s .\n",
    "  ?page schema:inLanguage \"%s\" .\n",
    "}\n",
    "\"\"\" % lang)\n",
    "url = shlex.quote(f\"https://query.wikidata.org/sparql?format=json&query={query}\")\n",
    "badent = f'{root}/badent.json'\n",
    "!wget -O $badent $url\n",
    "\n",
    "bindings = json.load(open(badent)).get('results', []).get('bindings', [])\n",
    "qids = [b.get('s', {}).get('value', '')[31:] for b in bindings]\n",
    "badent = f'{root}/badent.txt'\n",
    "with open(badent, 'w') as fw:\n",
    "    print('\\n'.join(qids), file=fw)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Counting entities...: 100%|███████████| 11447/11447 [00:00<00:00, 294410.67it/s]\n",
      "INFO:root:Removing 130 bad entities\n",
      "Loading labels...: 100%|██████████████| 34465/34465 [00:00<00:00, 196894.89it/s]\n",
      "Filtering names...: 100%|██████████████| 11387/11387 [00:00<00:00, 69138.81it/s]\n",
      "INFO:root:Filtering out 1 bad names\n",
      "INFO:root:Keeping 11386 good names\n",
      "INFO:root:Writing to wiki/iawiki-latest/clean.json\n"
     ]
    }
   ],
   "source": [
    "disambigfile = f'{root}/disambig.json'\n",
    "countfile = f'{root}/count.min2.json'\n",
    "!minimel -v clean -b $badent $outdb $disambigfile $countfile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:root:Vectorizing training examples for 286 ambiguous names\n",
      "INFO:root:Writing to wiki/iawiki-latest/vec.clean.dat.parts\n",
      "INFO:root:Finished in 1s#################] | 100% Completed |  1.1s\u001b[2K\n",
      "INFO:root:Wrote 34 partitions\n",
      "INFO:root:Concatenating to wiki/iawiki-latest/vec.clean.dat\n",
      "Concatenating: 100%|██████████████████████████| 34/34 [00:00<00:00, 4602.88it/s]\n"
     ]
    }
   ],
   "source": [
    "cleanfile = f'{root}/clean.json'\n",
    "!minimel -v vectorize $paragraphlinks $cleanfile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:root:Writing to wiki/iawiki-latest/model.20b.vw\n",
      "creating quadratic features for pairs: ls sf\n",
      "final_regressor = wiki/iawiki-latest/model.20b.vw\n",
      "creating cache_file = wiki/iawiki-latest/vec.clean.dat.cache\n",
      "Reading datafile = wiki/iawiki-latest/vec.clean.dat\n",
      "num sources = 1\n",
      "Num weight bits = 20\n",
      "learning rate = 0.5\n",
      "initial_t = 0\n",
      "power_t = 0.5\n",
      "decay_learning_rate = 1\n",
      "Enabled learners: gd, scorer-identity, csoaa_ldf-prob, shared_feature_merger\n",
      "Input label = CS\n",
      "Output pred = SCALARS\n",
      "average  since         example        example        current        current  current\n",
      "loss     last          counter         weight          label        predict features\n",
      "0.000000 0.000000            1            1.0        unknown              0     1330\n",
      "0.000000 0.000000            2            2.0        unknown              0      344\n",
      "0.000000 0.000000            4            4.0        unknown              0      364\n",
      "0.250000 0.500000            8            8.0          known            183      712\n",
      "0.187500 0.125000           16           16.0          known           1321       36\n",
      "0.125000 0.062500           32           32.0        unknown              0      272\n",
      "0.078125 0.031250           64           64.0        unknown              0      140\n",
      "0.125000 0.171875          128          128.0        unknown              0       84\n",
      "0.136719 0.148438          256          256.0        unknown              0      124\n",
      "0.117188 0.097656          512          512.0        unknown              0       36\n",
      "0.125000 0.132812         1024         1024.0        unknown              0      552\n",
      "0.113281 0.101562         2048         2048.0        unknown              0      104\n",
      "0.115723 0.118164         4096         4096.0          known          41304       24\n",
      "0.111938 0.108154         8192         8192.0        unknown              0      128\n",
      "unknown  unknown         16384        16384.0          known            652      108 h\n",
      "unknown  unknown         32768        32768.0        unknown              0       24 h\n",
      "\n",
      "finished run\n",
      "number of examples per pass = 10150\n",
      "passes used = 4\n",
      "weighted example sum = 40600.000000\n",
      "weighted label sum = 0.000000\n",
      "average loss = undefined (no holdout)\n",
      "average multiclass log loss = 0.396795 h\n",
      "total feature number = 16042808\n",
      "INFO:root:Wrote to model.20b.vw\n"
     ]
    }
   ],
   "source": [
    "vecfile = f'{root}/vec.clean.dat'\n",
    "!minimel -v train $vecfile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting: 100%|███████████████████████| 59134/59134 [00:16<00:00, 3681.70it/s]\n",
      "INFO:root:,,0\n",
      "micro,precision,0.9081185405247082\n",
      "micro,recall,0.9081185405247082\n",
      "micro,fscore,0.9081185405247082\n",
      "macro,precision,0.9232628882555252\n",
      "macro,recall,0.9058898794831253\n",
      "macro,fscore,0.911819107954825\n",
      ",support,191192.0\n",
      "\n"
     ]
    }
   ],
   "source": [
    "modelfile = f'{root}/model.20b.vw'\n",
    "!minimel -v run --evaluate -o /dev/null $dawg $cleanfile $modelfile $paragraphlinks/*"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "minimel",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.17"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}