Table.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import csv\n",
    "\n",
    "with open('input_file_wenhu.csv', mode='w') as fw:\n",
    "    with open('input_file_shiyang.csv', mode='w') as fs:\n",
    "        with open('input_file_yunkai.csv', mode='w') as fy:\n",
    "            \n",
    "            #output_wenhu = output[:5000]\n",
    "            #fileds = ['url1', 'q1', 'a1', 'url2', 'q2', 'a2', 'url3', 'q3', 'a3', 'url4', 'q4', 'a4', 'url5', 'q5', 'a5']\n",
    "            #writer = csv.DictWriter(fw, fieldnames=fileds)\n",
    "            #writer.writeheader()\n",
    "            #for i in range(0, len(output_wenhu) - 5, 5):\n",
    "            #    writer.writerow({'url1':output_wenhu[i]['url'], 'q1':output_wenhu[i]['q'], 'a1':output_wenhu[i]['a'],\n",
    "            #                    'url2':output_wenhu[i+1]['url'], 'q2':output_wenhu[i+1]['q'], 'a2':output_wenhu[i+1]['a'],\n",
    "            #                    'url3':output_wenhu[i+2]['url'], 'q3':output_wenhu[i+2]['q'], 'a3':output_wenhu[i+2]['a'],\n",
    "            #                    'url4':output_wenhu[i+3]['url'], 'q4':output_wenhu[i+3]['q'], 'a4':output_wenhu[i+3]['a'],\n",
    "            #                    'url5':output_wenhu[i+4]['url'], 'q5':output_wenhu[i+4]['q'], 'a5':output_wenhu[i+4]['a']})\n",
    "            \n",
    "            num = 10\n",
    "            \n",
    "            output_shiyang = output[5000:20000]\n",
    "            fields = []\n",
    "            for i in range(1, num+1):\n",
    "                fields.extend(['url{}'.format(i), 'q{}'.format(i), 'a{}'.format(i)])\n",
    "            writer = csv.DictWriter(fs, fieldnames=fields)\n",
    "            writer.writeheader()\n",
    "            for i in range(0, len(output_shiyang) - num, num):\n",
    "                elem = {}\n",
    "                for j in range(1, num+1):\n",
    "                    elem['url{}'.format(j)] = output_shiyang[i+j-1]['url']\n",
    "                    elem['q{}'.format(j)] = output_shiyang[i+j-1]['q']\n",
    "                    elem['a{}'.format(j)] = output_shiyang[i+j-1]['a']\n",
    "                \n",
    "                writer.writerow(elem) \n",
    "            \n",
    "            \n",
    "            output_yunkai = output[20000:]\n",
    "            fields = []\n",
    "            for i in range(1, num+1):\n",
    "                fields.extend(['url{}'.format(i), 'q{}'.format(i), 'a{}'.format(i)])\n",
    "            writer = csv.DictWriter(fy, fieldnames=fields)\n",
    "            writer.writeheader()\n",
    "            for i in range(0, len(output_yunkai) - num, num):\n",
    "                elem = {}\n",
    "                for j in range(1, num+1):\n",
    "                    elem['url{}'.format(j)] = output_yunkai[i+j-1]['url']\n",
    "                    elem['q{}'.format(j)] = output_yunkai[i+j-1]['q']\n",
    "                    elem['a{}'.format(j)] = output_yunkai[i+j-1]['a']\n",
    "                \n",
    "                writer.writerow(elem) \n",
    "                "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def regex(inp):\n",
    "    inp = re.sub(r\"> +\", r\">\", inp)\n",
    "    inp = re.sub(r\" +<\", r\"<\", inp)\n",
    "    inp = re.sub(r\">@\", r\">\", inp)\n",
    "    inp = re.sub(r\"\\*\", r\"\", inp)\n",
    "    inp = re.sub(r\"#\", r\"\", inp)\n",
    "    inp = re.sub(r\"‡\", r\"\", inp)\n",
    "    #inp = re.sub(r\".0([^0-9])\", r\"\\1\", inp)\n",
    "    for month, abb in [(\"January\", \"Jan\"), (\"February\", \"Feb\"), (\"March\", \"March\"), (\"April\", \"April\"), \n",
    "                   (\"May\", \"May\"), (\"June\",\"June\") , (\"July\", \"July\"), (\"August\", \"Aug\"), \n",
    "                   (\"September\", \"Sep\"), (\"October\", \"Oct\"), (\"November\", \"Nov\"), (\"December\", \"Dec\")]:\n",
    "        inp = re.sub(r\"({})([0-9]+)\".format(month), r\"\\1 \\2\", inp)\n",
    "        inp = re.sub(r\"({})([0-9]+)\".format(abb), r\"\\1 \\2\", inp)\n",
    "    return inp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# encoding=utf8\n",
    "import sys\n",
    "reload(sys)\n",
    "sys.setdefaultencoding('utf8')\n",
    "import jsonlines\n",
    "import csv\n",
    "import re\n",
    "\"\"\"\n",
    "with open('all_html_original.csv', 'w') as fw:\n",
    "    fields = ['url', 'content']\n",
    "    writer = csv.DictWriter(fw, fieldnames=fields)\n",
    "    writer.writeheader()\n",
    "\"\"\"\n",
    "with open('WikiTableQuestions/wikidata/train.tables.jsonl') as f:\n",
    "    data = jsonlines.Reader(f)\n",
    "    for d in data:\n",
    "        f = '<table class=\"wikitable\"><tr>'\n",
    "        for h in d['header']:\n",
    "            f += \"<th> {} </th>\".format(h)\n",
    "        f += \"</tr>\"\n",
    "        for r in d['rows']:\n",
    "            f += \"<tr>\"\n",
    "            for elem in r:\n",
    "                f += \"<td> {} </td>\".format(elem)\n",
    "            f += \"</tr>\"\n",
    "        f += \"</table>\"\n",
    "        #writer.writerow({'url': d['id'], \"content\": f})\n",
    "        with open('WikiTableQuestions/wikidata/all_html/{}.html'.format(d['id']), 'w') as fw:\n",
    "            print >> fw, regex(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "def regex_equation(line):\n",
    "    line = re.sub(r\"([^+-])([0-9]+)-([0-9]+)-([0-9]+)-([0-9]+)=\", r\"\\1\\2+\\3+\\4+\\5=\", line)\n",
    "    line = re.sub(r\"([^+-])([0-9]+)-([0-9]+)-([0-9]+)=\", r\"\\1\\2+\\3+\\4=\", line)\n",
    "    line = re.sub(r\"([^+-])([0-9]+)-([0-9]+)=\", r\"\\1\\2+\\3=\", line)\n",
    "    #line = re.sub(r\"([^+-])([0-9]+)-([0-9]+)+([0-9]+)=\", r\"\\1\\2+\\3+\\4=\", line)\n",
    "    #line = re.sub(r\"([^+-])([0-9]+)-([0-9]+)-([0-9]+)-([0-9]+)=\", r\"\\1+\\2+\\3+\\4=\", line)    \n",
    "    return line"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('/tmp/list.txt') as f:\n",
    "    for line in f:\n",
    "        with open('WikiTableQuestions/wikidata/all_html/' + line.strip(), 'r') as f1:\n",
    "            content = f1.readline().strip()\n",
    "        with open('WikiTableQuestions/wikidata/all_html/' + line.strip(), 'w') as f1:\n",
    "            print >> f1, regex_equation(content)\n",
    "        #print regex_equation(content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "with open('WikiTableQuestions/wikidata/train_gold.json') as f:\n",
    "    raw_output = json.load(f)\n",
    "    output = [\"/\".join([str(__) for __ in _]) for _ in raw_output]\n",
    "\n",
    "with open('WikiTableQuestions/wikidata/all_training.tsv', 'w') as fw:\n",
    "    print >> fw, \"id\\tutterance\\tcontext\\ttargetValue\"\n",
    "    with open('WikiTableQuestions/wikidata/train.jsonl') as f:\n",
    "        data = jsonlines.Reader(f)\n",
    "        idx = 1\n",
    "        for d, o in zip(data, output):\n",
    "            print >> fw, \"{}\\t{}\\t{}\\t{}\".format(idx, d['question'].replace('\\t', ''), d['table_id'], o)\n",
    "            idx += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas\n",
    "\n",
    "files = pandas.read_table('WikiTableQuestions/wikidata/all_training.tsv', delimiter=\"\\t\")\n",
    "\n",
    "output = []\n",
    "for f in os.listdir('WikiTableQuestions/wikidata/all_html/'):\n",
    "    if \"html\" in f:\n",
    "        numbering = f.split('.')[0]\n",
    "        results = files[files.context == numbering]\n",
    "        \n",
    "        for q, a in zip(results.utterance, results.targetValue):\n",
    "            tmp = {}\n",
    "            tmp[\"url\"] = 'https://raw.githubusercontent.com/wenhuchen/Interface/master/WikiTableQuestions/wikidata/all_html/{}.html'.format(numbering)\n",
    "            tmp[\"q\"] = q\n",
    "            tmp[\"a\"] = a\n",
    "            output.append(tmp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('WikiTableQuestions/data/all_training_new.tsv', 'w') as fw:\n",
    "    with open('WikiTableQuestions/data/all_training.tsv') as f:\n",
    "        for line in f:\n",
    "            if \"csv\" in line:\n",
    "                try:\n",
    "                    t1, t2, t3, t4 = line.strip().split('\\t')\n",
    "                except Exception:\n",
    "                    print line.strip()\n",
    "                    continue\n",
    "                    \n",
    "                try:\n",
    "                    f1, f2, f3 = t3.split('/')\n",
    "                except Exception:\n",
    "                    print t3.strip()\n",
    "                    continue\n",
    "                    \n",
    "                t3 = f2.split('-')[0] + \"-\" + f3\n",
    "                new_line = \"\\t\".join([t1, t2, t3, t4])\n",
    "                print >> fw, new_line\n",
    "            else:\n",
    "                print >> fw, line.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import json\n",
    "import csv\n",
    "import pandas\n",
    "\n",
    "result = pandas.read_csv('results_v2.csv')\n",
    "filtered = result[result.AssignmentStatus==\"Approved\"]\n",
    "\n",
    "items = []\n",
    "for i, r in filtered.iterrows():\n",
    "    if \"Input.url10\" in r:\n",
    "        num = 10\n",
    "    else:\n",
    "        num = 5\n",
    "    for i in range(1, num+1):\n",
    "        if r['Input.a{}'.format(i)] not in ['None', 'n/a'] and \" par \" not in r['Answer.d{}'.format(i)].lower():\n",
    "            items.append((r['Input.url{}'.format(i)], r['Input.q{}'.format(i)], r['Input.a{}'.format(i)], r['Answer.d{}'.format(i)]))\n",
    "\n",
    "with open('v2_results.json', 'w') as f:\n",
    "    json.dump(items, f, indent=2)\n",
    "    \n",
    "with open('v2_rewrite_input.csv', 'w') as f:\n",
    "    #json.dump(items, f, indent=2)\n",
    "    fields = []\n",
    "    for j in range(1, num + 1):\n",
    "        fields.append(\"url{}\".format(j))\n",
    "        fields.append(\"s{}\".format(j))\n",
    "        \n",
    "    writer = csv.DictWriter(f, fieldnames=fields)\n",
    "    writer.writeheader()\n",
    "    for i in range(0, len(items) - num, num):\n",
    "        elem = {}\n",
    "        for j in range(num):\n",
    "            elem[\"url{}\".format(j + 1)] = items[i + j][0]\n",
    "            elem[\"s{}\".format(j + 1)] = items[i + j][3]\n",
    "            \n",
    "        writer.writerow(elem)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import json\n",
    "import pandas\n",
    "\n",
    "with open('v2_results.json', 'r') as f:\n",
    "    data = json.load(f)\n",
    "\n",
    "keys = [_[1] for _ in data]\n",
    "result = pandas.read_csv('results_v2_new.csv')\n",
    "filtered = result[result.AssignmentStatus==\"Approved\"]\n",
    "\n",
    "items = []\n",
    "for i, r in filtered.iterrows():\n",
    "    if \"Input.url10\" in r:\n",
    "        num = 10\n",
    "    else:\n",
    "        num = 5\n",
    "        \n",
    "    for i in range(1, num+1):\n",
    "        if r['Input.a{}'.format(i)] not in ['None', 'n/a'] and \" par \" not in r['Answer.d{}'.format(i)].lower() and r['Input.q{}'.format(i)] not in keys:\n",
    "            items.append((r['Input.url{}'.format(i)], r['Input.q{}'.format(i)], r['Input.a{}'.format(i)], r['Answer.d{}'.format(i)]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "\n",
    "result = pandas.read_csv('results_v3.csv')\n",
    "\n",
    "filtered = result[result.AssignmentStatus==\"Approved\"]\n",
    "\n",
    "new_items = []\n",
    "for i, r in filtered.iterrows():\n",
    "    if \"Input.url10\" in r:\n",
    "        num = 10\n",
    "    else:\n",
    "        num = 5\n",
    "    for i in range(1, num + 1):\n",
    "        if r['Input.a{}'.format(i)] not in ['None', 'n/a'] and \" par \" not in str(r['Answer.d{}'.format(i)]).lower():\n",
    "            new_items.append((r['Input.url{}'.format(i)], r['Input.q{}'.format(i)], r['Input.a{}'.format(i)], r['Answer.d{}'.format(i)]))\n",
    "\n",
    "new_items = new_items + items\n",
    "\n",
    "with open('v3_results.json', 'w') as f:\n",
    "    json.dump(new_items, f, indent=2)\n",
    "    \n",
    "num = 5\n",
    "with open('v3_rewrite_input.csv', 'w') as f:\n",
    "    #json.dump(items, f, indent=2)\n",
    "    fields = []\n",
    "    for j in range(1, num + 1):\n",
    "        fields.append(\"url{}\".format(j))\n",
    "        fields.append(\"s{}\".format(j))\n",
    "        \n",
    "    writer = csv.DictWriter(f, fieldnames=fields)\n",
    "    writer.writeheader()\n",
    "    for i in range(0, len(new_items) - num, num):\n",
    "        elem = {}\n",
    "        for j in range(num):\n",
    "            elem[\"url{}\".format(j + 1)] = new_items[i + j][0]\n",
    "            elem[\"s{}\".format(j + 1)] = new_items[i + j][3]\n",
    "            \n",
    "        writer.writerow(elem)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "s = \"\"\n",
    "for month, abb in [(\"January\", \"Jan\"), (\"February\", \"Feb\"), (\"March\", \"March\"), (\"April\", \"April\"), \n",
    "                   (\"May\", \"May\"), (\"June\",\"June\") , (\"July\", \"July\"), (\"August\", \"Aug\"), \n",
    "                   (\"September\", \"Sep\"), (\"October\", \"Oct\"), (\"November\", \"Nov\"), (\"December\", \"Dec\")]:\n",
    "    s += \";s/({})([1-9]+)/\\\\1 \\\\2/g;s/({}),([1-9]+)//g\".format(month, abb)\n",
    "print s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas\n",
    "import simplejson\n",
    "\n",
    "result = pandas.read_csv('v2_refine.csv')\n",
    "filtered_result = result[result.AssignmentStatus==\"Approved\"]\n",
    "\n",
    "items = []\n",
    "num = 10\n",
    "for i, r in filtered_result.iterrows():\n",
    "    for i in range(1, num+1):\n",
    "        items.append((r[\"Input.url{}\".format(i)],\"-\" ,\"-\" , r[\"Answer.d{}\".format(i)]))\n",
    "\n",
    "with open('v2_rewrite.json', 'w') as f:    \n",
    "    simplejson.dump(items, f, encoding='utf-8', ignore_nan=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "#import pandas\n",
    "import re\n",
    "import csv\n",
    "\n",
    "def dealwithNum(inp):\n",
    "    inp = re.sub(r\"([^0-9.])(0)([0-9])%\", r\"\\1\\2.\\3%\", inp)\n",
    "    \n",
    "    inp = re.sub(r\"([^0-9.])([1-9])([0-9])([0-9])%\", r\"\\1\\2\\3.\\4%\", inp)\n",
    "    inp = re.sub(r\"([^0-9.])(0)([0-9])([0-9])%\", r\"\\1\\2.\\3\\4%\", inp)\n",
    "    \n",
    "    inp = re.sub(r\"([^0-9.])([1-9])([0-9])([0-9])([0-9])%\", r\"\\1\\2\\3.\\4\\5%\", inp)\n",
    "    inp = re.sub(r\"([^0-9.])(0)([0-9])([0-9])([0-9])%\", r\"\\1\\2.\\3\\4\\5%\", inp)\n",
    "    \n",
    "    inp = re.sub(r\"10000%\", r\"100.00%\", inp)\n",
    "    inp = re.sub(r\"1000%\", r\"100.0%\", inp)\n",
    "    \n",
    "    inp = re.sub(r\"([^0-9.])([1-9])([0-9])([0-9])([0-9])([0-9])%\", r\"\\1\\2\\3.\\4\\5\\6%\", inp)\n",
    "    inp = re.sub(r\"([^0-9.])(0)([0-9])([0-9])([0-9])([0-9])%\", r\"\\1\\2.\\3\\4\\5\\6%\", inp)\n",
    "    \n",
    "    inp = re.sub(r\",([0-9])%\", r\".\\1%\", inp)\n",
    "    inp = re.sub(r\",([0-9])([0-9])%\", r\".\\1\\2%\", inp)\n",
    "    inp = re.sub(r\",([0-9])([0-9])([0-9])%\", r\".\\1\\2\\3%\", inp)\n",
    "    \n",
    "    \n",
    "    #inp = re.sub(r\"([0-9]),([0-9])\", r\"\\1\\2\", inp)\n",
    "    return inp\n",
    "\n",
    "\n",
    "with open('all_html.csv', 'w') as f:\n",
    "    fields = [\"url\", \"content\"]\n",
    "    writer = csv.DictWriter(f, fieldnames=fields)\n",
    "    writer.writeheader()\n",
    "    for f in os.listdir('WikiTableQuestions/wikidata/all_html/'):\n",
    "        with open('WikiTableQuestions/wikidata/all_html/' + f, 'r') as fr:\n",
    "            string = fr.readline().strip()\n",
    "        \n",
    "        writer.writerow({\"url\": f, \"content\": dealwithNum(string)})\n",
    "\n",
    "#inp_s = \"<td>123% <td> 0123% <td>012% <td>1233%  <td>12333% <td>10000% <td>01333%\"\n",
    "#dealwithNum(inp_s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas\n",
    "\n",
    "table = pandas.read_csv('all_html.csv')\n",
    "\n",
    "for i, item in table.iterrows():\n",
    "    name = item['url']\n",
    "    content = item['content']\n",
    "    with open('WikiTableQuestions/wikidata/all_html/{}'.format(name), 'w') as f:\n",
    "        print >> f, content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import csv\n",
    "import pandas\n",
    "\n",
    "result = pandas.read_csv('results_v4_1.csv')\n",
    "\n",
    "filtered = result[result.AssignmentStatus==\"Approved\"]\n",
    "\n",
    "items = []\n",
    "for i, r in filtered.iterrows():\n",
    "    if \"Input.url10\" in r:\n",
    "        num = 10\n",
    "    else:\n",
    "        num = 5\n",
    "    for i in range(1, num+1):\n",
    "        if r['Input.a{}'.format(i)] not in ['None', 'n/a'] and \" par \" not in r['Answer.d{}'.format(i)].lower():\n",
    "            items.append((r['Input.url{}'.format(i)], r['Input.q{}'.format(i)], r['Input.a{}'.format(i)], r['Answer.d{}'.format(i)]))\n",
    "\n",
    "with open('v4_1_results.json', 'w') as f:\n",
    "    json.dump(items, f, indent=2)\n",
    "\n",
    "num = 5\n",
    "with open('v4_rewrite_input.csv', 'w') as f:\n",
    "    #json.dump(items, f, indent=2)\n",
    "    fields = []\n",
    "    for j in range(1, num + 1):\n",
    "        fields.append(\"url{}\".format(j))\n",
    "        fields.append(\"s{}\".format(j))\n",
    "    \n",
    "    #fields.extend([\"url11\", \"s11\"])\n",
    "    \n",
    "    writer = csv.DictWriter(f, fieldnames=fields)\n",
    "    writer.writeheader()\n",
    "    for i in range(0, len(items) - num, num):\n",
    "        elem = {}\n",
    "        for j in range(num):\n",
    "            elem[\"url{}\".format(j + 1)] = items[i + j][0]\n",
    "            elem[\"s{}\".format(j + 1)] = items[i + j][3]\n",
    "        \n",
    "        #elem['url11'] = \"https://raw.githubusercontent.com/wenhuchen/Interface/master/WikiTableQuestions/wikidata/all_html/2-1236260-1.html\"\n",
    "        #elem['s11'] = \"Total is the rank of total\"\n",
    "            \n",
    "        writer.writerow(elem)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import csv\n",
    "import pandas\n",
    "\n",
    "result1 = pandas.read_csv('data/blind1.csv')\n",
    "result2 = pandas.read_csv('data/blind2.csv')\n",
    "result = pandas.concat([result1, result2])\n",
    "\n",
    "filtered = result[result.AssignmentStatus==\"Approved\"]\n",
    "\n",
    "num = 5\n",
    "with open('blind_rewrite_input.csv', 'w') as f:\n",
    "    fields = []\n",
    "    fields.append(\"url1\")\n",
    "    fields.append(\"wiki1\")\n",
    "    fields.append(\"topic1\")\n",
    "    fields.extend(['s1', 's2', 's3', 's4', 's5'])\n",
    "        \n",
    "    writer = csv.DictWriter(f, fieldnames=fields)\n",
    "    writer.writeheader()\n",
    "    for i in range(0, len(filtered)):\n",
    "        elem = {'url1': filtered.iloc[i]['Input.url1'],\n",
    "                'wiki1': filtered.iloc[i]['Input.wiki1'],\n",
    "                'topic1': filtered.iloc[i]['Input.topic1']}\n",
    "        \n",
    "        for j in range(num):\n",
    "            elem[\"s{}\".format(j + 1)] = filtered.iloc[i]['Answer.d{}'.format(j + 1)].lower()\n",
    "            \n",
    "        writer.writerow(elem)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import pandas\n",
    "import csv\n",
    "import json\n",
    "import simplejson\n",
    "\n",
    "r1 = pandas.read_csv('partial_refine_v2.csv')\n",
    "r2 = pandas.read_csv('partial_refine_v3.csv')\n",
    "r1 = r1[r1.AssignmentStatus==\"Approved\"]\n",
    "r2 = r2[r2.AssignmentStatus==\"Approved\"]\n",
    "\n",
    "results = {}\n",
    "finished = []\n",
    "for i, r in r1.iterrows():\n",
    "    if \"Input.url10\" in r:\n",
    "        num = 10\n",
    "    else:\n",
    "        num = 5\n",
    "    for j in range(1, num + 1):\n",
    "        html_name = r['Input.url{}'.format(j)].split('/')[-1]\n",
    "        t = r['Answer.d{}'.format(j)]\n",
    "        if t and isinstance(t, str) and t.lower() not in [\"na\", \"none\", \"n/a\", \"no\"]:\n",
    "            if html_name not in results:\n",
    "                results[html_name] = {\"text\": [], \"label\": []}\n",
    "            results[html_name]['text'].append(t)\n",
    "            results[html_name]['label'].append(1)\n",
    "        finished.append(r['Input.s{}'.format(j)])\n",
    "        \n",
    "for i, r in r2.iterrows():\n",
    "    if \"Input.url10\" in r:\n",
    "        num = 10\n",
    "    else:\n",
    "        num = 5\n",
    "    for j in range(1, num + 1):\n",
    "        html_name = r['Input.url{}'.format(j)].split('/')[-1]\n",
    "        t = r['Answer.d{}'.format(j)]\n",
    "        if t and isinstance(t, str) and t.lower() not in [\"na\", \"none\", \"n/a\", \"no\"]:\n",
    "            if html_name not in results:\n",
    "                results[html_name] = {\"text\": [], \"label\": []}        \n",
    "            results[html_name]['text'].append(r['Answer.d{}'.format(j)])\n",
    "            results[html_name]['label'].append(1)\n",
    "        finished.append(r['Input.s{}'.format(j)])\n",
    "\n",
    "r1 = pandas.read_csv('partial_neg_v2.csv')\n",
    "r2 = pandas.read_csv('partial_neg_v3.csv')\n",
    "r1 = r1[r1.AssignmentStatus==\"Approved\"]\n",
    "r2 = r2[r2.AssignmentStatus==\"Approved\"]\n",
    "\n",
    "for i, r in r1.iterrows():\n",
    "    if \"Input.url10\" in r:\n",
    "        num = 10\n",
    "    else:\n",
    "        num = 5\n",
    "    for j in range(1, num + 1):\n",
    "        t = r['Answer.d{}'.format(j)]\n",
    "        if t and isinstance(t, str) and t.lower() not in [\"na\", \"none\", \"n/a\", \"no\"]:\n",
    "            html_name = r['Input.url{}'.format(j)].split('/')[-1]\n",
    "            if html_name not in results:\n",
    "                results[html_name] = {\"text\": [], \"label\": []}\n",
    "            if r['Answer.d{}'.format(j)] not in results[html_name]['text']:\n",
    "                results[html_name]['text'].append(r['Answer.d{}'.format(j)])\n",
    "                results[html_name]['label'].append(-1)\n",
    "\n",
    "for i, r in r2.iterrows():\n",
    "    if \"Input.url10\" in r:\n",
    "        num = 10\n",
    "    else:\n",
    "        num = 5\n",
    "    for j in range(1, num + 1):\n",
    "        t = r['Answer.d{}'.format(j)]\n",
    "        if t and isinstance(t, str) and t.lower() not in [\"na\", \"none\", \"n/a\", \"no\"]:\n",
    "            html_name = r['Input.url{}'.format(j)].split('/')[-1]\n",
    "            if html_name not in results:\n",
    "                results[html_name] = {\"text\": [], \"label\": []}        \n",
    "            results[html_name]['text'].append(r['Answer.d{}'.format(j)])\n",
    "            results[html_name]['label'].append(-1)\n",
    "            \n",
    "with open('READY/prelim.json', 'w') as f:\n",
    "    simplejson.dump(results, f, encoding='utf-8', ignore_nan=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Prepare results\n",
    "inp_v2 = pandas.read_csv('v2_rewrite_input.csv')\n",
    "inp_v3 = pandas.read_csv('v3_rewrite_input.csv')\n",
    "\n",
    "not_finished = []\n",
    "done = 0\n",
    "for i, r in inp_v2.iterrows():\n",
    "    if \"Input.url10\" in r:\n",
    "        num = 10\n",
    "    else:\n",
    "        num = 5\n",
    "    for j in range(1, num + 1):\n",
    "        if r['s{}'.format(j)] not in finished:\n",
    "            not_finished.append((r['url{}'.format(j)], r['s{}'.format(j)]))\n",
    "        else:\n",
    "            done += 1\n",
    "        \n",
    "for i, r in inp_v3.iterrows():\n",
    "    if \"Input.url10\" in r:\n",
    "        num = 10\n",
    "    else:\n",
    "        num = 5\n",
    "    for j in range(1, num + 1):\n",
    "        if r['s{}'.format(j)] not in finished:\n",
    "            not_finished.append((r['url{}'.format(j)], r['s{}'.format(j)]))\n",
    "        else:\n",
    "            done += 1\n",
    "\n",
    "num = 5\n",
    "with open('v23_left_rewrite_input.csv', 'w') as f:\n",
    "    #json.dump(items, f, indent=2)\n",
    "    fields = []\n",
    "    for j in range(1, num + 1):\n",
    "        fields.append(\"url{}\".format(j))\n",
    "        fields.append(\"s{}\".format(j))\n",
    "    \n",
    "    #fields.extend([\"url11\", \"s11\"])\n",
    "    \n",
    "    writer = csv.DictWriter(f, fieldnames=fields)\n",
    "    writer.writeheader()\n",
    "    for i in range(0, len(not_finished) - num, num):\n",
    "        elem = {}\n",
    "        for j in range(num):\n",
    "            elem[\"url{}\".format(j + 1)] = not_finished[i + j][0]\n",
    "            elem[\"s{}\".format(j + 1)] = not_finished[i + j][1]\n",
    "        writer.writerow(elem)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = pandas.read_csv(\"input_file_yunkai.csv\")\n",
    "keys = set()\n",
    "for i, r in result.iterrows():\n",
    "    if \"Input.url10\" in r:\n",
    "        num = 10\n",
    "    else:\n",
    "        num = 5\n",
    "    for j in range(1, num + 1):\n",
    "        keys.add(r[\"url{}\".format(j)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = pandas.read_csv(\"v4_rewrite_input.csv\")\n",
    "#keys = set()\n",
    "for i, r in result.iterrows():\n",
    "    if \"Input.url10\" in r:\n",
    "        num = 10\n",
    "    else:\n",
    "        num = 5\n",
    "    for j in range(1, num + 1):\n",
    "        if r[\"url{}\".format(j)] in keys:\n",
    "            keys.remove(r[\"url{}\".format(j)])\n",
    "unseen_tables = list(keys)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('READY/prelim.json') as f:\n",
    "    data = json.load(f)\n",
    "\n",
    "pos = []\n",
    "neg = []\n",
    "pos_length = 0\n",
    "neg_length = 0\n",
    "for k in data:\n",
    "    text = data[k]['text']\n",
    "    label = data[k]['label']\n",
    "    for t, l in zip(text, label):\n",
    "        if l == 1:\n",
    "            pos.append(t)\n",
    "            pos_length += len(t.split())\n",
    "        else:\n",
    "            neg.append(t)\n",
    "            neg_length += len(t.split())\n",
    "\n",
    "print len(pos), len(neg)\n",
    "print (pos_length + 0.0) / len(pos)\n",
    "print (neg_length + 0.0) / len(neg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "\n",
    "a = Counter()\n",
    "a.update([1,2,3])\n",
    "a.update([1,3,4])\n",
    "\n",
    "print a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import pandas\n",
    "import csv\n",
    "\n",
    "r1 = pandas.read_csv('/Users/wenhuchen/Downloads/Batch_3580679_batch_results.csv')\n",
    "r2 = pandas.read_csv('/Users/wenhuchen/Downloads/Batch_3584639_batch_results.csv')\n",
    "r1 = r1[r1.AssignmentStatus==\"Approved\"]\n",
    "r2 = r2[r2.AssignmentStatus==\"Approved\"]\n",
    "\n",
    "r3 = pandas.concat([r1, r2])\n",
    "finished = []\n",
    "\n",
    "r1_inp = pandas.read_csv('v23_left_rewrite_input.csv')\n",
    "r2_inp = pandas.read_csv('v4_rewrite_input.csv')\n",
    "r3_inp = pandas.concat([r1_inp, r2_inp])\n",
    "\n",
    "for i, r in r3.iterrows():\n",
    "    if \"Input.url10\" in r:\n",
    "        num = 10\n",
    "    else:\n",
    "        num = 5\n",
    "    for j in range(1, num + 1):\n",
    "        #if r[\"url{}\".format(j)] in keys:\n",
    "        finished.append(r[\"Input.s{}\".format(j)])\n",
    "\n",
    "print len(finished)\n",
    "unfinished = []\n",
    "done = 0\n",
    "finished_old = []\n",
    "for i,r in r3_inp.iterrows():\n",
    "    if \"url10\" in r:\n",
    "        num = 10\n",
    "    else:\n",
    "        num = 5\n",
    "    for j in range(1, num + 1):\n",
    "        #if r[\"url{}\".format(j)] in keys:\n",
    "        if r[\"s{}\".format(j)]  not in finished:\n",
    "            unfinished.append((r[\"url{}\".format(j)], r[\"s{}\".format(j)]))\n",
    "        else:\n",
    "            finished_old.append(r[\"s{}\".format(j)])\n",
    "            done += 1\n",
    "\n",
    "print done\n",
    "with open('v234_rest_rewrite_input.csv', 'w') as f:\n",
    "    #json.dump(items, f, indent=2)\n",
    "    num = 5\n",
    "    fields = []\n",
    "    for j in range(1, num + 1):\n",
    "        fields.append(\"url{}\".format(j))\n",
    "        fields.append(\"s{}\".format(j))\n",
    "        \n",
    "    writer = csv.DictWriter(f, fieldnames=fields)\n",
    "    writer.writeheader()\n",
    "    for i in range(0, len(unfinished) - num, num):\n",
    "        elem = {}\n",
    "        for j in range(num):\n",
    "            elem[\"url{}\".format(j + 1)] = unfinished[i + j][0]\n",
    "            elem[\"s{}\".format(j + 1)] = unfinished[i + j][1]\n",
    "            \n",
    "        writer.writerow(elem)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "r1 = pandas.read_csv('/Users/wenhuchen/Downloads/Batch_3583922_batch_results.csv')\n",
    "r2 = pandas.read_csv('/Users/wenhuchen/Downloads/Batch_3579309_batch_results.csv')\n",
    "r3 = pandas.read_csv('/Users/wenhuchen/Downloads/Batch_3579785_batch_results.csv')\n",
    "\n",
    "r1 = r1[r1.AssignmentStatus==\"Approved\"]\n",
    "r2 = r2[r2.AssignmentStatus==\"Approved\"]\n",
    "r3 = r3[r3.AssignmentStatus==\"Approved\"]\n",
    "\n",
    "r4 = pandas.concat([r1, r2, r3])\n",
    "finished = []\n",
    "\n",
    "\n",
    "for i, r in r4.iterrows():\n",
    "    if \"Input.url10\" in r:\n",
    "        num = 10\n",
    "    else:\n",
    "        num = 5\n",
    "    for j in range(1, num + 1):\n",
    "        finished.append(r[\"Input.s{}\".format(j)])\n",
    "print len(finished)\n",
    "\n",
    "r1_inp = pandas.read_csv('v2_rewrite_input.csv')\n",
    "r2_inp = pandas.read_csv('v3_rewrite_input.csv')\n",
    "r3_inp = pandas.read_csv('v4_rewrite_input.csv')\n",
    "r4_inp = pandas.concat([r1_inp, r2_inp, r3_inp])\n",
    "unfinished = []\n",
    "done = 0\n",
    "for i,r in r4_inp.iterrows():\n",
    "    if \"url10\" in r:\n",
    "        num = 10\n",
    "    else:\n",
    "        num = 5\n",
    "    for j in range(1, num + 1):\n",
    "        #if r[\"url{}\".format(j)] in keys:\n",
    "        if r[\"s{}\".format(j)]  not in finished:\n",
    "            unfinished.append((r[\"url{}\".format(j)], r[\"s{}\".format(j)]))\n",
    "        else:\n",
    "            done += 1\n",
    "\n",
    "print done\n",
    "with open('v234_rest_rewrite_fake_input.csv', 'w') as f:\n",
    "    num = 5\n",
    "    fields = []\n",
    "    for j in range(1, num + 1):\n",
    "        fields.append(\"url{}\".format(j))\n",
    "        fields.append(\"s{}\".format(j))\n",
    "        \n",
    "    writer = csv.DictWriter(f, fieldnames=fields)\n",
    "    writer.writeheader()\n",
    "    for i in range(0, len(unfinished) - num, num):\n",
    "        elem = {}\n",
    "        for j in range(num):\n",
    "            elem[\"url{}\".format(j + 1)] = unfinished[i + j][0]\n",
    "            elem[\"s{}\".format(j + 1)] = unfinished[i + j][1]\n",
    "            \n",
    "        writer.writerow(elem)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "string = \"faf*\\\\\"\n",
    "string = string.replace(r'*', '')\n",
    "string.replace('\\\\', '')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "he has n't done it\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "\n",
    "def substitute(string):\n",
    "    string = string.lower()\n",
    "    \n",
    "    string = string.replace(r\"\\n\", \"\")\n",
    "    string = string.replace(r\"\\\\\", \"\")\n",
    "    string = string.replace(r\"!$\", \"\")\n",
    "    string = string.replace(r\"#\", \"\")\n",
    "    string = string.replace('–', '-')\n",
    "    string = string.replace('−', '-')\n",
    "    string = string.replace('−', '-')\n",
    "    string = string.replace('â', '')\n",
    "    string = string.replace('“', '')\n",
    "    string = string.replace('€', '')    \n",
    "    string = string.replace(\"√\", \"\")\n",
    "    string = string.replace(r'\"', '')\n",
    "    string = string.replace(r'\\.0 ', '')\n",
    "    string = string.replace(r'¹', '')\n",
    "    string = string.replace(r'‡', '')\n",
    "    string = string.replace(r'«', '')\n",
    "    string = string.replace(r'»', '')\n",
    "    string = string.replace(r';', ',')\n",
    "    string = string.replace(r'$', '')\n",
    "    string = string.replace(r'£', '')\n",
    "    string = string.replace(r'¥', '')\n",
    "    string = string.replace(r'*', '')\n",
    "    string.replace('\\\\', '')\n",
    "    \n",
    "    string = string.replace(r\"'s\", \" 's\")\n",
    "    string = string.replace(r\"'re\", \" 're\")\n",
    "    string = string.replace(r\"'ve\", \" 've\")\n",
    "    #string = string.replace(r\"'nt\", \" 'nt\")\n",
    "    #string = string.replace(r\"'ll\", \" 'll\")\n",
    "    #string = string.replace(r\"\\b' \", \" 'nt\")\n",
    "    #string = string.replace(r\" ' \", \"'\")\n",
    "    string = string.replace(r'@', ' ')\n",
    "    string = string.replace(r'`', '')\n",
    "    string = string.replace(r'•', ' ')\n",
    "    string = string.replace(r'·', ' ')\n",
    "    string = string.replace(r'²', ' square')\n",
    "    string = string.replace(r'\\[', '')\n",
    "    string = string.replace(r'\\]', '')\n",
    "    string = string.replace(r'\\{', '')\n",
    "    string = string.replace(r'\\}', '')\n",
    "    string = string.replace(r'\\|', '')\n",
    "    string = string.replace(r'\\^', '')\n",
    "    string = string.replace(r'”', '')\n",
    "\n",
    "    string = re.sub(r'\\[.+\\]', ' ', string)\n",
    "    string = re.sub(r',([0-9]{3})(?![0-9])', r'\\1', string)\n",
    "\n",
    "    string = re.sub(r'([^,]),', r\"\\1 ,\", string)\n",
    "    string = re.sub(r',([^,])', r\", \\1\", string)\n",
    "    string = re.sub(r'([^(])\\(', r\"\\1 (\", string)\n",
    "    string = re.sub(r'\\)([^)])', r\") \\1\", string)\n",
    "    string = re.sub(r'\\.\\)', r\")\", string)\n",
    "    string = re.sub(r'([0-9]+)-([0-9]+)-([0-9]+)-([0-9]+)=', r\"\\1+\\2+\\3+\\4=\", string)\n",
    "    string = re.sub(r'([0-9]+)-([0-9]+)-([0-9]+)=', r\"\\1+\\2+\\3=\", string)\n",
    "    string = re.sub(r'([0-9]+)-([0-9]+)=', r\"\\1+\\2=\", string)\n",
    "    string = re.sub(r\"' +([0-9])\", r\"'\\1\", string)\n",
    "\n",
    "    string = re.sub(r\"([^ ])\\+\", r\"\\1 +\", string)\n",
    "    string = re.sub(r\"\\+([^ ])\", r\"+ \\1\", string)\n",
    "    string = re.sub(r\"([^ ])=\", r\"\\1 =\", string)\n",
    "    string = re.sub(r\"=([^ ])\", r\"= \\1\", string)\n",
    "\n",
    "    string = re.sub(r\"-([^- ])\", r\"- \\1\", string)\n",
    "    string = re.sub(r\"([^- ])-\", r\"\\1 -\", string)\n",
    "    string = re.sub(r\"-([^- ])\", r\"- \\1\", string)\n",
    "    string = re.sub(r\"/([^ ])\", r\"/ \\1\", string)\n",
    "    string = re.sub(r\"([^ ])/\", r\"\\1 /\", string)\n",
    "    string = string.replace(r'()', '')\n",
    "    string = re.sub(r'([^0-9])\\.', r'\\1', string)\n",
    "\n",
    "    string = re.sub(r'([^0-9 ]):', r\"\\1 :\", string)\n",
    "    string = re.sub(r':([^0-9 ])', r\": \\1\", string)\n",
    "    string = re.sub(r'([0-9])pm', r\"\\1 pm\", string)\n",
    "    string = re.sub(r'([0-9])am', r\"\\1 am\", string)\n",
    "    string = re.sub(r'([0-9])rpm', r\"\\1 rpm\", string)\n",
    "    string = re.sub(r'([0-9])km', r\"\\1 km\", string)\n",
    "    string = re.sub(r'([0-9])cm', r\"\\1 cm\", string)\n",
    "    string = re.sub(r'([0-9])m', r\"\\1 m\", string)\n",
    "    string = re.sub(r'([0-9])mm', r\"\\1 mm\", string)\n",
    "    string = re.sub(r'([0-9])kg', r\"\\1 kg\", string)\n",
    "    string = re.sub(r'([0-9])g', r\"\\1 g\", string)\n",
    "    string = re.sub(r'([0-9])kw', r\"\\1 kw\", string)\n",
    "    string = re.sub(r'([0-9])kv', r\"\\1 kv\", string)\n",
    "    string = re.sub(r'([0-9])mph', r\"\\1 mph\", string)\n",
    "    #string = re.sub(r'([0-9])@', r\"\\1 @\", string)\n",
    "    #string = re.sub(r'@([0-9])', r\"@ \\1\", string)\n",
    "    string = re.sub(r'category : articles with hcards', r\"\", string)\n",
    "    string = re.sub(r'category : articles with hcard', r\"\", string)\n",
    "    string = re.sub(r'category : articles without hcard', r\"\", string)\n",
    "    \n",
    "    string = re.sub(r' 1 time', r\" one time\", string)\n",
    "    string = re.sub(r' 2 time', r\" two time\", string)\n",
    "    string = re.sub(r' 3 time', r\" three time\", string)\n",
    "    string = re.sub(r' 4 time', r\" four time\", string)\n",
    "    string = re.sub(r' 5 time', r\" five time\", string)\n",
    "    string = re.sub(r' 6 time', r\" six time\", string)\n",
    "    string = re.sub(r' 7 time', r\" seven time\", string)\n",
    "    string = re.sub(r' 8 time', r\" eight time\", string)\n",
    "    string = re.sub(r' 9 time', r\" nine time\", string)\n",
    "    string = re.sub(r' 10 time', r\" ten time\", string)\n",
    "    string = re.sub(r' 11 time', r\" eleven time\", string)\n",
    "    string = re.sub(r' 12 time', r\" twelve time\", string)\n",
    "    string = re.sub(r' 13 time', r\" thirteen time\", string)\n",
    "    string = re.sub(r' 14 time', r\" fourteen time\", string)\n",
    "    string = re.sub(r' 15 time', r\" fifteen time\", string)\n",
    "    string = re.sub(r' 16 time', r\" sixteen time\", string)\n",
    "    string = re.sub(r' 17 time', r\" seventeen time\", string)\n",
    "    string = re.sub(r' 18 time', r\" eighteen time\", string)\n",
    "    string = re.sub(r' 19 time', r\" nineteen time\", string)\n",
    "    string = re.sub(r' 20 time', r\" twenty time\", string)\n",
    "    \n",
    "    string = re.sub(r'once', r\"one time\", string)\n",
    "    string = re.sub(r'twice', r\"two times\", string)  \n",
    "    string = re.sub(r\"\\.+$\", '', string)\n",
    "    string = re.sub(r',+$', '', string)    \n",
    "    string = re.sub(r'\\s+', ' ', string)\n",
    "    string = re.sub(r'\\.+','.',string)\n",
    "    string = re.sub(r',+',',',string)\n",
    "    string = re.sub(r'^ ', '', string)\n",
    "    string = re.sub(r' $', '', string)\n",
    "    #string = re.sub('70 - 76 - 68 - 214', '70 + 76 + 68 = 214', string)\n",
    "    string = re.sub('turkish cup was 56', 'turkish cup was 54', string)\n",
    "    #string = re.sub('brigido iriarte', 'brígido iriarte', string)\n",
    "    return string\n",
    "\n",
    "#print substitute(\"fa\\g##ga,/// she got(ff)ff...\")\n",
    "string = \"he hasn't done it\"\n",
    "print re.sub(r\"n't\", \" n't\", string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#substitute('the new york jets won in san diego againtst the chargers by 7 points (34-27) in september of 1969.')\n",
    "# encoding=utf8\n",
    "import sys\n",
    "reload(sys)\n",
    "sys.setdefaultencoding('utf8')\n",
    "\n",
    "import json\n",
    "with open('challenge/blind_test.json') as f:\n",
    "    data = json.load(f)\n",
    "new_data = {}\n",
    "for k, d in data.items():\n",
    "    new_data[k] = [substitute(d[0]), d[1], d[2]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('challenge/blind_test_tokenized.json', 'w') as f:\n",
    "    json.dump(new_data, f, indent=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def blacklist(string):\n",
    "    black = ['bye', \"no chart for\", \"km (mi)\", \"lb·ft\", \"ft (m)\", \"kg (lb)\", \n",
    "             \"a report of report\", \"report was report\", \"is 'report'\", \"?\"]\n",
    "    for b in black:\n",
    "        if b in string:\n",
    "            return True\n",
    "    return False\n",
    "\n",
    "with open('data/short_subset.txt') as f:\n",
    "    limit_length = [_.strip() for _ in f.readlines()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# encoding=utf8\n",
    "import sys\n",
    "import pandas\n",
    "\n",
    "r1_1 = pandas.read_csv(\"/Users/wenhuchen/Downloads/harvest/refine_0.csv\")\n",
    "r1_2 = pandas.read_csv(\"/Users/wenhuchen/Downloads/harvest/refine_1.csv\")\n",
    "\n",
    "r1 = pandas.concat([r1_1, r1_2])\n",
    "r1 = r1[r1.AssignmentStatus==\"Approved\"]\n",
    "\n",
    "f = open('/tmp/output.txt', 'w')\n",
    "\n",
    "index = 0\n",
    "finished = {}\n",
    "trash = []\n",
    "for i,r in r1.iterrows():\n",
    "    if \"Input.url10\" in r:\n",
    "        num = 10\n",
    "    else:\n",
    "        num = 5\n",
    "    for j in range(1, num + 1):\n",
    "        orig_input = r[\"Answer.d{}\".format(j)]\n",
    "        html_name = r['Input.url{}'.format(j)].split('/')[-1] + '.csv'\n",
    "        if html_name not in limit_length:\n",
    "            continue\n",
    "        if isinstance(orig_input, str) and orig_input.lower() in [\"na\", \"n/a\", \"no\"]:\n",
    "            #trash.append(html_name)\n",
    "            print \"error\"\n",
    "        if isinstance(orig_input, str) and len(orig_input.split(' ')) > 3 and not blacklist(orig_input):\n",
    "            replaced_sent = substitute(orig_input)\n",
    "            if 'hcard' not in replaced_sent:\n",
    "                print >> f, replaced_sent\n",
    "                index += 1\n",
    "                if html_name not in finished:\n",
    "                    finished[html_name] = [[replaced_sent], [1]]\n",
    "                else:\n",
    "                    finished[html_name][0].append(replaced_sent)\n",
    "                    finished[html_name][1].append(1)\n",
    "                    \n",
    "r2_1 = pandas.read_csv(\"/Users/wenhuchen/Downloads/harvest/refine_2.csv\")\n",
    "r2_2 = pandas.read_csv(\"/Users/wenhuchen/Downloads/harvest/refine_3.csv\")\n",
    "r2_3 = pandas.read_csv(\"/Users/wenhuchen/Downloads/harvest/refine_4.csv\")\n",
    "\n",
    "r2 = pandas.concat([r2_1, r2_2, r2_3])\n",
    "r2 = r2[r2.AssignmentStatus==\"Approved\"]\n",
    "\n",
    "for i,r in r2.iterrows():\n",
    "    if \"Input.url10\" in r:\n",
    "        num = 10\n",
    "    else:\n",
    "        num = 5\n",
    "    for j in range(1, num + 1):\n",
    "        orig_input = r[\"Answer.d{}\".format(j)]\n",
    "        html_name = r['Input.url{}'.format(j)].split('/')[-1] + '.csv'\n",
    "        if html_name not in limit_length:\n",
    "            continue     \n",
    "        if isinstance(orig_input, str) and orig_input.lower() in [\"na\", \"n/a\", \"no\"]:\n",
    "            print \"error\"\n",
    "        if isinstance(orig_input, str) and len(orig_input.split(' ')) > 3 and not blacklist(orig_input):\n",
    "            replaced_sent = substitute(orig_input)\n",
    "            if 'hcard' not in replaced_sent:\n",
    "                print >> f, replaced_sent\n",
    "                index += 1\n",
    "                if html_name not in finished:\n",
    "                    finished[html_name] = [[replaced_sent], [1]]\n",
    "                else:\n",
    "                    finished[html_name][0].append(replaced_sent)\n",
    "                    finished[html_name][1].append(1)\n",
    "\n",
    "\n",
    "r3_1 = pandas.read_csv(\"/Users/wenhuchen/Downloads/harvest/fake_0.csv\")\n",
    "r3_2 = pandas.read_csv(\"/Users/wenhuchen/Downloads/harvest/fake_1.csv\")\n",
    "r3_3 = pandas.read_csv(\"/Users/wenhuchen/Downloads/harvest/fake_2.csv\")\n",
    "r3_4 = pandas.read_csv(\"/Users/wenhuchen/Downloads/harvest/fake_3.csv\")\n",
    "\n",
    "r3 = pandas.concat([r3_1, r3_2, r3_3, r3_4])\n",
    "r3 = r3[r3.AssignmentStatus==\"Approved\"]\n",
    "\n",
    "f = open('/tmp/output.txt', 'w')\n",
    "\n",
    "index = 0\n",
    "for i,r in r3.iterrows():\n",
    "    if \"Input.url10\" in r:\n",
    "        num = 10\n",
    "    else:\n",
    "        num = 5\n",
    "    for j in range(1, num + 1):\n",
    "        orig_input = r[\"Answer.d{}\".format(j)]\n",
    "        html_name = r['Input.url{}'.format(j)].split('/')[-1] + '.csv'\n",
    "        if html_name not in limit_length:\n",
    "            continue\n",
    "        if isinstance(orig_input, str) and orig_input.lower() in [\"na\", \"n/a\", \"no\"]:\n",
    "            print \"error\"\n",
    "        if isinstance(orig_input, str) and len(orig_input.split(' ')) > 3 and not blacklist(orig_input):\n",
    "            replaced_sent = substitute(orig_input)\n",
    "            if 'hcard' not in replaced_sent:\n",
    "                print >> f, replaced_sent\n",
    "                index += 1\n",
    "                if html_name not in finished:\n",
    "                    finished[html_name] = [[replaced_sent], [0]]\n",
    "                else:\n",
    "                    finished[html_name][0].append(replaced_sent)\n",
    "                    finished[html_name][1].append(0)\n",
    "\n",
    "f.close()\n",
    "print index\n",
    "import json\n",
    "with open(\"READY/pre_training_all.json\", 'w') as f:\n",
    "    json.dump(finished, f, indent=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "print trash\n",
    "import os\n",
    "import shutil\n",
    "\n",
    "print trash\n",
    "for l in trash + [\"2-11916083-12.html\", \"2-1236260-1.html\"]:\n",
    "    try:\n",
    "        shutil.move(\"data/all_csv/\" + l + \".csv\", \"data/trash_csv/\" + l + \".csv\")\n",
    "    except Exception:\n",
    "        print \"error, {}\".format(\"data/all_csv/\" + l + \".csv\", \"data/trash_csv/\" + l + \".csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "import urllib2\n",
    "import csv\n",
    "\n",
    "url = open('WikiTableQuestions/wikidata/all_html/2-18178551-5.html').readline().strip()\n",
    "soup = BeautifulSoup(url, \"html.parser\")\n",
    "\n",
    "table = soup.findAll(\"table\", {\"class\":\"wikitable\"})[0]\n",
    "rows = table.findAll(\"tr\")\n",
    "\n",
    "with open('/tmp/editors.csv', \"w\") as f:\n",
    "    writer = csv.writer(f)\n",
    "    for row in rows:\n",
    "        csv_row = []\n",
    "        for cell in row.findAll([\"td\", \"th\"]):\n",
    "            csv_row.append(cell.get_text())\n",
    "        writer.writerow(csv_row)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# encoding=utf8\n",
    "import sys\n",
    "reload(sys)\n",
    "sys.setdefaultencoding('utf8')\n",
    "\n",
    "import csv\n",
    "import os\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "for filename in os.listdir('WikiTableQuestions/wikidata/all_html/'):\n",
    "    if filename.endswith(\".html\"):\n",
    "        url = open('WikiTableQuestions/wikidata/all_html/' + filename).readline().strip()\n",
    "        soup = BeautifulSoup(url, \"html.parser\")\n",
    "        table = soup.findAll(\"table\", {\"class\":\"wikitable\"})\n",
    "        if len(table) > 0:\n",
    "            table = table[0]\n",
    "            rows = table.findAll(\"tr\")\n",
    "            with open('WikiTableQuestions/wikidata/all_csv/' + filename + '.csv', 'w') as csvfile:\n",
    "                spamwriter = csv.writer(csvfile, delimiter='#')\n",
    "                for row in rows:\n",
    "                    csv_row = []\n",
    "                    for cell in row.findAll([\"td\", \"th\"]):\n",
    "                        print cell.get_text()\n",
    "                        csv_row.append(substitute(cell.get_text()))\n",
    "                        print substitute(cell.get_text())\n",
    "                        print \"\"\n",
    "                    spamwriter.writerow(csv_row)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# encoding=utf8\n",
    "import sys\n",
    "reload(sys)\n",
    "sys.setdefaultencoding('utf8')\n",
    "import csv\n",
    "import urllib\n",
    "import json\n",
    "import os\n",
    "\n",
    "with open('READY/r1_training_all.json') as f:\n",
    "    data = json.load(f)\n",
    "    \n",
    "files = data.keys()\n",
    "\n",
    "unseen = []\n",
    "with open('data/short_subset.txt') as fs:\n",
    "    for f in fs:\n",
    "        f = f.strip()\n",
    "        if f not in files and f in tiny_mapping:\n",
    "            unseen.append(f)\n",
    "with open(\"data/v5_unseen.json\", 'w') as f:\n",
    "    json.dump(unseen, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# encoding=utf8\n",
    "import sys\n",
    "reload(sys)\n",
    "sys.setdefaultencoding('utf8')\n",
    "import json\n",
    "import urllib\n",
    "import json\n",
    "import os \n",
    "import csv\n",
    "\n",
    "with open('data/table_to_page_new.json', 'r') as f:\n",
    "    tiny_mapping = json.load(f)\n",
    "    \n",
    "with open(\"data/v5_unseen.json\", 'r') as f:\n",
    "    unseen = json.load(f)\n",
    "    \n",
    "with open('v5_write_input.csv', 'w') as f:\n",
    "    fields = [\"url1\", \"wiki1\", \"topic1\"]\n",
    "    writer = csv.DictWriter(f, fieldnames=fields)\n",
    "    writer.writeheader()\n",
    "\n",
    "    for k in unseen:\n",
    "        if k in tiny_mapping:\n",
    "            writer.writerow({\"url1\": 'https://raw.githubusercontent.com/wenhuchen/Table-Fact-Checking/master/data/all_csv/' + k, \n",
    "                            \"wiki1\": tiny_mapping[k][1], \"topic1\": tiny_mapping[k][0]})\n",
    "        else:\n",
    "            writer.writerow({\"url1\": 'https://raw.githubusercontent.com/wenhuchen/Table-Fact-Checking/master/data/all_csv/' + k, \"wiki1\": \"javascript:void(0)\", \"topic1\": \"None\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('/tmp/table_to_page.json', 'r') as f:\n",
    "    old_tiny_mapping1 = json.load(f)\n",
    "with open('data/table_to_page.json', 'r') as f:\n",
    "    old_tiny_mapping2 = json.load(f)\n",
    "    \n",
    "new_tiny_mapping = {}\n",
    "for k, v in old_tiny_mapping2.iteritems():\n",
    "    new_tiny_mapping[k] = [substitute(old_tiny_mapping1[k].split('/')[-1].replace('_', ' ')), old_tiny_mapping2[k]]\n",
    "\n",
    "with open('data/table_to_page_new.json', 'w') as f:\n",
    "    json.dump(new_tiny_mapping, f, indent=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('/tmp/g.txt', 'w') as f:\n",
    "    x = \"https://en.wikipedia.org/wiki/2007%E2%80%9308_Scottish_Second_Division\"\n",
    "    print >> f, [\"topic\", urllib.unquote(x).split('/')[-1]]\n",
    "    x = 'https://en.wikipedia.org/wiki/Ana_Timoti%C4%87'\n",
    "    print >> f, {\"topic\": urllib.unquote(x).split('/')[-1]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import pandas\n",
    "\n",
    "name = files[0]\n",
    "tab = pandas.read_csv('data/all_csv/' + name + '.csv', delimiter='#')\n",
    "sent = data[name][0][0]\n",
    "label = data[name][1][0]\n",
    "\n",
    "tab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from fuzzywuzzy import fuzz\n",
    "\n",
    "activations = []\n",
    "for i, t in enumerate(tab.columns):\n",
    "    for j, v in enumerate(tab[t]):\n",
    "        if isinstance(v, str) and len(v.split(' ')) > 2:\n",
    "            if fuzz.partial_ratio(v, sent) > 95:\n",
    "                print t, j\n",
    "        else:\n",
    "            if str(v) in sent:\n",
    "                print t, j"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import pandas\n",
    "from collections import Counter\n",
    "import pprint\n",
    "\n",
    "t = pandas.read_csv(\"/Users/wenhuchen/Downloads/Batch_3598504_batch_results.csv\")\n",
    "t = t[t.AssignmentStatus==\"Approved\"]\n",
    "\n",
    "\n",
    "def transform(string):\n",
    "    if string == \"Problematic\":\n",
    "        return \"Neutral\"\n",
    "    else:\n",
    "        return string\n",
    "\n",
    "kv_pairs = {}\n",
    "num = 5\n",
    "for i, r in t.iterrows():\n",
    "    for j in range(1, num + 1):\n",
    "        csv_name = r['Input.url{}'.format(j)].split('/')[-1]\n",
    "        if (csv_name, r['Input.s{}'.format(j)]) not in kv_pairs:\n",
    "            kv_pairs[(csv_name, r['Input.s{}'.format(j)])] = [r[\"Input.o{}\".format(j)], r[\"Answer.A{}\".format(j)], \"None\"]\n",
    "        else:\n",
    "            kv_pairs[(csv_name, r['Input.s{}'.format(j)])][2] = r[\"Answer.A{}\".format(j)]\n",
    "\n",
    "#new_kv_pairs = {}\n",
    "#for k, v in kv_pairs.iteritems():\n",
    "#    if \"Problematic\" not in v:\n",
    "#        new_kv_pairs[k] = v\n",
    "        #if v[1] == v[2]:\n",
    "        #    new_kv_pairs[k][0] = v[1]\n",
    "            \n",
    "counter = Counter()\n",
    "p_c = [] \n",
    "for k, v in kv_pairs.iteritems():\n",
    "    tmp = set(v)\n",
    "    if len(tmp) == 1:\n",
    "        p_c.append(1)\n",
    "    elif len(tmp) == 2:\n",
    "        p_c.append(1/3.)\n",
    "    else:\n",
    "        p_c.append(0)\n",
    "    \n",
    "    counter.update(v)\n",
    "\n",
    "p_e = []\n",
    "for i, j in counter.items():\n",
    "    p_e.append(j + 0.0)\n",
    "\n",
    "p_e = [_ / sum(p_e) for _ in p_e]\n",
    "p_e = sum(_**2 for _ in p_e)\n",
    "\n",
    "p_c = sum(p_c) / len(p_c)\n",
    "\n",
    "kappa = (p_c - p_e) / (1 - p_e)\n",
    "print \"kappa = {}\".format(kappa)\n",
    "print counter\n",
    "for k, v in new_kv_pairs.iteritems():\n",
    "    if v[0] == 'Contradictory':\n",
    "        print k\n",
    "        print v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cleaned_pairs = {}\n",
    "\n",
    "def decide(s1, s2, s3):\n",
    "    s2int = {'Entailed': 1, 'Neutral': 0, 'Contradictory': -1}\n",
    "    avg_score = (s2int[s1] + s2int[s2] + s2int[s3]) // 3.\n",
    "    if avg_score > 0:\n",
    "        return \"Entailed\"\n",
    "    elif avg_score < 0:\n",
    "        return \"Contradictory\"\n",
    "    else:\n",
    "        return \"Neutral\"\n",
    "\n",
    "for k, v in new_kv_pairs.iteritems():\n",
    "    if k[0] not in cleaned_pairs:\n",
    "        cleaned_pairs[k[0]] = [[k[1]], [decide(*v)]]\n",
    "    else:\n",
    "        cleaned_pairs[k[0]][0].append(k[1])\n",
    "        cleaned_pairs[k[0]][1].append(decide(*v))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "with open('READY/cleaned.json', 'w') as f:\n",
    "    json.dump(cleaned_pairs, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas\n",
    "\n",
    "t = pandas.read_csv('/Users/wenhuchen/Downloads/clean/negative_partial.csv')\n",
    "t = t[t.AssignmentStatus==\"Approved\"]\n",
    "\n",
    "print len(t)\n",
    "finished = []\n",
    "for i, row in t.iterrows():\n",
    "    for j in range(1, 11):\n",
    "        finished.append(row['Input.s{}'.format(j)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# encoding=utf8\n",
    "import sys\n",
    "reload(sys)\n",
    "sys.setdefaultencoding('utf8')\n",
    "import json\n",
    "import csv\n",
    "import random\n",
    "\n",
    "with open('READY/r1_training_all.json') as f:\n",
    "    data = json.load(f)\n",
    "\n",
    "with open('data/table_to_page.json') as f:\n",
    "    mapping = json.load(f)\n",
    "\n",
    "dummy = [\n",
    "    # refuted\n",
    "    [\n",
    "        'https://raw.githubusercontent.com/wenhuchen/Table-Fact-Checking/master/data/all_csv/2-18847467-2.html.csv',\n",
    "        'chicago bears has won the game in september',\n",
    "        '1982 new orleans saints season',\n",
    "        'https://en.wikipedia.org/wiki/1982_New_Orleans_Saints_season'\n",
    "    ],\n",
    "    # erroneous\n",
    "    [\n",
    "        'https://raw.githubusercontent.com/wenhuchen/Table-Fact-Checking/master/data/all_csv/2-18847467-2.html.csv',\n",
    "        'chicago bears is the best team in the league',\n",
    "        '1982 new orleans saints season',\n",
    "        'https://en.wikipedia.org/wiki/1982_New_Orleans_Saints_season'\n",
    "    ],\n",
    "    [\n",
    "        'https://raw.githubusercontent.com/wenhuchen/Table-Fact-Checking/master/data/all_csv/2-18847467-2.html.csv',\n",
    "        'the ninth week against atlant falcons was a big win with 35 - 6',\n",
    "        '1982 new orleans saints season',\n",
    "        'https://en.wikipedia.org/wiki/1982_New_Orleans_Saints_season'\n",
    "    ],\n",
    "    [\n",
    "        'https://raw.githubusercontent.com/wenhuchen/Table-Fact-Checking/master/data/all_csv/2-18847467-2.html.csv',\n",
    "        'the result against st. lous cardinals was a lost',\n",
    "        '1982 new orleans saints season',\n",
    "        'https://en.wikipedia.org/wiki/1982_New_Orleans_Saints_season'\n",
    "    ],\n",
    "    [\n",
    "        'https://raw.githubusercontent.com/wenhuchen/Table-Fact-Checking/master/data/all_csv/2-18847467-2.html.csv',\n",
    "        'san francisco 49ers was competing on noveber 28 , 1982',\n",
    "        '1982 new orleans saints season',\n",
    "        'https://en.wikipedia.org/wiki/1982_New_Orleans_Saints_season'\n",
    "    ],\n",
    "        [\n",
    "        'https://raw.githubusercontent.com/wenhuchen/Table-Fact-Checking/master/data/all_csv/2-18847467-2.html.csv',\n",
    "        'san francisco is is opponent where week is number',\n",
    "        '1982 new orleans saints season',\n",
    "        'https://en.wikipedia.org/wiki/1982_New_Orleans_Saints_season'\n",
    "    ]\n",
    "]\n",
    "    \n",
    "num = 10\n",
    "count = 0\n",
    "with open('verify_inputs_fake.csv', 'w') as fs:\n",
    "    fields = []\n",
    "    for i in range(0, num + 1):\n",
    "        fields.extend(['wiki{}'.format(i), 'url{}'.format(i), 's{}'.format(i), 'topic{}'.format(i)])\n",
    "    csvwriter = csv.DictWriter(fs, fieldnames=fields)\n",
    "    csvwriter.writeheader()\n",
    "\n",
    "    seed = random.choice(range(0, len(dummy)))\n",
    "    buf = {'wiki0': dummy[seed][3], 'topic0': dummy[seed][2], 'url0': dummy[seed][0], 's0': dummy[seed][1]}\n",
    "    for k, v in data.iteritems():\n",
    "        entry = data[k]\n",
    "        for sent, lab in zip(entry[0], entry[1]):\n",
    "            if lab == 0:\n",
    "                cur = len(buf) // 4\n",
    "                if 'url10' in buf:\n",
    "                    csvwriter.writerow(buf)\n",
    "                    seed = random.choice(range(0, len(dummy)))\n",
    "                    buf = {'wiki0': dummy[seed][3], 'topic0': dummy[seed][2], 'url0': dummy[seed][0], 's0': dummy[seed][1]}\n",
    "                    cur = 1\n",
    "                    count += 1\n",
    "                if sent in finished:\n",
    "                    continue\n",
    "                buf['url{}'.format(cur)] = 'https://raw.githubusercontent.com/wenhuchen/Table-Fact-Checking/master/data/all_csv/' + k\n",
    "                buf['s{}'.format(cur)] = sent\n",
    "                if k in mapping:\n",
    "                    buf['wiki{}'.format(cur)] = mapping[k][1]\n",
    "                    buf['topic{}'.format(cur)] = mapping[k][0]\n",
    "                else:\n",
    "                    buf['wiki{}'.format(cur)] = \"#\"\n",
    "                    buf['topic{}'.format(cur)] = \"None\"\n",
    "        if count % 500 == 0:\n",
    "            print count\n",
    "        #if count > 30:\n",
    "        #    break\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# encoding=utf8\n",
    "import sys\n",
    "reload(sys)\n",
    "sys.setdefaultencoding('utf8')\n",
    "import json\n",
    "import csv\n",
    "\n",
    "with open('READY/r1_training_all.json') as f:\n",
    "    data = json.load(f)\n",
    "\n",
    "num = 11\n",
    "count = 0\n",
    "with open('verify_inputs_fake.csv', 'w') as fs:\n",
    "    fields = []\n",
    "    for i in range(0, num):\n",
    "        fields.extend(['wiki{}'.format(i), 'url{}'.format(i), 's{}'.format(i), 'o{}'.format(i)])\n",
    "    csvwriter = csv.DictWriter(fs, fieldnames=fields)\n",
    "    csvwriter.writeheader()\n",
    "\n",
    "    buf = {}\n",
    "    for k, v in data.iteritems():\n",
    "        entry = data[k]\n",
    "        for sent, lab in zip(entry[0], entry[1]):\n",
    "            cur = len(buf) // 3 + 1\n",
    "            if cur > num:\n",
    "                csvwriter.writerow(buf)\n",
    "                buf = {}\n",
    "                cur = 1\n",
    "                count += 1\n",
    "            buf['url{}'.format(cur)] = 'https://raw.githubusercontent.com/wenhuchen/Table-Fact-Checking/master/data/all_csv/' + k\n",
    "            buf['s{}'.format(cur)] = sent\n",
    "            if lab == 1:\n",
    "                buf['o{}'.format(cur)] = \"Entailed\"\n",
    "            else:\n",
    "                buf['o{}'.format(cur)] = \"Contradictory\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "data = []\n",
    "with open('all_sources/full.json', 'r') as f:\n",
    "    for line in f:\n",
    "        data.append(json.loads(line.strip()))\n",
    "    #mapping[str(d['goldAnnotation']['titleId']) + '-' + str(d['tableId'])] = d['goldAnnotation']['title']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mapping = {}\n",
    "for d in data:\n",
    "    mapping[str(d['pgId']) + '-' + str(d['tableId'])] = d['pgTitle']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "\n",
    "tiny_mapping = {}\n",
    "for f in os.listdir('data/all_csv/'):\n",
    "    if f.endswith('.csv'):\n",
    "        _, pageid, tableid = f.split('.')[0].split('-')\n",
    "        if pageid + '-' + tableid in mapping:\n",
    "            tiny_mapping[f] = \"https://en.wikipedia.org/wiki/\" + \"_\".join(mapping[pageid + '-' + tableid].split(' '))\n",
    "\n",
    "with open('data/table_to_page.json', 'w') as f:\n",
    "    json.dump(tiny_mapping, f, indent=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "with open('data/table_to_page.json', 'r') as f:\n",
    "    tiny_mapping = json.load(f)\n",
    "\n",
    "print len(tiny_mapping)\n",
    "print tiny_mapping['1-18974269-1.html.csv']\n",
    "#print tiny_mapping['1-1007688-1.html.csv']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# encoding=utf8\n",
    "import sys\n",
    "reload(sys)\n",
    "sys.setdefaultencoding('utf8')\n",
    "\n",
    "import urllib\n",
    "import json\n",
    "\n",
    "with open('data/table_to_page.json', 'r') as f:\n",
    "    tiny_mapping = json.load(f)\n",
    "    \n",
    "new_tiny_mapping = {}\n",
    "for k, v in tiny_mapping.iteritems():\n",
    "    new_tiny_mapping[k] = \"https://en.wikipedia.org/wiki/\" + urllib.quote(v[30:].encode('utf8'))\n",
    "\n",
    "with open('data/table_to_page_new.json', 'w') as f:\n",
    "    json.dump(new_tiny_mapping, f, indent=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import pandas\n",
    "\n",
    "t = pandas.read_csv('v5_write_input.csv')\n",
    "length = len(t)\n",
    "\n",
    "t = t.head(3000).tail(2000)\n",
    "#t = t.head(1000)\n",
    "\n",
    "t.to_csv('v5_write_input_1000_3000.csv', index=False)\n",
    "\n",
    "t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import pandas\n",
    "\n",
    "t = pandas.read_csv(\"/Users/wenhuchen/Downloads/harvest-v2/first1000.csv\")\n",
    "print len(t)\n",
    "t = t[t.AssignmentStatus==\"Approved\"]\n",
    "print len(t)\n",
    "index = 0\n",
    "num = 5\n",
    "finished = {}\n",
    "for i,r in t.iterrows():\n",
    "    html_name = r['Input.url1']\n",
    "    html_name = html_name.split('/')[-1]\n",
    "    for j in range(1, num + 1):\n",
    "        orig_input = r[\"Answer.d{}\".format(j)]\n",
    "        #if isinstance(orig_input, str) and orig_input.lower() in [\"na\", \"n/a\", \"no\"]:\n",
    "        #    print \"error\"\n",
    "        if isinstance(orig_input, str) and len(orig_input.split(' ')) > 3:\n",
    "            replaced_sent = substitute(orig_input)\n",
    "            #replaced_sent = orig_input\n",
    "            index += 1\n",
    "            if html_name not in finished:\n",
    "                finished[html_name] = [[replaced_sent], [1]]\n",
    "            else:\n",
    "                finished[html_name][0].append(replaced_sent)\n",
    "                finished[html_name][1].append(1)\n",
    "\n",
    "with open(\"READY/round2_first1000.json\",'w') as f:\n",
    "    json.dump(finished, f, indent=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import pandas\n",
    "\n",
    "t = pandas.read_csv(\"/Users/wenhuchen/Downloads/harvest-v2/last4000.csv\")\n",
    "print len(t)\n",
    "t = t[t.AssignmentStatus==\"Approved\"]\n",
    "print len(t)\n",
    "index = 0\n",
    "num = 5\n",
    "finished = {}\n",
    "for i,r in t.iterrows():\n",
    "    html_name = r['Input.url1']\n",
    "    html_name = html_name.split('/')[-1]\n",
    "    for j in range(1, num + 1):\n",
    "        orig_input = r[\"Answer.d{}\".format(j)]\n",
    "        #if isinstance(orig_input, str) and orig_input.lower() in [\"na\", \"n/a\", \"no\"]:\n",
    "        #    print \"error\"\n",
    "        if isinstance(orig_input, str) and len(orig_input.split(' ')) > 3:\n",
    "            replaced_sent = substitute(orig_input)\n",
    "            #replaced_sent = orig_input\n",
    "            index += 1\n",
    "            if html_name not in finished:\n",
    "                finished[html_name] = [[replaced_sent], [1]]\n",
    "            else:\n",
    "                finished[html_name][0].append(replaced_sent)\n",
    "                finished[html_name][1].append(1)\n",
    "\n",
    "with open(\"READY/round2_last4000.json\",'w') as f:\n",
    "    json.dump(finished, f, indent=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import pandas\n",
    "\n",
    "t = pandas.read_csv(\"/Users/wenhuchen/Downloads/harvest-v2/middle2000.csv\")\n",
    "print len(t)\n",
    "t = t[t.AssignmentStatus==\"Approved\"]\n",
    "print len(t)\n",
    "index = 0\n",
    "num = 5\n",
    "finished = {}\n",
    "for i,r in t.iterrows():\n",
    "    html_name = r['Input.url1']\n",
    "    html_name = html_name.split('/')[-1]\n",
    "    for j in range(1, num + 1):\n",
    "        orig_input = r[\"Answer.d{}\".format(j)]\n",
    "        #if isinstance(orig_input, str) and orig_input.lower() in [\"na\", \"n/a\", \"no\"]:\n",
    "        #    print \"error\"\n",
    "        if isinstance(orig_input, str) and len(orig_input.split(' ')) > 3:\n",
    "            replaced_sent = substitute(orig_input)\n",
    "            #replaced_sent = orig_input\n",
    "            index += 1\n",
    "            if html_name not in finished:\n",
    "                finished[html_name] = [[replaced_sent], [1]]\n",
    "            else:\n",
    "                finished[html_name][0].append(replaced_sent)\n",
    "                finished[html_name][1].append(1)\n",
    "\n",
    "with open(\"READY/round2_middle2000.json\",'w') as f:\n",
    "    json.dump(finished, f, indent=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# encoding=utf8\n",
    "import sys\n",
    "reload(sys)\n",
    "sys.setdefaultencoding('utf8')\n",
    "import csv\n",
    "\n",
    "with open(\"READY/round2_first1000.json\") as f:\n",
    "    data = json.load(f)\n",
    "\n",
    "with open(\"data/table_to_page.json\") as f:\n",
    "    mapping = json.load(f)\n",
    "    \n",
    "fields = ['url1', 'wiki1', 'topic1', 's1', 's2', 's3', 's4', 's5']\n",
    "index = 0\n",
    "with open(\"rewrite_fake_first1000.csv\", 'w') as f:\n",
    "    writer = csv.DictWriter(f, fieldnames=fields)\n",
    "    writer.writeheader()\n",
    "    for k, v in data.iteritems():\n",
    "        for i in range(0, len(v[0]), 5):\n",
    "            v_s = v[0][i:i+5]\n",
    "            field = {}\n",
    "            for j, s in enumerate(v_s):\n",
    "                field['s{}'.format(j + 1)] = s\n",
    "            field['url1'] = 'https://raw.githubusercontent.com/wenhuchen/Table-Fact-Checking/master/data/all_csv/' + k\n",
    "            field['wiki1'] = mapping[k][1]\n",
    "            field['topic1'] = mapping[k][0]\n",
    "            writer.writerow(field)\n",
    "            index += 1\n",
    "        #if index > 20:\n",
    "        #    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# encoding=utf8\n",
    "import sys\n",
    "reload(sys)\n",
    "sys.setdefaultencoding('utf8')\n",
    "import csv\n",
    "\n",
    "with open(\"READY/round2_middle2000.json\") as f:\n",
    "    data = json.load(f)\n",
    "\n",
    "with open(\"data/table_to_page.json\") as f:\n",
    "    mapping = json.load(f)\n",
    "    \n",
    "fields = ['url1', 'wiki1', 'topic1', 's1', 's2', 's3', 's4', 's5']\n",
    "index = 0\n",
    "with open(\"rewrite_fake_middle2000.csv\", 'w') as f:\n",
    "    writer = csv.DictWriter(f, fieldnames=fields)\n",
    "    writer.writeheader()\n",
    "    for k, v in data.iteritems():\n",
    "        for i in range(0, len(v[0]), 5):\n",
    "            v_s = v[0][i:i+5]\n",
    "            field = {}\n",
    "            for j, s in enumerate(v_s):\n",
    "                field['s{}'.format(j + 1)] = s\n",
    "            field['url1'] = 'https://raw.githubusercontent.com/wenhuchen/Table-Fact-Checking/master/data/all_csv/' + k\n",
    "            field['wiki1'] = mapping[k][1]\n",
    "            field['topic1'] = mapping[k][0]\n",
    "            writer.writerow(field)\n",
    "            index += 1\n",
    "        #if index > 20:\n",
    "        #    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# encoding=utf8\n",
    "import sys\n",
    "reload(sys)\n",
    "sys.setdefaultencoding('utf8')\n",
    "import csv\n",
    "\n",
    "with open(\"READY/round2_last4000.json\") as f:\n",
    "    data = json.load(f)\n",
    "\n",
    "files = data.keys()\n",
    "num = 1473\n",
    "shiyang_data = files[:num]\n",
    "wanghong_data = files[num:2*num]\n",
    "yunkai_data = files[2*num:]\n",
    "    \n",
    "with open(\"data/table_to_page.json\") as f:\n",
    "    mapping = json.load(f)\n",
    "    \n",
    "fields = ['url1', 'wiki1', 'topic1', 's1', 's2', 's3', 's4', 's5']\n",
    "\n",
    "with open(\"rewrite_fake_shiyang.csv\", 'w') as f:\n",
    "    writer = csv.DictWriter(f, fieldnames=fields)\n",
    "    writer.writeheader()\n",
    "    for k in shiyang_data:\n",
    "        v = data[k]\n",
    "        for i in range(0, len(v[0]), 5):\n",
    "            v_s = v[0][i:i+5]\n",
    "            field = {}\n",
    "            for j, s in enumerate(v_s):\n",
    "                field['s{}'.format(j + 1)] = s\n",
    "            field['url1'] = 'https://raw.githubusercontent.com/wenhuchen/Table-Fact-Checking/master/data/all_csv/' + k\n",
    "            field['wiki1'] = mapping[k][1]\n",
    "            field['topic1'] = mapping[k][0]\n",
    "            writer.writerow(field)\n",
    "            \n",
    "with open(\"rewrite_fake_yunkai.csv\", 'w') as f:\n",
    "    writer = csv.DictWriter(f, fieldnames=fields)\n",
    "    writer.writeheader()\n",
    "    for k in yunkai_data:\n",
    "        v = data[k]\n",
    "        for i in range(0, len(v[0]), 5):\n",
    "            v_s = v[0][i:i+5]\n",
    "            field = {}\n",
    "            for j, s in enumerate(v_s):\n",
    "                field['s{}'.format(j + 1)] = s\n",
    "            field['url1'] = 'https://raw.githubusercontent.com/wenhuchen/Table-Fact-Checking/master/data/all_csv/' + k\n",
    "            field['wiki1'] = mapping[k][1]\n",
    "            field['topic1'] = mapping[k][0]\n",
    "            writer.writerow(field)\n",
    "            \n",
    "with open(\"rewrite_fake_wanghong.csv\", 'w') as f:\n",
    "    writer = csv.DictWriter(f, fieldnames=fields)\n",
    "    writer.writeheader()\n",
    "    for k in wanghong_data:\n",
    "        v = data[k]\n",
    "        for i in range(0, len(v[0]), 5):\n",
    "            v_s = v[0][i:i+5]\n",
    "            field = {}\n",
    "            for j, s in enumerate(v_s):\n",
    "                field['s{}'.format(j + 1)] = s\n",
    "            field['url1'] = 'https://raw.githubusercontent.com/wenhuchen/Table-Fact-Checking/master/data/all_csv/' + k\n",
    "            field['wiki1'] = mapping[k][1]\n",
    "            field['topic1'] = mapping[k][0]\n",
    "            writer.writerow(field)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import pandas\n",
    "\n",
    "t = pandas.read_csv('/Users/wenhuchen/Downloads/harvest-v3/partial_positive.csv')\n",
    "t = t[t.AssignmentStatus==\"Approved\"]\n",
    "\n",
    "mapping = {}\n",
    "for i, r in t.iterrows():\n",
    "    for i in range(1, 6):\n",
    "        name = r['Input.url{}'.format(i)].split('/')[-1]\n",
    "        if name not in mapping:\n",
    "            mapping[name] = [[r['Input.s{}'.format(i)]], [r['Answer.A{}'.format(i)]]]\n",
    "        else:\n",
    "            mapping[name][0].append(r['Input.s{}'.format(i)])\n",
    "            mapping[name][1].append(r['Answer.A{}'.format(i)])\n",
    "import json\n",
    "\n",
    "with open('READY/verify.json', 'w') as f:\n",
    "    json.dump(mapping, f, indent=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import string\n",
    "import json\n",
    "stop_words = ['be', 'she', 'he', 'her', 'his', 'their', 'the', 'it', ',', '.', '-', 'also', 'will', 'would', 'this', 'that',\n",
    "             'these', 'those', 'well', 'with', 'on', 'at', 'and', 'as', 'for', 'from', 'in', 'its', 'of', 'to', 'a',\n",
    "             'an', 'where', 'when', 'by', 'not', \"'s\", \"'nt\", \"make\", 'who', 'have', 'within', 'without', 'what',\n",
    "             'during', 'than', 'then', 'if', 'when', 'while', 'time', 'appear', 'attend', 'every', 'one', 'two', 'over',\n",
    "             'both', 'above', 'only']\n",
    "\n",
    "def get_closest(string, indexes, tab):\n",
    "    dist = 10000\n",
    "    len_string = len(string)\n",
    "    for index in indexes:\n",
    "        len_tab = len(tab[index[0]][index[1]])\n",
    "        if abs(len_tab - len_string) == 0:\n",
    "            return index\n",
    "        elif abs(len_tab - len_string) < dist:\n",
    "            minimum = index\n",
    "            dist = abs(len_tab - len_string)\n",
    "    return minimum\n",
    "\n",
    "def postprocess(inp, backbone, tabs):\n",
    "    inp = re.sub(r'([^0-9])\\.', r'\\1', inp)\n",
    "    inp = re.sub(r'(\\b)one(\\b)', r'\\g<1>1\\g<2>', inp)\n",
    "    inp = re.sub(r'(\\b)is one(\\b)', r'\\g<1>is 1\\g<2>', inp)\n",
    "    inp = re.sub(r'(\\b)two(\\b)', '\\g<1>2\\g<2>', inp)\n",
    "    inp = re.sub(r'(\\b)three(\\b)', '\\g<1>3\\g<2>', inp)\n",
    "    inp = re.sub(r'(\\b)four(\\b)', '\\g<1>4\\g<2>', inp)\n",
    "    inp = re.sub(r'(\\b)five(\\b)', '\\g<1>5\\g<2>', inp)\n",
    "    inp = re.sub(r'(\\b)six(\\b)', '\\g<1>6\\g<2>', inp)\n",
    "    inp = re.sub(r'(\\b)seven(\\b)', '\\g<1>7\\g<2>', inp)\n",
    "    inp = re.sub(r'(\\b)eight(\\b)', '\\g<1>8\\g<2>', inp)\n",
    "    inp = re.sub(r'(\\b)nine(\\b)', '\\g<1>9\\g<2>', inp)\n",
    "    inp = re.sub(r'(\\b)ten(\\b)', '\\g<1>10\\g<2>', inp)\n",
    "    inp = re.sub(r'(\\b)eleven(\\b)', '\\g<1>11\\g<2>', inp)\n",
    "    inp = re.sub(r'(\\b)twelve(\\b)', '\\g<1>12\\g<2>', inp)\n",
    "    inp = re.sub(r'(\\b)thirteen(\\b)', '\\g<1>13\\g<2>', inp)\n",
    "    inp = re.sub(r'(\\b)fourteen(\\b)', '\\g<1>14\\g<2>', inp)\n",
    "    inp = re.sub(r'(\\b)fifteen(\\b)', '\\g<1>15\\g<2>', inp)\n",
    "    inp = re.sub(r'(\\b)sixteen(\\b)', '\\g<1>16\\g<2>', inp)\n",
    "    inp = re.sub(r'(\\b)seventeen(\\b)', '\\g<1>17\\g<2>', inp)\n",
    "    inp = re.sub(r'(\\b)eighteen(\\b)', '\\g<1>18\\g<2>', inp)\n",
    "    inp = re.sub(r'(\\b)nineteen(\\b)', '\\g<1>19\\g<2>', inp)\n",
    "    inp = re.sub(r'(\\b)twenty(\\b)', '\\g<1>20\\g<2>', inp)\n",
    "    new_str = []\n",
    "    new_tags = []\n",
    "    buf = \"\"\n",
    "    last = set()\n",
    "    inp, pos_tags = get_lemmatize(inp, True)\n",
    "    for w, p in zip(inp, pos_tags):\n",
    "        if w in backbone:\n",
    "            if buf == \"\":\n",
    "                last = set(backbone[w])\n",
    "                buf = w\n",
    "            else:\n",
    "                proposed = set(backbone[w]) & last\n",
    "                if not proposed:\n",
    "                    if buf not in stop_words:\n",
    "                        closest = get_closest(buf, last, tabs)\n",
    "                        buf = '#' + buf + '#' + json.dumps(closest)\n",
    "                    new_str.append(buf)\n",
    "                    new_tags.append('ENT')\n",
    "                    buf = w\n",
    "                    last = set(backbone[w])\n",
    "                else:\n",
    "                    buf += \" \" + w\n",
    "                    last = proposed\n",
    "        else:\n",
    "            if buf != \"\":\n",
    "                if buf not in stop_words:\n",
    "                    closest = get_closest(buf, last, tabs)\n",
    "                    buf = '#' + buf + '#' + json.dumps(closest)\n",
    "                new_str.append(buf)\n",
    "                new_tags.append('ENT')\n",
    "            buf = \"\"\n",
    "            last = set()\n",
    "            new_str.append(w)\n",
    "            new_tags.append(p)\n",
    "    \n",
    "    if buf != \"\":\n",
    "        if buf not in stop_words:\n",
    "            closest = get_closest(buf, last, tabs)\n",
    "            buf = '#' + buf + '#' + json.dumps(closest)\n",
    "        new_str.append(buf)\n",
    "        new_tags.append(\"ENT\")\n",
    "    #inp = re.sub(r'ʼ', '', inp)\n",
    "    #inp = re.sub(r'ʻ', '', inp)\n",
    "    #inp = re.sub(r' ?- ?', '-', inp)\n",
    "    #inp = re.sub(r' ?\\+ ?', '+', inp)\n",
    "    #inp = re.sub(r' ?/ ?', '/', inp)\n",
    "    #inp = re.sub(r\" ?'s\", \"'s\", inp)\n",
    "    #inp = re.sub(r\" ?'nt\", \"'nt\", inp)\n",
    "    #inp = inp\n",
    "    return \" \".join(new_str), \" \".join(new_tags)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# encoding=utf8\n",
    "import sys\n",
    "reload(sys)\n",
    "sys.setdefaultencoding('utf8')\n",
    "import pandas\n",
    "import json\n",
    "\n",
    "with open('data/short_subset.txt') as f:\n",
    "    limit_length = [_.strip() for _ in f.readlines()]\n",
    "    \n",
    "def is_ascii(s):\n",
    "    return all(ord(c) < 128 for c in s)\n",
    "\n",
    "with open('data/table_to_page.json', 'r') as f:\n",
    "    mapping = json.load(f)\n",
    "    \n",
    "t = pandas.read_csv('/Users/wenhuchen/Downloads/harvest/positive.csv')\n",
    "t = t[t.AssignmentStatus==\"Approved\"]\n",
    "\n",
    "print len(t) * 10\n",
    "num = 10\n",
    "results = {}\n",
    "count = 0\n",
    "for i, row in t.iterrows():\n",
    "    for j in range(1, num + 1):\n",
    "        if row['Answer.A{}'.format(j)] == \"Entailed\":\n",
    "            name = row['Input.url{}'.format(j)].split('/')[-1]\n",
    "            if name in limit_length:\n",
    "                count += 1\n",
    "                if name in results:\n",
    "                    sent = substitute(row['Input.s{}'.format(j)])\n",
    "                    results[name][0].append(sent)\n",
    "                    results[name][1].append(1)\n",
    "                else:\n",
    "                    sent = substitute(row['Input.s{}'.format(j)])\n",
    "                    results[name] = [[sent], [1], mapping.get(name, [\"none\", \"none\"])[0]]\n",
    "\n",
    "t1 = pandas.read_csv('/Users/wenhuchen/Downloads/harvest/negative_1.csv')\n",
    "t2 = pandas.read_csv('/Users/wenhuchen/Downloads/harvest/negative_2.csv')\n",
    "t = pandas.concat([t1, t2])\n",
    "t = t[t.AssignmentStatus==\"Approved\"]\n",
    "\n",
    "print len(t) * 10\n",
    "for i, row in t.iterrows():\n",
    "    for j in range(1, num + 1):\n",
    "        if row['Answer.A{}'.format(j)] == \"Refuted\":\n",
    "            name = row['Input.url{}'.format(j)].split('/')[-1]\n",
    "            if name in limit_length:\n",
    "                count += 1\n",
    "                if name in results:\n",
    "                    sent = substitute(row['Input.s{}'.format(j)])\n",
    "                    results[name][0].append(sent)\n",
    "                    results[name][1].append(0)\n",
    "                else:\n",
    "                    sent = substitute(row['Input.s{}'.format(j)])\n",
    "                    results[name] = [[sent], [0], mapping.get(name, [\"none\", \"none\"])[0]]\n",
    "print count\n",
    "\n",
    "with open('READY/r1_training_all.json', 'w') as f:\n",
    "    json.dump(results, f, indent=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas\n",
    "\n",
    "with open('data/short_subset.txt') as f:\n",
    "    limit_length = [_.strip() for _ in f.readlines()]\n",
    "\n",
    "for j, f in enumerate(limit_length):\n",
    "    print \"------------\", f, \"------------\"\n",
    "    t = pandas.read_csv('data/all_csv/' + f, '#')\n",
    "    for i in t.columns:\n",
    "        print i, \":\", t.at[0, i]\n",
    "    if j > 100:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "from nltk.corpus import wordnet\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "tag_dict = {\"JJ\": wordnet.ADJ,\n",
    "            \"NN\": wordnet.NOUN,\n",
    "            \"NNS\": wordnet.NOUN,\n",
    "            \"NNP\": wordnet.NOUN,\n",
    "            \"NNPS\": wordnet.NOUN,\n",
    "            \"VB\": wordnet.VERB,\n",
    "            \"VBD\": wordnet.VERB,\n",
    "            \"VBG\": wordnet.VERB,\n",
    "            \"VBN\": wordnet.VERB,\n",
    "            \"VBP\": wordnet.VERB,\n",
    "            \"VBZ\": wordnet.VERB,\n",
    "            \"RB\": wordnet.ADV,\n",
    "            \"RP\": wordnet.ADV}\n",
    "\n",
    "lemmatizer = WordNetLemmatizer()\n",
    "\n",
    "def get_lemmatize(words, return_pos):\n",
    "    #words = nltk.word_tokenize(words)\n",
    "    words = words.split(' ')\n",
    "    pos_tags = [_[1] for _ in nltk.pos_tag(words)]\n",
    "    word_roots = []\n",
    "    for w, p in zip(words, pos_tags):\n",
    "        if is_ascii(w) and p in tag_dict:\n",
    "            word_roots.append(lemmatizer.lemmatize(w, tag_dict[p]))\n",
    "        else:\n",
    "            word_roots.append(w)\n",
    "    if return_pos:\n",
    "        return word_roots, pos_tags\n",
    "    else:\n",
    "        return word_roots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas\n",
    "\n",
    "t1 = pandas.read_csv('/Users/wenhuchen/Downloads/clean/Batch_3625468_batch_results.csv')\n",
    "t2 = pandas.read_csv('/Users/wenhuchen/Downloads/clean/negative_partial.csv')\n",
    "\n",
    "t = pandas.concat([t1, t2])\n",
    "\n",
    "t.to_csv('/Users/wenhuchen/Downloads/clean/negative.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas\n",
    "\n",
    "t1 = pandas.read_csv('/Users/wenhuchen/Downloads/harvest-v2/Batch_3618313_batch_results.csv')\n",
    "t2 = pandas.read_csv('/Users/wenhuchen/Downloads/harvest-v2/Batch_3618582_batch_results.csv')\n",
    "t3 = pandas.read_csv('/Users/wenhuchen/Downloads/harvest-v2/Batch_3623032_batch_results.csv')\n",
    "t4 = pandas.read_csv('/Users/wenhuchen/Downloads/harvest-v2/Batch_3618599_batch_results.csv')\n",
    "\n",
    "t = pandas.concat([t1, t2, t3, t4])\n",
    "t.to_csv('/Users/wenhuchen/Downloads/harvest-v2/negative.csv')\n",
    "\n",
    "t1 = pandas.read_csv('/Users/wenhuchen/Downloads/harvest-v2/first1000.csv')\n",
    "t2 = pandas.read_csv('/Users/wenhuchen/Downloads/harvest-v2/middle2000.csv')\n",
    "t3 = pandas.read_csv('/Users/wenhuchen/Downloads/harvest-v2/last4000.csv')\n",
    "\n",
    "t = pandas.concat([t1, t2, t3])\n",
    "t.to_csv('/Users/wenhuchen/Downloads/harvest-v2/positive.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Extra data: line 1 column 8 - line 595 column 3302 (char 7 - 2142109)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-13-827550bbf0e5>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     57\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     58\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"/Users/wenhuchen/Downloads/harvest-v2/agreement.csv\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 59\u001b[0;31m     \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mjson\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     60\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     61\u001b[0m \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mAssignmentStatus\u001b[0m\u001b[0;34m==\u001b[0m\u001b[0;34m\"Approved\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/Cellar/python@2/2.7.16/Frameworks/Python.framework/Versions/2.7/lib/python2.7/json/__init__.pyc\u001b[0m in \u001b[0;36mload\u001b[0;34m(fp, encoding, cls, object_hook, parse_float, parse_int, parse_constant, object_pairs_hook, **kw)\u001b[0m\n\u001b[1;32m    289\u001b[0m         \u001b[0mparse_float\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mparse_float\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparse_int\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mparse_int\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    290\u001b[0m         \u001b[0mparse_constant\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mparse_constant\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobject_pairs_hook\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mobject_pairs_hook\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 291\u001b[0;31m         **kw)\n\u001b[0m\u001b[1;32m    292\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    293\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/Cellar/python@2/2.7.16/Frameworks/Python.framework/Versions/2.7/lib/python2.7/json/__init__.pyc\u001b[0m in \u001b[0;36mloads\u001b[0;34m(s, encoding, cls, object_hook, parse_float, parse_int, parse_constant, object_pairs_hook, **kw)\u001b[0m\n\u001b[1;32m    337\u001b[0m             \u001b[0mparse_int\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mparse_float\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m \u001b[0;32mand\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    338\u001b[0m             parse_constant is None and object_pairs_hook is None and not kw):\n\u001b[0;32m--> 339\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0m_default_decoder\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdecode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    340\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mcls\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    341\u001b[0m         \u001b[0mcls\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mJSONDecoder\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/Cellar/python@2/2.7.16/Frameworks/Python.framework/Versions/2.7/lib/python2.7/json/decoder.pyc\u001b[0m in \u001b[0;36mdecode\u001b[0;34m(self, s, _w)\u001b[0m\n\u001b[1;32m    365\u001b[0m         \u001b[0mend\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_w\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mend\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    366\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mend\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 367\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0merrmsg\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Extra data\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0ms\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mend\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    368\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    369\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: Extra data: line 1 column 8 - line 595 column 3302 (char 7 - 2142109)"
     ]
    }
   ],
   "source": [
    "# encoding=utf8\n",
    "import sys\n",
    "import pandas\n",
    "import json\n",
    "from collections import Counter\n",
    "\n",
    "r1_1 = pandas.read_csv('/Users/wenhuchen/Downloads/harvest-v2/first1000.csv')\n",
    "r1_2 = pandas.read_csv('/Users/wenhuchen/Downloads/harvest-v2/middle2000.csv')\n",
    "r1_3 = pandas.read_csv('/Users/wenhuchen/Downloads/harvest-v2/last4000.csv')\n",
    "\n",
    "r1 = pandas.concat([r1_1, r1_2, r1_3])\n",
    "r1 = r1[r1.AssignmentStatus==\"Approved\"]\n",
    "\n",
    "with open('data/table_to_page.json', 'r') as f:\n",
    "    mapping = json.load(f)\n",
    "    \n",
    "index = 0\n",
    "finished = {}\n",
    "trash = []\n",
    "for i,r in r1.iterrows():\n",
    "    num = 5\n",
    "    for j in range(1, num + 1):\n",
    "        html_name = r['Input.url1'].split('/')[-1]\n",
    "        if html_name in limit_length:\n",
    "            orig_input = r[\"Answer.d{}\".format(j)]   \n",
    "            if isinstance(orig_input, str) and len(orig_input.split(' ')) > 3:\n",
    "                replaced_sent = substitute(orig_input)\n",
    "                index += 1\n",
    "                if html_name not in finished:\n",
    "                    finished[html_name] = [[replaced_sent], [1], mapping.get(html_name, [\"none\", \"none\"])[0]]\n",
    "                else:\n",
    "                    finished[html_name][0].append(replaced_sent)\n",
    "                    finished[html_name][1].append(1)\n",
    "                    \n",
    "r2_1 = pandas.read_csv('/Users/wenhuchen/Downloads/harvest-v2/Batch_3618313_batch_results.csv')\n",
    "r2_2 = pandas.read_csv('/Users/wenhuchen/Downloads/harvest-v2/Batch_3618582_batch_results.csv')\n",
    "r2_3 = pandas.read_csv('/Users/wenhuchen/Downloads/harvest-v2/Batch_3623032_batch_results.csv')\n",
    "r2_4 = pandas.read_csv('/Users/wenhuchen/Downloads/harvest-v2/Batch_3618599_batch_results.csv')\n",
    "\n",
    "r2 = pandas.concat([r2_1, r2_2, r2_3, r2_4])\n",
    "r2 = r2[r2.AssignmentStatus==\"Approved\"]\n",
    "\n",
    "for i,r in r2.iterrows():\n",
    "    num = 5\n",
    "    for j in range(1, num + 1):\n",
    "        html_name = r['Input.url1'].split('/')[-1]\n",
    "        if html_name in limit_length:\n",
    "            orig_input = r[\"Answer.d{}\".format(j)]   \n",
    "            if isinstance(orig_input, str) and len(orig_input.split(' ')) > 3:\n",
    "                replaced_sent = substitute(orig_input)\n",
    "                index += 1\n",
    "                if html_name not in finished:\n",
    "                    finished[html_name] = [[replaced_sent], [0], mapping.get(html_name, [\"none\", \"none\"])[0]]\n",
    "                else:\n",
    "                    finished[html_name][0].append(replaced_sent)\n",
    "                    finished[html_name][1].append(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"/Users/wenhuchen/Downloads/harvest-v2/agreement.csv\") as f:\n",
    "    data = pandas.read_csv(f)\n",
    "\n",
    "data = data[data.AssignmentStatus==\"Approved\"]\n",
    "\n",
    "succ, fail = 0, 0\n",
    "fields = 10\n",
    "sent = {}\n",
    "\n",
    "tp, tn, fp, fn = 0, 0, 0, 0\n",
    "orig_corr, orig_fals = 0, 0\n",
    "pred_corr, pred_fals = 0, 0\n",
    "for i, row in data.iterrows():\n",
    "    for j in range(1, fields + 1):\n",
    "        if row['Input.label{}'.format(j)] == \"Entailed\" and row['Answer.A{}'.format(j)] == \"Entailed\":\n",
    "            tp += 1\n",
    "        elif row['Input.label{}'.format(j)] == \"Entailed\" and row['Answer.A{}'.format(j)] == \"Refuted\":\n",
    "            fn += 1\n",
    "        elif row['Input.label{}'.format(j)] == \"Refuted\" and row['Answer.A{}'.format(j)] == \"Entailed\":\n",
    "            fp += 1\n",
    "        else:\n",
    "            tn += 1\n",
    "        \n",
    "        label = row['Answer.A{}'.format(j)]\n",
    "        if row['Input.s{}'.format(j)] in sent:\n",
    "            sent[row['Input.s{}'.format(j)]].append(label)\n",
    "        else:\n",
    "            sent[row['Input.s{}'.format(j)]] = [label]\n",
    "\n",
    "for k, v in finished.iteritems():\n",
    "    for i in range(len(v[0])):\n",
    "        if v[0][i] in sent:\n",
    "            counter = Counter(sent[v[0][i]])\n",
    "            if counter['Entailed'] > counter['Refuted']:\n",
    "                v[1][i] = 1\n",
    "            else:\n",
    "                v[1][i] = 0\n",
    "\n",
    "with open(\"READY/r2_training_all.json\", 'w') as f:\n",
    "    json.dump(finished, f, indent=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"READY/full_cleaned.json\") as f:\n",
    "    data = json.load(f)\n",
    "\n",
    "fs = data.keys()\n",
    "\n",
    "val = []\n",
    "test = []\n",
    "train = []\n",
    "for f in fs:\n",
    "    label = data[f][1]\n",
    "    if abs(sum(label) * 2 - len(label)) <= 1:\n",
    "        if len(val) < 1700:\n",
    "            val.append(f)\n",
    "        elif len(test) < 1700:\n",
    "            test.append(f)\n",
    "        else:\n",
    "            train.append(f)\n",
    "    else:\n",
    "        train.append(f) \n",
    "\n",
    "print len(train)\n",
    "print len(val)\n",
    "print len(test)\n",
    "\n",
    "all_label = 0\n",
    "pos_label = 0\n",
    "for fn in train:\n",
    "    label = data[fn][1]\n",
    "    pos_label += sum(label)\n",
    "    all_label += len(label)\n",
    "\n",
    "print \"pos: {}/neg: {}\".format(pos_label, all_label - pos_label)\n",
    "\n",
    "\n",
    "all_label = 0\n",
    "pos_label = 0\n",
    "for fn in val:\n",
    "    label = data[fn][1]\n",
    "    pos_label += sum(label)\n",
    "    all_label += len(label)\n",
    "\n",
    "print \"pos: {}/neg: {}\".format(pos_label, all_label - pos_label)\n",
    "\n",
    "\n",
    "all_label = 0\n",
    "pos_label = 0\n",
    "for fn in test:\n",
    "    label = data[fn][1]\n",
    "    pos_label += sum(label)\n",
    "    all_label += len(label)\n",
    "\n",
    "print \"pos: {}/neg: {}\".format(pos_label, all_label - pos_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('data/train_id.json', 'w') as f:\n",
    "    json.dump(train, f, indent=2)\n",
    "with open('data/val_id.json', 'w') as f:\n",
    "    json.dump(val, f, indent=2)\n",
    "with open('data/test_id.json', 'w') as f:\n",
    "    json.dump(test, f, indent=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "sent='four of the cultural interest fraternities and sororities were fraternities while three are a sorority'\n",
    "postprocess(sent, [], [])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "get_lemmatize(\"crying isn't the best way\", True)\n",
    "#lemmatizer.lemmatize(\"cryin'\", pos=\"v\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "get_lemmatize(\"crying isn't the best way\", True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import json\n",
    "with open('READY/r1_training_all.json') as f:\n",
    "    data = json.load(f)\n",
    "\n",
    "words = 0\n",
    "entities = 0\n",
    "for k in data:\n",
    "    for sent in data[k][0]:\n",
    "        words += len(sent.split(' '))\n",
    "        entities += sent.count('#') // 2\n",
    "print 'words:', words\n",
    "\n",
    "with open('READY/r1_training_cleaned.json') as f:\n",
    "    data = json.load(f)\n",
    "entities = 0\n",
    "for k in data:\n",
    "    for sent in data[k][0]:\n",
    "        entities += sent.count('#') // 2\n",
    "print 'linked:', entities\n",
    "with open('READY/r2_training_all.json') as f:\n",
    "    data = json.load(f)\n",
    "\n",
    "words = 0\n",
    "for k in data:\n",
    "    for sent in data[k][0]:\n",
    "        words += len(sent.split(' '))\n",
    "print 'words:', words\n",
    "\n",
    "with open('READY/r2_training_cleaned.json') as f:\n",
    "    data = json.load(f)\n",
    "entities = 0\n",
    "for k in data:\n",
    "    for sent in data[k][0]:\n",
    "        entities += sent.count('#') // 2\n",
    "print 'linked:', entities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def charposition(string, char):\n",
    "    pos = [] #list to store positions for each 'char' in 'string'\n",
    "    for n in range(len(string)):\n",
    "        if string[n] == '#':\n",
    "            pos.append(n)\n",
    "    return pos\n",
    "\n",
    "import json\n",
    "with open('READY/full_cleaned.json') as f:\n",
    "    data = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas\n",
    "import numpy\n",
    "\n",
    "t = pandas.read_csv('data/all_csv/2-18847467-2.html.csv', '#')\n",
    "\n",
    "t.dtypes[0] in [numpy.dtype('int64'), numpy.dtype('int32'), numpy.dtype('float32'), numpy.dtype('float64')]\n",
    "\n",
    "print t.columns[1]\n",
    "\n",
    "t.at[1, t.columns[1]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import multiprocessing\n",
    "from itertools import product\n",
    "from pathos.multiprocessing import ProcessingPool as Pool\n",
    "\n",
    "def f(x, y, z):\n",
    "    return x * y * z\n",
    "\n",
    "cores = multiprocessing.cpu_count()\n",
    "#with multiprocessing.Pool(processes=cores) as pool:\n",
    "#    results = pool.starmap(merge_names, )\n",
    "res = Pool().map(f, range(5), range(5), range(5))\n",
    "    \n",
    "# method 1: map\n",
    "print res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "#create sample data\n",
    "data = {'model': ['Lisa', 'Lisa 2', 'Macintosh 128K', 'Macintosh 512K'],\n",
    "        'launched': [1983,1984,1984,1984],\n",
    "        'discontinued': [1986, 1985, 1984, 1986]}\n",
    "\n",
    "df = pd.DataFrame(data, columns = ['model', 'launched', 'discontinued'])\n",
    "df\n",
    "df[df['discontinued'] == 1986]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "\n",
    "fail = 0\n",
    "success = 0\n",
    "for fn in os.listdir('data/all_program/'):\n",
    "    if f.endswith('json'):\n",
    "        with open('data/all_program/' + fn) as f:\n",
    "            print f\n",
    "            data = json.load(f)\n",
    "            if len(data[-1]) == 0:\n",
    "                fail += 1\n",
    "            else:\n",
    "                success += 1\n",
    "\n",
    "print \"{}/{}\".format(sucess, fail)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "with open('READY/r1_training_all.json') as f:\n",
    "    r1_data = json.load(f)\n",
    "\n",
    "with open('READY/r1_training_all.json') as f:\n",
    "    r_data = json.load(f)\n",
    "\n",
    "r_data.update(r1_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "line = 0\n",
    "word_counter = Counter()\n",
    "for k, v in r_data.iteritems():\n",
    "    for r in v[0]:\n",
    "        text = r.split(' ')\n",
    "        word_counter.update(text)\n",
    "        line += 1\n",
    "\n",
    "new = ['be', 'she', 'he', 'her', 'his', 'their', 'the', 'it', ',', '.', '-', 'also', 'will', 'would', 'this', 'that',\n",
    "         'these', 'those', 'well', 'with', 'on', 'at', 'and', 'as', 'for', 'from', 'in', 'its', 'of', 'to', 'a',\n",
    "         'an', 'where', 'when', 'by', 'not', \"'s\", \"'nt\", \"make\", 'who', 'have', 'within', 'without', 'what',\n",
    "         'during', 'than', 'then', 'if', 'when', 'while', 'time', 'appear', 'attend', 'every', 'one', 'two', 'over',\n",
    "         'both', 'above', 'only', \",\", \".\", \"(\", \")\", \"&\", \":\"]\n",
    "#word_counter.update(new)\n",
    "with open('/tmp/stop_words.json', 'w') as f:\n",
    "    json.dump([_[0] for _ in word_counter.most_common(100)] + new, f, indent=2)\n",
    "#print len(word_counter)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# encoding=utf8\n",
    "import sys\n",
    "reload(sys)\n",
    "sys.setdefaultencoding('utf8')\n",
    "import json\n",
    "import csv\n",
    "import re\n",
    "import random\n",
    "import pandas\n",
    "\n",
    "num = 10\n",
    "num_field = 5\n",
    "\n",
    "with open('READY/r2_training_all.json') as f:\n",
    "    data = json.load(f)\n",
    "\n",
    "with open('data/test_id.json') as f:    \n",
    "    test_ids = json.load(f)\n",
    "    \n",
    "with open('data/table_to_page.json', 'r') as f:\n",
    "    mapping = json.load(f)\n",
    "    \n",
    "with open('agreement_file.csv', mode='w') as fw:\n",
    "    fields = []\n",
    "    for i in range(1, num + 1):\n",
    "        fields.extend(['wiki{}'.format(i), 'url{}'.format(i), 's{}'.format(i), 'topic{}'.format(i), 'label{}'.format(i)])\n",
    "        \n",
    "    csvwriter = csv.DictWriter(fw, fieldnames=fields)\n",
    "    csvwriter.writeheader()\n",
    "    \n",
    "    small_test_ids = []\n",
    "    index = 0\n",
    "    buffers = {}\n",
    "    for k, entry in data.iteritems():\n",
    "        if k in test_ids:\n",
    "            small_test_ids.append(k)\n",
    "            idxs = range(len(entry[0]))\n",
    "            random.shuffle(idxs)\n",
    "            for idx in idxs:\n",
    "                r = entry[0][idx]\n",
    "                l = entry[1][idx]\n",
    "                r = re.sub(r';[^#]+#', '', r)\n",
    "                r = re.sub(r'#', '', r)\n",
    "                print r\n",
    "                if len(buffers) // num_field == num:\n",
    "                    csvwriter.writerow(buffers)\n",
    "                    index += 1\n",
    "                    buffers = {}\n",
    "                cur = len(buffers) // num_field\n",
    "                buffers[\"url{}\".format(cur + 1)] = 'https://raw.githubusercontent.com/wenhuchen/Table-Fact-Checking/master/data/all_csv/' + k\n",
    "                buffers[\"s{}\".format(cur + 1)] = r\n",
    "                buffers[\"label{}\".format(cur + 1)] = \"Entailed\" if l == 1 else \"Refuted\"\n",
    "                if k in mapping:\n",
    "                    buffers[\"wiki{}\".format(cur + 1)] = mapping[k][1]\n",
    "                    buffers['topic{}'.format(cur + 1)] = mapping[k][0]\n",
    "                else:\n",
    "                    buffers['wiki{}'.format(cur + 1)] = \"#\"\n",
    "                    buffers['topic{}'.format(cur + 1)] = \"None\"\n",
    "            \n",
    "            if index > 100:\n",
    "                break\n",
    "    #with open('data/small_test_id.json', 'w') as f:\n",
    "    #    json.dump(small_test_ids, f, indent=2)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# encoding=utf8\n",
    "import sys\n",
    "reload(sys)\n",
    "sys.setdefaultencoding('utf8')\n",
    "import json\n",
    "import csv\n",
    "import re\n",
    "import random\n",
    "import pandas\n",
    "\n",
    "num = 10\n",
    "num_field = 5\n",
    "\n",
    "with open('READY/r1_training_all.json') as f:\n",
    "    data = json.load(f)\n",
    "\n",
    "with open('data/test_id.json') as f:    \n",
    "    test_ids = json.load(f)\n",
    "    \n",
    "with open('data/table_to_page.json', 'r') as f:\n",
    "    mapping = json.load(f)\n",
    "    \n",
    "with open('agreement_file.csv', mode='w') as fw:\n",
    "    fields = []\n",
    "    for i in range(1, num + 1):\n",
    "        fields.extend(['wiki{}'.format(i), 'url{}'.format(i), 's{}'.format(i), 'topic{}'.format(i), 'label{}'.format(i)])\n",
    "        \n",
    "    csvwriter = csv.DictWriter(fw, fieldnames=fields)\n",
    "    csvwriter.writeheader()\n",
    "    \n",
    "    index = 0\n",
    "    buffers = {}\n",
    "    for k, entry in data.iteritems():\n",
    "        if k in test_ids:\n",
    "            small_test_ids.append(k)\n",
    "            idxs = range(len(entry[0]))\n",
    "            random.shuffle(idxs)\n",
    "            for idx in idxs:\n",
    "                r = entry[0][idx]\n",
    "                l = entry[1][idx]\n",
    "                r = re.sub(r';[^#]+#', '', r)\n",
    "                r = re.sub(r'#', '', r)\n",
    "                print r\n",
    "                if len(buffers) // num_field == num:\n",
    "                    csvwriter.writerow(buffers)\n",
    "                    index += 1\n",
    "                    buffers = {}\n",
    "                cur = len(buffers) // num_field\n",
    "                buffers[\"url{}\".format(cur + 1)] = 'https://raw.githubusercontent.com/wenhuchen/Table-Fact-Checking/master/data/all_csv/' + k\n",
    "                buffers[\"s{}\".format(cur + 1)] = r\n",
    "                buffers[\"label{}\".format(cur + 1)] = \"Entailed\" if l == 1 else \"Refuted\"\n",
    "                if k in mapping:\n",
    "                    buffers[\"wiki{}\".format(cur + 1)] = mapping[k][1]\n",
    "                    buffers['topic{}'.format(cur + 1)] = mapping[k][0]\n",
    "                else:\n",
    "                    buffers['wiki{}'.format(cur + 1)] = \"#\"\n",
    "                    buffers['topic{}'.format(cur + 1)] = \"None\"\n",
    "            \n",
    "            if index > 100:\n",
    "                break\n",
    "    with open('data/small_test_id.json', 'w') as f:\n",
    "        json.dump(small_test_ids, f, indent=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TP: 457, FP: 51, FN: 31, TN: 468\n",
      "Accuracy: 0.91857000993\n",
      "Precision: 0.899606299213 Recall: 0.936475409836 F1: 0.917670682731\n",
      "0.875670307845 0.500256828808\n",
      "kappa = 0.751212824262\n"
     ]
    }
   ],
   "source": [
    "from collections import Counter\n",
    "import pandas\n",
    "\n",
    "data = pandas.read_csv('/Users/wenhuchen/Downloads/harvest-v2/agreement.csv')\n",
    "\n",
    "data = data[data.AssignmentStatus==\"Approved\"]\n",
    "\n",
    "succ, fail = 0, 0\n",
    "fields = 10\n",
    "\n",
    "sent = {}\n",
    "gt = {}\n",
    "\n",
    "tp, tn, fp, fn = 0, 0, 0, 0\n",
    "for i, row in data.iterrows():\n",
    "    for j in range(1, fields + 1):\n",
    "        \"\"\"\n",
    "        if row['Input.label{}'.format(j)] == \"Entailed\" and row['Answer.A{}'.format(j)] == \"Entailed\":\n",
    "            tp += 1\n",
    "        elif row['Input.label{}'.format(j)] == \"Entailed\" and row['Answer.A{}'.format(j)] == \"Refuted\":\n",
    "            fn += 1\n",
    "        elif row['Input.label{}'.format(j)] == \"Refuted\" and row['Answer.A{}'.format(j)] == \"Entailed\":\n",
    "            fp += 1\n",
    "        else:\n",
    "            tn += 1\n",
    "        \"\"\"\n",
    "        label = row['Answer.A{}'.format(j)]\n",
    "        if row['Input.s{}'.format(j)] in sent:\n",
    "            sent[row['Input.s{}'.format(j)]].append(label)\n",
    "        else:\n",
    "            sent[row['Input.s{}'.format(j)]] = [label]\n",
    "\n",
    "        label = row['Input.label{}'.format(j)]\n",
    "        gt[row['Input.s{}'.format(j)]] = label\n",
    "\n",
    "\n",
    "for s in sent:\n",
    "    stat = Counter(sent[s])\n",
    "    if stat['Entailed'] > stat['Refuted'] and gt[s] == \"Entailed\":\n",
    "        tp += 1\n",
    "    elif stat['Entailed'] > stat['Refuted'] and gt[s] == \"Refuted\":\n",
    "        fn += 1\n",
    "    elif stat['Entailed'] < stat['Refuted'] and gt[s] == \"Refuted\":\n",
    "        tn += 1\n",
    "    else:\n",
    "        fp += 1\n",
    "    \n",
    "#print \"original correct: {}, original false: {}\".format(orig_corr, orig_fals)\n",
    "#print \"pred correct: {}, pred false: {}\".format(pred_corr, pred_fals)\n",
    "print \"TP: {}, FP: {}, FN: {}, TN: {}\".format(tp, fp, fn, tn)\n",
    "print \"Accuracy: {}\".format((tp + tn) / (tp + fp + fn + tn + 0.))\n",
    "prec = tp / (tp + fp + 0.)\n",
    "recall = tp / (tp + fn + 0.)\n",
    "F1 = 2 * prec * recall / (prec + recall)\n",
    "print \"Precision: {} Recall: {} F1: {}\".format(prec, recall, F1)\n",
    "\n",
    "#for s in sent:\n",
    "#    if len(sent[s]) > 1:\n",
    "#        print sent[s]\n",
    "p_c = []\n",
    "N_ent = 0\n",
    "N_ref = 0\n",
    "for _, v in sent.iteritems():\n",
    "    counter = Counter(v)\n",
    "    total = counter['Entailed'] + counter['Refuted']\n",
    "    tmp = counter['Entailed'] ** 2 + counter['Refuted'] ** 2 - total\n",
    "    tmp = tmp / (total * (total - 1) + 0.0)\n",
    "    p_c.append(tmp)\n",
    "    N_ent += counter['Entailed']\n",
    "    N_ref += counter['Refuted']\n",
    "\n",
    "p_c = sum(p_c) / len(p_c)\n",
    "\n",
    "p_ent = N_ent / (N_ent + N_ref + 0.)\n",
    "p_ref = N_ref / (N_ent + N_ref + 0.)\n",
    "\n",
    "p_e = p_ent ** 2 + p_ref ** 2\n",
    "\n",
    "print p_c, p_e\n",
    "\n",
    "kappa = (p_c - p_e) / (1 - p_e)\n",
    "print \"kappa = {}\".format(kappa)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# encoding=utf8\n",
    "import sys\n",
    "reload(sys)\n",
    "sys.setdefaultencoding('utf8')\n",
    "\n",
    "import json\n",
    "import csv\n",
    "import random\n",
    "import os\n",
    "\n",
    "with open('data/table_to_page.json', 'r') as f:\n",
    "    tiny_mapping = json.load(f)\n",
    "    \n",
    "with open('data/simple_ids.json', 'r') as f:\n",
    "    unseen = json.load(f)\n",
    "\n",
    "random.shuffle(unseen)\n",
    "\n",
    "count = 0\n",
    "with open('blind_input.csv', 'w') as f:\n",
    "    fields = [\"url1\", \"wiki1\", \"topic1\"]\n",
    "    writer = csv.DictWriter(f, fieldnames=fields)\n",
    "    writer.writeheader()\n",
    "\n",
    "    for k in unseen:\n",
    "        if os.path.exists('data/all_csv/{}'.format(k)):\n",
    "            if k in tiny_mapping:\n",
    "                writer.writerow({\"url1\": 'https://raw.githubusercontent.com/wenhuchen/Table-Fact-Checking/master/data/all_csv/' + k, \n",
    "                                \"wiki1\": tiny_mapping[k][1], \"topic1\": tiny_mapping[k][0]})\n",
    "                count += 1\n",
    "\n",
    "            if count == 980:\n",
    "                break"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}