subset_data_ner-conll2003.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "import pickle as pkl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import Dict, List, Sequence, Iterable\n",
    "import itertools\n",
    "import logging\n",
    "\n",
    "from overrides import overrides\n",
    "\n",
    "from allennlp.common.checks import ConfigurationError\n",
    "from allennlp.common.file_utils import cached_path\n",
    "from allennlp.data.dataset_readers.dataset_reader import DatasetReader\n",
    "from allennlp.data.dataset_readers.dataset_utils import to_bioul\n",
    "from allennlp.data.fields import TextField, SequenceLabelField, Field, MetadataField\n",
    "from allennlp.data.instance import Instance\n",
    "from allennlp.data.token_indexers import TokenIndexer, SingleIdTokenIndexer,TokenCharactersIndexer\n",
    "from allennlp.data.tokenizers import Token\n",
    "\n",
    "logger = logging.getLogger(__name__)  # pylint: disable=invalid-name\n",
    "\n",
    "def _is_divider(line: str) -> bool:\n",
    "    empty_line = line.strip() == ''\n",
    "    if empty_line:\n",
    "        return True\n",
    "    else:\n",
    "        first_token = line.split()[0]\n",
    "        if first_token == \"-DOCSTART-\":  # pylint: disable=simplifiable-if-statement\n",
    "            return True\n",
    "        else:\n",
    "            return False\n",
    "\n",
    "\n",
    "\n",
    "class Conll2003DatasetReader(DatasetReader):\n",
    "    \"\"\"\n",
    "    Reads instances from a pretokenised file where each line is in the following format:\n",
    "\n",
    "    WORD POS-TAG CHUNK-TAG NER-TAG\n",
    "\n",
    "    with a blank line indicating the end of each sentence\n",
    "    and '-DOCSTART- -X- -X- O' indicating the end of each article,\n",
    "    and converts it into a ``Dataset`` suitable for sequence tagging.\n",
    "\n",
    "    Each ``Instance`` contains the words in the ``\"tokens\"`` ``TextField``.\n",
    "    The values corresponding to the ``tag_label``\n",
    "    values will get loaded into the ``\"tags\"`` ``SequenceLabelField``.\n",
    "    And if you specify any ``feature_labels`` (you probably shouldn't),\n",
    "    the corresponding values will get loaded into their own ``SequenceLabelField`` s.\n",
    "\n",
    "    This dataset reader ignores the \"article\" divisions and simply treats\n",
    "    each sentence as an independent ``Instance``. (Technically the reader splits sentences\n",
    "    on any combination of blank lines and \"DOCSTART\" tags; in particular, it does the right\n",
    "    thing on well formed inputs.)\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    token_indexers : ``Dict[str, TokenIndexer]``, optional (default=``{\"tokens\": SingleIdTokenIndexer()}``)\n",
    "        We use this to define the input representation for the text.  See :class:`TokenIndexer`.\n",
    "    tag_label: ``str``, optional (default=``ner``)\n",
    "        Specify `ner`, `pos`, or `chunk` to have that tag loaded into the instance field `tag`.\n",
    "    feature_labels: ``Sequence[str]``, optional (default=``()``)\n",
    "        These labels will be loaded as features into the corresponding instance fields:\n",
    "        ``pos`` -> ``pos_tags``, ``chunk`` -> ``chunk_tags``, ``ner`` -> ``ner_tags``\n",
    "        Each will have its own namespace: ``pos_tags``, ``chunk_tags``, ``ner_tags``.\n",
    "        If you want to use one of the tags as a `feature` in your model, it should be\n",
    "        specified here.\n",
    "    coding_scheme: ``str``, optional (default=``IOB1``)\n",
    "        Specifies the coding scheme for ``ner_labels`` and ``chunk_labels``.\n",
    "        Valid options are ``IOB1`` and ``BIOUL``.  The ``IOB1`` default maintains\n",
    "        the original IOB1 scheme in the CoNLL 2003 NER data.\n",
    "        In the IOB1 scheme, I is a token inside a span, O is a token outside\n",
    "        a span and B is the beginning of span immediately following another\n",
    "        span of the same type.\n",
    "    label_namespace: ``str``, optional (default=``labels``)\n",
    "        Specifies the namespace for the chosen ``tag_label``.\n",
    "    \"\"\"\n",
    "    _VALID_LABELS = {'ner', 'pos', 'chunk'}\n",
    "\n",
    "    def __init__(self,\n",
    "                 token_indexers: Dict[str, TokenIndexer] = None,\n",
    "                 tag_label: str = \"ner\",\n",
    "                 feature_labels: Sequence[str] = (),\n",
    "                 lazy: bool = False,\n",
    "                 coding_scheme: str = \"IOB1\",\n",
    "                 label_namespace: str = \"labels\") -> None:\n",
    "        super().__init__(lazy)\n",
    "        self._token_indexers = token_indexers or {'tokens': SingleIdTokenIndexer()}\n",
    "        if tag_label is not None and tag_label not in self._VALID_LABELS:\n",
    "            raise ConfigurationError(\"unknown tag label type: {}\".format(tag_label))\n",
    "        for label in feature_labels:\n",
    "            if label not in self._VALID_LABELS:\n",
    "                raise ConfigurationError(\"unknown feature label type: {}\".format(label))\n",
    "        if coding_scheme not in (\"IOB1\", \"BIOUL\",\"BIO\"):\n",
    "            raise ConfigurationError(\"unknown coding_scheme: {}\".format(coding_scheme))\n",
    "\n",
    "        self.tag_label = tag_label\n",
    "        self.feature_labels = set(feature_labels)\n",
    "        self.coding_scheme = coding_scheme\n",
    "        self.label_namespace = label_namespace\n",
    "        self._original_coding_scheme = \"IOB1\"\n",
    "\n",
    "    @overrides\n",
    "    def _read(self, file_path: str) -> Iterable[Instance]:\n",
    "        # if `file_path` is a URL, redirect to the cache\n",
    "        file_path = cached_path(file_path)\n",
    "\n",
    "        with open(file_path, \"r\", encoding='ISO-8859-1') as data_file:\n",
    "            logger.info(\"Reading instances from lines in file at: %s\", file_path)\n",
    "\n",
    "            # Group into alternative divider / sentence chunks.\n",
    "            for is_divider, lines in itertools.groupby(data_file, _is_divider):\n",
    "                # Ignore the divider chunks, so that `lines` corresponds to the words\n",
    "                # of a single sentence.\n",
    "                if not is_divider:\n",
    "                    fields = [line.strip().split() for line in lines]\n",
    "                    # unzipping trick returns tuples, but our Fields need lists\n",
    "                    fields = [list(field) for field in zip(*fields)]\n",
    "                    if len(fields) == 4 : tokens_, pos_tags, chunk_tags, ner_tags = fields\n",
    "                    elif len(fields) == 3  :\n",
    "                        tokens_,_,ner_tags = fields\n",
    "                        pos_tags = None\n",
    "                        chunk_tags = None\n",
    "                    elif len(fields) == 5  :\n",
    "                        tokens_, lemma, pos_tags, chunk_tags, ner_tags = fields\n",
    "                    else :\n",
    "                        raise Exception(\"could not parse line\")\n",
    "                    # TextField requires ``Token`` objects\n",
    "                    tokens = [Token(token) for token in tokens_]\n",
    "\n",
    "                    yield self.text_to_instance(tokens, pos_tags, chunk_tags, ner_tags)\n",
    "\n",
    "    def text_to_instance(self, # type: ignore\n",
    "                         tokens: List[Token],\n",
    "                         pos_tags: List[str] = None,\n",
    "                         chunk_tags: List[str] = None,\n",
    "                         ner_tags: List[str] = None) -> Instance:\n",
    "        \"\"\"\n",
    "        We take `pre-tokenized` input here, because we don't have a tokenizer in this class.\n",
    "        \"\"\"\n",
    "        # pylint: disable=arguments-differ\n",
    "        sequence = TextField(tokens, self._token_indexers)\n",
    "        instance_fields: Dict[str, Field] = {'tokens': sequence}\n",
    "        instance_fields[\"metadata\"] = MetadataField({\"words\": [x.text for x in tokens]})\n",
    "\n",
    "        # Recode the labels if necessary.\n",
    "        if self.coding_scheme == \"BIOUL\":\n",
    "            coded_chunks = to_bioul(chunk_tags,\n",
    "                                    encoding=self._original_coding_scheme) if chunk_tags is not None else None\n",
    "            coded_ner = to_bioul(ner_tags,\n",
    "                                 encoding=self._original_coding_scheme) if ner_tags is not None else None\n",
    "        else:\n",
    "            # the default IOB1\n",
    "            coded_chunks = chunk_tags\n",
    "            coded_ner = ner_tags\n",
    "\n",
    "        # Add \"feature labels\" to instance\n",
    "        if 'pos' in self.feature_labels:\n",
    "            if pos_tags is None:\n",
    "                raise ConfigurationError(\"Dataset reader was specified to use pos_tags as \"\n",
    "                                         \"features. Pass them to text_to_instance.\")\n",
    "            instance_fields['pos_tags'] = SequenceLabelField(pos_tags, sequence, \"pos_tags\")\n",
    "        if 'chunk' in self.feature_labels:\n",
    "            if coded_chunks is None:\n",
    "                raise ConfigurationError(\"Dataset reader was specified to use chunk tags as \"\n",
    "                                         \"features. Pass them to text_to_instance.\")\n",
    "            instance_fields['chunk_tags'] = SequenceLabelField(coded_chunks, sequence, \"chunk_tags\")\n",
    "        if 'ner' in self.feature_labels:\n",
    "            if coded_ner is None:\n",
    "                raise ConfigurationError(\"Dataset reader was specified to use NER tags as \"\n",
    "                                         \" features. Pass them to text_to_instance.\")\n",
    "            instance_fields['ner_tags'] = SequenceLabelField(coded_ner, sequence, \"ner_tags\")\n",
    "\n",
    "        # Add \"tag label\" to instance\n",
    "        if self.tag_label == 'ner' and coded_ner is not None:\n",
    "            instance_fields['tags'] = SequenceLabelField(coded_ner, sequence,\n",
    "                                                         self.label_namespace)\n",
    "        elif self.tag_label == 'pos' and pos_tags is not None:\n",
    "            instance_fields['tags'] = SequenceLabelField(pos_tags, sequence,\n",
    "                                                         self.label_namespace)\n",
    "        elif self.tag_label == 'chunk' and coded_chunks is not None:\n",
    "            instance_fields['tags'] = SequenceLabelField(coded_chunks, sequence,\n",
    "                                                         self.label_namespace)\n",
    "\n",
    "        return Instance(instance_fields)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "mtix = {\n",
    "      \"tokens\": SingleIdTokenIndexer(lowercase_tokens = True),\n",
    "      \"token_characters\": TokenCharactersIndexer()\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pack_data_into_pkl(domain, quantity=1000, method = 'iid'):\n",
    "    reader = Conll2003DatasetReader(token_indexers=mtix,coding_scheme=\"BIO\")\n",
    "    train_data_gen = reader._read('datasets/ner_xling/{0}.train'.format(domain))\n",
    "    train_data = list(train_data_gen)\n",
    "    random.seed(quantity)\n",
    "    random.shuffle(train_data)\n",
    "    ktrain_data = train_data[:quantity]\n",
    "    return ktrain_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = pack_data_into_pkl(domain='deu', quantity=1000, method = 'iid')\n",
    "pkl.dump(train_data, open('datasets/ner_xling/deu_1k.pkl','wb'))\n",
    "\n",
    "train_data = pack_data_into_pkl(domain='deu', quantity=5000, method = 'iid')\n",
    "pkl.dump(train_data, open('datasets/ner_xling/deu_5k.pkl','wb'))\n",
    "\n",
    "train_data = pack_data_into_pkl(domain='deu', quantity=10000, method = 'iid')\n",
    "pkl.dump(train_data, open('datasets/ner_xling/deu_10k.pkl','wb'))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}