Text_Classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Text Classification Example\n",
    "Copyright (c) 2021 Robert Bosch GmbH\n",
    "\n",
    "This program is free software: you can redistribute it and/or modify\n",
    "it under the terms of the GNU Affero General Public License as published\n",
    "by the Free Software Foundation, either version 3 of the License, or\n",
    "(at your option) any later version.\n",
    "\n",
    "This program is distributed in the hope that it will be useful,\n",
    "but WITHOUT ANY WARRANTY; without even the implied warranty of\n",
    "MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n",
    "GNU Affero General Public License for more details.\n",
    "\n",
    "You should have received a copy of the GNU Affero General Public icense\n",
    "along with this program.  If not, see <https://www.gnu.org/licenses/>."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from flair.datasets import ClassificationCorpus\n",
    "from flair.tokenization import SpaceTokenizer\n",
    "\n",
    "# Check out the following page for more information on corpora in flair\n",
    "# https://github.com/flairNLP/flair/blob/master/resources/docs/TUTORIAL_6_CORPUS.md\n",
    "\n",
    "corpus = ClassificationCorpus(\n",
    "    'path/to/dataset/',           # path where train/dev/test files are stored (they have to be in the same directory)\n",
    "    train_file = 'name_of_train.bio',\n",
    "    test_file = 'name_of_test.bio',\n",
    "    dev_file = 'name_of_dev.bio', # will be sampled from train if no file is provided\n",
    "    label_type = 'question_type',\n",
    "    tokenizer = SpaceTokenizer(), # you can remove this line and a \"real\" tokenizer is used\n",
    "    memory_mode ='full'           # only if you can keep everything in memory; but required for FAME-features\n",
    ")\n",
    "lang = 'en'\n",
    "\n",
    "label_dict = corpus.make_label_dictionary()\n",
    "print(label_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from flair.embeddings import CharacterEmbeddings, WordEmbeddings, TransformerWordEmbeddings\n",
    "from src.embeddings import AveragingBytePairEmbeddings\n",
    "\n",
    "# Check out the following page for more information on embeddings in flair\n",
    "# https://github.com/flairNLP/flair/blob/master/resources/docs/TUTORIAL_3_WORD_EMBEDDING.md\n",
    "char = CharacterEmbeddings()\n",
    "word = WordEmbeddings(lang)\n",
    "bpemb = AveragingBytePairEmbeddings(language=lang)\n",
    "\n",
    "# You should be able to use any huggingface transformer with these embeddings\n",
    "transformer = TransformerWordEmbeddings(\n",
    "    'xlm-roberta-large',\n",
    "    layers = '-1',\n",
    "    fine_tune = False,   # There is also the option to finetune these models\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from src.features import add_features\n",
    "\n",
    "features = add_features(corpus, use_frequencies_from_file='path/to/fasttext/files/cc.{lang}.300.vec')\n",
    "feature_flags, features_dimensions, idx2shape, idx2word = features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from src.embeddings import MetaEmbeddings\n",
    "from src.models import FeatureEmbedding\n",
    "\n",
    "shape_hidden_size = 25  # Embedding size of shape embeddings\n",
    "feature_flags = 'fslb'  # Indicate which features you want to use\n",
    "                        # - (f)requency\n",
    "                        # - (s)hape embedding\n",
    "                        # - (l)ength\n",
    "                        # - (b)asic shape information like HasCapitalLetter (12 one-hot encoded features)\n",
    "\n",
    "\n",
    "feature_model = FeatureEmbedding(\n",
    "    use_features=feature_flags, \n",
    "    feature_dims=features_dimensions, \n",
    "    idx2shape=idx2shape, \n",
    "    shape_dim=shape_hidden_size\n",
    ")\n",
    "\n",
    "# This is the drop-in replacement for the flair.embeddings.StackedEmbeddings\n",
    "meta_embeddings = MetaEmbeddings(\n",
    "    [char, bpemb, word, transformer],\n",
    "    use_average=True,    # use averages of embeddings instead of concatenation\n",
    "    use_attention=True,  # use attention to compute weighted averages\n",
    "    use_features=True,   # You can also include word features in the attention function\n",
    "                         # as proposed in our EMNLP paper\n",
    "    feature_model=feature_model\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from src.embeddings import DocumentRNNEmbeddings\n",
    "\n",
    "# This will create document representations from the input embeddings\n",
    "# based on a bidirectional LSTM\n",
    "# There are some other DocumentEmbeddings available. Check them out if you want.\n",
    "doc_embeds = DocumentRNNEmbeddings(\n",
    "    meta_embeddings,\n",
    "    bidirectional = True,\n",
    "    rnn_type=\"LSTM\",\n",
    "    reproject_words=False,\n",
    "    hidden_size=256,\n",
    "    rnn_layers=1, \n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from flair.models import TextClassifier\n",
    "\n",
    "# This will create the actual text classification model\n",
    "# All previous modules (MetaEmbeddings, DocumentEmbeddings, ...) are part of this\n",
    "classifier = TextClassifier(doc_embeds, label_dictionary=label_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from src.models import DomainClassifier\n",
    "\n",
    "# Create the discrminator for adversarial training\n",
    "domain_c = DomainClassifier(\n",
    "    meta_embeddings.embedding_length,\n",
    "    meta_embeddings.embedding_length,\n",
    "    meta_embeddings.num_embeddings,\n",
    "    dropout = 0.2,\n",
    "    lambd = 1.0e-4\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.optim import AdamW\n",
    "from src.trainer import AdversarialModelTrainer\n",
    "\n",
    "# Create our Trainer that does NER and adversarial training and start the training\n",
    "trainer = AdversarialModelTrainer(classifier, corpus, D=domain_c, optimizer=AdamW)\n",
    "\n",
    "trainer.train(\n",
    "    'taggers/name/',\n",
    "    learning_rate=5.0e-6,\n",
    "    mini_batch_size=64,\n",
    "    max_epochs=100,\n",
    "    train_with_dev=True,\n",
    "    adversarial_learning_k=10,\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:.conda-meta-embeddings]",
   "language": "python",
   "name": "conda-env-.conda-meta-embeddings-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}