NLP with pretrained models - spaCy and StanfordNLP.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NLP with pretrained models - spaCy and StanfordNLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy\n",
    "\n",
    "en = spacy.load(\"en\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Donald John Trump (born June 14, 1946) is the 45th and current president of the United States.  Before entering politics, he was a businessman and television personality.\n"
     ]
    }
   ],
   "source": [
    "text = (\"Donald John Trump (born June 14, 1946) is the 45th and current president of \"\n",
    "        \"the United States.  Before entering politics, he was a businessman and television personality.\")\n",
    "print(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc_en = en(text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First spaCy splits your document into sentences, and the sentences in tokens."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Donald John Trump (born June 14, 1946) is the 45th and current president of the United States.  ,\n",
       " Before entering politics, he was a businessman and television personality.]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(doc_en.sents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<tbody>\n",
       "<tr><td>Donald     </td></tr>\n",
       "<tr><td>John       </td></tr>\n",
       "<tr><td>Trump      </td></tr>\n",
       "<tr><td>(          </td></tr>\n",
       "<tr><td>born       </td></tr>\n",
       "<tr><td>June       </td></tr>\n",
       "<tr><td>14         </td></tr>\n",
       "<tr><td>,          </td></tr>\n",
       "<tr><td>1946       </td></tr>\n",
       "<tr><td>)          </td></tr>\n",
       "<tr><td>is         </td></tr>\n",
       "<tr><td>the        </td></tr>\n",
       "<tr><td>45th       </td></tr>\n",
       "<tr><td>and        </td></tr>\n",
       "<tr><td>current    </td></tr>\n",
       "<tr><td>president  </td></tr>\n",
       "<tr><td>of         </td></tr>\n",
       "<tr><td>the        </td></tr>\n",
       "<tr><td>United     </td></tr>\n",
       "<tr><td>States     </td></tr>\n",
       "<tr><td>.          </td></tr>\n",
       "<tr><td>           </td></tr>\n",
       "<tr><td>Before     </td></tr>\n",
       "<tr><td>entering   </td></tr>\n",
       "<tr><td>politics   </td></tr>\n",
       "<tr><td>,          </td></tr>\n",
       "<tr><td>he         </td></tr>\n",
       "<tr><td>was        </td></tr>\n",
       "<tr><td>a          </td></tr>\n",
       "<tr><td>businessman</td></tr>\n",
       "<tr><td>and        </td></tr>\n",
       "<tr><td>television </td></tr>\n",
       "<tr><td>personality</td></tr>\n",
       "<tr><td>.          </td></tr>\n",
       "</tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from IPython.display import HTML, display\n",
    "import tabulate\n",
    "\n",
    "tokens = [[token] for token in doc_en]\n",
    "display(HTML(tabulate.tabulate(tokens, tablefmt='html')))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In addition, spaCy also identifies a number of linguistic features for every token. The most basic of these are the lemma, and two types of parts-of-speech tags: the `pos_` attribute contains the [Universal POS tags](https://universaldependencies.org/u/pos/) from the [Universal Dependencies](https://universaldependencies.org/), while the `tag_` attribute contains more fine-grained, language-specific part-of-speech tags."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<tbody>\n",
       "<tr><td>Donald     </td><td>donald     </td><td>PROPN</td><td>NNP  </td></tr>\n",
       "<tr><td>John       </td><td>john       </td><td>PROPN</td><td>NNP  </td></tr>\n",
       "<tr><td>Trump      </td><td>trump      </td><td>PROPN</td><td>NNP  </td></tr>\n",
       "<tr><td>(          </td><td>(          </td><td>PUNCT</td><td>-LRB-</td></tr>\n",
       "<tr><td>born       </td><td>bear       </td><td>VERB </td><td>VBN  </td></tr>\n",
       "<tr><td>June       </td><td>june       </td><td>PROPN</td><td>NNP  </td></tr>\n",
       "<tr><td>14         </td><td>14         </td><td>NUM  </td><td>CD   </td></tr>\n",
       "<tr><td>,          </td><td>,          </td><td>PUNCT</td><td>,    </td></tr>\n",
       "<tr><td>1946       </td><td>1946       </td><td>NUM  </td><td>CD   </td></tr>\n",
       "<tr><td>)          </td><td>)          </td><td>PUNCT</td><td>-RRB-</td></tr>\n",
       "<tr><td>is         </td><td>be         </td><td>VERB </td><td>VBZ  </td></tr>\n",
       "<tr><td>the        </td><td>the        </td><td>DET  </td><td>DT   </td></tr>\n",
       "<tr><td>45th       </td><td>45th       </td><td>ADJ  </td><td>JJ   </td></tr>\n",
       "<tr><td>and        </td><td>and        </td><td>CCONJ</td><td>CC   </td></tr>\n",
       "<tr><td>current    </td><td>current    </td><td>ADJ  </td><td>JJ   </td></tr>\n",
       "<tr><td>president  </td><td>president  </td><td>NOUN </td><td>NN   </td></tr>\n",
       "<tr><td>of         </td><td>of         </td><td>ADP  </td><td>IN   </td></tr>\n",
       "<tr><td>the        </td><td>the        </td><td>DET  </td><td>DT   </td></tr>\n",
       "<tr><td>United     </td><td>united     </td><td>PROPN</td><td>NNP  </td></tr>\n",
       "<tr><td>States     </td><td>states     </td><td>PROPN</td><td>NNP  </td></tr>\n",
       "<tr><td>.          </td><td>.          </td><td>PUNCT</td><td>.    </td></tr>\n",
       "<tr><td>           </td><td>           </td><td>SPACE</td><td>     </td></tr>\n",
       "<tr><td>Before     </td><td>before     </td><td>ADP  </td><td>IN   </td></tr>\n",
       "<tr><td>entering   </td><td>enter      </td><td>VERB </td><td>VBG  </td></tr>\n",
       "<tr><td>politics   </td><td>politic    </td><td>NOUN </td><td>NNS  </td></tr>\n",
       "<tr><td>,          </td><td>,          </td><td>PUNCT</td><td>,    </td></tr>\n",
       "<tr><td>he         </td><td>-PRON-     </td><td>PRON </td><td>PRP  </td></tr>\n",
       "<tr><td>was        </td><td>be         </td><td>VERB </td><td>VBD  </td></tr>\n",
       "<tr><td>a          </td><td>a          </td><td>DET  </td><td>DT   </td></tr>\n",
       "<tr><td>businessman</td><td>businessman</td><td>NOUN </td><td>NN   </td></tr>\n",
       "<tr><td>and        </td><td>and        </td><td>CCONJ</td><td>CC   </td></tr>\n",
       "<tr><td>television </td><td>television </td><td>NOUN </td><td>NN   </td></tr>\n",
       "<tr><td>personality</td><td>personality</td><td>NOUN </td><td>NN   </td></tr>\n",
       "<tr><td>.          </td><td>.          </td><td>PUNCT</td><td>.    </td></tr>\n",
       "</tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "features = [[t.orth_, t.lemma_, t.pos_, t.tag_] for t  in doc_en]\n",
    "display(HTML(tabulate.tabulate(features, tablefmt='html')))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Next, spaCy also offers pre-trained models for named entity recognition. Their results can be found on the `ent_iob_` and `ent_type` attributes. The `ent_type` attribute tells us what type of entity the token refers to. In the English models, these entity types follow the [OntoNotes standard](https://catalog.ldc.upenn.edu/docs/LDC2013T19/OntoNotes-Release-5.0.pdf). In our example, we see that `Donald John Trump` refers to a person, `June 14, 1946` to a date, `45th` to an ordinal number, and `the United States` to a geo-political entity (GPE). \n",
    "\n",
    "The letters on the `ent_iob_` attribute give the position of the token in the entity. `O` means the token is outside of an entity, `B` means the token is at the beginning of an entity, and `I` means it is inside an entity (at any position except for the beginning). In this way, we can tell apart several entities of the same type that immediately follow each other. Together these letters form the so-called `BIO` tagging scheme. There are other tagging schemes, such as `BILUO`, which also has letters for the last position and single (unique) tokens in an entity, but the BIO scheme gives you all the information you need.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<tbody>\n",
       "<tr><td>Donald     </td><td>B</td><td>PERSON </td></tr>\n",
       "<tr><td>John       </td><td>I</td><td>PERSON </td></tr>\n",
       "<tr><td>Trump      </td><td>I</td><td>PERSON </td></tr>\n",
       "<tr><td>(          </td><td>O</td><td>       </td></tr>\n",
       "<tr><td>born       </td><td>O</td><td>       </td></tr>\n",
       "<tr><td>June       </td><td>B</td><td>DATE   </td></tr>\n",
       "<tr><td>14         </td><td>I</td><td>DATE   </td></tr>\n",
       "<tr><td>,          </td><td>I</td><td>DATE   </td></tr>\n",
       "<tr><td>1946       </td><td>I</td><td>DATE   </td></tr>\n",
       "<tr><td>)          </td><td>O</td><td>       </td></tr>\n",
       "<tr><td>is         </td><td>O</td><td>       </td></tr>\n",
       "<tr><td>the        </td><td>O</td><td>       </td></tr>\n",
       "<tr><td>45th       </td><td>B</td><td>ORDINAL</td></tr>\n",
       "<tr><td>and        </td><td>O</td><td>       </td></tr>\n",
       "<tr><td>current    </td><td>O</td><td>       </td></tr>\n",
       "<tr><td>president  </td><td>O</td><td>       </td></tr>\n",
       "<tr><td>of         </td><td>O</td><td>       </td></tr>\n",
       "<tr><td>the        </td><td>B</td><td>GPE    </td></tr>\n",
       "<tr><td>United     </td><td>I</td><td>GPE    </td></tr>\n",
       "<tr><td>States     </td><td>I</td><td>GPE    </td></tr>\n",
       "<tr><td>.          </td><td>O</td><td>       </td></tr>\n",
       "<tr><td>           </td><td>O</td><td>       </td></tr>\n",
       "<tr><td>Before     </td><td>O</td><td>       </td></tr>\n",
       "<tr><td>entering   </td><td>O</td><td>       </td></tr>\n",
       "<tr><td>politics   </td><td>O</td><td>       </td></tr>\n",
       "<tr><td>,          </td><td>O</td><td>       </td></tr>\n",
       "<tr><td>he         </td><td>O</td><td>       </td></tr>\n",
       "<tr><td>was        </td><td>O</td><td>       </td></tr>\n",
       "<tr><td>a          </td><td>O</td><td>       </td></tr>\n",
       "<tr><td>businessman</td><td>O</td><td>       </td></tr>\n",
       "<tr><td>and        </td><td>O</td><td>       </td></tr>\n",
       "<tr><td>television </td><td>O</td><td>       </td></tr>\n",
       "<tr><td>personality</td><td>O</td><td>       </td></tr>\n",
       "<tr><td>.          </td><td>O</td><td>       </td></tr>\n",
       "</tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "entities = [(t.orth_, t.ent_iob_, t.ent_type_) for t in doc_en]\n",
    "display(HTML(tabulate.tabulate(entities, tablefmt='html')))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You can also access the entities directly on the `ents` attribute of the document: "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('Donald John Trump', 'PERSON'), ('June 14, 1946', 'DATE'), ('45th', 'ORDINAL'), ('the United States', 'GPE')]\n"
     ]
    }
   ],
   "source": [
    "print([(ent.text, ent.label_) for ent in doc_en.ents])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finally, spaCy also contains a dependency parser, which analyzes the grammatical relations between the tokens. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<tbody>\n",
       "<tr><td>Donald     </td><td>compound</td><td>Trump      </td></tr>\n",
       "<tr><td>John       </td><td>compound</td><td>Trump      </td></tr>\n",
       "<tr><td>Trump      </td><td>nsubj   </td><td>is         </td></tr>\n",
       "<tr><td>(          </td><td>punct   </td><td>Trump      </td></tr>\n",
       "<tr><td>born       </td><td>acl     </td><td>Trump      </td></tr>\n",
       "<tr><td>June       </td><td>npadvmod</td><td>born       </td></tr>\n",
       "<tr><td>14         </td><td>nummod  </td><td>June       </td></tr>\n",
       "<tr><td>,          </td><td>punct   </td><td>June       </td></tr>\n",
       "<tr><td>1946       </td><td>nummod  </td><td>June       </td></tr>\n",
       "<tr><td>)          </td><td>punct   </td><td>Trump      </td></tr>\n",
       "<tr><td>is         </td><td>ROOT    </td><td>is         </td></tr>\n",
       "<tr><td>the        </td><td>det     </td><td>president  </td></tr>\n",
       "<tr><td>45th       </td><td>amod    </td><td>president  </td></tr>\n",
       "<tr><td>and        </td><td>cc      </td><td>45th       </td></tr>\n",
       "<tr><td>current    </td><td>conj    </td><td>45th       </td></tr>\n",
       "<tr><td>president  </td><td>attr    </td><td>is         </td></tr>\n",
       "<tr><td>of         </td><td>prep    </td><td>president  </td></tr>\n",
       "<tr><td>the        </td><td>det     </td><td>States     </td></tr>\n",
       "<tr><td>United     </td><td>compound</td><td>States     </td></tr>\n",
       "<tr><td>States     </td><td>pobj    </td><td>of         </td></tr>\n",
       "<tr><td>.          </td><td>punct   </td><td>is         </td></tr>\n",
       "<tr><td>           </td><td>        </td><td>.          </td></tr>\n",
       "<tr><td>Before     </td><td>prep    </td><td>was        </td></tr>\n",
       "<tr><td>entering   </td><td>pcomp   </td><td>Before     </td></tr>\n",
       "<tr><td>politics   </td><td>dobj    </td><td>entering   </td></tr>\n",
       "<tr><td>,          </td><td>punct   </td><td>was        </td></tr>\n",
       "<tr><td>he         </td><td>nsubj   </td><td>was        </td></tr>\n",
       "<tr><td>was        </td><td>ROOT    </td><td>was        </td></tr>\n",
       "<tr><td>a          </td><td>det     </td><td>personality</td></tr>\n",
       "<tr><td>businessman</td><td>nmod    </td><td>personality</td></tr>\n",
       "<tr><td>and        </td><td>cc      </td><td>businessman</td></tr>\n",
       "<tr><td>television </td><td>conj    </td><td>businessman</td></tr>\n",
       "<tr><td>personality</td><td>attr    </td><td>was        </td></tr>\n",
       "<tr><td>.          </td><td>punct   </td><td>was        </td></tr>\n",
       "</tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "syntax = [[token.text, token.dep_, token.head.text ] for token in doc_en]\n",
    "display(HTML(tabulate.tabulate(syntax, tablefmt='html')))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Multingual NLP"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "SpaCy doesn't only have models for English, but also for many other languages. Here's an example of a Dutch sentence, which means \"Charles Michel is the prime minister of Belgium\"."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "nl = spacy.load(\"nl\")\n",
    "text_nl = \"Charles Michel is de eerste minister van België.\"\n",
    "doc_nl = nl(text_nl)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The tokens in the Dutch document have the same attributes as those in the English one. Take care, however, because the functionality of the models can differ across languages. Here are three main differences between the English and the Dutch model: \n",
    "\n",
    "- The Dutch model does not offer lemmatization: the lemma_ attribute is identical to the orth_ attribute.\n",
    "- The Dutch model has a very different fine-grained part-of-speech tags on the tag_ attribute.\n",
    "- The Dutch model has different entity types (PER, LOC and ORG) than the English one. \n",
    "\n",
    "This is a result of the training corpora that were used to build the models, whose annotation guidelines may be very different."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<tbody>\n",
       "<tr><td>Charles </td><td>Charles </td><td>NOUN </td><td>N_N|eigen|ev|neut_eigen|ev|neut___                                                     </td><td>B</td><td>PER</td></tr>\n",
       "<tr><td>Michel  </td><td>Michel  </td><td>PROPN</td><td>PROPN___                                                                               </td><td>I</td><td>PER</td></tr>\n",
       "<tr><td>is      </td><td>is      </td><td>VERB </td><td>V|hulpofkopp|ott|3|ev__Aspect=Imp|Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin</td><td>O</td><td>   </td></tr>\n",
       "<tr><td>de      </td><td>de      </td><td>DET  </td><td>Art|bep|zijdofmv|neut__Definite=Def|PronType=Art                                       </td><td>O</td><td>   </td></tr>\n",
       "<tr><td>eerste  </td><td>eerste  </td><td>NUM  </td><td>Num|rang|bep|attr|onverv__Definite=Def|NumType=Ord                                     </td><td>O</td><td>   </td></tr>\n",
       "<tr><td>minister</td><td>minister</td><td>NOUN </td><td>N|soort|ev|neut__Number=Sing                                                           </td><td>O</td><td>   </td></tr>\n",
       "<tr><td>van     </td><td>van     </td><td>ADP  </td><td>Prep|voor__AdpType=Prep                                                                </td><td>O</td><td>   </td></tr>\n",
       "<tr><td>België  </td><td>België  </td><td>NOUN </td><td>N|eigen|ev|neut__Number=Sing                                                           </td><td>B</td><td>LOC</td></tr>\n",
       "<tr><td>.       </td><td>.       </td><td>PUNCT</td><td>Punc|punt__PunctType=Peri                                                              </td><td>O</td><td>   </td></tr>\n",
       "</tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "info = [(t.orth_, t.lemma_, t.pos_, t.tag_, t.ent_iob_, t.ent_type_) for t in doc_nl]\n",
    "display(HTML(tabulate.tabulate(info, tablefmt='html')))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## StanfordNLP"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Another library whose functionality overlaps with that of spaCy is StanfordNLP. [StanfordNLP](https://stanfordnlp.github.io/stanfordnlp/), not to be confused with Stanford's Java [CoreNLP](https://stanfordnlp.github.io/CoreNLP/) library, is a [Python library](https://github.com/stanfordnlp/stanfordnlp) built on top of PyTorch that offers a fully neural pipeline with tokenization (including multi-word units), lemmatization, part-of-speech tagging (including morphological features) and dependency parsing. These components were built and trained for the [CoNLL-2018 shared task](https://nlp.stanford.edu/pubs/qi2018universal.pdf). There are no named entities, but the quality of the dependency parsing is state of the art. On top of that, it also offers a Python interface to CoreNLP. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Its API is very similar to that of spaCy:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Use device: cpu\n",
      "---\n",
      "Loading: tokenize\n",
      "With settings: \n",
      "{'model_path': '/Users/yvespeirsman/stanfordnlp_resources/nl_alpino_models/nl_alpino_tokenizer.pt', 'lang': 'nl', 'shorthand': 'nl_alpino', 'mode': 'predict'}\n",
      "---\n",
      "Loading: pos\n",
      "With settings: \n",
      "{'model_path': '/Users/yvespeirsman/stanfordnlp_resources/nl_alpino_models/nl_alpino_tagger.pt', 'pretrain_path': '/Users/yvespeirsman/stanfordnlp_resources/nl_alpino_models/nl_alpino.pretrain.pt', 'lang': 'nl', 'shorthand': 'nl_alpino', 'mode': 'predict'}\n",
      "---\n",
      "Loading: lemma\n",
      "With settings: \n",
      "{'model_path': '/Users/yvespeirsman/stanfordnlp_resources/nl_alpino_models/nl_alpino_lemmatizer.pt', 'lang': 'nl', 'shorthand': 'nl_alpino', 'mode': 'predict'}\n",
      "Building an attentional Seq2Seq model...\n",
      "Using a Bi-LSTM encoder\n",
      "Using soft attention for LSTM.\n",
      "Finetune all embeddings.\n",
      "[Running seq2seq lemmatizer with edit classifier]\n",
      "---\n",
      "Loading: depparse\n",
      "With settings: \n",
      "{'model_path': '/Users/yvespeirsman/stanfordnlp_resources/nl_alpino_models/nl_alpino_parser.pt', 'pretrain_path': '/Users/yvespeirsman/stanfordnlp_resources/nl_alpino_models/nl_alpino.pretrain.pt', 'lang': 'nl', 'shorthand': 'nl_alpino', 'mode': 'predict'}\n",
      "Done loading processors!\n",
      "---\n"
     ]
    }
   ],
   "source": [
    "import stanfordnlp\n",
    "\n",
    "#stanfordnlp.download('nl')\n",
    "nl_stanford = stanfordnlp.Pipeline(lang=\"nl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc_nl_stanford = nl_stanford(text_nl)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The `text` and `lemma` properties speak for themselves. The `upos` attribute contains the universal dependencies we also find on spaCy's `pos_` attribute; the `xpos` attribute corresponds to spaCy's `tag_` attribute and contains the fine-grained tags with morphological properties. The `governor` attribute contains the (1-based) index of the head of each token; `dependency_relation` contains the grammatical relation between the two. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "stanford_info = []\n",
    "for sentence in doc_nl_stanford.sentences:\n",
    "    for token in sentence.tokens:\n",
    "        for word in token.words:\n",
    "            stanford_info.append((len(stanford_info)+1, word.text, word.lemma, word.upos, word.xpos, word.dependency_relation, word.governor))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<tbody>\n",
       "<tr><td style=\"text-align: right;\">1</td><td>Charles </td><td>Charles </td><td>PROPN</td><td>SPEC|deeleigen            </td><td>nsubj    </td><td style=\"text-align: right;\">6</td></tr>\n",
       "<tr><td style=\"text-align: right;\">2</td><td>Michel  </td><td>Michel  </td><td>PROPN</td><td>SPEC|deeleigen            </td><td>flat:name</td><td style=\"text-align: right;\">1</td></tr>\n",
       "<tr><td style=\"text-align: right;\">3</td><td>is      </td><td>zijn    </td><td>AUX  </td><td>WW|pv|tgw|ev              </td><td>cop      </td><td style=\"text-align: right;\">6</td></tr>\n",
       "<tr><td style=\"text-align: right;\">4</td><td>de      </td><td>de      </td><td>DET  </td><td>LID|bep|stan|rest         </td><td>det      </td><td style=\"text-align: right;\">6</td></tr>\n",
       "<tr><td style=\"text-align: right;\">5</td><td>eerste  </td><td>één     </td><td>ADJ  </td><td>TW|rang|prenom|stan       </td><td>amod     </td><td style=\"text-align: right;\">6</td></tr>\n",
       "<tr><td style=\"text-align: right;\">6</td><td>minister</td><td>minister</td><td>NOUN </td><td>N|soort|ev|basis|zijd|stan</td><td>root     </td><td style=\"text-align: right;\">0</td></tr>\n",
       "<tr><td style=\"text-align: right;\">7</td><td>van     </td><td>van     </td><td>ADP  </td><td>VZ|init                   </td><td>case     </td><td style=\"text-align: right;\">8</td></tr>\n",
       "<tr><td style=\"text-align: right;\">8</td><td>België  </td><td>België  </td><td>PROPN</td><td>N|eigen|ev|basis|onz|stan </td><td>nmod     </td><td style=\"text-align: right;\">6</td></tr>\n",
       "<tr><td style=\"text-align: right;\">9</td><td>.       </td><td>.       </td><td>PUNCT</td><td>LET                       </td><td>punct    </td><td style=\"text-align: right;\">6</td></tr>\n",
       "</tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(HTML(tabulate.tabulate(stanford_info, tablefmt='html')))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Combining spaCy and StanfordNLP"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If you can't choose between spaCy and StanfordNLP, you can also combine the two. Yes, sometimes you can have the best of both worlds. Thanks to the `spacy_stanfordnlp` wrapper, we can plug a Stanford NLP model into a spaCy pipeline, and get its annotations in a spaCy document. Suddenly we have Dutch lemmatization and state-of-the-art part-of-speech tagging and dependency parsing."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<tbody>\n",
       "<tr><td>Charles </td><td>Charles </td><td>PROPN</td><td>SPEC|deeleigen            </td></tr>\n",
       "<tr><td>Michel  </td><td>Michel  </td><td>PROPN</td><td>SPEC|deeleigen            </td></tr>\n",
       "<tr><td>is      </td><td>zijn    </td><td>AUX  </td><td>WW|pv|tgw|ev              </td></tr>\n",
       "<tr><td>de      </td><td>de      </td><td>DET  </td><td>LID|bep|stan|rest         </td></tr>\n",
       "<tr><td>eerste  </td><td>één     </td><td>ADJ  </td><td>TW|rang|prenom|stan       </td></tr>\n",
       "<tr><td>minister</td><td>minister</td><td>NOUN </td><td>N|soort|ev|basis|zijd|stan</td></tr>\n",
       "<tr><td>van     </td><td>van     </td><td>ADP  </td><td>VZ|init                   </td></tr>\n",
       "<tr><td>België  </td><td>België  </td><td>PROPN</td><td>N|eigen|ev|basis|onz|stan </td></tr>\n",
       "<tr><td>.       </td><td>.       </td><td>PUNCT</td><td>LET                       </td></tr>\n",
       "</tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from spacy_stanfordnlp import StanfordNLPLanguage\n",
    "\n",
    "nl_combined = StanfordNLPLanguage(nl_stanford)\n",
    "\n",
    "doc_nl_combined = nl_combined(text_nl)\n",
    "\n",
    "info = [(t.orth_, t.lemma_, t.pos_, t.tag_) for t in doc_nl_combined]\n",
    "display(HTML(tabulate.tabulate(info, tablefmt='html')))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Additionally, we can mix and match the strengths of the two libraries, for example by extending our pipeline with spaCy's Named Entity Recognition."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<tbody>\n",
       "<tr><td>Charles </td><td>Charles </td><td>PROPN</td><td>SPEC|deeleigen            </td><td>B</td><td>PER</td></tr>\n",
       "<tr><td>Michel  </td><td>Michel  </td><td>PROPN</td><td>SPEC|deeleigen            </td><td>I</td><td>PER</td></tr>\n",
       "<tr><td>is      </td><td>zijn    </td><td>AUX  </td><td>WW|pv|tgw|ev              </td><td>O</td><td>   </td></tr>\n",
       "<tr><td>de      </td><td>de      </td><td>DET  </td><td>LID|bep|stan|rest         </td><td>O</td><td>   </td></tr>\n",
       "<tr><td>eerste  </td><td>één     </td><td>ADJ  </td><td>TW|rang|prenom|stan       </td><td>O</td><td>   </td></tr>\n",
       "<tr><td>minister</td><td>minister</td><td>NOUN </td><td>N|soort|ev|basis|zijd|stan</td><td>O</td><td>   </td></tr>\n",
       "<tr><td>van     </td><td>van     </td><td>ADP  </td><td>VZ|init                   </td><td>O</td><td>   </td></tr>\n",
       "<tr><td>België  </td><td>België  </td><td>PROPN</td><td>N|eigen|ev|basis|onz|stan </td><td>B</td><td>LOC</td></tr>\n",
       "<tr><td>.       </td><td>.       </td><td>PUNCT</td><td>LET                       </td><td>O</td><td>   </td></tr>\n",
       "</tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "nl_combined = StanfordNLPLanguage(nl_stanford)\n",
    "nl_ner = nl.get_pipe(\"ner\")\n",
    "nl_combined.add_pipe(nl_ner)\n",
    "nl_combined.vocab.strings.add(\"PER\")\n",
    "\n",
    "doc_nl_combined = nl_combined(text_nl)\n",
    "\n",
    "info = [(t.orth_, t.lemma_, t.pos_, t.tag_, t.ent_iob_, t.ent_type_) for t in doc_nl_combined]\n",
    "display(HTML(tabulate.tabulate(info, tablefmt='html')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}