Merge pull request #46 from chrisjbryant/errant-dev

Add spaCy 3 support

.gitignore

@@ -0,0 +1,161 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+errant_env/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/

CHANGELOG.md

@@ -2,6 +2,14 @@
 
 This log describes all the changes made to ERRANT since its release.
 
+## v3.0.0 (04-11-23)
+
+1. Finally updated ERRANT to support Spacy 3! 
+    * I specifically tested Spacy 3.2 - 3.7 and found a negligible difference in performance on the BEA19 dev set. 
+    * This update also comes with an unexpected 10-20% speed gain.
+
+2. Added a `.gitignore` file. [#39](https://github.com/chrisjbryant/errant/issues/39)
+
 ## v2.3.3 (14-04-22)
 
 1. Missed one case of changing Levenshtein to rapidfuzz... Now fixed.

LICENSE.md

@@ -1,6 +1,6 @@
 # MIT License
 
-Copyright (c) 2017 Christopher Bryant, Mariano Felice
+Copyright (c) 2023 Christopher Bryant, Mariano Felice
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -1,4 +1,4 @@
-# ERRANT v2.3.3
+# ERRANT v3.0.0
 
 This repository contains the grammatical ERRor ANnotation Toolkit (ERRANT) described in:
 
@@ -30,30 +30,21 @@ A "noop" edit is a special kind of edit that explicitly indicates an annotator/s
 
 ## Pip Install
 
-The easiest way to install ERRANT and its dependencies is using `pip`. We also recommend installing it in a clean virtual environment (e.g. with `venv`). The latest version of ERRANT only supports Python >= 3.6.
+The easiest way to install ERRANT and its dependencies is using `pip`. We also recommend installing it in a clean virtual environment (e.g. with `venv`). The latest version of ERRANT only supports Python >= 3.7.
 ```
 python3 -m venv errant_env
 source errant_env/bin/activate
-pip3 install -U pip setuptools wheel
-pip3 install errant
-python3 -m spacy download en
+pip install -U pip setuptools wheel
+pip install errant
+python3 -m spacy download en_core_web_sm
 ```
 This will create and activate a new python3 environment called `errant_env` in the current directory. `pip` will then update some setup tools and install ERRANT, [spaCy](https://spacy.io/), [rapidfuzz](https://pypi.org/project/rapidfuzz/) and spaCy's default English model in this environment. You can deactivate the environment at any time by running `deactivate`, but must remember to activate it again whenever you want to use ERRANT.  
 
-#### ERRANT and spaCy
-
-ERRANT was originally designed to work with spaCy v1.9.0 and works best with this version. SpaCy v1.9.0 does not work with Python >= 3.7 however, and so we were forced to update ERRANT to be compatible with spaCy 2. Since spaCy 2 uses a neural system to trade speed for accuracy, this means ERRANT v2.2 is **~4x slower** than ERRANT v2.1. We have not yet extended ERRANT to work with spaCy 3, but preliminary tests suggest ERRANT will become even slower. 
-
-Consequently, we recommend ERRANT v2.1.0 if speed is a priority and you can use Python < 3.7.  
-```
-pip3 install errant==2.1.0
-```
-
 #### BEA-2019 Shared Task
 
-ERRANT v2.0.0 was designed to be fully compatible with the [BEA-2019 Shared Task](https://www.cl.cam.ac.uk/research/nl/bea2019st/). If you want to directly compare against the results in the shared task, you should make sure to install ERRANT v2.0.0 as newer versions may produce slightly different scores. You can also use [Codalab](https://competitions.codalab.org/competitions/20228) to evaluate anonymously on the shared task datasets. ERRANT v2.0.0 is not compatible with Python >= 3.7.  
+ERRANT v2.0.0 was designed to be fully compatible with the [BEA-2019 Shared Task](https://www.cl.cam.ac.uk/research/nl/bea2019st/). If you want to directly compare against the results in the shared task, you may want to install ERRANT v2.0.0 as newer versions may produce slightly different scores. You can also use [Codalab](https://codalab.lisn.upsaclay.fr/competitions/4057) to evaluate anonymously on the shared task datasets. ERRANT v2.0.0 is not compatible with Python >= 3.7.  
 ```
-pip3 install errant==2.0.0
+pip install errant==2.0.0
 ```
 
 ## Source Install
@@ -64,9 +55,9 @@ git clone https://github.com/chrisjbryant/errant.git
 cd errant
 python3 -m venv errant_env
 source errant_env/bin/activate
-pip3 install -U pip setuptools wheel
-pip3 install -e .
-python3 -m spacy download en
+pip install -U pip setuptools wheel
+pip install -e .
+python3 -m spacy download en_core_web_sm
 ```
 This will clone the github ERRANT source into the current directory, build and activate a python environment inside it, and then install ERRANT and all its dependencies. If you wish to modify ERRANT code, this is the recommended way to install it.
 

errant/__init__.py

@@ -3,23 +3,23 @@
 from errant.annotator import Annotator
 
 # ERRANT version
-__version__ = '2.3.3'
+__version__ = '3.0.0'
 
 # Load an ERRANT Annotator object for a given language
 def load(lang, nlp=None):
     # Make sure the language is supported
     supported = {"en"}
     if lang not in supported:
-        raise Exception("%s is an unsupported or unknown language" % lang)
+        raise Exception(f"{lang} is an unsupported or unknown language")
 
-    # Load spacy
-    nlp = nlp or spacy.load(lang, disable=["ner"])
+    # Load spacy (small model if no model supplied)
+    nlp = nlp or spacy.load(f"{lang}_core_web_sm", disable=["ner"])
 
     # Load language edit merger
-    merger = import_module("errant.%s.merger" % lang)
+    merger = import_module(f"errant.{lang}.merger")
 
     # Load language edit classifier
-    classifier = import_module("errant.%s.classifier" % lang)
+    classifier = import_module(f"errant.{lang}.classifier")
     # The English classifier needs spacy
     if lang == "en": classifier.nlp = nlp
 

errant/annotator.py

@@ -19,12 +19,11 @@ def __init__(self, lang, nlp=None, merger=None, classifier=None):
     # Input 2: A flag for word tokenisation
     # Output: The input string parsed by spacy
     def parse(self, text, tokenise=False):
-        if tokenise:
-            text = self.nlp(text)
-        else:
+        # Create Doc object from pretokenised text
+        if not tokenise:
             text = Doc(self.nlp.vocab, text.split())
-            self.nlp.tagger(text)
-            self.nlp.parser(text)
+        # POS tag and parse
+        text = self.nlp(text)
         return text
 
     # Input 1: An original text string parsed by spacy

setup.py

@@ -10,7 +10,7 @@
 
 setup(
     name = "errant",
-    version = "2.3.3",
+    version = "3.0.0",
     license = "MIT",
     description = "The ERRor ANnotation Toolkit (ERRANT). Automatically extract and classify edits in parallel sentences.",
     long_description = readme,
@@ -19,8 +19,8 @@
     author_email = "christopher.bryant@cl.cam.ac.uk",
     url = "https://github.com/chrisjbryant/errant",    
     keywords = ["automatic annotation", "grammatical errors", "natural language processing"],
-    python_requires = ">= 3.6",
-    install_requires = ["spacy>=2.2.0,<3", "rapidfuzz>=2.0.0"],
+    python_requires = ">= 3.7",
+    install_requires = ["spacy>=3.2.0,<4", "rapidfuzz>=3.4.0"],
     packages = find_packages(),    
     include_package_data=True,
     entry_points = {
@@ -35,10 +35,11 @@
         "License :: OSI Approved :: MIT License",
         "Natural Language :: English",
         "Operating System :: OS Independent",
-        'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: 3.9',
+        'Programming Language :: Python :: 3.10',
+        'Programming Language :: Python :: 3.11',
         "Topic :: Education",
         "Topic :: Scientific/Engineering :: Artificial Intelligence",
         "Topic :: Scientific/Engineering :: Information Analysis",