Merge pull request #2153 from purificant/pep479

PEP 479 amends

nltk/toolbox.py

@@ -69,7 +69,12 @@ def raw_fields(self):
         # need to get first line outside the loop for correct handling
         # of the first marker if it spans multiple lines
         file_iter = iter(self._file)
-        line = next(file_iter)
+        # PEP 479, prevent RuntimeError when StopIteration is raised inside generator
+        try:
+            line = next(file_iter)
+        except StopIteration:
+            # no more data is available, terminate the generator
+            return
         mobj = re.match(first_line_pat, line)
         mkr, line_value = mobj.groups()
         value_lines = [line_value,]

nltk/util.py

@@ -464,7 +464,13 @@ def ngrams(sequence, n, pad_left=False, pad_right=False,
 
     history = []
     while n > 1:
-        history.append(next(sequence))
+        # PEP 479, prevent RuntimeError from being raised when StopIteration bubbles out of generator
+        try:
+            next_item = next(sequence)
+        except StopIteration:
+            # no more data, terminate the generator
+            return
+        history.append(next_item)
         n -= 1
     for item in sequence:
         history.append(item)