-
Notifications
You must be signed in to change notification settings - Fork 1
/
stemmer.py
617 lines (561 loc) · 25.9 KB
/
stemmer.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
"""
This was copied from the NLTK source:
https://github.com/nltk/nltk/blob/7e06fcb2be41a7dbc23bf0b4f666aef7b915d402/nltk/stem/porter.py
It was modified slightly to run outside NLTK.
"""
"""
Porter Stemmer
This is the Porter stemming algorithm. It follows the algorithm
presented in
Porter, M. "An algorithm for suffix stripping." Program 14.3 (1980): 130-137.
with some optional deviations that can be turned on or off with the
`mode` argument to the constructor.
Martin Porter, the algorithm's inventor, maintains a web page about the
algorithm at
http://www.tartarus.org/~martin/PorterStemmer/
which includes another Python implementation and other implementations
in many languages.
"""
__docformat__ = 'plaintext'
class PorterStemmer:
"""
A word stemmer based on the Porter stemming algorithm.
Porter, M. "An algorithm for suffix stripping."
Program 14.3 (1980): 130-137.
See http://www.tartarus.org/~martin/PorterStemmer/ for the homepage
of the algorithm.
Martin Porter has endorsed several modifications to the Porter
algorithm since writing his original paper, and those extensions are
included in the implementations on his website. Additionally, others
have proposed further improvements to the algorithm, including NLTK
contributors. There are thus three modes that can be selected by
passing the appropriate constant to the class constructor's `mode`
attribute:
PorterStemmer.ORIGINAL_ALGORITHM
- Implementation that is faithful to the original paper.
Note that Martin Porter has deprecated this version of the
algorithm. Martin distributes implementations of the Porter
Stemmer in many languages, hosted at:
http://www.tartarus.org/~martin/PorterStemmer/
and all of these implementations include his extensions. He
strongly recommends against using the original, published
version of the algorithm; only use this mode if you clearly
understand why you are choosing to do so.
PorterStemmer.MARTIN_EXTENSIONS
- Implementation that only uses the modifications to the
algorithm that are included in the implementations on Martin
Porter's website. He has declared Porter frozen, so the
behaviour of those implementations should never change.
PorterStemmer.NLTK_EXTENSIONS (default)
- Implementation that includes further improvements devised by
NLTK contributors or taken from other modified implementations
found on the web.
For the best stemming, you should use the default NLTK_EXTENSIONS
version. However, if you need to get the same results as either the
original algorithm or one of Martin Porter's hosted versions for
compability with an existing implementation or dataset, you can use
one of the other modes instead.
"""
# Modes the Stemmer can be instantiated in
NLTK_EXTENSIONS = 'NLTK_EXTENSIONS'
MARTIN_EXTENSIONS = 'MARTIN_EXTENSIONS'
ORIGINAL_ALGORITHM = 'ORIGINAL_ALGORITHM'
def __init__(self, mode=NLTK_EXTENSIONS):
if mode not in (
self.NLTK_EXTENSIONS,
self.MARTIN_EXTENSIONS,
self.ORIGINAL_ALGORITHM
):
raise ValueError(
"Mode must be one of PorterStemmer.NLTK_EXTENSIONS, "
"PorterStemmer.MARTIN_EXTENSIONS, or "
"PorterStemmer.ORIGINAL_ALGORITHM"
)
self.mode = mode
if self.mode == self.NLTK_EXTENSIONS:
# This is a table of irregular forms. It is quite short,
# but still reflects the errors actually drawn to Martin
# Porter's attention over a 20 year period!
irregular_forms = {
"sky": ["sky", "skies"],
"die": ["dying"],
"lie": ["lying"],
"tie": ["tying"],
"news": ["news"],
"inning": ["innings", "inning"],
"outing": ["outings", "outing"],
"canning": ["cannings", "canning"],
"howe": ["howe"],
"proceed": ["proceed"],
"exceed": ["exceed"],
"succeed": ["succeed"],
}
self.pool = {}
for key in irregular_forms:
for val in irregular_forms[key]:
self.pool[val] = key
self.vowels = frozenset(['a', 'e', 'i', 'o', 'u'])
def _is_consonant(self, word, i):
"""Returns True if word[i] is a consonant, False otherwise
A consonant is defined in the paper as follows:
A consonant in a word is a letter other than A, E, I, O or
U, and other than Y preceded by a consonant. (The fact that
the term `consonant' is defined to some extent in terms of
itself does not make it ambiguous.) So in TOY the consonants
are T and Y, and in SYZYGY they are S, Z and G. If a letter
is not a consonant it is a vowel.
"""
if word[i] in self.vowels:
return False
if word[i] == 'y':
if i == 0:
return True
else:
return (not self._is_consonant(word, i - 1))
return True
def _measure(self, stem):
"""Returns the 'measure' of stem, per definition in the paper
From the paper:
A consonant will be denoted by c, a vowel by v. A list
ccc... of length greater than 0 will be denoted by C, and a
list vvv... of length greater than 0 will be denoted by V.
Any word, or part of a word, therefore has one of the four
forms:
CVCV ... C
CVCV ... V
VCVC ... C
VCVC ... V
These may all be represented by the single form
[C]VCVC ... [V]
where the square brackets denote arbitrary presence of their
contents. Using (VC){m} to denote VC repeated m times, this
may again be written as
[C](VC){m}[V].
m will be called the \measure\ of any word or word part when
represented in this form. The case m = 0 covers the null
word. Here are some examples:
m=0 TR, EE, TREE, Y, BY.
m=1 TROUBLE, OATS, TREES, IVY.
m=2 TROUBLES, PRIVATE, OATEN, ORRERY.
"""
cv_sequence = ''
# Construct a string of 'c's and 'v's representing whether each
# character in `stem` is a consonant or a vowel.
# e.g. 'falafel' becomes 'cvcvcvc',
# 'architecture' becomes 'vcccvcvccvcv'
for i in range(len(stem)):
if self._is_consonant(stem, i):
cv_sequence += 'c'
else:
cv_sequence += 'v'
# Count the number of 'vc' occurences, which is equivalent to
# the number of 'VC' occurrences in Porter's reduced form in the
# docstring above, which is in turn equivalent to `m`
return cv_sequence.count('vc')
def _has_positive_measure(self, stem):
return self._measure(stem) > 0
def _contains_vowel(self, stem):
"""Returns True if stem contains a vowel, else False"""
for i in range(len(stem)):
if not self._is_consonant(stem, i):
return True
return False
def _ends_double_consonant(self, word):
"""Implements condition *d from the paper
Returns True if word ends with a double consonant
"""
return (
len(word) >= 2 and
word[-1] == word[-2] and
self._is_consonant(word, len(word) - 1)
)
def _ends_cvc(self, word):
"""Implements condition *o from the paper
From the paper:
*o - the stem ends cvc, where the second c is not W, X or Y
(e.g. -WIL, -HOP).
"""
return (
len(word) >= 3 and
self._is_consonant(word, len(word) - 3) and
not self._is_consonant(word, len(word) - 2) and
self._is_consonant(word, len(word) - 1) and
word[-1] not in ('w', 'x', 'y')
) or (
self.mode == self.NLTK_EXTENSIONS and
len(word) == 2 and
not self._is_consonant(word, 0) and
self._is_consonant(word, 1)
)
def _replace_suffix(self, word, suffix, replacement):
"""Replaces `suffix` of `word` with `replacement"""
assert word.endswith(suffix), "Given word doesn't end with given suffix"
if suffix == '':
return word + replacement
else:
return word[:-len(suffix)] + replacement
def _apply_rule_list(self, word, rules):
"""Applies the first applicable suffix-removal rule to the word
Takes a word and a list of suffix-removal rules represented as
3-tuples, with the first element being the suffix to remove,
the second element being the string to replace it with, and the
final element being the condition for the rule to be applicable,
or None if the rule is unconditional.
"""
for rule in rules:
suffix, replacement, condition = rule
if suffix == '*d' and self._ends_double_consonant(word):
stem = word[:-2]
if condition is None or condition(stem):
return stem + replacement
else:
# Don't try any further rules
return word
if word.endswith(suffix):
stem = self._replace_suffix(word, suffix, '')
if condition is None or condition(stem):
return stem + replacement
else:
# Don't try any further rules
return word
return word
def _step1a(self, word):
"""Implements Step 1a from "An algorithm for suffix stripping"
From the paper:
SSES -> SS caresses -> caress
IES -> I ponies -> poni
ties -> ti
SS -> SS caress -> caress
S -> cats -> cat
"""
# this NLTK-only rule extends the original algorithm, so
# that 'flies'->'fli' but 'dies'->'die' etc
if self.mode == self.NLTK_EXTENSIONS:
if word.endswith('ies') and len(word) == 4:
return self._replace_suffix(word, 'ies', 'ie')
return self._apply_rule_list(word, [
('sses', 'ss', None), # SSES -> SS
('ies', 'i', None), # IES -> I
('ss', 'ss', None), # SS -> SS
('s', '', None), # S ->
])
def _step1b(self, word):
"""Implements Step 1b from "An algorithm for suffix stripping"
From the paper:
(m>0) EED -> EE feed -> feed
agreed -> agree
(*v*) ED -> plastered -> plaster
bled -> bled
(*v*) ING -> motoring -> motor
sing -> sing
If the second or third of the rules in Step 1b is successful,
the following is done:
AT -> ATE conflat(ed) -> conflate
BL -> BLE troubl(ed) -> trouble
IZ -> IZE siz(ed) -> size
(*d and not (*L or *S or *Z))
-> single letter
hopp(ing) -> hop
tann(ed) -> tan
fall(ing) -> fall
hiss(ing) -> hiss
fizz(ed) -> fizz
(m=1 and *o) -> E fail(ing) -> fail
fil(ing) -> file
The rule to map to a single letter causes the removal of one of
the double letter pair. The -E is put back on -AT, -BL and -IZ,
so that the suffixes -ATE, -BLE and -IZE can be recognised
later. This E may be removed in step 4.
"""
# this NLTK-only block extends the original algorithm, so that
# 'spied'->'spi' but 'died'->'die' etc
if self.mode == self.NLTK_EXTENSIONS:
if word.endswith('ied'):
if len(word) == 4:
return self._replace_suffix(word, 'ied', 'ie')
else:
return self._replace_suffix(word, 'ied', 'i')
# (m>0) EED -> EE
if word.endswith('eed'):
stem = self._replace_suffix(word, 'eed', '')
if self._measure(stem) > 0:
return stem + 'ee'
else:
return word
rule_2_or_3_succeeded = False
for suffix in ['ed', 'ing']:
if word.endswith(suffix):
intermediate_stem = self._replace_suffix(word, suffix, '')
if self._contains_vowel(intermediate_stem):
rule_2_or_3_succeeded = True
break
if not rule_2_or_3_succeeded:
return word
return self._apply_rule_list(intermediate_stem, [
('at', 'ate', None), # AT -> ATE
('bl', 'ble', None), # BL -> BLE
('iz', 'ize', None), # IZ -> IZE
# (*d and not (*L or *S or *Z))
# -> single letter
(
'*d',
intermediate_stem[-1],
lambda stem: intermediate_stem[-1] not in ('l', 's', 'z')
),
# (m=1 and *o) -> E
(
'',
'e',
lambda stem: (self._measure(stem) == 1 and
self._ends_cvc(stem))
),
])
def _step1c(self, word):
"""Implements Step 1c from "An algorithm for suffix stripping"
From the paper:
Step 1c
(*v*) Y -> I happy -> happi
sky -> sky
"""
def nltk_condition(stem):
"""
This has been modified from the original Porter algorithm so
that y->i is only done when y is preceded by a consonant,
but not if the stem is only a single consonant, i.e.
(*c and not c) Y -> I
So 'happy' -> 'happi', but
'enjoy' -> 'enjoy' etc
This is a much better rule. Formerly 'enjoy'->'enjoi' and
'enjoyment'->'enjoy'. Step 1c is perhaps done too soon; but
with this modification that no longer really matters.
Also, the removal of the contains_vowel(z) condition means
that 'spy', 'fly', 'try' ... stem to 'spi', 'fli', 'tri' and
conflate with 'spied', 'tried', 'flies' ...
"""
return len(stem) > 1 and self._is_consonant(stem, len(stem) - 1)
def original_condition(stem):
return self._contains_vowel(stem)
return self._apply_rule_list(word, [
(
'y',
'i',
nltk_condition if self.mode == self.NLTK_EXTENSIONS
else original_condition
)
])
def _step2(self, word):
"""Implements Step 2 from "An algorithm for suffix stripping"
From the paper:
Step 2
(m>0) ATIONAL -> ATE relational -> relate
(m>0) TIONAL -> TION conditional -> condition
rational -> rational
(m>0) ENCI -> ENCE valenci -> valence
(m>0) ANCI -> ANCE hesitanci -> hesitance
(m>0) IZER -> IZE digitizer -> digitize
(m>0) ABLI -> ABLE conformabli -> conformable
(m>0) ALLI -> AL radicalli -> radical
(m>0) ENTLI -> ENT differentli -> different
(m>0) ELI -> E vileli - > vile
(m>0) OUSLI -> OUS analogousli -> analogous
(m>0) IZATION -> IZE vietnamization -> vietnamize
(m>0) ATION -> ATE predication -> predicate
(m>0) ATOR -> ATE operator -> operate
(m>0) ALISM -> AL feudalism -> feudal
(m>0) IVENESS -> IVE decisiveness -> decisive
(m>0) FULNESS -> FUL hopefulness -> hopeful
(m>0) OUSNESS -> OUS callousness -> callous
(m>0) ALITI -> AL formaliti -> formal
(m>0) IVITI -> IVE sensitiviti -> sensitive
(m>0) BILITI -> BLE sensibiliti -> sensible
"""
if self.mode == self.NLTK_EXTENSIONS:
# Instead of applying the ALLI -> AL rule after '(a)bli' per
# the published algorithm, instead we apply it first, and,
# if it succeeds, run the result through step2 again.
if (
word.endswith('alli') and
self._has_positive_measure(
self._replace_suffix(word, 'alli', '')
)
):
return self._step2(
self._replace_suffix(word, 'alli', 'al')
)
bli_rule = ('bli', 'ble', self._has_positive_measure)
abli_rule = ('abli', 'able', self._has_positive_measure)
rules = [
('ational', 'ate', self._has_positive_measure),
('tional', 'tion', self._has_positive_measure),
('enci', 'ence', self._has_positive_measure),
('anci', 'ance', self._has_positive_measure),
('izer', 'ize', self._has_positive_measure),
abli_rule if self.mode == self.ORIGINAL_ALGORITHM else bli_rule,
('alli', 'al', self._has_positive_measure),
('entli', 'ent', self._has_positive_measure),
('eli', 'e', self._has_positive_measure),
('ousli', 'ous', self._has_positive_measure),
('ization', 'ize', self._has_positive_measure),
('ation', 'ate', self._has_positive_measure),
('ator', 'ate', self._has_positive_measure),
('alism', 'al', self._has_positive_measure),
('iveness', 'ive', self._has_positive_measure),
('fulness', 'ful', self._has_positive_measure),
('ousness', 'ous', self._has_positive_measure),
('aliti', 'al', self._has_positive_measure),
('iviti', 'ive', self._has_positive_measure),
('biliti', 'ble', self._has_positive_measure),
]
if self.mode == self.NLTK_EXTENSIONS:
rules.append(
('fulli', 'ful', self._has_positive_measure)
)
# The 'l' of the 'logi' -> 'log' rule is put with the stem,
# so that short stems like 'geo' 'theo' etc work like
# 'archaeo' 'philo' etc.
rules.append((
"logi",
"log",
lambda stem: self._has_positive_measure(word[:-3])
))
if self.mode == self.MARTIN_EXTENSIONS:
rules.append(
("logi", "log", self._has_positive_measure)
)
return self._apply_rule_list(word, rules)
def _step3(self, word):
"""Implements Step 3 from "An algorithm for suffix stripping"
From the paper:
Step 3
(m>0) ICATE -> IC triplicate -> triplic
(m>0) ATIVE -> formative -> form
(m>0) ALIZE -> AL formalize -> formal
(m>0) ICITI -> IC electriciti -> electric
(m>0) ICAL -> IC electrical -> electric
(m>0) FUL -> hopeful -> hope
(m>0) NESS -> goodness -> good
"""
return self._apply_rule_list(word, [
('icate', 'ic', self._has_positive_measure),
('ative', '', self._has_positive_measure),
('alize', 'al', self._has_positive_measure),
('iciti', 'ic', self._has_positive_measure),
('ical', 'ic', self._has_positive_measure),
('ful', '', self._has_positive_measure),
('ness', '', self._has_positive_measure),
])
def _step4(self, word):
"""Implements Step 4 from "An algorithm for suffix stripping"
Step 4
(m>1) AL -> revival -> reviv
(m>1) ANCE -> allowance -> allow
(m>1) ENCE -> inference -> infer
(m>1) ER -> airliner -> airlin
(m>1) IC -> gyroscopic -> gyroscop
(m>1) ABLE -> adjustable -> adjust
(m>1) IBLE -> defensible -> defens
(m>1) ANT -> irritant -> irrit
(m>1) EMENT -> replacement -> replac
(m>1) MENT -> adjustment -> adjust
(m>1) ENT -> dependent -> depend
(m>1 and (*S or *T)) ION -> adoption -> adopt
(m>1) OU -> homologou -> homolog
(m>1) ISM -> communism -> commun
(m>1) ATE -> activate -> activ
(m>1) ITI -> angulariti -> angular
(m>1) OUS -> homologous -> homolog
(m>1) IVE -> effective -> effect
(m>1) IZE -> bowdlerize -> bowdler
The suffixes are now removed. All that remains is a little
tidying up.
"""
measure_gt_1 = lambda stem: self._measure(stem) > 1
return self._apply_rule_list(word, [
('al', '', measure_gt_1),
('ance', '', measure_gt_1),
('ence', '', measure_gt_1),
('er', '', measure_gt_1),
('ic', '', measure_gt_1),
('able', '', measure_gt_1),
('ible', '', measure_gt_1),
('ant', '', measure_gt_1),
('ement', '', measure_gt_1),
('ment', '', measure_gt_1),
('ent', '', measure_gt_1),
# (m>1 and (*S or *T)) ION ->
(
'ion',
'',
lambda stem: self._measure(stem) > 1 and stem[-1] in ('s', 't')
),
('ou', '', measure_gt_1),
('ism', '', measure_gt_1),
('ate', '', measure_gt_1),
('iti', '', measure_gt_1),
('ous', '', measure_gt_1),
('ive', '', measure_gt_1),
('ize', '', measure_gt_1),
])
def _step5a(self, word):
"""Implements Step 5a from "An algorithm for suffix stripping"
From the paper:
Step 5a
(m>1) E -> probate -> probat
rate -> rate
(m=1 and not *o) E -> cease -> ceas
"""
# Note that Martin's test vocabulary and reference
# implementations are inconsistent in how they handle the case
# where two rules both refer to a suffix that matches the word
# to be stemmed, but only the condition of the second one is
# true.
# Earlier in step2b we had the rules:
# (m>0) EED -> EE
# (*v*) ED ->
# but the examples in the paper included "feed"->"feed", even
# though (*v*) is true for "fe" and therefore the second rule
# alone would map "feed"->"fe".
# However, in THIS case, we need to handle the consecutive rules
# differently and try both conditions (obviously; the second
# rule here would be redundant otherwise). Martin's paper makes
# no explicit mention of the inconsistency; you have to infer it
# from the examples.
# For this reason, we can't use _apply_rule_list here.
if word.endswith('e'):
stem = self._replace_suffix(word, 'e', '')
if self._measure(stem) > 1:
return stem
if self._measure(stem) == 1 and not self._ends_cvc(stem):
return stem
return word
def _step5b(self, word):
"""Implements Step 5a from "An algorithm for suffix stripping"
From the paper:
Step 5b
(m > 1 and *d and *L) -> single letter
controll -> control
roll -> roll
"""
return self._apply_rule_list(word, [
('ll', 'l', lambda stem: self._measure(word[:-1]) > 1)
])
def stem(self, word):
stem = word.lower()
if self.mode == self.NLTK_EXTENSIONS and word in self.pool:
return self.pool[word]
if self.mode != self.ORIGINAL_ALGORITHM and len(word) <= 2:
# With this line, strings of length 1 or 2 don't go through
# the stemming process, although no mention is made of this
# in the published algorithm.
return word
stem = self._step1a(stem)
stem = self._step1b(stem)
stem = self._step1c(stem)
stem = self._step2(stem)
stem = self._step3(stem)
stem = self._step4(stem)
stem = self._step5a(stem)
stem = self._step5b(stem)
return stem
def __repr__(self):
return '<PorterStemmer>'