Merge pull request #765 from dimazest/wsd-tests

Make nltk.wsd.lesk() deterministic.

nltk/test/wsd.doctest

@@ -20,12 +20,13 @@ a Synset with the highest number of overlapping words between the context
 sentence and different definitions from each Synset.
 
     >>> from nltk.wsd import lesk
-    >>> from nltk import word_tokenize
-    >>> sent = word_tokenize("I went to the bank to deposit money.")
-    >>> word = "bank"
-    >>> pos = "n"
-    >>> print(lesk(sent, word, pos))
-    Synset('depository_financial_institution.n.01')
+    >>> sent = ['I', 'went', 'to', 'the', 'bank', 'to', 'deposit', 'money', '.']
+
+    >>> print(lesk(sent, 'bank', 'n'))
+    Synset('savings_bank.n.02')
+
+    >>> print(lesk(sent, 'bank'))
+    Synset('savings_bank.n.02')
 
 The definitions for "bank" are:
 

nltk/wsd.py

@@ -6,59 +6,41 @@
 # URL: <http://nltk.org/>
 # For license information, see LICENSE.TXT
 
-from nltk.corpus import wordnet as wn
+from nltk.corpus import wordnet
 
-############################################################
-# Lesk Algorithm
-############################################################
 
-def _compare_overlaps_greedy(context, synsets_signatures, pos=None):
-    """
-    Calculate overlaps between the context sentence and the synset_signature
-    and returns the synset with the highest overlap.
+def lesk(context_sentence, ambiguous_word, pos=None, synsets=None):
+    """Return a synset for an ambigous word.
 
-    :param context: ``context_sentence`` The context sentence where the ambiguous word occurs.
-    :param synsets_signatures: ``dictionary`` A list of words that 'signifies' the ambiguous word.
-    :param pos: ``pos`` A specified Part-of-Speech (POS).
+    :param context_sentence: The context sentence where the ambiguous word occurs.
+    :param ambiguous_word: The ambiguous word that requires WSD.
+    :param pos: A specified Part-of-Speech (POS).
+    :param iter sysnsets: Possible synsets of the ambiguous word.
     :return: ``lesk_sense`` The Synset() object with the highest signature overlaps.
-    """
-    max_overlaps = 0
-    lesk_sense = None
-    for ss in synsets_signatures:
-        if pos and str(ss.pos()) != pos: # Skips different POS.
-            continue
-        overlaps = set(synsets_signatures[ss]).intersection(context)
-        if len(overlaps) > max_overlaps:
-            lesk_sense = ss
-            max_overlaps = len(overlaps)  
-    return lesk_sense
 
-def lesk(context_sentence, ambiguous_word, pos=None, dictionary=None):
-    """
-    This function is the implementation of the original Lesk algorithm (1986).
-    It requires a dictionary which contains the definition of the different
-    sense of each word. See http://goo.gl/8TB15w
+    This function is an implementation of the original Lesk algorithm (1986) [1].
+
+    Usage example::
 
-        >>> from nltk import word_tokenize
-        >>> sent = word_tokenize("I went to the bank to deposit money.")
-        >>> word = "bank"
-        >>> pos = "n"
-        >>> lesk(sent, word, pos)
+        >>> lesk(['I', 'went', 'to', 'the', 'bank', 'to', 'deposit', 'money', '.'], 'bank', 'n')
         Synset('savings_bank.n.02')
 
-    :param context_sentence: The context sentence where the ambiguous word occurs.
-    :param ambiguous_word: The ambiguous word that requires WSD.
-    :param pos: A specified Part-of-Speech (POS).
-    :param dictionary: A list of words that 'signifies' the ambiguous word.
-    :return: ``lesk_sense`` The Synset() object with the highest signature overlaps.
+    [1] Lesk, Michael. "Automatic sense disambiguation using machine readable
+    dictionaries: how to tell a pine cone from an ice cream cone." Proceedings
+    of the 5th annual international conference on Systems documentation. ACM,
+    1986. http://dl.acm.org/citation.cfm?id=318728
+
     """
-    if not dictionary:
-        dictionary = {}
-        for ss in wn.synsets(ambiguous_word):
-            dictionary[ss] = ss.definition().split()
-    best_sense = _compare_overlaps_greedy(context_sentence,
-                                       dictionary, pos)
-    return best_sense
+    context = set(context_sentence)
+    if not synsets:
+        synsets = wordnet.synsets(ambiguous_word)
+
+    _, sense = max(
+        (len(context.intersection(ss.definition().split())), ss)
+        for ss in synsets if pos is None or str(ss.pos()) == pos
+    )
+
+    return sense
 
 
 if __name__ == "__main__":