from words import *
from nltk.wordnet import *
from operator import itemgetter
import nltk
import re
from string import join

def build_word_associations():
    cfd = nltk.ConditionalFreqDist()

    # get a list of all English stop words
    stopwords_list = nltk.corpus.stopwords.words('english')

    # count words that occur within a window of size 5 ahead of other words
    for sentence in nltk.corpus.brown.tagged_sents():
        sentence = [(token.lower(), tag) for (token, tag) in sentence if token.lower() not in stopwords_list]
        for (index, (token, tag)) in enumerate(sentence):
            if token not in stopwords_list:
                window = sentence[index+1:index+5]
                for (window_token, window_tag) in window:
                    if window_token not in stopwords_list and window_tag[0] is tag[0]:
                        cfd[token].inc(window_token)
    return cfd

def associate():
    while True:
        word = raw_input("Enter a word: ")
        for i in range(100):
            next = cfd[word].max()
            if next:
                print "->", next,
                word = next
            else:
                break
        print

def build_word_contexts(words):
    contexts_to_words = {}
    words = [w.lower() for w in words]
    for i in range(1,len(words)-1):
        context = words[i-1]+"_"+words[i+1]
        if context not in contexts_to_words:
            contexts_to_words[context] = []
        contexts_to_words[context].append(words[i])
    # inverted structure, tracking frequency
    words_to_contexts = {}
    for context in contexts_to_words:
        for word in contexts_to_words[context]:
            if word not in words_to_contexts:
                words_to_contexts[word] = []
            words_to_contexts[word].append(context)
    return words_to_contexts, contexts_to_words

def search_contexts(words):
    words_to_contexts, contexts_to_words = build_word_contexts(words)
    while True:
        hits = []
        word = raw_input("word> ")
        if word not in words_to_contexts:
            print "Word not found"
            continue
        contexts = words_to_contexts[word]
        for w in words_to_contexts:  # all words
            for context in words_to_contexts[w]:
                if context in contexts:
                    hits.append(w)
        hit_freqs = count_words(hits).items()
        sorted_hits = sorted(hit_freqs, key=itemgetter(1), reverse=True)
        words = [word for (word, count) in sorted_hits[1:] if count > 1]
        print join(words)
        
def lookup(word):
    for category in [N, V, ADJ, ADV]:
        if word in category:
            for synset in category[word]:
                print category[word], ":", synset.gloss

############################################
# Simple Tagger
############################################

# map brown pos tags
# http://khnt.hit.uib.no/icame/manuals/brown/INDEX.HTM

def map1(tag):
    tag = re.sub(r'fw-', '', tag)     # foreign words
    tag = re.sub(r'-[th]l', '', tag)  # headlines, titles
    tag = re.sub(r'-nc', '', tag)     # cited
    tag = re.sub(r'ber?', 'vb', tag)  # verb "to be"
    tag = re.sub(r'hv', 'vb', tag)    # verb "to have"
    tag = re.sub(r'do', 'vb', tag)    # verb "to do"
    tag = re.sub(r'nc', 'nn', tag)    # cited word
    tag = re.sub(r'z', '', tag)       # third-person singular
    return tag

def map2(tag):
    tag = re.sub(r'\bj[^-+]*', 'J', tag)  # adjectives
    tag = re.sub(r'\bp[^-+]*', 'P', tag)  # pronouns
    tag = re.sub(r'\bm[^-+]*', 'M', tag)  # modals
    tag = re.sub(r'\bq[^-+]*', 'Q', tag)  # qualifiers
    tag = re.sub(r'\babl',     'Q', tag)  # qualifiers
    tag = re.sub(r'\bab[nx]',  'D', tag)  # determiners
    tag = re.sub(r'\bap',      'D', tag)  # determiners
    tag = re.sub(r'\bd[^-+]*', 'D', tag)  # determiners
    tag = re.sub(r'\bat',      'D', tag)  # determiners
    tag = re.sub(r'\bw[^-+]*', 'W', tag)  # wh words
    tag = re.sub(r'\br[^-+]*', 'R', tag)  # adverbs
    tag = re.sub(r'\bto',      'T', tag)  # "to"
    tag = re.sub(r'\bc[cs]',   'C', tag)  # conjunctions
    tag = re.sub(r's',         '',  tag)  # plurals
    tag = re.sub(r'\bin',      'I', tag)  # prepositions
    tag = re.sub(r'\buh',      'U', tag)  # interjections (uh)
    tag = re.sub(r'\bex',      'E', tag)  # existential "there"
    tag = re.sub(r'\bvbn',     'VN', tag) # past participle
    tag = re.sub(r'\bvbd',     'VD', tag) # past tense
    tag = re.sub(r'\bvbg',     'VG', tag) # gerund
    tag = re.sub(r'\bvb',      'V', tag)  # verb
    tag = re.sub(r'\bnn',      'N', tag)  # noun
    tag = re.sub(r'\bnp',      'NP', tag) # proper noun
    tag = re.sub(r'\bnr',      'NR', tag) # adverbial noun
    tag = re.sub(r'\bex',      'E', tag)  # existential "there"
    tag = re.sub(r'\bod',      'OD', tag) # ordinal
    tag = re.sub(r'\bcd',      'CD', tag) # cardinal
    tag = re.sub(r'-t',        '', tag)   # misc
    tag = re.sub(r'[a-z\*]',   '', tag)   # misc
    return tag

def map(tag):
    return map2(map1(tag.lower()))

# print sorted(set(map2(map1(tag)) for s in brown.tagged() for w,tag in s))

def load_brown_corpus(sections):
    global map
    corpus = nltk.corpus.brown.tagged_sents(tuple(sections))
    return [[(w.lower(), map(t)) for w, t in sent] for sent in corpus]

def train_tagger(corpus):
    t0 = nltk.tag.Default('N')
    t1 = nltk.tag.Unigram(cutoff=0, backoff=t0)
    t2 = nltk.tag.Bigram(cutoff=0, backoff=t1)
    t3 = nltk.tag.Trigram(cutoff=1, backoff=t2)
    
    t1.train(corpus, verbose=True)
    t2.train(corpus, verbose=True)
    t3.train(corpus, verbose=True)
    return t3

def tag(corpus):
    print "Training tagger..."
    tagger = train_tagger(corpus)
    while True:
        text = raw_input("sentence> ")
        words = text.split()
        print join(word+"/"+tag for word, tag in tagger.tag(words))

WORD_OR_TAG = '[^/ ]+'
BOUNDARY = r'\b'

def process(pattern):
    new = []
    for term in pattern.split():
        if re.match('[A-Z]+$', term):
            new.append(BOUNDARY + WORD_OR_TAG + '/' + term + BOUNDARY)
        elif '/' in term:
            new.append(BOUNDARY + term + BOUNDARY)
        else:
            new.append(BOUNDARY + term + '/' + WORD_OR_TAG + BOUNDARY)
    return join(new)

def search(corpus, num=25):
    print "Loading corpus..."
    strings = [join(w+'/'+t for (w,t) in sent) for sent in corpus]
    while True:
        pattern = ""
        while not pattern:
            pattern = raw_input("search> ")
        pattern = process(pattern)
        i = 0
        for sent in strings:
            m = re.search(pattern, sent)
            if m:
                sent = ' '*35 + sent + ' '*45
                print sent[m.start():m.start()+80]
                i += 1
                if i > num:
                    break

############################################
# Wordnet Browser
# now incorporated into NLTK as wordnet.browse
############################################

############################################
# Mad Libs
############################################

madlib = """Britney Spears will meet up with her %(NP)s label for
crisis talks about the future of her %(N)s this week reports Digital Spy.
%(NP)s Records plan to tell Spears to stop %(VG)s and take more
care of her %(J)s image if she wants to avoid being %(VD)s by the noun.
The news %(V)s shortly after Britney posted a message on her
website promising a new album and tour.  The last couple of years
have been quite a ride for me, the media has criticized %(P)s every
noun %(C)s printed a skewed perception of who I really am as a human
being, she wrote in a letter posted %(NR)s."""

# mapping = {}
# mapping['NP'] =
# mapping['N'] =
# mapping['VG'] =
# mapping['J'] =
# mapping['VD'] =
# mapping['V'] =
# mapping['P'] =
# mapping['C'] =
# mapping['NR'] =

# print madlib % mapping

