Skip to content

E
ease
Commit 09b8c904 by Vik Paruchuri
Options

adding documentation

parent ead03fa6
Showing with 168 additions and 147 deletions
Readme.md
Project to integrate machine learning based essay scoring with xserver. Aspell must be installed and added to path to run. numpy, scipy and sklearn also need to be installed.
Project to integrate machine learning based essay scoring with xserver. Aspell must be installed and added to path to run. numpy, scipy, sklearn, and nltk also need to be installed.
Runnable files:
......
create_test_models.py
......@@ -4,14 +4,13 @@ import os
import sys
import argparse
base_path = os.path.dirname( __file__ )
base_path = os.path.dirname(__file__)
sys.path.append(base_path)
import model_creator
def main(argv):
parser = argparse.ArgumentParser(description="Generate model from test data files")
parser.add_argument('train_file')
parser.add_argument('prompt_file')
......@@ -19,11 +18,11 @@ def main(argv):
args = parser.parse_args(argv)
score,text=model_creator.read_in_test_data(args.train_file)
prompt_string=model_creator.read_in_test_prompt(args.prompt_file)
e_set=model_creator.create_essay_set(text,score,prompt_string)
feature_ext,classifier=model_creator.extract_features_and_generate_model(e_set)
model_creator.dump_model_to_file(prompt_string,feature_ext,classifier,args.model_path)
score, text = model_creator.read_in_test_data(args.train_file)
prompt_string = model_creator.read_in_test_prompt(args.prompt_file)
e_set = model_creator.create_essay_set(text, score, prompt_string)
feature_ext, classifier = model_creator.extract_features_and_generate_model(e_set)
model_creator.dump_model_to_file(prompt_string, feature_ext, classifier, args.model_path)
if __name__=="__main__":
if __name__ == "__main__":
main(sys.argv[1:])
essay_set.py
......@@ -13,75 +13,94 @@ import util_functions
class essay_set:
def __init__(self, type="train"):
if(type!="train" and type!="test"):
type="train"
"""
Initialize variables and check essay set type
"""
if(type != "train" and type != "test"):
type = "train"
self._type = type
self._score,self._text,self._id,self._clean_text,self._tokens,self._pos,\
self._clean_stem_text,self._generated=[],[],[],[],[],[],[],[]
self._prompt=""
self._score, self._text, self._id, self._clean_text, self._tokens, self._pos,\
self._clean_stem_text, self._generated = [], [], [], [], [], [], [], []
self._prompt = ""
#Add new (essay_text,essay_score) pair to the essay set
def add_essay(self,essay_text,essay_score,essay_generated=0):
def add_essay(self, essay_text, essay_score, essay_generated=0):
"""
Add new (essay_text,essay_score) pair to the essay set.
essay_text must be a string.
essay_score must be an int.
essay_generated should not be changed by the user.
Returns a confirmation that essay was added.
"""
#Get maximum current essay id, or set to 0 if this is the first essay added
if(len(self._id)>0):
max_id=max(self._id)
else :
max_id=0
#Verify that essay_score is an int, essay_text is a string, and essay_generated equals 0 or 1
if type(essay_score)==type(0) and type(essay_text)==type("text") \
and (essay_generated==0 or essay_generated==1):
self._id.append(max_id+1)
if(len(self._id) > 0):
max_id = max(self._id)
else:
max_id = 0
#Verify that essay_score is an int, essay_text is a string, and essay_generated equals 0 or 1
if type(essay_score) == type(0) and type(essay_text) == type("text")\
and (essay_generated == 0 or essay_generated == 1):
self._id.append(max_id + 1)
self._score.append(essay_score)
#Clean text by removing non digit/work/punctuation characters
self._text.append(util_functions.sub_chars(essay_text).lower())
#Spell correct text using aspell
self._clean_text.append(util_functions.spell_correct(self._text[len(self._text)-1]))
self._clean_text.append(util_functions.spell_correct(self._text[len(self._text) - 1]))
#Tokenize text
self._tokens.append(nltk.word_tokenize(self._clean_text[len(self._clean_text)-1]))
self._tokens.append(nltk.word_tokenize(self._clean_text[len(self._clean_text) - 1]))
#Part of speech tag text
self._pos.append(nltk.pos_tag(self._tokens[len(self._tokens)-1]))
self._pos.append(nltk.pos_tag(self._tokens[len(self._tokens) - 1]))
self._generated.append(essay_generated)
#Stem spell corrected text
porter = nltk.PorterStemmer()
por_toks=" ".join([porter.stem(w) for w in self._tokens[len(self._tokens)-1]])
por_toks = " ".join([porter.stem(w) for w in self._tokens[len(self._tokens) - 1]])
self._clean_stem_text.append(por_toks)
ret="text: " + self._text[len(self._text)-1] + " score: " + str(essay_score)
ret = "text: " + self._text[len(self._text) - 1] + " score: " + str(essay_score)
else:
raise util_functions.InputError(essay_text,"arguments need to be in format "
"(text,score). text needs to be string,"
" score needs to be int.")
raise util_functions.InputError(essay_text, "arguments need to be in format "
"(text,score). text needs to be string,"
" score needs to be int.")
return ret
#Update the default prompt string, which is ""
def update_prompt(self,prompt_text):
if(type(prompt_text)==type("text")):
self._prompt=util_functions.sub_chars(prompt_text)
ret=self._prompt
def update_prompt(self, prompt_text):
"""
Update the default prompt string, which is "".
prompt_text should be a string.
Returns the prompt as a confirmation.
"""
if(type(prompt_text) == type("text")):
self._prompt = util_functions.sub_chars(prompt_text)
ret = self._prompt
else:
raise util_functions.InputError(prompt_text,"Invalid prompt. Need to enter a string value.")
raise util_functions.InputError(prompt_text, "Invalid prompt. Need to enter a string value.")
return ret
#Substitute synonyms to generate extra essays from existing ones
def generate_additional_essays(self,e_text,e_score,dict=None,max_syns=3):
def generate_additional_essays(self, e_text, e_score, dict=None, max_syns=3):
"""
Substitute synonyms to generate extra essays from existing ones.
This is done to increase the amount of training data.
Should only be used with lowest scoring essays.
e_text is the text of the original essay.
e_score is the score of the original essay.
dict is a fixed dictionary (list) of words to replace.
max_syns defines the maximum number of additional essays to generate. Do not set too high.
"""
random.seed(1)
e_toks=nltk.word_tokenize(e_text)
all_syns=[]
e_toks = nltk.word_tokenize(e_text)
all_syns = []
for word in e_toks:
synonyms=util_functions.get_wordnet_syns(word)
if(len(synonyms)>max_syns):
synonyms=random.sample(synonyms,max_syns)
synonyms = util_functions.get_wordnet_syns(word)
if(len(synonyms) > max_syns):
synonyms = random.sample(synonyms, max_syns)
all_syns.append(synonyms)
new_essays=[]
for i in range(0,max_syns):
syn_toks=e_toks
for z in range(0,len(e_toks)):
if len(all_syns[z])>i and (dict==None or e_toks[z] in dict):
syn_toks[z]=all_syns[z][i]
new_essays = []
for i in range(0, max_syns):
syn_toks = e_toks
for z in range(0, len(e_toks)):
if len(all_syns[z]) > i and (dict == None or e_toks[z] in dict):
syn_toks[z] = all_syns[z][i]
new_essays.append(" ".join(syn_toks))
for z in xrange(0,len(new_essays)):
self.add_essay(new_essays[z],e_score,1)
\ No newline at end of file
for z in xrange(0, len(new_essays)):
self.add_essay(new_essays[z], e_score, 1)
\ No newline at end of file
feature_extractor.py
......@@ -16,92 +16,93 @@ import util_functions
class feature_extractor:
def __init__(self):
self._good_pos_ngrams=self.get_good_pos_ngrams()
self.dict_initialized=False
def initialize_dictionaries(self,e_set):
self._good_pos_ngrams = self.get_good_pos_ngrams()
self.dict_initialized = False
def initialize_dictionaries(self, e_set):
if(hasattr(e_set, '_type')):
if(e_set._type=="train"):
nvocab=util_functions.get_vocab(e_set._text,e_set._score)
svocab=util_functions.get_vocab(e_set._clean_stem_text,e_set._score)
self._normal_dict=CountVectorizer(min_n=1,max_n=2,vocabulary=nvocab)
self._stem_dict=CountVectorizer(min_n=1,max_n=2,vocabulary=svocab)
self.dict_initialized=True
ret="ok"
if(e_set._type == "train"):
nvocab = util_functions.get_vocab(e_set._text, e_set._score)
svocab = util_functions.get_vocab(e_set._clean_stem_text, e_set._score)
self._normal_dict = CountVectorizer(min_n=1, max_n=2, vocabulary=nvocab)
self._stem_dict = CountVectorizer(min_n=1, max_n=2, vocabulary=svocab)
self.dict_initialized = True
ret = "ok"
else:
raise util_functions.InputError(e_set,"needs to be an essay set of the train type.")
raise util_functions.InputError(e_set, "needs to be an essay set of the train type.")
else:
raise util_functions.InputError(e_set,"wrong input. need an essay set object")
raise util_functions.InputError(e_set, "wrong input. need an essay set object")
return ret
def get_good_pos_ngrams(self):
if(os.path.isfile("good_pos_ngrams.p")):
good_pos_ngrams=pickle.load(open('good_pos_ngrams.p', 'rb'))
else :
essay_corpus=open("essaycorpus.txt").read()
essay_corpus=util_functions.sub_chars(essay_corpus)
good_pos_ngrams=util_functions.regenerate_good_tokens(essay_corpus)
good_pos_ngrams = pickle.load(open('good_pos_ngrams.p', 'rb'))
else:
essay_corpus = open("essaycorpus.txt").read()
essay_corpus = util_functions.sub_chars(essay_corpus)
good_pos_ngrams = util_functions.regenerate_good_tokens(essay_corpus)
pickle.dump(good_pos_ngrams, open('good_pos_ngrams.p', 'wb'))
return good_pos_ngrams
def gen_length_feats(self,e_set):
text=e_set._text
lengths=[len(e) for e in text]
word_counts=[len(t) for t in e_set._tokens]
comma_count=[e.count(",") for e in text]
ap_count=[e.count("'") for e in text]
punc_count=[e.count(".")+e.count("?")+e.count("!") for e in text]
chars_per_word=[lengths[m]/float(word_counts[m]) for m in xrange(0,len(text))]
good_pos_tags=[]
for i in xrange(0,len(text)) :
pos_seq=[tag[1] for tag in e_set._pos[i]]
pos_ngrams=util_functions.ngrams(pos_seq,2,4)
overlap_ngrams=[i for i in pos_ngrams if i in self._good_pos_ngrams]
def gen_length_feats(self, e_set):
text = e_set._text
lengths = [len(e) for e in text]
word_counts = [len(t) for t in e_set._tokens]
comma_count = [e.count(",") for e in text]
ap_count = [e.count("'") for e in text]
punc_count = [e.count(".") + e.count("?") + e.count("!") for e in text]
chars_per_word = [lengths[m] / float(word_counts[m]) for m in xrange(0, len(text))]
good_pos_tags = []
for i in xrange(0, len(text)):
pos_seq = [tag[1] for tag in e_set._pos[i]]
pos_ngrams = util_functions.ngrams(pos_seq, 2, 4)
overlap_ngrams = [i for i in pos_ngrams if i in self._good_pos_ngrams]
good_pos_tags.append(len(overlap_ngrams))
good_pos_tag_prop=[good_pos_tags[m]/float(word_counts[m]) for m in xrange(0,len(text))]
length_arr=numpy.array((lengths,word_counts,comma_count,ap_count,punc_count,chars_per_word,good_pos_tags,good_pos_tag_prop)).transpose()
good_pos_tag_prop = [good_pos_tags[m] / float(word_counts[m]) for m in xrange(0, len(text))]
length_arr = numpy.array((
lengths, word_counts, comma_count, ap_count, punc_count, chars_per_word, good_pos_tags,
good_pos_tag_prop)).transpose()
return length_arr.copy()
def gen_bag_feats(self,e_set):
def gen_bag_feats(self, e_set):
if(hasattr(self, '_stem_dict')):
sfeats=self._stem_dict.transform(e_set._clean_stem_text)
nfeats=self._normal_dict.transform(e_set._text)
bag_feats=numpy.concatenate((sfeats.toarray(),nfeats.toarray()),axis=1)
sfeats = self._stem_dict.transform(e_set._clean_stem_text)
nfeats = self._normal_dict.transform(e_set._text)
bag_feats = numpy.concatenate((sfeats.toarray(), nfeats.toarray()), axis=1)
else:
raise util_functions.InputError(self,"Dictionaries must be initialized prior to generating bag features.")
raise util_functions.InputError(self, "Dictionaries must be initialized prior to generating bag features.")
return bag_feats.copy()
def gen_feats(self,e_set):
bag_feats=self.gen_bag_feats(e_set)
length_feats=self.gen_length_feats(e_set)
prompt_feats=self.gen_prompt_feats(e_set)
overall_feats=numpy.concatenate((length_feats,prompt_feats,bag_feats),axis=1)
overall_feats=overall_feats.copy()
def gen_feats(self, e_set):
bag_feats = self.gen_bag_feats(e_set)
length_feats = self.gen_length_feats(e_set)
prompt_feats = self.gen_prompt_feats(e_set)
overall_feats = numpy.concatenate((length_feats, prompt_feats, bag_feats), axis=1)
overall_feats = overall_feats.copy()
return overall_feats
def gen_prompt_feats(self,e_set):
prompt_toks=nltk.word_tokenize(e_set._prompt)
expand_syns=[]
def gen_prompt_feats(self, e_set):
prompt_toks = nltk.word_tokenize(e_set._prompt)
expand_syns = []
for word in prompt_toks:
synonyms=util_functions.get_wordnet_syns(word)
synonyms = util_functions.get_wordnet_syns(word)
expand_syns.append(synonyms)
expand_syns=list(chain.from_iterable(expand_syns))
prompt_overlap=[]
prompt_overlap_prop=[]
expand_syns = list(chain.from_iterable(expand_syns))
prompt_overlap = []
prompt_overlap_prop = []
for j in e_set._tokens:
prompt_overlap.append(len([i for i in j if i in prompt_toks]))
prompt_overlap_prop.append(prompt_overlap[len(prompt_overlap)-1]/float(len(j)))
expand_overlap=[]
expand_overlap_prop=[]
prompt_overlap_prop.append(prompt_overlap[len(prompt_overlap) - 1] / float(len(j)))
expand_overlap = []
expand_overlap_prop = []
for j in e_set._tokens:
expand_overlap.append(len([i for i in j if i in expand_syns]))
expand_overlap_prop.append(expand_overlap[len(expand_overlap)-1]/float(len(j)))
prompt_arr=numpy.array((prompt_overlap,prompt_overlap_prop,expand_overlap,expand_overlap_prop)).transpose()
expand_overlap_prop.append(expand_overlap[len(expand_overlap) - 1] / float(len(j)))
prompt_arr = numpy.array((prompt_overlap, prompt_overlap_prop, expand_overlap, expand_overlap_prop)).transpose()
return prompt_arr.copy()
\ No newline at end of file
model_creator.py
......@@ -10,7 +10,7 @@ import os
import sklearn.ensemble
from itertools import chain
base_path = os.path.dirname( __file__ )
base_path = os.path.dirname(__file__)
sys.path.append(base_path)
from essay_set import essay_set
......@@ -18,30 +18,31 @@ import util_functions
import feature_extractor
def read_in_test_data(filename):
id,e_set,score,score2,text=[],[],[],[],[]
combined_raw=open(filename).read()
raw_lines=combined_raw.splitlines()
for row in xrange(1,len(raw_lines)):
id1,set1,score1,score12,text1 = raw_lines[row].strip().split("\t")
id, e_set, score, score2, text = [], [], [], [], []
combined_raw = open(filename).read()
raw_lines = combined_raw.splitlines()
for row in xrange(1, len(raw_lines)):
id1, set1, score1, score12, text1 = raw_lines[row].strip().split("\t")
id.append(int(id1))
text.append(text1)
e_set.append(int(set1))
score.append(int(score1))
score2.append(int(score12))
return score,text
return score, text
def read_in_test_prompt(filename):
prompt_string=open(filename).read()
prompt_string = open(filename).read()
return prompt_string
#Create an essay set. text and score should be lists of strings and ints, respectively.
def create_essay_set(text,score,prompt_string,generate_additional=True):
x=essay_set()
for i in xrange(0,len(text)):
x.add_essay(text[i],score[i])
if score[i]==min(score) and generate_additional==True:
x.generate_additional_essays(x._clean_text[len(x._clean_text)-1],score[i])
def create_essay_set(text, score, prompt_string, generate_additional=True):
x = essay_set()
for i in xrange(0, len(text)):
x.add_essay(text[i], score[i])
if score[i] == min(score) and generate_additional == True:
x.generate_additional_essays(x._clean_text[len(x._clean_text) - 1], score[i])
x.update_prompt(prompt_string)
......@@ -49,22 +50,22 @@ def create_essay_set(text,score,prompt_string,generate_additional=True):
#Feed in an essay set to get feature vector and classifier
def extract_features_and_generate_model(essays):
f=feature_extractor.feature_extractor()
f = feature_extractor.feature_extractor()
f.initialize_dictionaries(essays)
train_feats=f.gen_feats(essays)
train_feats = f.gen_feats(essays)
clf = sklearn.ensemble.GradientBoostingClassifier(n_estimators=100, learn_rate=.05,
max_depth=4, random_state=1,
min_samples_leaf=3)
max_depth=4, random_state=1,
min_samples_leaf=3)
model=util_functions.gen_model(clf,train_feats,essays._score)
model = util_functions.gen_model(clf, train_feats, essays._score)
return f,clf
return f, clf
#Writes out model to pickle file
def dump_model_to_file(prompt_string,feature_ext,classifier,model_path):
model_file={'prompt': prompt_string, 'extractor' : feature_ext, 'model' : classifier}
pickle.dump(model_file,file=open(model_path,"w"))
def dump_model_to_file(prompt_string, feature_ext, classifier, model_path):
model_file = {'prompt': prompt_string, 'extractor': feature_ext, 'model': classifier}
pickle.dump(model_file, file=open(model_path, "w"))
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment