Performance tests

perf/grade.py

+import sys
+import time
+import pickle
+from ease.grade import grade
+
+
+NUM_TRIALS = 5
+
+
+def main(submission_path):
+
+    print "Loading submission..."
+    with open(submission_path) as sub_file:
+        submission = sub_file.read()
+
+    print "Loading model..."
+    with open('model') as model_file:
+        model = pickle.load(model_file)
+
+    print "Loading feature extractor..."
+    with open('feature_extractor') as feature_ext_file:
+        feature_ext = pickle.load(feature_ext_file)
+
+    grader_data = {
+        'model': model,
+        'extractor': feature_ext,
+        'prompt': 'Test prompt'
+    }
+
+    print "Grading submission..."
+    times = []
+    for trial in range(NUM_TRIALS):
+        print "-- trial #{}".format(trial)
+        start = time.time()
+        result = grade(grader_data, submission)
+        end = time.time()
+        times.append(end - start)
+        print "-- took {} seconds".format(times[-1])
+
+    avg = sum(times) / len(times)
+    variance = sum((avg - t) ** 2 for t in times) / len(times)
+    print "== Took {} seconds (avg) with {} variance to grade".format(avg, variance)
+
+    if not result['success']:
+        print "Error occurred!"
+        for error in result['errors']:
+            print error
+
+if __name__ == "__main__":
+    main(sys.argv[1])

perf/submission.txt

+I had seen little of Holmes lately. My marriage had drifted us
+away from each other. My own complete happiness, and the
+home-centred interests which rise up around the man who first
+finds himself master of his own establishment, were sufficient to
+absorb all my attention, while Holmes, who loathed every form of
+society with his whole Bohemian soul, remained in our lodgings in
+Baker Street, buried among his old books, and alternating from
+week to week between cocaine and ambition, the drowsiness of the
+drug, and the fierce energy of his own keen nature. He was still,
+as ever, deeply attracted by the study of crime, and occupied his
+immense faculties and extraordinary powers of observation in
+following out those clues, and clearing up those mysteries which
+had been abandoned as hopeless by the official police. From time
+to time I heard some vague account of his doings: of his summons
+to Odessa in the case of the Trepoff murder, of his clearing up
+of the singular tragedy of the Atkinson brothers at Trincomalee,
+and finally of the mission which he had accomplished so
+delicately and successfully for the reigning family of Holland.
+Beyond these signs of his activity, however, which I merely
+shared with all the readers of the daily press, I knew little of
+my former friend and companion.
+
+"Not at all. The 'G' with the small 't' stands for
+'Gesellschaft,' which is the German for 'Company.' It is a
+customary contraction like our 'Co.' 'P,' of course, stands for
+'Papier.' Now for the 'Eg.' Let us glance at our Continental
+Gazetteer." He took down a heavy brown volume from his shelves.
+"Eglow, Eglonitz--here we are, Egria. It is in a German-speaking
+country--in Bohemia, not far from Carlsbad. 'Remarkable as being
+the scene of the death of Wallenstein, and for its numerous
+glass-factories and paper-mills.' Ha, ha, my boy, what do you
+make of that?" His eyes sparkled, and he sent up a great blue
+triumphant cloud from his cigarette.
+
+A man entered who could hardly have been less than six feet six
+inches in height, with the chest and limbs of a Hercules. His
+dress was rich with a richness which would, in England, be looked
+upon as akin to bad taste. Heavy bands of astrakhan were slashed
+across the sleeves and fronts of his double-breasted coat, while
+the deep blue cloak which was thrown over his shoulders was lined
+with flame-coloured silk and secured at the neck with a brooch
+which consisted of a single flaming beryl. Boots which extended
+halfway up his calves, and which were trimmed at the tops with
+rich brown fur, completed the impression of barbaric opulence
+which was suggested by his whole appearance. He carried a
+broad-brimmed hat in his hand, while he wore across the upper
+part of his face, extending down past the cheekbones, a black
+vizard mask, which he had apparently adjusted that very moment,
+for his hand was still raised to it as he entered. From the lower
+part of the face he appeared to be a man of strong character,
+with a thick, hanging lip, and a long, straight chin suggestive
+of resolution pushed to the length of obstinacy.
+
+It was close upon four before the door opened, and a
+drunken-looking groom, ill-kempt and side-whiskered, with an
+inflamed face and disreputable clothes, walked into the room.
+Accustomed as I was to my friend's amazing powers in the use of
+disguises, I had to look three times before I was certain that it
+was indeed he. With a nod he vanished into the bedroom, whence he
+emerged in five minutes tweed-suited and respectable, as of old.
+Putting his hands into his pockets, he stretched out his legs in
+front of the fire and laughed heartily for some minutes.

perf/train.py

+import time
+import sys
+import random
+import pickle
+from ease.create import create
+
+MIN_SCORE = 0
+MAX_SCORE = 5
+NUM_LINES_PER_ESSAY = 100
+NUM_TRIALS = 5
+
+
+def main(num_essays):
+    essays = []
+    print "Loading essays..."
+    with open('essaycorpus.txt') as corpus:
+        lines = []
+        for line in range(NUM_LINES_PER_ESSAY):
+            line = corpus.readline()
+            if line == '':
+                corpus.seek(0)
+            lines.append(line)
+
+        for _ in range(num_essays):
+            essays.append(''.join(lines))
+
+    scores = [
+        random.randint(MIN_SCORE, MAX_SCORE)
+        for _ in range(num_essays)
+    ]
+
+    print "Training model..."
+    times = []
+    for trial in range(NUM_TRIALS):
+        print "-- trial #{}".format(trial)
+        start = time.time()
+        results = create(essays, scores, "")
+        end = time.time()
+        times.append(end - start)
+        print "-- took {} seconds".format(times[-1])
+
+    avg = sum(times) / len(times)
+    variance = sum((avg - t) ** 2 for t in times) / len(times)
+    print "== Took {} seconds (avg) with {} variance to train on {} essays".format(
+        avg, variance, num_essays
+    )
+
+    if not results['success']:
+        print "Errors occurred!"
+        for error in results['errors']:
+            print error
+    else:
+        print "Saving model..."
+        with open('feature_extractor', 'w') as feature_ext_file:
+            pickle.dump(results['feature_ext'], feature_ext_file)
+        with open('model', 'w') as model_file:
+            pickle.dump(results['classifier'], model_file)
+
+
+if __name__ == "__main__":
+    main(int(sys.argv[1]))