Merge pull request #366 from edx/will/ai-grading-s3-bucket-prefix

Will/ai grading s3 bucket prefix

apps/openassessment/assessment/models/ai.py

@@ -3,6 +3,7 @@ Database models for AI assessment.
 """
 from uuid import uuid4
 import json
+from django.conf import settings
 from django.core.files.base import ContentFile
 from django.db import models, transaction
 from django.utils.timezone import now
@@ -140,9 +141,10 @@ class AIClassifierSet(models.Model):
             try:
                 classifier.classifier_data.save(filename, contents)
             except Exception as ex:
+                full_filename = upload_to_path(classifier, filename)
                 msg = (
                     u"Could not upload classifier data to {filename}: {ex}"
-                ).format(filename=filename, ex=ex)
+                ).format(filename=full_filename, ex=ex)
                 raise ClassifierUploadError(msg)
 
         return classifier_set
@@ -168,6 +170,42 @@ class AIClassifierSet(models.Model):
             }
 
 
+# Directory in which classifiers will be stored
+# For instance, if we're using the default file system storage backend
+# for local development, this will be a subdirectory.
+# If using an S3 storage backend, this will be a subdirectory in
+# an AWS S3 bucket.
+AI_CLASSIFIER_STORAGE = "ora2_ai_classifiers"
+
+def upload_to_path(instance, filename):    # pylint:disable=W0613
+    """
+    Calculate the file path where classifiers should be uploaded.
+    Optionally prepends the path with a prefix (determined by Django settings).
+    This allows us to put classifiers from different environments
+    (stage / prod) in different directories within the same S3 bucket.
+
+    Args:
+        instance (AIClassifier): Not used.
+        filename (unicode): The filename provided when saving the file.
+
+    Returns:
+        unicode
+
+    """
+    prefix = getattr(settings, 'ORA2_FILE_PREFIX', None)
+    if prefix is not None:
+        return u"{prefix}/{root}/{filename}".format(
+            prefix=prefix,
+            root=AI_CLASSIFIER_STORAGE,
+            filename=filename
+        )
+    else:
+        return u"{root}/{filename}".format(
+            root=AI_CLASSIFIER_STORAGE,
+            filename=filename
+        )
+
+
 class AIClassifier(models.Model):
     """
     A trained classifier (immutable).
@@ -176,13 +214,6 @@ class AIClassifier(models.Model):
     class Meta:
         app_label = "assessment"
 
-    # Directory in which classifiers will be stored
-    # For instance, if we're using the default file system storage backend
-    # for local development, this will be a subdirectory.
-    # If using an S3 storage backend, this will be a subdirectory in
-    # an AWS S3 bucket.
-    AI_CLASSIFIER_STORAGE = "ora2_ai_classifiers"
-
     # The set of classifiers this classifier belongs to
     classifier_set = models.ForeignKey(AIClassifierSet, related_name="classifiers")
 
@@ -192,7 +223,7 @@ class AIClassifier(models.Model):
     # The serialized classifier
     # Because this may be large, we store it using a Django `FileField`,
     # which allows us to plug in different storage backends (such as S3)
-    classifier_data = models.FileField(upload_to=AI_CLASSIFIER_STORAGE)
+    classifier_data = models.FileField(upload_to=upload_to_path)
 
     def download_classifier_data(self):
         """

apps/openassessment/assessment/test/test_ai_models.py

+# coding=utf-8
+"""
+Test AI Django models.
+"""
+from django.test.utils import override_settings
+from openassessment.test_utils import CacheResetTest
+from openassessment.assessment.models import (
+    AIClassifierSet, AIClassifier, AI_CLASSIFIER_STORAGE
+)
+from openassessment.assessment.serializers import rubric_from_dict
+from .constants import RUBRIC
+
+
+class AIClassifierTest(CacheResetTest):
+    """
+    Tests for the AIClassifier model.
+    """
+    CLASSIFIERS_DICT = {
+        u"vøȼȺƀᵾłȺɍɏ": "test data",
+        u"ﻭɼค๓๓คɼ": "more test data"
+    }
+
+    def test_upload_to_path_default(self):
+        # No path prefix provided in the settings
+        classifier = self._create_classifier()
+        components = classifier.classifier_data.name.split(u'/')
+        self.assertEqual(len(components), 2)
+        self.assertEqual(components[0], AI_CLASSIFIER_STORAGE)
+        self.assertGreater(len(components[1]), 0)
+
+    @override_settings(ORA2_FILE_PREFIX=u"ƒιℓє_ρяєƒιχ")
+    def test_upload_to_path_with_prefix(self):
+        classifier = self._create_classifier()
+        components = classifier.classifier_data.name.split(u'/')
+        self.assertEqual(len(components), 3)
+        self.assertEqual(components[0], u"ƒιℓє_ρяєƒιχ")
+        self.assertEqual(components[1], AI_CLASSIFIER_STORAGE)
+        self.assertGreater(len(components[2]), 0)
+
+    def _create_classifier(self):
+        """
+        Create and return an AIClassifier.
+        """
+        rubric = rubric_from_dict(RUBRIC)
+        classifier_set = AIClassifierSet.create_classifier_set(
+            self.CLASSIFIERS_DICT, rubric, "test_algorithm"
+        )
+        return AIClassifier.objects.filter(classifier_set=classifier_set)[0]