Stage environment

VEDA/settings/stage.py

+"""
+Stage environment settings.
+"""
+from VEDA.settings.base import *
+from VEDA.utils import get_config
+from VEDA.settings.utils import get_logger_config
+
+DEBUG = False
+TEMPLATE_DEBUG = DEBUG
+DEFATULT_SERVICE_VARIANT_NAME = 'video-pipeline'
+
+ALLOWED_HOSTS = ['*']
+
+CONFIG_DATA = get_config()
+
+LOGGING = get_logger_config(service_variant=CONFIG_DATA.get('SERVICE_VARIANT_NAME', DEFATULT_SERVICE_VARIANT_NAME))
+
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.sqlite3',
+        'NAME': os.path.join(ROOT_DIR, 'stage.db'),
+        'USER': '',
+        'PASSWORD': '',
+        'HOST': '',
+        'PORT': '',
+    }
+}
+
+JWT_AUTH = {
+    'JWT_SECRET_KEY': CONFIG_DATA['val_secret_key'],
+    'JWT_ISSUER': '{}/oauth2'.format(CONFIG_DATA['lms_base_url'].rstrip('/')),
+    'JWT_AUDIENCE': CONFIG_DATA['val_client_id'],
+    'JWT_VERIFY_AUDIENCE': True,
+}

bin/ingest

@@ -64,8 +64,8 @@ class IngestCli(object):
         ))
         os.system(runcommand)
 
-        E1 = EmailAlert(message='Ingest Daemon Crash', subject='Ingest Daemon')
-        E1.email()
+        E1 = EmailAlert(body='Ingest Daemon Crash', subject='Ingest Daemon')
+        E1.email_fault()
 
 
 

bin/youtubecallback

@@ -75,8 +75,8 @@ class YoutubeCallbackCli(object):
             '-y'
         ))
         os.system(runcommand)
-        E1 = EmailAlert(message='Youtube Callback Daemon Crash', subject='Youtube Callback Daemon')
-        E1.email()
+        E1 = EmailAlert(body='Youtube Callback Daemon Crash', subject='Youtube Callback Daemon')
+        E1.email_fault()
 
 
 def main():

control/veda_file_discovery.py

@@ -32,6 +32,7 @@ boto.config.set('Boto', 'http_socket_timeout', '100')
 
 logging.basicConfig(level=logging.INFO)
 logging.getLogger("requests").setLevel(logging.WARNING)
+logging.getLogger("boto").setLevel(logging.CRITICAL)
 LOGGER = logging.getLogger(__name__)
 
 
@@ -43,6 +44,12 @@ class FileDiscovery(object):
         self.bucket = None
         self.node_work_directory = kwargs.get('node_work_directory', WORK_DIRECTORY)
 
+        # In stage, a course could possibly not exist in the local database
+        # but remain referenced by edx-platform.
+        # If the course doesn't exist but a course ID and hex is supplied,
+        # create the course anyway.
+        self.create_course_override = self.auth_dict['environment'] == "stage"
+
     def about_video_ingest(self):
         """
         Crawl VEDA Upload bucket
@@ -182,19 +189,45 @@ class FileDiscovery(object):
                     course.local_storedir = ','.join(course_runs)
                     course.save()
             else:
+                self._create_course(course_key, course_id)
+        else:
+            try:
+                course = Course.objects.get(studio_hex=course_hex)
+            except Course.DoesNotExist:
+                if self.create_course_override:
+                    try:
+                        course_key = CourseKey.from_string(course_id)
+                    except InvalidKeyError:
+                        return
+
+                    self._create_course(course_key, course_id, course_hex)
+                else:
+                    return
+
+        return course
+
+    def _create_course(self, course_key, course_id, studio_hex=None):
+        """
+        Creates a course with the specified parameters.
+        If another class needs to create a course, use get_or_create_course
+        instead of this method.
+
+        Arguments:
+            -  course_key
+            -  course_id
+            -  studio_hex
+        """
         course_name = '{org} {number}'.format(org=course_key.org, number=course_key.course)
         course = Course.objects.create(
             course_name=course_name,
             institution=course_key.org,
             edx_classid=course_key.course,
             local_storedir=course_id,
-                    yt_proc=False,
+            yt_proc=False
         )
-        else:
-            try:
-                course = Course.objects.get(studio_hex=course_hex)
-            except Course.DoesNotExist:
-                return
+
+        if studio_hex:
+            course['studio_hex'] = studio_hex
 
         return course
 

control/veda_utils.py

@@ -18,26 +18,33 @@ class EmailAlert(object):
     """
     def __init__(self, **kwargs):
         self.auth_dict = get_config()
-        self.message = kwargs.get('message', None)
+        self.body = kwargs.get('body', None)
         self.subject = kwargs.get('subject', None)
+        self.additional_recipients = kwargs.get('additional_recipients', [])
+        self.production_environment = self.auth_dict['environment'] == "production"
 
-    def email(self):
-        email_subject = '[ VEDA ALERTING ]'
-        email_subject += ' : ' + self.subject
+    def email_fault(self):
+        self.subject = '[ VEDA ALERTING ] : ' + str(self.subject)
+        self.body = 'There has been a fault:' + str(self.body)
+        self.email()
 
-        email_body = 'There has been a fault:'
-        email_body += self.message
+    def email(self):
+        if self.production_environment:
             try:
                 conn = boto.ses.connect_to_region('us-east-1')
             except boto.exception.NoAuthHandlerFound:
                 return
 
+            recipients = self.additional_recipients.append(self.auth_dict['admin_email'])
+
             conn.send_email(
                 self.auth_dict['veda_noreply_email'],
-            email_subject,
-            email_body,
-            [self.auth_dict['admin_email']]
+                self.subject,
+                self.body,
+                recipients
             )
+        else:
+            pass
 
 
 class Output(object):
@@ -149,8 +156,7 @@ class Report(object):
                 youtube_id=self.youtube_id
             )
 
-        email_subject = 'VEDA / edX About Video Status Update : '
-        email_subject += final_success
+        email_subject = 'VEDA / edX About Video Status Update : ' + str(final_success)
 
         email_body = (
             'This is an auto generated message:\n\n'
@@ -167,17 +173,12 @@ class Report(object):
             'edX Studio Course URL : ' + v1[0].edx_studio_url + '\n\n'
             'Please do not reply to this email.\n\n <<EOM'
         )
-        try:
-            conn = boto.ses.connect_to_region('us-east-1')
-        except boto.exception.NoAuthHandlerFound:
-            return
 
-        conn.send_email(
-            self.auth_dict['veda_noreply_email'],
-            email_subject,
-            email_body,
-            [v1[0].status_email, self.auth_dict['admin_email']]
-        )
+        EmailAlert(
+            body=email_body,
+            subject=email_subject,
+            additional_recipients = [v1[0].status_email]
+        ).email()
 
 
 class VideoProto(object):