bin scripts

bin/deliver

+#!/usr/bin/env python
+import os
+import sys
+import argparse
+import yaml
+
+project_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+if project_path not in sys.path:
+    sys.path.append(project_path)
+
+
+class DeliverCli:
+    """
+    Deliver
+
+    Command Line Interface
+    """
+
+    def __init__(self, **kwargs):
+        self.args = None
+        self.test = False
+        self.logging = kwargs.get('logging', True)
+
+        self.ROOT_DIR = os.path.dirname(os.path.dirname(
+            os.path.abspath(__file__)
+        ))
+        self.auth_yaml = os.path.join(self.ROOT_DIR, 'instance_config.yaml')
+        self.new_relic_config = os.path.join(self.ROOT_DIR, 'veda_newrelic.ini')
+        self.celery_daemon = os.path.join(self.ROOT_DIR, 'control', 'celeryapp.py')
+
+    def get_args(self):
+        parser = argparse.ArgumentParser()
+        parser.usage = '''
+        {cmd} -l List
+        [-l ]
+        Use --help to see all options.
+        '''.format(cmd=sys.argv[0])
+
+        parser.add_argument(
+            '-l', '--list',
+            help='Unused, Exit',
+            action='store_true'
+            )
+
+        self.args = parser.parse_args()
+        self._parse_args()
+
+    def _parse_args(self):
+        self.list = self.args.list
+
+    def run(self):
+        """
+        Launch Celery Delivery Worker
+
+        """
+        with open(self.auth_yaml, 'r') as stream:
+            try:
+                auth_dict = yaml.load(stream)
+            except yaml.YAMLError as exc:
+                auth_dict = None
+
+        if auth_dict is not None:
+            os.system(
+                ' '.join((
+                    'NEW_RELIC_CONFIG_FILE=' + self.new_relic_config,
+                    'newrelic-admin run-program python',
+                    self.celery_daemon,
+                    'worker',
+                    '--loglevel=info',
+                    '--concurrency=' + str(auth_dict['celery_threads']),
+                    '-Q ' + auth_dict['celery_stat_queue'],
+                    '-n deliver.%h'
+                ))
+            )
+
+
+def main():
+    deliverinstance = DeliverCli()
+    deliverinstance.get_args()
+    deliverinstance.run()
+
+
+if __name__ == '__main__':
+    sys.exit(main())

bin/heal

+#!/usr/bin/env python
+import os
+import sys
+import argparse
+import datetime
+from datetime import timedelta
+import pytz
+
+project_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+if project_path not in sys.path:
+    sys.path.append(project_path)
+
+from control.celeryapp import maintainer_healer
+from control.veda_heal import VedaHeal
+from VEDA_OS01.models import Video
+
+"""
+Deliver
+
+Command Line Interface
+"""
+
+
+class HealCli:
+
+    def __init__(self, **kwargs):
+        self.logging = kwargs.get('logging', True)
+
+        self.binscript = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'heal')
+
+    def schedule(self):
+        go_time = datetime.datetime.now(pytz.timezone("America/New_York")) \
+                      .replace(hour=0, minute=0, second=0, microsecond=0) \
+                      .astimezone(pytz.utc) + timedelta(days=1)
+
+        maintainer_healer.apply_async((self.binscript,), queue='transcode_stat', eta=go_time)
+
+
+def main():
+    """
+    Maintenance Daemon + ETA dialer for healing
+
+    """
+    parser = argparse.ArgumentParser()
+    parser.usage = '''
+        {cmd} -i veda_id
+        {cmd} -c course_id
+        {cmd} -s schedule
+        [-i -c -s]
+        Use --help to see all options.
+        '''.format(cmd=sys.argv[0])
+
+    parser.add_argument(
+        '-i', '--veda_id', default=None,
+        help='VEDA ID'
+        )
+
+    parser.add_argument(
+        '-c', '--course_id',
+        help='Course ID',
+        )
+
+    parser.add_argument(
+        '-s', '--schedule',
+        help='Trigger Scheduler',
+        action='store_true'
+        )
+
+    args = parser.parse_args()
+
+    veda_id = args.veda_id
+    course_id = args.course_id
+    schedule = args.schedule
+
+    print '%s - %s: %s' % ('Healing', 'VEDA ID', veda_id)
+    print '%s - %s: %s' % ('Healing', 'Course', course_id)
+
+    if veda_id is None and course_id is None and schedule is False:
+        VH = VedaHeal()
+        VH.discovery()
+        VH.purge()
+        HC = HealCli()
+        HC.schedule()
+        return None
+
+    if veda_id is not None:
+        VH = VedaHeal(
+            video_query=Video.objects.filter(
+                edx_id=veda_id.strip()
+                )
+            )
+        VH.send_encodes()
+        return None
+
+    if course_id is not None:
+        VH = VedaHeal(
+            video_query=Video.objects.filter(
+                inst_class=Course.objects.filter(
+                    institution=course_id[0:3],
+                    edx_classid=course_id[3:8]
+                    )
+                )
+            )
+        VH.send_encodes()
+        return None
+
+    if schedule is True:
+        HC = HealCli()
+        HC.schedule()
+        return None
+
+
+if __name__ == '__main__':
+    sys.exit(main())

bin/ingest

+#!/usr/bin/env python
+import os
+import sys
+import argparse
+
+project_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+if project_path not in sys.path:
+    sys.path.append(project_path)
+
+from control.veda_utils import EmailAlert
+
+"""
+Ingest
+
+Command Line Interface
+"""
+
+class IngestCli():
+
+    def __init__(self, **kwargs):
+        self.args = None
+        self.test = False
+        self.logging = kwargs.get('logging', True)
+
+        self.course_list = []
+
+    def get_args(self):
+        parser = argparse.ArgumentParser()
+        parser.usage = '''
+        {cmd} -l List
+        [-l ]
+        Use --help to see all options.
+        '''.format(cmd=sys.argv[0])
+
+        parser.add_argument(
+            '-l', '--list',
+            help='List Eligible Courses, Exit',
+            action='store_true'
+            )
+        parser.add_argument(
+            '-c', '--courseid',
+            help='List Eligible Courses, Exit',
+            action='store_true'
+            )
+
+        self.args = parser.parse_args()
+        self._parse_args()
+
+    def _parse_args(self):
+        self.course_id = self.args.courseid
+        self.list = self.args.list
+
+    def run(self):
+        """
+        Loop, constant video retreival from Remotes
+
+        :return: If fault: Email admin list
+        """
+        runcommand = ' '.join((
+            'python',
+            os.path.join(os.path.dirname(os.path.abspath(__file__)), 'loop.py'),
+            '-i'
+        ))
+        os.system(runcommand)
+
+        E1 = EmailAlert(message='Ingest Daemon Crash', subject='Ingest Daemon')
+        E1.email()
+
+
+
+def main():
+    IC = IngestCli()
+    IC.get_args()
+    IC.run()
+
+
+if __name__ == '__main__':
+    sys.exit(main())
+

bin/loop.py

+#!/usr/bin/env python
+
+import os
+import sys
+import argparse
+from django.db import reset_queries
+import resource
+import time
+
+project_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+if project_path not in sys.path:
+    sys.path.append(project_path)
+
+"""
+This is a cheapo way to get a pager (using SES)
+
+"""
+from control.veda_file_discovery import FileDiscovery
+from youtube_callback.daemon import generate_course_list
+from youtube_callback.sftp_id_retrieve import callfunction
+
+
+class DaemonCli:
+
+    def __init__(self):
+        self.args = None
+        self.ingest = False
+        self.youtube = False
+        self.course_list = []
+
+    def get_args(self):
+        parser = argparse.ArgumentParser()
+        parser.usage = '''
+        {cmd} -ingest IngestDaemon
+        {cmd} -youtube YoutubeCallbackDaemon
+        [-i -y]
+        Use --help to see all options.
+        '''.format(cmd=sys.argv[0])
+
+        parser.add_argument(
+            '-i', '--ingest',
+            help='Activate alerted ingest daemon',
+            action='store_true'
+        )
+        parser.add_argument(
+            '-y', '--youtube',
+            help='Activate alerted youtube callback daemon',
+            action='store_true'
+        )
+
+        self.args = parser.parse_args()
+        self.ingest = self.args.ingest
+        self.youtube = self.args.youtube
+
+    def run(self):
+        """
+        actually run the function
+        """
+        if self.ingest is True:
+            self.ingest_daemon()
+
+        if self.youtube is True:
+            self.youtube_daemon()
+
+    def ingest_daemon(self):
+        x = 0
+        while True:
+            node_work_directory = os.path.join(
+                os.path.dirname(os.path.dirname(os.path.dirname(
+                    os.path.abspath(__file__)
+                ))),
+                'VEDA_WORKING'
+            )
+            FD = FileDiscovery(
+                node_work_directory=node_work_directory
+            )
+
+            FD.studio_s3_ingest()
+            FD.about_video_ingest()
+            reset_queries()
+            x += 1
+            if x >= 100:
+                print 'Memory usage: %s (kb)' % resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
+                x = 0
+
+    def youtube_daemon(self):
+        x = 0
+        while True:
+            self.course_list = generate_course_list()
+            for course in self.course_list:
+                print "%s%s: Callback" % (course.institution, course.edx_classid)
+                callfunction(course)
+
+            x += 1
+            if x >= 100:
+                print 'Memory usage: %s (kb)' % resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
+                x = 0
+
+            reset_queries()
+            self.course_list = []
+            time.sleep(10)
+
+
+def main():
+
+    DC = DaemonCli()
+    DC.get_args()
+    DC.run()
+
+
+if __name__ == '__main__':
+    sys.exit(main())

bin/youtubecallback

+#!/usr/bin/env python
+import os
+import sys
+import argparse
+
+project_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+if project_path not in sys.path:
+    sys.path.append(project_path)
+
+from control.veda_utils import EmailAlert
+from youtube_callback.daemon import generate_course_list, get_course
+from youtube_callback.sftp_id_retrieve import callfunction
+
+"""
+Youtube Callback
+
+Command Line Interface
+"""
+
+
+class YoutubeCallbackCli():
+
+    def __init__(self, **kwargs):
+        self.args = None
+        self.test = False
+        self.logging = kwargs.get('logging', True)
+
+        self.course_list = []
+
+    def get_args(self):
+        parser = argparse.ArgumentParser()
+        parser.usage = '''
+        {cmd} -l List
+        {cmd} -c course
+        [-l -c ]
+        Use --help to see all options.
+        '''.format(cmd=sys.argv[0])
+
+        parser.add_argument(
+            '-l', '--list',
+            help='List Eligible Courses, Exit',
+            action='store_true'
+            )
+
+        parser.add_argument(
+            '-c', '--courseid',
+            default=None,
+            help='Parse Specific Course ID, Exit',
+            )
+
+        self.args = parser.parse_args()
+
+        self._parse_args()
+
+    def _parse_args(self):
+        self.course_id = self.args.courseid
+        self.list = self.args.list
+
+    def run(self):
+
+        if self.list is True:
+            self.listcourses()
+        else:
+            self.loop()
+
+    def loop(self):
+        """
+        Daemon Loop
+        """
+        if self.course_id is not None:
+            course = get_course(course_id=self.course_id)
+            if course is not None:
+                callfunction(course)
+            return None
+
+        runcommand = ' '.join((
+            'python',
+            os.path.join(os.path.dirname(os.path.abspath(__file__)), 'loop.py'),
+            '-y'
+        ))
+        os.system(runcommand)
+        E1 = EmailAlert(message='Youtube Callback Daemon Crash', subject='Youtube Callback Daemon')
+        E1.email()
+
+    def listcourses(self):
+        """
+        list and exit
+        :return:
+        """
+        self.course_list = generate_course_list()
+        for course in self.course_list:
+            print course.institution
+            print course.edx_classid
+
+
+def main():
+    YTCC = YoutubeCallbackCli()
+    YTCC.get_args()
+    YTCC.run()
+
+if __name__ == '__main__':
+    sys.exit(main())