veda_file_ingest.py 12.2 KB
Newer Older
Gregory Martin committed
1 2 3 4
"""
Discovered file ingest/insert/job triggering

"""
5 6 7 8 9 10 11

import datetime
import logging
import subprocess

from django.db.utils import DatabaseError

12
from control_env import *
13 14
from VEDA.utils import get_config
from veda_heal import VedaHeal
Gregory Martin committed
15
from veda_hotstore import Hotstore
16
from VEDA_OS01.models import TranscriptStatus
17 18 19
from veda_utils import Report
from veda_val import VALAPICall
from veda_video_validation import Validation
20

21 22
LOGGER = logging.getLogger(__name__)

Gregory Martin committed
23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41
'''
V = VideoProto(
    s3_filename=edx_filename,
    client_title=client_title,
    file_extension=file_extension,
    platform_course_url=platform_course_url
    )

I = VedaIngest(
    course_id=course_query[0],
    video_proto=V
    )
I.insert()

if I.complete is False:
    return None
'''


42
class VideoProto(object):
Gregory Martin committed
43 44 45 46 47 48 49

    def __init__(self, **kwargs):
        self.s3_filename = kwargs.get('s3_filename', None)
        self.client_title = kwargs.get('client_title', None)
        self.file_extension = kwargs.get('file_extension', None)
        self.platform_course_url = kwargs.get('platform_course_url', None)
        self.abvid_serial = kwargs.get('abvid_serial', None)
50 51 52 53 54 55 56 57

        # Transcription Process related Attributes
        self.process_transcription = kwargs.get('process_transcription', False)
        self.provider = kwargs.get('provider', None)
        self.three_play_turnaround = kwargs.get('three_play_turnaround', None)
        self.cielo24_turnaround = kwargs.get('cielo24_turnaround', None)
        self.cielo24_fidelity = kwargs.get('cielo24_fidelity', None)
        self.preferred_languages = kwargs.get('preferred_languages', [])
58
        self.source_language = kwargs.get('source_language', None)
59

60
        # Determined Videofile Attributes
Gregory Martin committed
61 62 63 64 65 66 67 68
        self.valid = False
        self.filesize = 0
        self.duration = 0
        self.bitrate = None
        self.resolution = None
        self.veda_id = None


69
class VedaIngest(object):
Gregory Martin committed
70 71 72 73

    def __init__(self, course_object, video_proto, **kwargs):
        self.course_object = course_object
        self.video_proto = video_proto
74
        self.auth_dict = get_config()
Gregory Martin committed
75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92
        self.node_work_directory = kwargs.get('node_work_directory', WORK_DIRECTORY)
        self.full_filename = kwargs.get('full_filename', None)
        self.complete = False
        self.archived = False

    def insert(self):
        self.database_record()
        self.val_insert()
        self.rename()
        self.archived = self.store()

        if self.video_proto.valid is False:
            self.abvid_report()
            self.complete = True
            if self.archived is True:
                os.remove(self.full_filename)
            return None

93 94 95
        LOGGER.info('[VIDEO_INGEST : Ingested] {video_id} : {datetime}'.format(
            video_id=self.video_proto.veda_id,
            datetime=str(datetime.datetime.utcnow()))
Gregory Martin committed
96
        )
97 98

        self.queue_job()
Gregory Martin committed
99 100 101 102 103 104 105 106 107 108
        Course.objects.filter(
            pk=self.course_object.pk
        ).update(
            previous_statechange=datetime.datetime.utcnow().replace(tzinfo=utc)
        )
        if self.archived is True:
            os.remove(self.full_filename)
        self.complete = True

    def queue_job(self):
109 110 111
        # TODO: Break heal method listed here out into helper util
        encode_instance = VedaHeal(
            video_query=Video.objects.filter(
112 113 114
                edx_id=self.video_proto.veda_id.strip()
            ),
            val_status='transcode_queue'
Gregory Martin committed
115
        )
116
        encode_instance.send_encodes()
Gregory Martin committed
117

118
    def _gather_metadata(self):
Gregory Martin committed
119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159
        """
        use st filesize for filesize
        Use "ffprobe" for other metadata
        """
        self.video_proto.filesize = os.stat(self.full_filename).st_size

        ff_command = ' '.join((
            FFPROBE,
            "\'" + self.full_filename + "\'"
        ))
        p = subprocess.Popen(ff_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True)

        for line in iter(p.stdout.readline, b''):
            if "Duration: " in line:

                self.video_proto.duration = line.split(',')[0].split(' ')[-1]
                try:
                    bitrate = line.split(',')[2].split(' :')[-1].strip()
                    self.video_proto.bitrate = bitrate.replace('bitrate: ', '')
                except:
                    pass

            elif "Stream #" in line:
                if " Video: " in line:
                    vid_breakout = line.split(',')
                    vid_reso_break = vid_breakout[2].strip().split(' ')
                    for v in vid_reso_break:
                        if "x" in v:
                            self.video_proto.resolution = v.strip()
                    if self.video_proto.resolution is None:
                        self.video_proto.resolution = vid_breakout[3].strip()
                    if ')' in self.video_proto.resolution.strip():
                        if ')' not in vid_breakout[4].strip():
                            self.video_proto.resolution = vid_breakout[4].strip()
                        elif ')' not in vid_breakout[5].strip():
                            self.video_proto.resolution = vid_breakout[5].strip()
                        else:
                            self.video_proto.resolution = '1920x1080'

    def database_record(self):
        """
160
        Start DB Inserts, Get Basic File name information
Gregory Martin committed
161
        """
162
        if self.video_proto.s3_filename:
Gregory Martin committed
163 164 165 166
            self.full_filename = '/'.join((
                self.node_work_directory,
                self.video_proto.s3_filename
            ))
167
        if self.video_proto.abvid_serial:
Gregory Martin committed
168 169 170 171
            self.full_filename = '/'.join((
                self.node_work_directory,
                self.video_proto.client_title
            ))
172 173
            if len(self.video_proto.file_extension) > 2:
                self.full_filename += "." + self.video_proto.file_extension
Gregory Martin committed
174

175
        if not self.full_filename:
Gregory Martin committed
176 177 178 179 180 181
            self.full_filename = '/'.join((
                self.node_work_directory,
                self.video_proto.client_title
            ))

        if not os.path.exists(self.full_filename):
182 183
            LOGGER.exception('[VIDEO_INGEST] File Not Found %s', self.video_proto.veda_id)
            return
Gregory Martin committed
184 185 186 187 188 189 190 191 192

        """
        Validate File
        """
        VV = Validation(videofile=self.full_filename)

        self.video_proto.valid = VV.validate()

        if self.video_proto.valid is True:
193
            self._gather_metadata()
194 195 196 197 198 199

        # DB Inserts
        if self.video_proto.s3_filename:
            video = Video.objects.filter(studio_id=self.video_proto.s3_filename).first()
            if video:
                # Protect against crash/duplicate inserts, won't insert object
Gregory Martin committed
200 201
                self.video_proto.veda_id = video.edx_id
                self.video_proto.video_orig_duration = video.video_orig_duration
202 203 204
                self.complete = True
                return

Gregory Martin committed
205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257
        v1 = Video(inst_class=self.course_object)
        """
        Generate veda_id / update course record
        * Note: defensive against the possibility of later passing in an ID
        """
        if self.video_proto.veda_id is None:
            lsid = self.course_object.last_vid_number + 100
            self.video_proto.veda_id = self.course_object.institution
            self.video_proto.veda_id += self.course_object.edx_classid
            self.video_proto.veda_id += self.course_object.semesterid
            self.video_proto.veda_id += "-V" + str(lsid).zfill(6)

            """
            Update Course Record
            """
            self.course_object.last_vid_number = lsid
            self.course_object.save()

        v1.edx_id = self.video_proto.veda_id

        v1.video_orig_extension = self.video_proto.file_extension
        v1.studio_id = self.video_proto.s3_filename
        v1.client_title = self.video_proto.client_title
        v1.abvid_serial = self.video_proto.abvid_serial

        if self.video_proto.valid is False:
            """
            Invalid File, Save, exit
            """
            v1.video_trans_status = 'Corrupt File'
            v1.video_active = False
            try:
                v1.save()
            except:
                """
                decode to ascii
                """
                char_string = self.video_proto.client_title
                string_len = len(char_string)
                s1 = 0
                final_string = ""
                while string_len > s1:
                    try:
                        char_string[s1].decode('ascii')
                        final_string += char_string[s1]
                    except:
                        final_string += "?"
                    s1 += 1
                v1.client_title = final_string
                v1.save()
            self.complete = True
            return None

258 259 260
        # Update transcription preferences for the Video
        if self.video_proto.process_transcription:
            v1.process_transcription = self.video_proto.process_transcription
261
            v1.transcript_status = TranscriptStatus.PENDING
262 263 264 265 266
            v1.provider = self.video_proto.provider
            v1.three_play_turnaround = self.video_proto.three_play_turnaround
            v1.cielo24_turnaround = self.video_proto.cielo24_turnaround
            v1.cielo24_fidelity = self.video_proto.cielo24_fidelity
            v1.preferred_languages = self.video_proto.preferred_languages
267
            v1.source_language = self.video_proto.source_language
268

Gregory Martin committed
269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288
        """
        Files Below are all valid
        """
        v1.video_orig_filesize = self.video_proto.filesize
        v1.video_orig_duration = self.video_proto.duration
        v1.video_orig_bitrate = self.video_proto.bitrate
        v1.video_orig_resolution = self.video_proto.resolution

        """
        Ready for Task Fire
        """
        v1.video_active = True
        v1.video_trans_status = 'Ingest'
        v1.video_trans_start = datetime.datetime.utcnow().replace(tzinfo=utc)

        """
        Save / Decode / Update Course
        """
        try:
            v1.save()
289 290
        except DatabaseError:
            # in case if the client title's length is too long
Gregory Martin committed
291 292 293 294 295 296 297 298 299 300 301 302 303 304
            char_string = self.video_proto.client_title
            string_len = len(char_string)
            s1 = 0
            final_string = ""
            while string_len > s1:
                try:
                    char_string[s1].decode('ascii')
                    final_string += char_string[s1]
                except:
                    final_string += "?"
                s1 += 1
            v1.client_title = final_string
            v1.save()

305 306
        except Exception:
            # Log the exception and raise.
307
            LOGGER.exception('[VIDEO_INGEST] - Cataloging of video=%s failed.', self.video_proto.veda_id)
308 309
            raise

Gregory Martin committed
310
    def val_insert(self):
311
        if self.video_proto.abvid_serial:
Gregory Martin committed
312 313 314 315 316 317 318 319 320 321
            return None

        if self.video_proto.valid is False:
            val_status = 'file_corrupt'
        else:
            val_status = 'ingest'

        VAC = VALAPICall(
            video_proto=self.video_proto,
            val_status=val_status,
Gregory Martin committed
322
            platform_course_url=""  # Empty record for initial status update
Gregory Martin committed
323 324 325 326 327 328 329
        )
        VAC.call()

    def abvid_report(self):
        if self.video_proto.abvid_serial is None:
            return None

330
        email_report = Report(
Gregory Martin committed
331 332 333 334
            status="File Corrupt on Ingest",
            upload_serial=self.video_proto.abvid_serial,
            youtube_id=''
        )
335
        email_report.upload_status()
Gregory Martin committed
336 337 338 339 340
        self.complete = True

    def rename(self):
        """
        Rename to VEDA ID,
341

Gregory Martin committed
342 343 344
        """
        if self.video_proto.veda_id is None:
            self.video_proto.valid = False
345
            return
Gregory Martin committed
346

347 348 349 350 351
        veda_filename = self.video_proto.veda_id
        if self.video_proto.file_extension:
            veda_filename += '.{ext}'.format(ext=self.video_proto.file_extension)
        os.rename(
            self.full_filename, os.path.join(
Gregory Martin committed
352
                self.node_work_directory,
353
                veda_filename
Gregory Martin committed
354
            )
355 356
        )
        self.full_filename = os.path.join(self.node_work_directory, veda_filename)
Gregory Martin committed
357
        os.system('chmod ugo+rwx ' + self.full_filename)
358
        return
Gregory Martin committed
359 360 361 362 363 364 365 366 367 368

    def store(self):
        """
        Ingest File Backup / Archive Policy
        """
        H1 = Hotstore(
            video_proto=self.video_proto,
            upload_filepath=self.full_filename
        )
        return H1.upload()