Merge pull request #6352 from edx/gprice/video-upload-csv

Add CSV download button to Studio video upload page

cms/djangoapps/contentstore/admin.py

+"""
+Admin site bindings for contentstore
+"""
+
+from django.contrib import admin
+
+from config_models.admin import ConfigurationModelAdmin
+from contentstore.models import VideoUploadConfig
+
+admin.site.register(VideoUploadConfig, ConfigurationModelAdmin)

cms/djangoapps/contentstore/migrations/0001_initial.py

+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding model 'VideoUploadConfig'
+        db.create_table('contentstore_videouploadconfig', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('change_date', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)),
+            ('changed_by', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'], null=True, on_delete=models.PROTECT)),
+            ('enabled', self.gf('django.db.models.fields.BooleanField')(default=False)),
+            ('profile_whitelist', self.gf('django.db.models.fields.TextField')(blank=True)),
+            ('status_whitelist', self.gf('django.db.models.fields.TextField')(blank=True)),
+        ))
+        db.send_create_signal('contentstore', ['VideoUploadConfig'])
+
+        if not db.dry_run:
+            orm.VideoUploadConfig.objects.create(
+                profile_whitelist="desktop_mp4,desktop_webm,mobile_low,youtube",
+                status_whitelist="Uploading,In Progress,Complete,Failed,Invalid Token,Unknown"
+            )
+
+    def backwards(self, orm):
+        # Deleting model 'VideoUploadConfig'
+        db.delete_table('contentstore_videouploadconfig')
+
+    models = {
+        'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        'auth.permission': {
+            'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        'auth.user': {
+            'Meta': {'object_name': 'User'},
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        'contentstore.videouploadconfig': {
+            'Meta': {'object_name': 'VideoUploadConfig'},
+            'change_date': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'changed_by': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True', 'on_delete': 'models.PROTECT'}),
+            'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'profile_whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'status_whitelist': ('django.db.models.fields.TextField', [], {'blank': 'True'})
+        },
+        'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        }
+    }
+
+    complete_apps = ['contentstore']

cms/djangoapps/contentstore/models.py

+"""
+Models for contentstore
+"""
+# pylint: disable=no-member
+
+from django.db.models.fields import TextField
+
+from config_models.models import ConfigurationModel
+
+
+class VideoUploadConfig(ConfigurationModel):
+    """Configuration for the video upload feature."""
+    profile_whitelist = TextField(
+        blank=True,
+        help_text="A comma-separated list of names of profiles to include in video encoding downloads."
+    )
+    status_whitelist = TextField(
+        blank=True,
+        help_text=(
+            "A comma-separated list of Studio status values;" +
+            " only videos with these status values will be included in video encoding downloads."
+        )
+    )
+
+    @classmethod
+    def get_profile_whitelist(cls):
+        """Get the list of profiles to include in the encoding download"""
+        return [profile for profile in cls.current().profile_whitelist.split(",") if profile]
+
+    @classmethod
+    def get_status_whitelist(cls):
+        """
+        Get the list of status values to include files for in the encoding
+        download
+        """
+        return [status for status in cls.current().status_whitelist.split(",") if status]

cms/djangoapps/contentstore/views/videos.py

@@ -2,16 +2,20 @@
 Views related to the video upload feature
 """
 from boto import s3
+import csv
 from uuid import uuid4
 
 from django.conf import settings
 from django.contrib.auth.decorators import login_required
-from django.http import HttpResponseNotFound
-from django.views.decorators.http import require_http_methods
+from django.http import HttpResponse, HttpResponseNotFound
+from django.utils.translation import ugettext as _
+from django.views.decorators.http import require_GET, require_http_methods
+import rfc6266
 
 from edxval.api import create_video, get_videos_for_ids
 from opaque_keys.edx.keys import CourseKey
 
+from contentstore.models import VideoUploadConfig
 from contentstore.utils import reverse_course_url
 from edxmako.shortcuts import render_to_response
 from util.json_request import expect_json, JsonResponse
@@ -21,7 +25,7 @@ from xmodule.modulestore.django import modulestore
 from .course import get_course_and_check_access
 
 
-__all__ = ["videos_handler"]
+__all__ = ["videos_handler", "video_encodings_download"]
 
 
 # String constant used in asset keys to identify video assets.
@@ -31,6 +35,43 @@ VIDEO_ASSET_TYPE = "video"
 KEY_EXPIRATION_IN_SECONDS = 86400
 
 
+class StatusDisplayStrings(object):
+    """
+    Enum of display strings for Video Status presented in Studio (e.g., in UI and in CSV download).
+    """
+    # Translators: This is the status of an active video upload
+    UPLOADING = _("Uploading")
+    # Translators: This is the status for a video that the servers are currently processing
+    IN_PROGRESS = _("In Progress")
+    # Translators: This is the status for a video that the servers have successfully processed
+    COMPLETE = _("Complete")
+    # Translators: This is the status for a video that the servers have failed to process
+    FAILED = _("Failed"),
+    # Translators: This is the status for a video for which an invalid
+    # processing token was provided in the course settings
+    INVALID_TOKEN = _("Invalid Token"),
+    # Translators: This is the status for a video that is in an unknown state
+    UNKNOWN = _("Unknown")
+
+
+def status_display_string(val_status):
+    """
+    Converts VAL status string to Studio status string.
+    """
+    status_map = {
+        "upload": StatusDisplayStrings.UPLOADING,
+        "ingest": StatusDisplayStrings.IN_PROGRESS,
+        "transcode_queue": StatusDisplayStrings.IN_PROGRESS,
+        "transcode_active": StatusDisplayStrings.IN_PROGRESS,
+        "file_delivered": StatusDisplayStrings.COMPLETE,
+        "file_complete": StatusDisplayStrings.COMPLETE,
+        "file_corrupt": StatusDisplayStrings.FAILED,
+        "pipeline_error": StatusDisplayStrings.FAILED,
+        "invalid_token": StatusDisplayStrings.INVALID_TOKEN
+    }
+    return status_map.get(val_status, StatusDisplayStrings.UNKNOWN)
+
+
 @expect_json
 @login_required
 @require_http_methods(("GET", "POST"))
@@ -48,18 +89,9 @@ def videos_handler(request, course_key_string):
             to this endpoint but rather PUT to the respective upload_url values
             contained in the response
     """
-    course_key = CourseKey.from_string(course_key_string)
-
-    # For now, assume all studio users that have access to the course can upload videos.
-    # In the future, we plan to add a new org-level role for video uploaders.
-    course = get_course_and_check_access(course_key, request.user)
+    course = _get_and_validate_course(course_key_string, request.user)
 
-    if (
-            not settings.FEATURES["ENABLE_VIDEO_UPLOAD_PIPELINE"] or
-            not getattr(settings, "VIDEO_UPLOAD_PIPELINE", None) or
-            not course or
-            not course.video_pipeline_configured
-    ):
+    if not course:
         return HttpResponseNotFound()
 
     if request.method == "GET":
@@ -71,21 +103,141 @@ def videos_handler(request, course_key_string):
         return videos_post(course, request)
 
 
+@login_required
+@require_GET
+def video_encodings_download(request, course_key_string):
+    """
+    Returns a CSV report containing the encoded video URLs for video uploads
+    in the following format:
+
+    Video ID,Name,Status,Profile1 URL,Profile2 URL
+    aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaaaa,video.mp4,Complete,http://example.com/prof1.mp4,http://example.com/prof2.mp4
+    """
+    course = _get_and_validate_course(course_key_string, request.user)
+
+    if not course:
+        return HttpResponseNotFound()
+
+    def get_profile_header(profile):
+        """Returns the column header string for the given profile's URLs"""
+        # Translators: This is the header for a CSV file column
+        # containing URLs for video encodings for the named profile
+        # (e.g. desktop, mobile high quality, mobile low quality)
+        return _("{profile_name} URL").format(profile_name=profile)
+
+    profile_whitelist = VideoUploadConfig.get_profile_whitelist()
+    status_whitelist = VideoUploadConfig.get_status_whitelist()
+
+    videos = list(_get_videos(course))
+    name_col = _("Name")
+    duration_col = _("Duration")
+    added_col = _("Date Added")
+    video_id_col = _("Video ID")
+    status_col = _("Status")
+    profile_cols = [get_profile_header(profile) for profile in profile_whitelist]
+
+    def make_csv_dict(video):
+        """
+        Makes a dictionary suitable for writing CSV output. This involves
+        extracting the required items from the original video dict and
+        converting all keys and values to UTF-8 encoded string objects,
+        because the CSV module doesn't play well with unicode objects.
+        """
+        # Translators: This is listed as the duration for a video that has not
+        # yet reached the point in its processing by the servers where its
+        # duration is determined.
+        duration_val = str(video["duration"]) if video["duration"] > 0 else _("Pending")
+        ret = dict(
+            [
+                (name_col, video["client_video_id"]),
+                (duration_col, duration_val),
+                (added_col, video["created"].isoformat()),
+                (video_id_col, video["edx_video_id"]),
+                (status_col, video["status"]),
+            ] +
+            [
+                (get_profile_header(encoded_video["profile"]), encoded_video["url"])
+                for encoded_video in video["encoded_videos"]
+                if encoded_video["profile"] in profile_whitelist
+            ]
+        )
+        return {
+            key.encode("utf-8"): value.encode("utf-8")
+            for key, value in ret.items()
+        }
+
+    response = HttpResponse(content_type="text/csv")
+    # Translators: This is the suggested filename when downloading the URL
+    # listing for videos uploaded through Studio
+    filename = _("{course}_video_urls").format(course=course.id.course)
+    # See https://tools.ietf.org/html/rfc6266#appendix-D
+    response["Content-Disposition"] = rfc6266.build_header(
+        filename + ".csv",
+        filename_compat="video_urls.csv"
+    )
+    writer = csv.DictWriter(
+        response,
+        [name_col, duration_col, added_col, video_id_col, status_col] + profile_cols,
+        dialect=csv.excel
+    )
+    writer.writeheader()
+    for video in videos:
+        if video["status"] in status_whitelist:
+            writer.writerow(make_csv_dict(video))
+    return response
+
+
+def _get_and_validate_course(course_key_string, user):
+    """
+    Given a course key, return the course if it exists, the given user has
+    access to it, and it is properly configured for video uploads
+    """
+    course_key = CourseKey.from_string(course_key_string)
+
+    # For now, assume all studio users that have access to the course can upload videos.
+    # In the future, we plan to add a new org-level role for video uploaders.
+    course = get_course_and_check_access(course_key, user)
+
+    if (
+            settings.FEATURES["ENABLE_VIDEO_UPLOAD_PIPELINE"] and
+            getattr(settings, "VIDEO_UPLOAD_PIPELINE", None) and
+            course and
+            course.video_pipeline_configured
+    ):
+        return course
+    else:
+        return None
+
+
 def _get_videos(course):
     """
     Retrieves the list of videos from VAL corresponding to the videos listed in
-    the asset metadata store and returns the needed subset of fields
+    the asset metadata store.
     """
     edx_videos_ids = [
         v.asset_id.path
         for v in modulestore().get_all_asset_metadata(course.id, VIDEO_ASSET_TYPE)
     ]
+
+    videos = list(get_videos_for_ids(edx_videos_ids))
+
+    # convert VAL's status to studio's Video Upload feature status.
+    for video in videos:
+        video["status"] = status_display_string(video["status"])
+
+    return videos
+
+
+def _get_index_videos(course):
+    """
+    Returns the information about each video upload required for the video list
+    """
     return list(
         {
             attr: video[attr]
             for attr in ["edx_video_id", "client_video_id", "created", "duration", "status"]
         }
-        for video in get_videos_for_ids(edx_videos_ids)
+        for video in _get_videos(course)
     )
 
 
@@ -98,7 +250,8 @@ def videos_index_html(course):
         {
             "context_course": course,
             "post_url": reverse_course_url("videos_handler", unicode(course.id)),
-            "previous_uploads": _get_videos(course),
+            "encodings_download_url": reverse_course_url("video_encodings_download", unicode(course.id)),
+            "previous_uploads": _get_index_videos(course),
             "concurrent_upload_limit": settings.VIDEO_UPLOAD_PIPELINE.get("CONCURRENT_UPLOAD_LIMIT", 0),
         }
     )
@@ -117,7 +270,7 @@ def videos_index_json(course):
         }]
     }
     """
-    return JsonResponse({"videos": _get_videos(course)}, status=200)
+    return JsonResponse({"videos": _get_index_videos(course)}, status=200)
 
 
 def videos_post(course, request):

cms/static/js/factories/videos_index.js

@@ -2,7 +2,14 @@ define(
     ["jquery", "backbone", "js/views/active_video_upload_list", "js/views/previous_video_upload_list"],
     function ($, Backbone, ActiveVideoUploadListView, PreviousVideoUploadListView) {
         "use strict";
-        var VideosIndexFactory = function($contentWrapper, postUrl, concurrentUploadLimit, uploadButton, previousUploads) {
+        var VideosIndexFactory = function(
+            $contentWrapper,
+            postUrl,
+            encodingsDownloadUrl,
+            concurrentUploadLimit,
+            uploadButton,
+            previousUploads
+        ) {
             var activeView = new ActiveVideoUploadListView({
                 postUrl: postUrl,
                 concurrentUploadLimit: concurrentUploadLimit,
@@ -10,7 +17,10 @@ define(
             });
             $contentWrapper.append(activeView.render().$el);
             var previousCollection = new Backbone.Collection(previousUploads);
-            var previousView = new PreviousVideoUploadListView({collection: previousCollection});
+            var previousView = new PreviousVideoUploadListView({
+                collection: previousCollection,
+                encodingsDownloadUrl: encodingsDownloadUrl
+            });
             $contentWrapper.append(previousView.render().$el);
         };
 

cms/static/js/spec/views/previous_video_upload_spec.js

@@ -67,16 +67,12 @@ define(
 
             _.each(
                 [
-                    {status: "upload", expected: "Uploading"},
-                    {status: "ingest", expected: "In Progress"},
-                    {status: "transcode_queue", expected: "In Progress"},
-                    {status: "transcode_active", expected: "In Progress"},
-                    {status: "file_delivered", expected: "Complete"},
-                    {status: "file_complete", expected: "Complete"},
-                    {status: "file_corrupt", expected: "Failed"},
-                    {status: "pipeline_error", expected: "Failed"},
-                    {status: "invalid_token", expected: "Invalid Token"},
-                    {status: "unexpected_status_string", expected: "Unknown"}
+                    {status: "Uploading", expected: "Uploading"},
+                    {status: "In Progress", expected: "In Progress"},
+                    {status: "Complete", expected: "Complete"},
+                    {status: "Failed", expected: "Failed"},
+                    {status: "Invalid Token", expected: "Invalid Token"},
+                    {status: "Unknown", expected: "Unknown"}
                 ],
                 function(caseInfo) {
                     it("should render " + caseInfo.status + " status correctly", function() {

cms/static/js/views/previous_video_upload.js

@@ -3,38 +3,6 @@ define(
     function(gettext, DateUtils, BaseView) {
         "use strict";
 
-        var statusDisplayStrings = {
-            // Translators: This is the status of an active video upload
-            UPLOADING: gettext("Uploading"),
-            // Translators: This is the status for a video that the servers
-            // are currently processing
-            IN_PROGRESS: gettext("In Progress"),
-            // Translators: This is the status for a video that the servers
-            // have successfully processed
-            COMPLETE: gettext("Complete"),
-            // Translators: This is the status for a video that the servers
-            // have failed to process
-            FAILED: gettext("Failed"),
-            // Translators: This is the status for a video for which an invalid
-            // processing token was provided in the course settings
-            INVALID_TOKEN: gettext("Invalid Token"),
-            // Translators: This is the status for a video that is in an unknown
-            // state
-            UNKNOWN: gettext("Unknown")
-        };
-
-        var statusMap = {
-            "upload": statusDisplayStrings.UPLOADING,
-            "ingest": statusDisplayStrings.IN_PROGRESS,
-            "transcode_queue": statusDisplayStrings.IN_PROGRESS,
-            "transcode_active": statusDisplayStrings.IN_PROGRESS,
-            "file_delivered": statusDisplayStrings.COMPLETE,
-            "file_complete": statusDisplayStrings.COMPLETE,
-            "file_corrupt": statusDisplayStrings.FAILED,
-            "pipeline_error": statusDisplayStrings.FAILED,
-            "invalid_token": statusDisplayStrings.INVALID_TOKEN
-        };
-
         var PreviousVideoUploadView = BaseView.extend({
             tagName: "tr",
 
@@ -57,7 +25,7 @@ define(
                     // the servers where its duration is determined.
                     duration: duration > 0 ? this.renderDuration(duration) : gettext("Pending"),
                     created: DateUtils.renderDate(this.model.get("created")),
-                    status: statusMap[this.model.get("status")] || statusDisplayStrings.UNKNOWN
+                    status: this.model.get("status")
                 };
                 this.$el.html(
                     this.template(_.extend({}, this.model.attributes, renderedAttributes))

cms/static/js/views/previous_video_upload_list.js

@@ -6,8 +6,9 @@ define(
             tagName: "section",
             className: "wrapper-assets",
 
-            initialize: function() {
+            initialize: function(options) {
                 this.template = this.loadTemplate("previous-video-upload-list");
+                this.encodingsDownloadUrl = options.encodingsDownloadUrl;
                 this.itemViews = this.collection.map(function(model) {
                     return new PreviousVideoUploadView({model: model});
                 });
@@ -15,7 +16,7 @@ define(
 
             render: function() {
                 var $el = this.$el;
-                $el.html(this.template());
+                $el.html(this.template({encodingsDownloadUrl: this.encodingsDownloadUrl}));
                 var $tabBody = $el.find(".js-table-body");
                 _.each(this.itemViews, function(view) {
                     $tabBody.append(view.render().$el);

cms/static/sass/views/_video-upload.scss

@@ -90,4 +90,17 @@
     @extend %ui-btn-non;
   }
 
+  .assets-library {
+    .assets-title {
+      display: inline-block;
+      width: flex-grid(5, 9);
+      @include margin-right(flex-gutter());
+    }
+
+    .wrapper-encodings-download {
+      display: inline-block;
+      width: flex-grid(4, 9);
+      text-align: right;
+    }
+  }
 }

cms/templates/js/previous-video-upload-list.underscore

 <div class="assets-library">
     <h3 class="assets-title"><%- gettext("Previous Uploads") %></h3>
+    <div class="wrapper-encodings-download">
+      <a href="<%- encodingsDownloadUrl %>">
+        <%- gettext("Download available encodings (.csv)") %>
+      </a>
+    </div>
     <table class="assets-table">
         <thead>
             <tr>

cms/templates/videos_index.html

@@ -24,6 +24,7 @@
         VideosIndexFactory(
             $contentWrapper,
             "${post_url}",
+            "${encodings_download_url}",
             ${concurrent_upload_limit},
             $(".nav-actions .upload-button"),
             $contentWrapper.data("previous-uploads")

cms/urls.py

@@ -92,6 +92,7 @@ urlpatterns += patterns(
     url(r'^textbooks/{}$'.format(settings.COURSE_KEY_PATTERN), 'textbooks_list_handler'),
     url(r'^textbooks/{}/(?P<textbook_id>\d[^/]*)$'.format(settings.COURSE_KEY_PATTERN), 'textbooks_detail_handler'),
     url(r'^videos/{}$'.format(settings.COURSE_KEY_PATTERN), 'videos_handler'),
+    url(r'^video_encodings_download/{}$'.format(settings.COURSE_KEY_PATTERN), 'video_encodings_download'),
     url(r'^group_configurations/{}$'.format(settings.COURSE_KEY_PATTERN), 'group_configurations_list_handler'),
     url(r'^group_configurations/{}/(?P<group_configuration_id>\d+)/?$'.format(settings.COURSE_KEY_PATTERN),
         'group_configurations_detail_handler'),

requirements/edx/base.txt

@@ -73,6 +73,7 @@ pysrt==0.4.7
 PyYAML==3.10
 requests==2.3.0
 requests-oauthlib==0.4.1
+rfc6266==0.0.4
 scipy==0.14.0
 Shapely==1.2.16
 singledispatch==3.4.0.2