Merge branch 'master' into feature/jsa/ds-reindex

Conflicts:
	Rakefile

LICENSE.txt

@@ -659,3 +659,13 @@ specific requirements.
 if any, to sign a "copyright disclaimer" for the program, if necessary.
 For more information on this, and how to apply and follow the GNU AGPL, see
 <http://www.gnu.org/licenses/>.
+
+EdX Inc. wishes to state, in clarification of the above license terms, that
+any public, independently available web service offered over the network and
+communicating with edX's copyrighted works by any form of inter-service
+communication, including but not limited to Remote Procedure Call (RPC)
+interfaces, is not a work based on our copyrighted work within the meaning
+of the license. "Corresponding Source" of this work, or works based on this
+work, as defined by the terms of this license do not include source code
+files for programs used solely to provide those public, independently
+available web services.

Rakefile

@@ -133,8 +133,8 @@ namespace :db do
 
   task :generate_comments, [:commentable_id, :num_threads, :num_top_comments, :num_subcomments] => :environment do |t, args|
     args.with_defaults(:num_threads => THREADS_PER_COMMENTABLE,
-     :num_top_comments=>TOP_COMMENTS_PER_THREAD,
-     :num_subcomments=> ADDITIONAL_COMMENTS_PER_THREAD)
+                       :num_top_comments=>TOP_COMMENTS_PER_THREAD,
+                       :num_subcomments=> ADDITIONAL_COMMENTS_PER_THREAD)
     generate_comments_for(args[:commentable_id], args[:num_threads], args[:num_top_comments], args[:num_subcomments])
 
   end
@@ -152,54 +152,54 @@ namespace :db do
     coll = db.collection("contents")
     args[:num].to_i.times do
       doc = {"_type" => "CommentThread", "anonymous" => [true, false].sample, "at_position_list" => [],
-        "tags_array" => [],
-        "comment_count" => 0, "title" => Faker::Lorem.sentence(6), "author_id" => rand(1..10).to_s,
-        "body" => Faker::Lorem.paragraphs.join("\n\n"), "course_id" => COURSE_ID, "created_at" => Time.now,
-        "commentable_id" => COURSE_ID, "closed" => [true, false].sample, "updated_at" => Time.now, "last_activity_at" => Time.now,
-        "votes" => {"count" => 0, "down" => [], "down_count" => 0, "point" => 0, "up" => [], "up_count" => []}}
-        coll.insert(doc)
-      end
-      binding.pry
-      Tire.index('comment_threads').delete
-      CommentThread.create_elasticsearch_index
-      Tire.index('comment_threads') { import CommentThread.all }
+          "tags_array" => [],
+          "comment_count" => 0, "title" => Faker::Lorem.sentence(6), "author_id" => rand(1..10).to_s,
+          "body" => Faker::Lorem.paragraphs.join("\n\n"), "course_id" => COURSE_ID, "created_at" => Time.now,
+          "commentable_id" => COURSE_ID, "closed" => [true, false].sample, "updated_at" => Time.now, "last_activity_at" => Time.now,
+          "votes" => {"count" => 0, "down" => [], "down_count" => 0, "point" => 0, "up" => [], "up_count" => []}}
+      coll.insert(doc)
     end
+    binding.pry
+    Tire.index('comment_threads').delete
+    CommentThread.create_elasticsearch_index
+    Tire.index('comment_threads') { import CommentThread.all }
+  end
 
-    task :seed_fast => :environment do
-      ADDITIONAL_COMMENTS_PER_THREAD = 20
-
-      config = YAML.load_file("config/mongoid.yml")[Sinatra::Base.environment]["sessions"]["default"]
-      connnection = Mongo::Connection.new(config["hosts"][0].split(":")[0], config["hosts"][0].split(":")[1])
-      db = Mongo::Connection.new.db(config["database"])
-      coll = db.collection("contents")
-      Comment.delete_all
-      CommentThread.each do |thread|
-        ADDITIONAL_COMMENTS_PER_THREAD.times do
-          doc = {"_type" => "Comment", "anonymous" => false, "at_position_list" => [],
-            "author_id" => rand(1..10).to_s, "body" => Faker::Lorem.paragraphs.join("\n\n"),
-            "comment_thread_id" => BSON::ObjectId.from_string(thread.id.to_s), "course_id" => COURSE_ID,
-            "created_at" => Time.now,
-            "endorsed" => [true, false].sample, "parent_ids" => [], "updated_at" => Time.now,
-            "votes" => {"count" => 0, "down" => [], "down_count" => 0, "point" => 0, "up" => [], "up_count" => []}}
-            coll.insert(doc)
-          end
-        end
+  task :seed_fast => :environment do
+    ADDITIONAL_COMMENTS_PER_THREAD = 20
+
+    config = YAML.load_file("config/mongoid.yml")[Sinatra::Base.environment]["sessions"]["default"]
+    connnection = Mongo::Connection.new(config["hosts"][0].split(":")[0], config["hosts"][0].split(":")[1])
+    db = Mongo::Connection.new.db(config["database"])
+    coll = db.collection("contents")
+    Comment.delete_all
+    CommentThread.each do |thread|
+      ADDITIONAL_COMMENTS_PER_THREAD.times do
+        doc = {"_type" => "Comment", "anonymous" => false, "at_position_list" => [],
+          "author_id" => rand(1..10).to_s, "body" => Faker::Lorem.paragraphs.join("\n\n"),
+          "comment_thread_id" => BSON::ObjectId.from_string(thread.id.to_s), "course_id" => COURSE_ID,
+          "created_at" => Time.now,
+          "endorsed" => [true, false].sample, "parent_ids" => [], "updated_at" => Time.now,
+          "votes" => {"count" => 0, "down" => [], "down_count" => 0, "point" => 0, "up" => [], "up_count" => []}}
+        coll.insert(doc)
       end
+    end
+  end
 
-      task :seed => :environment do
+  task :seed => :environment do
 
-        Comment.delete_all
-        CommentThread.delete_all
-        CommentThread.recalculate_all_context_tag_weights!
-        User.delete_all
-        Notification.delete_all
-        Subscription.delete_all
-        Tire.index 'comment_threads' do delete end
-          CommentThread.create_elasticsearch_index
+    Comment.delete_all
+    CommentThread.delete_all
+    CommentThread.recalculate_all_context_tag_weights!
+    User.delete_all
+    Notification.delete_all
+    Subscription.delete_all
+    Tire.index 'comment_threads' do delete end
+    CommentThread.create_elasticsearch_index
 
-          beginning_time = Time.now
+    beginning_time = Time.now
 
-          users = (1..10).map {|id| create_test_user(id)}
+    users = (1..10).map {|id| create_test_user(id)}
     # 3.times do
     #   other_user = users[1..9].sample
     #   users.first.subscribe(other_user)
@@ -224,7 +224,6 @@ namespace :db do
   end
 
   task :reindex_search => :environment do
-
     logger = Logger.new(STDERR)
     cutoff_dt = DateTime.now
     klasses = [Comment, CommentThread]

api/notifications.rb

+post "#{APIPREFIX}/notifications" do
+  # get all notifications for a set of users and a range of dates
+  # for example
+  # http://localhost:4567/api/v1/notifications?api_key=PUT_YOUR_API_KEY_HERE
+  # with POST params 
+  # user_ids=1217716,196353
+  # from=2013-03-18+13%3A52%3A47+-0400
+  # to=2013-03-19+13%3A53%3A11+-0400
+  # note this takes date format 8601 year-month-day-(hours:minutes:seconds difference from UTC
+  notifications_by_date_range_and_user_ids(CGI.unescape(params[:from]).to_time, CGI.unescape(params[:to]).to_time,params[:user_ids].split(','))
+end

app.rb

@@ -22,6 +22,10 @@ if ["staging", "production", "loadtest", "edgestage","edgeprod"].include? enviro
   require 'newrelic_rpm'
 end
 
+if ENV["ENABLE_GC_PROFILER"]
+  GC::Profiler.enable
+end
+
 set :cache, Dalli::Client.new
 
 application_yaml = ERB.new(File.read("config/application.yml")).result()
@@ -48,8 +52,6 @@ DEFAULT_PER_PAGE = 20
 
 if RACK_ENV.to_s != "test" # disable api_key auth in test environment
   before do
-    #duct tape to avoid 401 on deep search performance test
-    #error 401 unless params[:api_key] == CommentService.config[:api_key] or true
     error 401 unless params[:api_key] == CommentService.config[:api_key]
   end
 end
@@ -70,6 +72,7 @@ require './api/votes'
 require './api/flags'
 require './api/pins'
 require './api/notifications_and_subscriptions'
+require './api/notifications'
 
 if RACK_ENV.to_s == "development"
   get "#{APIPREFIX}/clean" do

config/mongoid.yml

 development:
   sessions:
-   default:
-     database: cs_comments_service_development
-     hosts:
-       - localhost:27017
-
+    default:
+      database: cs_comments_service_development
+      hosts:
+        - localhost:27017
 test:
   sessions:
     default:
@@ -22,6 +21,7 @@ production:
     default:
       hosts:
         - hurley.member0.mongohq.com:10000
+        - hurley.member1.mongohq.com:10000
       username: <%= ENV['MONGOHQ_USER'] %>
       password: <%= ENV['MONGOHQ_PASS'] %>
       database: app6929933

lib/helpers.rb

@@ -203,4 +203,74 @@ helpers do
     end.compact
   end
 
+  def notifications_by_date_range_and_user_ids start_date_time, end_date_time, user_ids  
+    #given a date range and a user, find all of the notifiable content
+    #key by thread id, and return notification messages for each user
+
+    #first, find the subscriptions for the users
+    subscriptions = Subscription.where(:subscriber_id.in => user_ids)
+
+    #get the thhread ids
+    thread_ids = subscriptions.collect{|t| t.source_id}.uniq
+
+    #find all the comments
+    comments = Comment.by_date_range_and_thread_ids start_date_time, end_date_time, thread_ids
+
+    #and get the threads too, b/c we'll need them for the title
+    thread_map = Hash[CommentThread.where(:_id.in => thread_ids).all.map { |t| [t.id, t] }]
+
+    #now build a thread to users subscription map
+    subscriptions_map = {}
+    subscriptions.each do |s|
+      if not subscriptions_map.keys.include? s.source_id.to_s
+        subscriptions_map[s.source_id.to_s] = []
+      end
+      subscriptions_map[s.source_id] << s.subscriber_id
+    end
+
+    #notification map will be user => course => thread => [comment bodies]
+
+    notification_map = {}
+
+    comments.each do |c|
+      current_thread = thread_map[c.comment_thread_id]
+
+      #do not include threads or comments who have current or historical abuse flags
+      if current_thread.abuse_flaggers.to_a.empty? and 
+        current_thread.historical_abuse_flaggers.to_a.empty? and 
+        c.abuse_flaggers.to_a.empty? and 
+        c.historical_abuse_flaggers.to_a.empty?
+        
+          user_ids = subscriptions_map[c.comment_thread_id.to_s]
+          user_ids.each do |u|
+            if not notification_map.keys.include? u
+              notification_map[u] = {}
+            end
+
+            if not notification_map[u].keys.include? c.course_id
+              notification_map[u][c.course_id] = {}
+            end
+
+            if not notification_map[u][c.course_id].include? c.comment_thread_id.to_s
+              t = notification_map[u][c.course_id][c.comment_thread_id.to_s] = {}
+              t["content"] = []
+              t["title"] = current_thread.title
+              t["commentable_id"] = current_thread.commentable_id
+            else
+              t = notification_map[u][c.course_id][c.comment_thread_id.to_s]
+            end
+
+            content_obj = {}
+            content_obj["username"] = c.author_with_anonymity(:username, "(anonymous)")
+            content_obj["updated_at"] = c.updated_at
+            content_obj["body"] = c.body
+            t["content"] << content_obj
+          end
+      end
+    end
+
+    notification_map.to_json
+
+  end
+
 end

models/comment.rb

@@ -109,6 +109,14 @@ class Comment < Content
       end
     end
   end
+
+  def self.by_date_range_and_thread_ids from_when, to_when, thread_ids
+     #return all content between from_when and to_when
+
+     self.where(:created_at.gte => (from_when)).where(:created_at.lte => (to_when)).
+       where(:comment_thread_id.in => thread_ids)
+  end
+  
 private
 
   def set_thread_last_activity_at

models/comment_thread.rb

@@ -277,8 +277,8 @@ class CommentThread < Content
       #   unread count
       if last_read_time
         unread_count = self.comments.where(
-        :updated_at => {:$gte => last_read_time},
-        :author_id => {:$ne => params[:user_id]},
+            :updated_at => {:$gte => last_read_time},
+            :author_id => {:$ne => params[:user_id]},
         ).count
         read = last_read_time >= self.updated_at
       else
@@ -292,8 +292,8 @@ class CommentThread < Content
     end
 
     doc = doc.merge("unread_comments_count" => unread_count)
-    .merge("read" => read)
-    .merge("comments_count" => comments_count)
+             .merge("read" => read)
+             .merge("comments_count" => comments_count)
 
     doc
 
@@ -303,12 +303,13 @@ class CommentThread < Content
     !!(tag =~ RE_TAG)
   end
 
-  private
   def comment_thread_id
     #so that we can use the comment thread id as a common attribute for flagging
     self.id
   end  
   
+private
+
   RE_HEADCHAR = /[a-z0-9]/
   RE_ENDONLYCHAR = /\+/
   RE_ENDCHAR = /[a-z0-9\#]/

spec/api/notifications_spec.rb

+require 'spec_helper'
+
+describe "app" do
+  describe "notifications" do
+    before(:each) { init_without_subscriptions }
+    describe "POST /api/v1/notifications" do
+      it "returns notifications by class and user" do
+        start_time = Time.now
+        user = User.create(:email => "test@example.com",:external_id => 1,:username => "example")
+        commentable = Commentable.new("question_1")
+        random_string = (0...8).map{ ('a'..'z').to_a[rand(26)] }.join
+        thread = CommentThread.new(title: "Test title", body: "elephant otter", course_id: "1", commentable_id: commentable.id, comments_text_dummy: random_string)
+        thread.author = user
+        thread.save!
+
+        subscription = Subscription.create({:subscriber_id => user._id.to_s, :source_id => thread._id.to_s})
+
+        dummy = random_string = (0..5).map{ ('a'..'z').to_a[rand(26)] }.join
+        comment = Comment.new
+        comment.comment_thread_id = thread.id
+        comment.body = dummy
+        comment.author_id = user.id
+        comment.course_id = 'test course'
+        comment.save!
+
+        sleep 1
+
+        end_time = Time.now
+
+        post "/api/v1/notifications", from: CGI::escape(start_time.to_s), to: CGI::escape(end_time.to_s), user_ids: subscription.subscriber_id
+        
+        last_response.should be_ok
+        last_response.body.to_s.include?(dummy).should == true
+      end
+
+      it "returns only threads subscribed to by user" do
+
+        # first make a dummy thread and comment and a subscription
+        commentable = Commentable.new("question_1")
+        user = User.create(:email => "test@example.com",:external_id => 1,:username => "example")
+        random_string = (0...8).map{ ('a'..'z').to_a[rand(26)] }.join
+
+        thread = CommentThread.new(title: "Test title", body: "elephant otter", course_id: "1", commentable_id: commentable.id, comments_text_dummy: random_string)
+        thread.author = user
+        thread.save!
+
+        subscription = Subscription.create({:subscriber_id => user._id.to_s, :source_id => thread._id.to_s})
+
+        comment = Comment.new(body: random_string, course_id: "1", commentable_id: commentable.id)
+        comment.author = user
+        comment.comment_thread = thread
+        comment.save!
+
+        start_time = Date.today - 100.days
+
+        sleep 1
+
+        end_time = Time.now + 5.seconds
+
+        post "/api/v1/notifications", from: CGI::escape(start_time.to_s), to: CGI::escape(end_time.to_s), user_ids: user.id
+
+        last_response.should be_ok
+        payload = JSON.parse last_response.body
+        courses = payload[user.id.to_s]
+        thread_ids = []
+        courses.each do |k,v|
+            v.each do |kk,vv|
+                thread_ids << kk
+            end
+        end
+        #now make sure the threads are a subset of the user's subscriptions
+        subscriptions = Subscription.where(:subscriber_id => user.id.to_s)
+        subscribed_thread_ids = subscriptions.collect{|s| s.source_id}
+
+        (subscribed_thread_ids.to_set.superset? thread_ids.to_set).should == true
+
+      end
+
+        it "returns only unflagged threads" do
+        start_time = Date.today - 100.days
+       
+
+        user = User.create(:email => "test@example.com",:external_id => 1,:username => "example")
+
+        sleep 1
+
+        end_time = Time.now + 5.seconds
+
+        post "/api/v1/notifications", from: CGI::escape(start_time.to_s), to: CGI::escape(end_time.to_s), user_ids: user.id
+
+        last_response.should be_ok
+        payload = JSON.parse last_response.body
+        courses = payload[user.id.to_s]
+        thread_ids = []
+        courses.each do |k,v|
+            v.each do |kk,vv|
+                thread_ids << kk
+            end
+        end
+        #now flag the first thread
+        thread = CommentThread.find thread_ids.first
+        thread.historical_abuse_flaggers << ["1"]
+
+        sleep 1
+        
+        end_time = Time.now + 5.seconds
+        
+        post "/api/v1/notifications", from: CGI::escape(start_time.to_s), to: CGI::escape(end_time.to_s), user_ids: user.id
+        last_response.should be_ok
+        payload = JSON.parse last_response.body
+        courses = payload[user.id.to_s]
+        new_thread_ids = []
+        courses.each do |k,v|
+            v.each do |kk,vv|
+                new_thread_ids << kk
+            end
+        end
+
+        (new_thread_ids.include? thread.id).should == false
+
+      end
+
+    end
+  end
+end