Merge pull request #52 from edx/feature/jsa/postfilter

add simple filtering on inbound content based on a collection of forbidden content bodies

api/comment_threads.rb

@@ -25,7 +25,7 @@ end
 
 put "#{APIPREFIX}/threads/:thread_id" do |thread_id|
   thread.update_attributes(params.slice(*%w[title body closed commentable_id group_id]))
-
+  filter_blocked_content thread
   if params["tags"]
     thread.tags = params["tags"]
     thread.save
@@ -44,6 +44,7 @@ post "#{APIPREFIX}/threads/:thread_id/comments" do |thread_id|
   comment.anonymous_to_peers = bool_anonymous_to_peers || false
   comment.author = user
   comment.comment_thread = thread
+  filter_blocked_content comment
   comment.save
   if comment.errors.any?
     error 400, comment.errors.full_messages.to_json

api/commentables.rb

@@ -26,6 +26,7 @@ post "#{APIPREFIX}/:commentable_id/threads" do |commentable_id|
   end
   
   thread.author = user
+  filter_blocked_content thread
   thread.save
   if thread.errors.any?
     error 400, thread.errors.full_messages.to_json

api/comments.rb

@@ -4,6 +4,7 @@ end
 
 put "#{APIPREFIX}/comments/:comment_id" do |comment_id|
   comment.update_attributes(params.slice(*%w[body endorsed]))
+  filter_blocked_content comment
   if comment.errors.any?
     error 400, comment.errors.full_messages.to_json
   else
@@ -17,6 +18,7 @@ post "#{APIPREFIX}/comments/:comment_id" do |comment_id|
   sub_comment.anonymous_to_peers = bool_anonymous_to_peers || false
   sub_comment.author = user
   sub_comment.comment_thread = comment.comment_thread
+  filter_blocked_content sub_comment
   sub_comment.save
   if sub_comment.errors.any?
     error 400, sub_comment.errors.full_messages.to_json

app.rb

@@ -13,7 +13,10 @@ environment = env_arg || ENV["SINATRA_ENV"] || "development"
 
 RACK_ENV = environment
 module CommentService
-  class << self; attr_accessor :config; end
+  class << self
+    attr_accessor :config
+    attr_accessor :blocked_hashes
+  end
   API_VERSION = 'v1'
   API_PREFIX = "/api/#{API_VERSION}"
 end
@@ -92,3 +95,6 @@ end
 error ArgumentError do
   error 400, [env['sinatra.error'].message].to_json
 end
+
+CommentService.blocked_hashes = Content.mongo_session[:blocked_hash].find.select(hash: 1).each.map {|d| d["hash"]}
+

lib/helpers.rb

@@ -273,4 +273,18 @@ helpers do
 
   end
 
+  def filter_blocked_content c
+    begin
+      normalized_body = c.body.strip.downcase.gsub(/[^a-z ]/, '').gsub(/\s+/, ' ')
+      hash = Digest::MD5.hexdigest(normalized_body)
+    rescue
+      # body was nil, or the hash function failed somehow - never mind
+      return
+    end  
+    if CommentService.blocked_hashes.include? hash then
+      logger.warn "blocked content with body hash [#{hash}]"
+      error 503
+    end
+  end
+
 end

spec/api/comment_spec.rb

@@ -46,6 +46,11 @@ describe "app" do
         put "/api/v1/comments/does_not_exist", body: "new body", endorsed: true
         last_response.status.should == 400
       end
+      it "returns 503 when the post hash is blocked" do
+        comment = Comment.first
+        put "/api/v1/comments/#{comment.id}", body: "BLOCKED POST", endorsed: true
+        last_response.status.should == 503
+      end
     end
     describe "POST /api/v1/comments/:comment_id" do
       it "create a sub comment to the comment" do
@@ -63,6 +68,12 @@ describe "app" do
         post "/api/v1/comments/does_not_exist", body: "new comment", course_id: "1", user_id: User.first.id
         last_response.status.should == 400
       end
+      it "returns 503 when the post hash is blocked" do
+        comment = Comment.first.to_hash(recursive: true)
+        user = User.first
+        post "/api/v1/comments/#{comment["id"]}", body: "BLOCKED POST", course_id: "1", user_id: User.first.id
+        last_response.status.should == 503
+      end
     end
     describe "DELETE /api/v1/comments/:comment_id" do
       it "delete the comment and its sub comments" do

spec/api/comment_thread_spec.rb

@@ -91,6 +91,13 @@ describe "app" do
         put "/api/v1/threads/does_not_exist", body: "new body", title: "new title"
         last_response.status.should == 400
       end
+      it "returns 503 if the post body has been blocked" do
+        thread = CommentThread.first
+        put "/api/v1/threads/#{thread.id}", body: "BLOCKED POST", title: "new title", commentable_id: "new_commentable_id"
+        last_response.status.should == 503
+        put "/api/v1/threads/#{thread.id}", body: "blocked,   post...", title: "new title", commentable_id: "new_commentable_id"
+        last_response.status.should == 503
+      end
       it "updates tag of comment thread" do
         thread = CommentThread.first
         put "/api/v1/threads/#{thread.id}", tags: "haha, hoho, huhu"
@@ -145,6 +152,10 @@ describe "app" do
         post "/api/v1/threads/#{CommentThread.first.id}/comments", default_params.merge(body: "    \n      \n  ")
         last_response.status.should == 400
       end
+      it "returns 503 when the post body has been blocked" do
+        post "/api/v1/threads/#{CommentThread.first.id}/comments", default_params.merge(body: "BLOCKED POST")
+        last_response.status.should == 503
+      end
     end
     describe "DELETE /api/v1/threads/:thread_id" do
       it "delete the comment thread and its comments" do

spec/api/commentable_spec.rb

@@ -94,6 +94,10 @@ describe "app" do
         post '/api/v1/question_1/threads', default_params.merge(body: "     \n    \n")
         last_response.status.should == 400
       end
+      it "returns 503 when the post content is blocked" do
+        post '/api/v1/question_1/threads', default_params.merge(body: "BLOCKED POST")
+        last_response.status.should == 503
+      end
       it "create a new comment thread with tag" do
         post '/api/v1/question_1/threads', default_params.merge(tags: "a, b, c")
         last_response.should be_ok

spec/spec_helper.rb

@@ -41,6 +41,7 @@ end
 def init_without_subscriptions
 
   [Comment, CommentThread, User, Notification, Subscription, Activity, Delayed::Backend::Mongoid::Job].each(&:delete_all).each(&:remove_indexes).each(&:create_indexes)
+  Content.mongo_session[:blocked_hash].drop
   Tire.index 'comment_threads' do delete end
   CommentThread.create_elasticsearch_index
   
@@ -105,6 +106,10 @@ def init_without_subscriptions
     users.each {|user| user.vote(c, [:up, :down].sample)}
   end
 
+  Content.mongo_session[:blocked_hash].insert(hash: Digest::MD5.hexdigest("blocked post"))
+  # reload the global holding the blocked hashes
+  CommentService.blocked_hashes = Content.mongo_session[:blocked_hash].find.select(hash: 1).each.map {|d| d["hash"]}
+
 end
 
 def init_with_subscriptions