Merge pull request #77 from edx/gprice/unicode-test-data

Add tests for non-ASCII characters in comments

Gemfile

@@ -48,6 +48,7 @@ group :test do
   gem 'guard'
   gem 'guard-unicorn'
   gem 'simplecov', :require => false
+  gem 'database_cleaner'
 end
 
 gem 'newrelic_rpm'

Gemfile.lock

@@ -63,6 +63,7 @@ GEM
     builder (3.0.4)
     coderay (1.0.7)
     dalli (2.1.0)
+    database_cleaner (1.2.0)
     delayed_job (3.0.3)
       activesupport (~> 3.0)
     diff-lcs (1.1.3)
@@ -166,6 +167,7 @@ DEPENDENCIES
   bson_ext
   bundler
   dalli
+  database_cleaner
   delayed_job
   delayed_job_mongoid!
   faker

spec/api/comment_spec.rb

 require 'spec_helper'
+require 'unicode_shared_examples'
 
 describe "app" do
   describe "comments" do
@@ -39,6 +40,16 @@ describe "app" do
         last_response.status.should == 400
         parse(last_response.body).first.should == I18n.t(:requested_object_not_found)
       end
+
+      def test_unicode_data(text)
+        comment = make_comment(User.first, CommentThread.first, text)
+        get "/api/v1/comments/#{comment.id}"
+        last_response.should be_ok
+        retrieved = parse last_response.body
+        retrieved["body"].should == text
+      end
+
+      include_examples "unicode data"
     end
     describe "PUT /api/v1/comments/:comment_id" do
       it "update information of the comment" do
@@ -60,6 +71,15 @@ describe "app" do
         last_response.status.should == 503
         parse(last_response.body).first.should == I18n.t(:blocked_content_with_body_hash, :hash => Digest::MD5.hexdigest("blocked post"))
       end
+
+      def test_unicode_data(text)
+        comment = Comment.first
+        put "/api/v1/comments/#{comment.id}", body: text
+        last_response.should be_ok
+        comment.body.should == text
+      end
+
+      include_examples "unicode data"
     end
     describe "POST /api/v1/comments/:comment_id" do
       it "create a sub comment to the comment" do
@@ -85,6 +105,15 @@ describe "app" do
         last_response.status.should == 503
         parse(last_response.body).first.should == I18n.t(:blocked_content_with_body_hash, :hash => Digest::MD5.hexdigest("blocked post"))
       end
+
+      def test_unicode_data(text)
+        parent = Comment.first
+        post "/api/v1/comments/#{parent.id}", body: text, course_id: parent.course_id, user_id: User.first.id
+        last_response.should be_ok
+        parent.children.where(body: text).should_not be_empty
+      end
+
+      include_examples "unicode data"
     end
     describe "DELETE /api/v1/comments/:comment_id" do
       it "delete the comment and its sub comments" do

spec/api/comment_thread_spec.rb

 require 'spec_helper'
+require 'unicode_shared_examples'
 
 describe "app" do
   describe "comment threads" do
@@ -302,6 +303,15 @@ describe "app" do
         
       end
 
+      def test_unicode_data(text)
+        course_id = "unicode_course"
+        thread = make_thread(User.first, text, course_id, "unicode_commentable")
+        make_comment(User.first, thread, text)
+        result = thread_result(course_id: course_id, recursive: true).first
+        check_thread_result(nil, thread, result, true)
+      end
+
+      include_examples "unicode data"
     end
 
     describe "GET /api/v1/threads/:thread_id" do
@@ -372,6 +382,16 @@ describe "app" do
         parse(last_response.body).first.should == I18n.t(:requested_object_not_found)
       end
 
+      def test_unicode_data(text)
+        thread = make_thread(User.first, text, "unicode_course", "unicode_commentable")
+        make_comment(User.first, thread, text)
+        get "/api/v1/threads/#{thread.id}", recursive: true
+        last_response.should be_ok
+        result = parse last_response.body
+        check_thread_result(nil, thread, result, true)
+      end
+
+      include_examples "unicode data"
     end
     describe "PUT /api/v1/threads/:thread_id" do
 
@@ -399,6 +419,16 @@ describe "app" do
         put "/api/v1/threads/#{thread.id}", body: "blocked,   post...", title: "new title", commentable_id: "new_commentable_id"
         last_response.status.should == 503
       end
+
+      def test_unicode_data(text)
+        thread = CommentThread.first
+        put "/api/v1/threads/#{thread.id}", body: text, title: text
+        last_response.should be_ok
+        thread.body.should == text
+        thread.title.should == text
+      end
+
+      include_examples "unicode data"
     end
     describe "POST /api/v1/threads/:thread_id/comments" do
 
@@ -448,8 +478,18 @@ describe "app" do
         post "/api/v1/threads/#{CommentThread.first.id}/comments", default_params.merge(body: "BLOCKED POST")
         last_response.status.should == 503
       end
+
+      def test_unicode_data(text)
+        thread = CommentThread.first
+        post "/api/v1/threads/#{thread.id}/comments", default_params.merge(body: text)
+        last_response.should be_ok
+        thread.comments.where(body: text).should_not be_empty
+      end
+
+      include_examples "unicode data"
     end
     describe "DELETE /api/v1/threads/:thread_id" do
+      before(:each) { init_without_subscriptions }
       it "delete the comment thread and its comments" do
         thread = CommentThread.first.to_hash
         delete "/api/v1/threads/#{thread['id']}"

spec/api/commentable_spec.rb

 require 'spec_helper'
+require 'unicode_shared_examples'
 
 describe "app" do
   describe "commentables" do
@@ -51,9 +52,24 @@ describe "app" do
         threads = response['collection']
         threads.length.should == 0
       end
+
+      def test_unicode_data(text)
+        commentable_id = "unicode_commentable"
+        thread = make_thread(User.first, text, "unicode_course", commentable_id)
+        make_comment(User.first, thread, text)
+        get "/api/v1/#{commentable_id}/threads", recursive: true
+        last_response.should be_ok
+        result = parse(last_response.body)["collection"]
+        result.should_not be_empty
+        check_thread_result(nil, thread, result.first, true)
+      end
+
+      include_examples "unicode data"
     end
     describe "POST /api/v1/:commentable_id/threads" do
-      default_params = {title: "Interesting question", body: "cool", course_id: "1", user_id: "1"}
+      let(:default_params) do
+        {title: "Interesting question", body: "cool", course_id: "1", user_id: "1"}
+      end
       it "create a new comment thread for the commentable object" do
         old_count = CommentThread.count
         post '/api/v1/question_1/threads', default_params
@@ -101,6 +117,15 @@ describe "app" do
         last_response.status.should == 503
         parse(last_response.body).first.should == I18n.t(:blocked_content_with_body_hash, :hash => Digest::MD5.hexdigest("blocked post"))
       end
+
+      def test_unicode_data(text)
+        commentable_id = "unicode_commentable"
+        post "/api/v1/#{commentable_id}/threads", default_params.merge(body: text, title: text)
+        last_response.should be_ok
+        CommentThread.where(commentable_id: commentable_id, body: text, title: text).should_not be_empty
+      end
+
+      include_examples "unicode data"
     end
   end
 end

spec/api/query_spec.rb

 require 'spec_helper'
 
 describe "app" do
+	let(:author) { create_test_user(1) }
 	describe "thread search" do
 		describe "GET /api/v1/search/threads" do
 			it "returns thread with query match" do
-				user = User.find 1
-				if user.nil?
-					user = create_test_user(1)
-				end
-
 				commentable = Commentable.new("question_1")
 
 				random_string = (0...8).map{ ('a'..'z').to_a[rand(26)] }.join
 
 				thread = CommentThread.new(title: "Test title", body: random_string, course_id: "1", commentable_id: commentable.id)
-				thread.author = user
+				thread.author = author
 				thread.save!
 
 				sleep 3
@@ -31,23 +27,18 @@ describe "app" do
 	describe "comment search" do
 		describe "GET /api/v1/search/threads" do
 			it "returns thread with comment query match" do
-				user = User.find 1
-				if user.nil?
-					user = create_test_user(1)
-				end
-				
 				commentable = Commentable.new("question_1")
 
 				random_string = (0...8).map{ ('a'..'z').to_a[rand(26)] }.join
 
 				thread = CommentThread.new(title: "Test title", body: "elephant otter", course_id: "1", commentable_id: commentable.id)
-				thread.author = user
+				thread.author = author
 				thread.save!
 
 				sleep 3
 
 				comment = Comment.new(body: random_string, course_id: "1", commentable_id: commentable.id)
-				comment.author = user
+				comment.author = author
 				comment.comment_thread = thread
 				comment.save!
 

spec/api/search_spec.rb

+require 'spec_helper'
+require 'unicode_shared_examples'
+
+describe "app" do
+  describe "search" do
+    let(:author) { create_test_user(42) }
+
+    describe "GET /api/v1/search/threads" do
+      def test_unicode_data(text)
+        # Elasticsearch may not be able to handle searching for non-ASCII text,
+        # so prepend the text with an ASCII term we can search for.
+        search_term = "artichoke"
+        course_id = "unicode_course"
+        thread = make_thread(author, "#{search_term} #{text}", course_id, "unicode_commentable")
+        make_comment(author, thread, text)
+        # Elasticsearch does not necessarily make newly indexed content
+        # available immediately, so we must explicitly refresh the index
+        CommentThread.tire.index.refresh
+        get "/api/v1/search/threads", course_id: course_id, text: search_term, recursive: true
+        last_response.should be_ok
+        result = parse(last_response.body)["collection"]
+        result.length.should == 1
+        check_thread_result(nil, thread, result.first, true, true)
+      end
+
+      include_examples "unicode data"
+    end
+  end
+end

spec/api/user_spec.rb

 require 'spec_helper'
+require 'unicode_shared_examples'
 
 describe "app" do
   describe "users" do
@@ -194,6 +195,18 @@ describe "app" do
           actual_order.should == expected_order
         end
       end
+
+      def test_unicode_data(text)
+        user = User.first
+        course_id = "unicode_course"
+        thread = make_thread(user, text, course_id, "unicode_commentable")
+        make_comment(user, thread, text)
+        result = thread_result(user.id, course_id: course_id)
+        result.length.should == 1
+        check_thread_result(nil, thread, result.first)
+      end
+
+      include_examples "unicode data"
     end
   end
 end

spec/models/comment_spec.rb

+require 'spec_helper'
+require 'unicode_shared_examples'
+
+describe Comment do
+  let(:author) do
+    create_test_user(42)
+  end
+
+  let(:thread) do
+    make_thread(author, "Test thread", "test_course", "test_commentable")
+  end
+
+  def test_unicode_data(text)
+    comment = make_comment(author, thread, text)
+    retrieved = Comment.find(comment._id)
+    retrieved.body.should == text
+  end
+
+  include_examples "unicode data"
+end

spec/models/comment_thread_spec.rb

 require 'spec_helper'
+require 'unicode_shared_examples'
 
 describe CommentThread do
+  let(:author) do
+    create_test_user(42)
+  end
 
   context "endorsed?" do
 
@@ -77,5 +81,13 @@ describe CommentThread do
     end
   end
 
+  def test_unicode_data(text)
+    thread = make_thread(author, text, "unicode_course", commentable_id: "unicode_commentable")
+    retrieved = CommentThread.find(thread._id)
+    retrieved.title.should == text
+    retrieved.body.should == text
+  end
+
+  include_examples "unicode data"
 end
 

spec/spec_helper.rb

@@ -7,6 +7,7 @@ require File.join(File.dirname(__FILE__), '..', 'app')
 require 'sinatra'
 require 'rack/test'
 require 'yajl'
+require 'database_cleaner'
 
 # setup test environment
 set :environment, :test
@@ -25,7 +26,14 @@ RSpec.configure do |config|
   config.treat_symbols_as_metadata_keys_with_true_values = true
   config.filter_run focus: true
   config.run_all_when_everything_filtered = true
-  config.before(:each) { Mongoid::IdentityMap.clear }
+  config.before(:each) do
+    Mongoid::IdentityMap.clear
+    DatabaseCleaner.clean
+    [CommentThread, Comment].each do |class_|
+      class_.tire.index.delete
+      class_.create_elasticsearch_index
+    end
+  end
 end
 
 Mongoid.configure do |config|

spec/unicode_shared_examples.rb

+# encoding: utf-8
+
+shared_examples "unicode data" do
+  it "can handle ASCII data" do
+    test_unicode_data("This post contains ASCII.")
+  end
+
+  it "can handle Latin-1 data" do
+    test_unicode_data("Thís pøst çòñtáins Lätin-1 tæxt")
+  end
+
+  it "can handle CJK data" do
+    test_unicode_data("ｲんﾉ丂 ｱo丂ｲ co刀ｲﾑﾉ刀丂 cﾌズ")
+  end
+
+  it "can handle non-BMP data" do
+    pending("circumventing a bug in ActiveSupport's JSON encoding of non-BMP characters")
+    test_unicode_data("𝕋𝕙𝕚𝕤 𝕡𝕠𝕤𝕥 𝕔𝕠𝕟𝕥𝕒𝕚𝕟𝕤 𝕔𝕙𝕒𝕣𝕒𝕔𝕥𝕖𝕣𝕤 𝕠𝕦𝕥𝕤𝕚𝕕𝕖 𝕥𝕙𝕖 𝔹𝕄ℙ")
+  end
+
+  it "can handle special chars" do
+    test_unicode_data(
+      "\" This , post > contains < delimiter ] and [ other } " +
+      "special { characters ; that & may ' break things"
+    )
+  end
+
+  it "can handle string interpolation syntax" do
+    test_unicode_data("This string contains %s string interpolation #\{syntax}")
+  end
+end