Add tests for non-ASCII characters in comments

Due to a bug[1] in ActiveRecord's JSON serialization, tests for characters
outside the BMP are marked as pending.

[1]: https://github.com/rails/rails/issues/3727

spec/api/comment_spec.rb

 require 'spec_helper'
+require 'unicode_shared_examples'
 
 describe "app" do
   describe "comments" do
@@ -39,6 +40,16 @@ describe "app" do
         last_response.status.should == 400
         parse(last_response.body).first.should == I18n.t(:requested_object_not_found)
       end
+
+      def test_unicode_data(text)
+        comment = make_comment(User.first, CommentThread.first, text)
+        get "/api/v1/comments/#{comment.id}"
+        last_response.should be_ok
+        retrieved = parse last_response.body
+        retrieved["body"].should == text
+      end
+
+      include_examples "unicode data"
     end
     describe "PUT /api/v1/comments/:comment_id" do
       it "update information of the comment" do
@@ -60,6 +71,15 @@ describe "app" do
         last_response.status.should == 503
         parse(last_response.body).first.should == I18n.t(:blocked_content_with_body_hash, :hash => Digest::MD5.hexdigest("blocked post"))
       end
+
+      def test_unicode_data(text)
+        comment = Comment.first
+        put "/api/v1/comments/#{comment.id}", body: text
+        last_response.should be_ok
+        comment.body.should == text
+      end
+
+      include_examples "unicode data"
     end
     describe "POST /api/v1/comments/:comment_id" do
       it "create a sub comment to the comment" do
@@ -85,6 +105,15 @@ describe "app" do
         last_response.status.should == 503
         parse(last_response.body).first.should == I18n.t(:blocked_content_with_body_hash, :hash => Digest::MD5.hexdigest("blocked post"))
       end
+
+      def test_unicode_data(text)
+        parent = Comment.first
+        post "/api/v1/comments/#{parent.id}", body: text, course_id: parent.course_id, user_id: User.first.id
+        last_response.should be_ok
+        parent.children.where(body: text).should_not be_empty
+      end
+
+      include_examples "unicode data"
     end
     describe "DELETE /api/v1/comments/:comment_id" do
       it "delete the comment and its sub comments" do

spec/api/comment_thread_spec.rb

 require 'spec_helper'
+require 'unicode_shared_examples'
 
 describe "app" do
   describe "comment threads" do
@@ -302,6 +303,15 @@ describe "app" do
         
       end
 
+      def test_unicode_data(text)
+        course_id = "unicode_course"
+        thread = make_thread(User.first, text, course_id, "unicode_commentable")
+        make_comment(User.first, thread, text)
+        result = thread_result(course_id: course_id, recursive: true).first
+        check_thread_result(nil, thread, result, true)
+      end
+
+      include_examples "unicode data"
     end
 
     describe "GET /api/v1/threads/:thread_id" do
@@ -372,6 +382,16 @@ describe "app" do
         parse(last_response.body).first.should == I18n.t(:requested_object_not_found)
       end
 
+      def test_unicode_data(text)
+        thread = make_thread(User.first, text, "unicode_course", "unicode_commentable")
+        make_comment(User.first, thread, text)
+        get "/api/v1/threads/#{thread.id}", recursive: true
+        last_response.should be_ok
+        result = parse last_response.body
+        check_thread_result(nil, thread, result, true)
+      end
+
+      include_examples "unicode data"
     end
     describe "PUT /api/v1/threads/:thread_id" do
 
@@ -399,6 +419,16 @@ describe "app" do
         put "/api/v1/threads/#{thread.id}", body: "blocked,   post...", title: "new title", commentable_id: "new_commentable_id"
         last_response.status.should == 503
       end
+
+      def test_unicode_data(text)
+        thread = CommentThread.first
+        put "/api/v1/threads/#{thread.id}", body: text, title: text
+        last_response.should be_ok
+        thread.body.should == text
+        thread.title.should == text
+      end
+
+      include_examples "unicode data"
     end
     describe "POST /api/v1/threads/:thread_id/comments" do
 
@@ -448,6 +478,15 @@ describe "app" do
         post "/api/v1/threads/#{CommentThread.first.id}/comments", default_params.merge(body: "BLOCKED POST")
         last_response.status.should == 503
       end
+
+      def test_unicode_data(text)
+        thread = CommentThread.first
+        post "/api/v1/threads/#{thread.id}/comments", default_params.merge(body: text)
+        last_response.should be_ok
+        thread.comments.where(body: text).should_not be_empty
+      end
+
+      include_examples "unicode data"
     end
     describe "DELETE /api/v1/threads/:thread_id" do
       before(:each) { init_without_subscriptions }

spec/api/commentable_spec.rb

 require 'spec_helper'
+require 'unicode_shared_examples'
 
 describe "app" do
   describe "commentables" do
@@ -51,9 +52,24 @@ describe "app" do
         threads = response['collection']
         threads.length.should == 0
       end
+
+      def test_unicode_data(text)
+        commentable_id = "unicode_commentable"
+        thread = make_thread(User.first, text, "unicode_course", commentable_id)
+        make_comment(User.first, thread, text)
+        get "/api/v1/#{commentable_id}/threads", recursive: true
+        last_response.should be_ok
+        result = parse(last_response.body)["collection"]
+        result.should_not be_empty
+        check_thread_result(nil, thread, result.first, true)
+      end
+
+      include_examples "unicode data"
     end
     describe "POST /api/v1/:commentable_id/threads" do
-      default_params = {title: "Interesting question", body: "cool", course_id: "1", user_id: "1"}
+      let(:default_params) do
+        {title: "Interesting question", body: "cool", course_id: "1", user_id: "1"}
+      end
       it "create a new comment thread for the commentable object" do
         old_count = CommentThread.count
         post '/api/v1/question_1/threads', default_params
@@ -101,6 +117,15 @@ describe "app" do
         last_response.status.should == 503
         parse(last_response.body).first.should == I18n.t(:blocked_content_with_body_hash, :hash => Digest::MD5.hexdigest("blocked post"))
       end
+
+      def test_unicode_data(text)
+        commentable_id = "unicode_commentable"
+        post "/api/v1/#{commentable_id}/threads", default_params.merge(body: text, title: text)
+        last_response.should be_ok
+        CommentThread.where(commentable_id: commentable_id, body: text, title: text).should_not be_empty
+      end
+
+      include_examples "unicode data"
     end
   end
 end

spec/api/search_spec.rb

+require 'spec_helper'
+require 'unicode_shared_examples'
+
+describe "app" do
+  describe "search" do
+    let(:author) { create_test_user(42) }
+
+    describe "GET /api/v1/search/threads" do
+      def test_unicode_data(text)
+        # Elasticsearch may not be able to handle searching for non-ASCII text,
+        # so prepend the text with an ASCII term we can search for.
+        search_term = "artichoke"
+        course_id = "unicode_course"
+        thread = make_thread(author, "#{search_term} #{text}", course_id, "unicode_commentable")
+        make_comment(author, thread, text)
+        # Elasticsearch does not necessarily make newly indexed content
+        # available immediately, so we must explicitly refresh the index
+        CommentThread.tire.index.refresh
+        get "/api/v1/search/threads", course_id: course_id, text: search_term, recursive: true
+        last_response.should be_ok
+        result = parse(last_response.body)["collection"]
+        result.length.should == 1
+        check_thread_result(nil, thread, result.first, true, true)
+      end
+
+      include_examples "unicode data"
+    end
+  end
+end

spec/api/user_spec.rb

 require 'spec_helper'
+require 'unicode_shared_examples'
 
 describe "app" do
   describe "users" do
@@ -194,6 +195,18 @@ describe "app" do
           actual_order.should == expected_order
         end
       end
+
+      def test_unicode_data(text)
+        user = User.first
+        course_id = "unicode_course"
+        thread = make_thread(user, text, course_id, "unicode_commentable")
+        make_comment(user, thread, text)
+        result = thread_result(user.id, course_id: course_id)
+        result.length.should == 1
+        check_thread_result(nil, thread, result.first)
+      end
+
+      include_examples "unicode data"
     end
   end
 end

spec/models/comment_spec.rb

+require 'spec_helper'
+require 'unicode_shared_examples'
+
+describe Comment do
+  let(:author) do
+    create_test_user(42)
+  end
+
+  let(:thread) do
+    make_thread(author, "Test thread", "test_course", "test_commentable")
+  end
+
+  def test_unicode_data(text)
+    comment = make_comment(author, thread, text)
+    retrieved = Comment.find(comment._id)
+    retrieved.body.should == text
+  end
+
+  include_examples "unicode data"
+end

spec/models/comment_thread_spec.rb

 require 'spec_helper'
+require 'unicode_shared_examples'
 
 describe CommentThread do
+  let(:author) do
+    create_test_user(42)
+  end
 
   context "endorsed?" do
 
@@ -77,5 +81,13 @@ describe CommentThread do
     end
   end
 
+  def test_unicode_data(text)
+    thread = make_thread(author, text, "unicode_course", commentable_id: "unicode_commentable")
+    retrieved = CommentThread.find(thread._id)
+    retrieved.title.should == text
+    retrieved.body.should == text
+  end
+
+  include_examples "unicode data"
 end
 

spec/unicode_shared_examples.rb

+# encoding: utf-8
+
+shared_examples "unicode data" do
+  it "can handle ASCII data" do
+    test_unicode_data("This post contains ASCII.")
+  end
+
+  it "can handle Latin-1 data" do
+    test_unicode_data("Thís pøst çòñtáins Lätin-1 tæxt")
+  end
+
+  it "can handle CJK data" do
+    test_unicode_data("ｲんﾉ丂 ｱo丂ｲ co刀ｲﾑﾉ刀丂 cﾌズ")
+  end
+
+  it "can handle non-BMP data" do
+    pending("circumventing a bug in ActiveSupport's JSON encoding of non-BMP characters")
+    test_unicode_data("𝕋𝕙𝕚𝕤 𝕡𝕠𝕤𝕥 𝕔𝕠𝕟𝕥𝕒𝕚𝕟𝕤 𝕔𝕙𝕒𝕣𝕒𝕔𝕥𝕖𝕣𝕤 𝕠𝕦𝕥𝕤𝕚𝕕𝕖 𝕥𝕙𝕖 𝔹𝕄ℙ")
+  end
+
+  it "can handle special chars" do
+    test_unicode_data(
+      "\" This , post > contains < delimiter ] and [ other } " +
+      "special { characters ; that & may ' break things"
+    )
+  end
+
+  it "can handle string interpolation syntax" do
+    test_unicode_data("This string contains %s string interpolation #\{syntax}")
+  end
+end