Merge branch 'master' into bbeggs/revert-context-migration

1f6978cc · Brian Beggs · GitHub · 265ceeab · f471f96d · 1f6978cc
Commit 1f6978cc authored Jul 12, 2016 by Brian Beggs Committed by GitHub Jul 12, 2016
53 changed files
--- a/.gitignore
+++ b/.gitignore
@@ -26,10 +26,8 @@ doc/
 config/benchmark.yml
 benchmark_log

-# bundler binstubs
-bin/
-
 log/
 #redcar
 .redcar/
 /nbproject
+.idea/
--- a/.ruby-gemset
+++ b/.ruby-gemset
+cs_comments_service
--- a/.rvmrc
+++ b/.rvmrc
-rvm 1.9.3@cs_comments_service --create
-
--- a/.travis.yml
+++ b/.travis.yml
+sudo: false
+
 language: ruby
+
 rvm:
  - "1.9.3"
-services:
-  - elasticsearch
+
+cache: bundler
+
 before_install:
-  - wget https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-0.90.11.deb && sudo dpkg --force-confnew -i elasticsearch-0.90.11.deb && sudo service elasticsearch restart
-  # Install mongo 2.6.4 according to http://docs.mongodb.org/manual/tutorial/install-mongodb-on-ubuntu/
-  # TODO: This won't be necessary when travis switches to 2.6 by default - see https://github.com/travis-ci/travis-ci/issues/2246
-  - sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 7F0CEB10
-  - echo 'deb http://downloads-distro.mongodb.org/repo/ubuntu-upstart dist 10gen' | sudo tee /etc/apt/sources.list.d/mongodb.list
-  - sudo apt-get update -q
-  - sudo apt-get install -y mongodb-org=2.6.4 mongodb-org-server=2.6.4 mongodb-org-shell=2.6.4 mongodb-org-mongos=2.6.4 mongodb-org-tools=2.6.4
-  - mongo --version
+  - gem update bundler # Ensure we use the latest version of bundler. Travis' default version of outdated.
+
+  # Run Elasticsearch as a daemon
+  - curl -O https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-0.90.13.zip
+  - unzip elasticsearch-0.90.13.zip
+  - elasticsearch-0.90.13/bin/elasticsearch
+
+  # Run MongoDB as a daemon
+  - curl -O https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-2.6.11.tgz
+  - tar -zxf mongodb-linux-x86_64-2.6.11.tgz
+  - export PATH=mongodb-linux-x86_64-2.6.11/bin:$PATH
+  - mkdir -p ./mongo/db
+  - mkdir -p ./mongo/log
+  - mongod --fork --dbpath ./mongo/db --logpath ./mongo/log/mongodb.log
+
 script: bundle exec rspec
--- a/AUTHORS
+++ b/AUTHORS
@@ -22,3 +22,5 @@ Alan Boudreault <alan@alanb.ca>
 Matjaz Gregoric <mtyaka@gmail.com>
 Ben McMorran <ben.mcmorran@gmail.com>
 Bill DeRusha <bill@edx.org>
+Brian Beggs <macdiesel@gmail.com>
+Clinton Blackburn <cblackburn@edx.org>
--- a/Gemfile
+++ b/Gemfile
@@ -14,27 +14,22 @@ gem 'sinatra'
 gem 'yajl-ruby'


-gem 'ampex'
-
-gem 'mongo'
-gem 'moped', "1.5.1"
-gem 'mongoid', "3.0.15"
+gem 'mongoid', '~> 5.0.0'
+gem 'bson', '~>3.1'
 gem 'bson_ext'
+gem 'protected_attributes'

 gem 'delayed_job'
-gem 'delayed_job_mongoid', :git => 'https://github.com/dementrock/delayed_job_mongoid.git'
-
-gem "enumerize", "~>0.8.0"
-gem 'mongoid-tree', :git => 'https://github.com/dementrock/mongoid-tree.git'
-gem 'voteable_mongo', :git => 'https://github.com/dementrock/voteable_mongo.git'
-gem 'mongoid_magic_counter_cache', :git => 'https://github.com/dementrock/mongoid-magic-counter-cache.git'
+gem 'delayed_job_mongoid'

-gem 'kaminari', :require => 'kaminari/sinatra', :git => 'https://github.com/dementrock/kaminari.git'
+gem "enumerize"
+gem 'mongoid-tree', :git => 'https://github.com/macdiesel/mongoid-tree'
+gem 'rs_voteable_mongo', :git => 'https://github.com/navneet35371/voteable_mongo.git'
+gem 'mongoid_magic_counter_cache'

-gem 'faker'
-gem 'will_paginate_mongoid'
+gem 'will_paginate_mongoid', "~>2.0"
 gem 'rdiscount'
-gem 'nokogiri'
+gem 'nokogiri', "~>1.6.8"

 gem 'tire', "0.6.2"
 gem 'tire-contrib'
@@ -44,17 +39,19 @@ gem 'dalli'
 gem 'rest-client'

 group :test do
-  gem 'rspec'
-  gem 'rack-test', :require => "rack/test"
+  gem 'codecov', :require => false
+  gem 'database_cleaner', '~> 1.5.1'
+  gem 'factory_girl', '~> 4.0'
+  gem 'faker', '~> 1.6'
  gem 'guard'
  gem 'guard-unicorn'
-  gem 'simplecov', :require => false
-  gem 'database_cleaner'
+  gem 'rack-test', :require => 'rack/test'
+  gem 'rspec', '~> 2.11.0'
 end

 gem 'newrelic_rpm'
-gem 'newrelic_moped'
 gem 'unicorn'
-gem "rack-timeout", "0.1.0beta3"
+gem "rack-timeout"
 gem "i18n"
 gem "rack-contrib", :git => 'https://github.com/rack/rack-contrib.git', :ref => '6ff3ca2b2d988911ca52a2712f6a7da5e064aa27'
+
--- a/Gemfile.lock
+++ b/Gemfile.lock
 GIT
-  remote: https://github.com/dementrock/delayed_job_mongoid.git
-  revision: 48b1420d59bc01e0b1aba1c2ad66bda4a5e04b9a
+  remote: https://github.com/macdiesel/mongoid-tree
+  revision: b381dd56f1b3b061df8f4b4181d5440dea1602d1
  specs:
-    delayed_job_mongoid (1.0.8)
-      delayed_job (~> 3.0.0)
-      mongoid (>= 3.0.0.rc)
+    mongoid-tree (2.0.0)
+      mongoid (>= 4.0, <= 5.0)

 GIT
-  remote: https://github.com/dementrock/kaminari.git
-  revision: 82a38e07db1ca1598c8daf073a8f6be22ae714d6
+  remote: https://github.com/navneet35371/voteable_mongo.git
+  revision: 55fcfe76705ab5da1c9e5670594331b33954c545
  specs:
-    kaminari (0.13.0)
-      actionpack (>= 3.0.0)
-      activesupport (>= 3.0.0)
-
-GIT
-  remote: https://github.com/dementrock/mongoid-magic-counter-cache.git
-  revision: 28bc5e617cab19187b323e7d97d49fe73a7de68a
-  specs:
-    mongoid_magic_counter_cache (0.1.1)
-      mongoid (>= 3.0)
-      rake
-
-GIT
-  remote: https://github.com/dementrock/mongoid-tree.git
-  revision: 5aa7a4ee16cd90dbbcac3ad702446d2119e971df
-  specs:
-    mongoid-tree (1.0.0)
-      mongoid (>= 3.0, <= 4.0)
-
-GIT
-  remote: https://github.com/dementrock/voteable_mongo.git
-  revision: 538e86856daa1c180ba80b7c6f2805e531ba420c
-  specs:
-    voteable_mongo (0.9.3)
+    rs_voteable_mongo (1.0.2)
+      mongoid (>= 3.0, <= 5.0)

 GIT
  remote: https://github.com/rack/rack-contrib.git
@@ -46,88 +23,100 @@ GIT
 GEM
  remote: https://rubygems.org/
  specs:
-    actionpack (3.2.8)
-      activemodel (= 3.2.8)
-      activesupport (= 3.2.8)
-      builder (~> 3.0.0)
-      erubis (~> 2.7.0)
-      journey (~> 1.0.4)
-      rack (~> 1.4.0)
-      rack-cache (~> 1.2)
-      rack-test (~> 0.6.1)
-      sprockets (~> 2.1.3)
-    activemodel (3.2.8)
-      activesupport (= 3.2.8)
-      builder (~> 3.0.0)
-    activesupport (3.2.8)
-      i18n (~> 0.6)
-      multi_json (~> 1.0)
-    ampex (2.0.0)
-      blankslate
-    ansi (1.4.3)
-    blankslate (2.1.2.4)
-    bson (1.6.4)
-    bson_ext (1.6.4)
-      bson (~> 1.6.4)
-    builder (3.0.4)
+    activemodel (4.2.4)
+      activesupport (= 4.2.4)
+      builder (~> 3.1)
+    activesupport (4.2.4)
+      i18n (~> 0.7)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.3, >= 0.3.4)
+      tzinfo (~> 1.1)
+    ansi (1.5.0)
+    bson (3.2.4)
+    bson_ext (1.5.1)
+    builder (3.2.2)
+    codecov (0.1.2)
+      json
+      simplecov
+      url
    coderay (1.0.7)
    dalli (2.1.0)
-    database_cleaner (1.2.0)
-    delayed_job (3.0.3)
-      activesupport (~> 3.0)
+    database_cleaner (1.5.1)
+    delayed_job (4.1.1)
+      activesupport (>= 3.0, < 5.0)
+    delayed_job_mongoid (2.2.0)
+      delayed_job (>= 3.0, < 5)
+      mongoid (>= 3.0, < 6)
+      mongoid-compatibility
    diff-lcs (1.1.3)
-    enumerize (0.8.0)
+    docile (1.1.5)
+    domain_name (0.5.24)
+      unf (>= 0.0.5, < 1.0.0)
+    enumerize (0.11.0)
      activesupport (>= 3.2)
-    erubis (2.7.0)
-    faker (1.0.1)
-      i18n (~> 0.4)
+    factory_girl (4.5.0)
+      activesupport (>= 3.0.0)
+    faker (1.6.1)
+      i18n (~> 0.5)
    guard (1.3.2)
      listen (>= 0.4.2)
      thor (>= 0.14.6)
    guard-unicorn (0.0.7)
      guard (>= 1.1)
    hashr (0.0.22)
-    hike (1.2.1)
-    i18n (0.6.9)
-    journey (1.0.4)
-    kgio (2.7.4)
+    http-cookie (1.0.2)
+      domain_name (~> 0.5)
+    i18n (0.7.0)
+    json (1.8.3)
+    kgio (2.10.0)
    listen (0.5.0)
    method_source (0.8)
-    mime-types (2.2)
-    mongo (1.6.4)
-      bson (~> 1.6.4)
-    mongoid (3.0.15)
-      activemodel (~> 3.1)
-      moped (~> 1.1)
-      origin (~> 1.0)
-      tzinfo (~> 0.3.22)
-    moped (1.5.1)
-    multi_json (1.10.0)
-    newrelic_moped (1.0.0)
-      moped
-      newrelic_rpm (>= 3.7)
-    newrelic_rpm (3.11.2.286)
-    nokogiri (1.5.5)
-    origin (1.1.0)
+    mime-types (2.6.1)
+    mini_portile2 (2.1.0)
+    minitest (5.8.1)
+    mongo (2.1.1)
+      bson (~> 3.0)
+    mongoid (5.0.0)
+      activemodel (~> 4.0)
+      mongo (~> 2.1)
+      origin (~> 2.1)
+      tzinfo (>= 0.3.37)
+    mongoid-compatibility (0.3.1)
+      activesupport
+      mongoid (>= 2.0)
+    mongoid_magic_counter_cache (1.1.1)
+      mongoid
+      rake
+    multi_json (1.11.2)
+    netrc (0.10.3)
+    newrelic_rpm (3.15.0.314)
+    nokogiri (1.6.8)
+      mini_portile2 (~> 2.1.0)
+      pkg-config (~> 1.1.7)
+    origin (2.1.1)
+    pkg-config (1.1.7)
+    protected_attributes (1.1.3)
+      activemodel (>= 4.0.1, < 5.0)
    pry (0.9.10)
      coderay (~> 1.0.5)
      method_source (~> 0.8)
      slop (~> 3.3.1)
    pry-nav (0.2.2)
      pry (~> 0.9.10)
-    rack (1.4.1)
-    rack-cache (1.2)
-      rack (>= 0.4)
+    rack (1.6.4)
    rack-protection (1.2.0)
      rack
-    rack-test (0.6.1)
+    rack-test (0.6.3)
      rack (>= 1.0)
-    rack-timeout (0.1.0beta3)
-    raindrops (0.10.0)
-    rake (10.3.1)
+    rack-timeout (0.3.2)
+    raindrops (0.15.0)
+    rake (10.4.2)
    rdiscount (1.6.8)
-    rest-client (1.6.7)
-      mime-types (>= 1.16)
+    rest-client (1.8.0)
+      http-cookie (>= 1.0.2, < 2.0)
+      mime-types (>= 1.16, < 3.0)
+      netrc (~> 0.7)
    rspec (2.11.0)
      rspec-core (~> 2.11.0)
      rspec-expectations (~> 2.11.0)
@@ -136,20 +125,18 @@ GEM
    rspec-expectations (2.11.2)
      diff-lcs (~> 1.1.3)
    rspec-mocks (2.11.2)
-    simplecov (0.7.1)
-      multi_json (~> 1.0)
-      simplecov-html (~> 0.7.1)
-    simplecov-html (0.7.1)
+    simplecov (0.11.1)
+      docile (~> 1.1.0)
+      json (~> 1.8)
+      simplecov-html (~> 0.10.0)
+    simplecov-html (0.10.0)
    sinatra (1.3.3)
      rack (~> 1.3, >= 1.3.6)
      rack-protection (~> 1.2)
      tilt (~> 1.3, >= 1.3.3)
    slop (3.3.2)
-    sprockets (2.1.3)
-      hike (~> 1.2)
-      rack (~> 1.0)
-      tilt (~> 1.1, != 1.3.0)
    thor (0.16.0)
+    thread_safe (0.3.5)
    tilt (1.3.3)
    tire (0.6.2)
      activemodel (>= 3.0)
@@ -161,56 +148,59 @@ GEM
      rest-client (~> 1.6)
    tire-contrib (0.1.1)
      tire
-    tzinfo (0.3.38)
-    unicorn (4.3.1)
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.7.1)
+    unicorn (4.9.0)
      kgio (~> 2.6)
      rack
      raindrops (~> 0.7)
-    will_paginate (3.0.4)
-    will_paginate_mongoid (1.1.0)
-      mongoid (>= 2.4)
+    url (0.3.2)
+    will_paginate (3.0.7)
+    will_paginate_mongoid (2.0.1)
+      mongoid
      will_paginate (~> 3.0)
-    yajl-ruby (1.1.0)
+    yajl-ruby (1.2.1)

 PLATFORMS
  ruby

 DEPENDENCIES
-  ampex
+  bson (~> 3.1)
  bson_ext
  bundler
+  codecov
  dalli
-  database_cleaner
+  database_cleaner (~> 1.5.1)
  delayed_job
-  delayed_job_mongoid!
-  enumerize (~> 0.8.0)
-  faker
+  delayed_job_mongoid
+  enumerize
+  factory_girl (~> 4.0)
+  faker (~> 1.6)
  guard
  guard-unicorn
  i18n
-  kaminari!
-  mongo
-  mongoid (= 3.0.15)
+  mongoid (~> 5.0.0)
  mongoid-tree!
-  mongoid_magic_counter_cache!
-  moped (= 1.5.1)
-  newrelic_moped
+  mongoid_magic_counter_cache
  newrelic_rpm
-  nokogiri
+  nokogiri (~> 1.6.8)
+  protected_attributes
  pry
  pry-nav
  rack-contrib!
  rack-test
-  rack-timeout (= 0.1.0beta3)
+  rack-timeout
  rake
  rdiscount
  rest-client
-  rspec
-  simplecov
+  rs_voteable_mongo!
+  rspec (~> 2.11.0)
  sinatra
  tire (= 0.6.2)
  tire-contrib
  unicorn
-  voteable_mongo!
-  will_paginate_mongoid
+  will_paginate_mongoid (~> 2.0)
  yajl-ruby
--- a/README.rst
+++ b/README.rst
@@ -2,37 +2,46 @@ Part of `edX code`__.

 __ http://code.edx.org/

-comment_as_a_service
-====================
+edX Comments Service/Forums   |Travis|_ |Codecov|_
+==================================================
+.. |Travis| image:: https://travis-ci.org/edx/cs_comments_service.svg?branch=master
+.. _Travis: https://travis-ci.org/edx/cs_comments_service
+
+.. |Codecov| image:: http://codecov.io/github/edx/cs_comments_service/coverage.svg?branch=master
+.. _Codecov: http://codecov.io/github/edx/cs_comments_service?branch=master

 An independent comment system which supports voting and nested comments. It
 also supports features including instructor endorsement for education-aimed
 discussion platforms.

-Running The Server
----
-Elasticsearch and MongoDB servers need to be available, and correctly referenced
-in config/application.yml and config/mongoid.yml, respectively.
-
-Before the server is first run, ensure gems are installed by doing ``bundle install``.
-
-To run the server, do ``ruby app.rb [-p PORT]`` where PORT defaults to 4567.

+Running the Server
+------------------
 If you are running cs_comments_service as part of edx-platform__ development under
 devstack, it is strongly recommended to read `those setup documents`__ first.  Note that
-devstack will take care of just about all of the installation, configuration, and 
-service management on your behalf.
+devstack will take care of just about all of the installation, configuration, and
+service management on your behalf. If running outside of devstack, continue reading below.

 __ https://github.com/edx/edx-platform
 __ https://github.com/edx/configuration/wiki/edX-Developer-Stack

+This service relies on Elasticsearch and MongoDB. By default the service will use the Elasticsearch server available at
+`http://localhost:9200` and the MongoDB server available at `localhost:27017`. This is suitable for local development;
+however, if you wish to change these values, refer to `config/application.yml` and `config/mongoid.yml` for the
+environment variables that can be set to override the defaults.
+
+Before the server is first run, ensure gems are installed by doing ``bundle install``.
+
+To run the server, do ``ruby app.rb [-p PORT]`` where PORT defaults to 4567.
+
+
 Running Tests
----
+-------------
 To run tests, do ``bundle exec rspec``.  Append ``--help`` or see rspec documentation
 for additional options to this command.

-Internationalization and Localization
----
+Internationalization (i18n) and Localization (l10n)
+---------------------------------------------------

 To run the comments service in a language other than English, set the
 ``SERVICE_LANGUAGE`` environment variable to the `language code` for the

--- a/Rakefile
+++ b/Rakefile
--- a/api/comment_threads.rb
+++ b/api/comment_threads.rb
@@ -27,9 +27,12 @@ get "#{APIPREFIX}/threads/:thread_id" do |thread_id|
    error 404, [t(:requested_object_not_found)].to_json
  end

-  if params["user_id"] and bool_mark_as_read
+  # user is required to return user-specific fields, such as "read" (even if bool_mark_as_read is False)
+  if params["user_id"]
    user = User.only([:id, :username, :read_states]).find_by(external_id: params["user_id"])
-    user.mark_as_read(thread) if user
+  end
+  if user and bool_mark_as_read
+    user.mark_as_read(thread)
  end

  presenter = ThreadPresenter.factory(thread, user || nil)
@@ -47,7 +50,7 @@ get "#{APIPREFIX}/threads/:thread_id" do |thread_id|
  else
    resp_limit = nil
  end
-  presenter.to_hash(true, resp_skip, resp_limit).to_json
+  presenter.to_hash(true, resp_skip, resp_limit, bool_recursive).to_json
 end

 put "#{APIPREFIX}/threads/:thread_id" do |thread_id|
@@ -69,6 +72,7 @@ post "#{APIPREFIX}/threads/:thread_id/comments" do |thread_id|
  comment.anonymous_to_peers = bool_anonymous_to_peers || false
  comment.author = user
  comment.comment_thread = thread
+  comment.child_count = 0
  comment.save
  if comment.errors.any?
    error 400, comment.errors.full_messages.to_json

--- a/api/commentables.rb
+++ b/api/commentables.rb
@@ -47,6 +47,8 @@ post "#{APIPREFIX}/:commentable_id/threads" do |commentable_id|
  else
    user.subscribe(thread) if bool_auto_subscribe
    presenter = ThreadPresenter.factory(thread, nil)
-    presenter.to_hash.to_json
+    thread = presenter.to_hash
+    thread["resp_total"] = 0
+    thread.to_json
  end
 end
--- a/api/comments.rb
+++ b/api/comments.rb
 get "#{APIPREFIX}/comments/:comment_id" do |comment_id|
-  comment.to_hash(recursive: bool_recursive).to_json
+  @comment = comment
+  comment_hash = @comment.to_hash(recursive: bool_recursive)
+  verify_or_fix_cached_comment_count(@comment, comment_hash)
+  comment_hash.to_json
 end

 put "#{APIPREFIX}/comments/:comment_id" do |comment_id|
@@ -8,7 +11,11 @@ put "#{APIPREFIX}/comments/:comment_id" do |comment_id|
  if params.has_key?("endorsed")
    new_endorsed_val = Boolean.mongoize(params["endorsed"])
    if new_endorsed_val != comment.endorsed
-      endorsement = {:user_id => params["endorsement_user_id"], :time => DateTime.now}
+      if params["endorsement_user_id"].nil?
+        endorsement = nil
+      else
+        endorsement = {:user_id => params["endorsement_user_id"], :time => DateTime.now}
+      end
      updated_content["endorsement"] = new_endorsed_val ? endorsement : nil
    end
  end
@@ -27,16 +34,31 @@ post "#{APIPREFIX}/comments/:comment_id" do |comment_id|
  sub_comment.anonymous_to_peers = bool_anonymous_to_peers || false
  sub_comment.author = user
  sub_comment.comment_thread = comment.comment_thread
+  sub_comment.child_count = 0
  sub_comment.save
  if sub_comment.errors.any?
    error 400, sub_comment.errors.full_messages.to_json
  else
-    user.subscribe(comment.comment_thread) if bool_auto_subscribe
-    sub_comment.to_hash.to_json
+    comment.update_cached_child_count
+    if comment.errors.any?
+      error 400, comment.errors.full_messages.to_json
+    else
+      user.subscribe(comment.comment_thread) if bool_auto_subscribe
+      sub_comment.to_hash.to_json
+    end
  end
 end

 delete "#{APIPREFIX}/comments/:comment_id" do |comment_id|
+  parent_id = comment.parent_id
  comment.destroy
+  unless parent_id.nil?
+    begin
+      parent_comment = Comment.find(parent_id)
+      parent_comment.update_cached_child_count
+    rescue Mongoid::Errors::DocumentNotFound
+      pass
+    end
+  end
  comment.to_hash.to_json
 end
--- a/api/search.rb
+++ b/api/search.rb
-require 'new_relic/agent/method_tracer'
-
 get "#{APIPREFIX}/search/threads" do
  local_params = params # Necessary for params to be available inside blocks
  group_ids = get_group_ids_from_params(local_params)

--- a/api/users.rb
+++ b/api/users.rb
-require 'new_relic/agent/method_tracer'
-
 post "#{APIPREFIX}/users" do
  user = User.new(external_id: params["id"])
  user.username = params["username"]
@@ -76,3 +74,8 @@ put "#{APIPREFIX}/users/:user_id" do |user_id|
    user.to_hash.to_json
  end
 end
+
+post "#{APIPREFIX}/users/:user_id/read" do |user_id|
+  user.mark_as_read(source)
+  user.reload.to_hash.to_json
+end
--- a/app.rb
+++ b/app.rb
@@ -19,29 +19,6 @@ module CommentService
  API_PREFIX = "/api/#{API_VERSION}"
 end

-if ["staging", "production", "loadtest", "edgestage","edgeprod"].include? environment
-  require 'newrelic_rpm'
-  require 'new_relic/agent/method_tracer'
-  Moped::Session.class_eval do
-    include NewRelic::Agent::MethodTracer
-    add_method_tracer :new
-    add_method_tracer :use
-    add_method_tracer :login
-  end
-  Moped::Cluster.class_eval do
-    include NewRelic::Agent::MethodTracer
-    add_method_tracer :with_primary
-    add_method_tracer :nodes
-  end
-  Moped::Node.class_eval do
-    include NewRelic::Agent::MethodTracer
-    add_method_tracer :command
-    add_method_tracer :connect
-    add_method_tracer :flush
-    add_method_tracer :refresh
-  end
-end
-
 if ENV["ENABLE_GC_PROFILER"]
  GC::Profiler.enable
 end
@@ -56,11 +33,12 @@ end

 Mongoid.load!("config/mongoid.yml", environment)
 Mongoid.logger.level = Logger::INFO
-Moped.logger.level = ENV["ENABLE_MOPED_DEBUGGING"] ? Logger::DEBUG : Logger::INFO
+Mongo::Logger.logger.level = ENV["ENABLE_MONGO_DEBUGGING"] ? Logger::DEBUG : Logger::INFO

 # set up i18n
 I18n.load_path += Dir[File.join(File.dirname(__FILE__), 'locale', '*.yml').to_s]
 I18n.default_locale = CommentService.config[:default_locale]
+I18n.enforce_available_locales = false
 I18n::Backend::Simple.send(:include, I18n::Backend::Fallbacks)
 use Rack::Locale

@@ -97,27 +75,6 @@ before do
  content_type "application/json"
 end

-if ENV["ENABLE_IDMAP_LOGGING"]
-
-  after do
-    idmap = Mongoid::Threaded.identity_map
-    vals = {
-      "pid" => Process.pid,
-      "dyno" => ENV["DYNO"],
-      "request_id" => params[:request_id]
-    }
-    idmap.each {|k, v| vals["idmap_count_#{k.to_s}"] = v.size }
-    logger.info vals.map{|e| e.join("=") }.join(" ")
-  end
-
-end
-
-# Enable the identity map. The middleware ensures that the identity map is
-# cleared for every request.
-Mongoid.identity_map_enabled = true
-use Rack::Mongoid::Middleware::IdentityMap
-
-
 # use yajl implementation for to_json.
 # https://github.com/brianmario/yajl-ruby#json-gem-compatibility-api
 #
@@ -128,16 +85,27 @@ require 'yajl/json_gem'

 # patch json serialization of ObjectIds to work properly with yajl.
 # See https://groups.google.com/forum/#!topic/mongoid/MaXFVw7D_4s
-module Moped
-  module BSON
-    class ObjectId
-      def to_json
-        self.to_s.to_json
-      end
+# Note that BSON was moved from Moped::BSON::ObjectId to BSON::ObjectId
+module BSON
+  class ObjectId
+    def to_json
+      self.to_s.to_json
    end
  end
 end

+# Patch json serialization of Time Objects
+class Time
+  # Returns a hash, that will be turned into a JSON object and represent this
+  # object.
+  # Note that this was done to prevent milliseconds from showing up in the JSON response thus breaking
+  # API compatibility for downstream clients.
+  def to_json(*)
+    '"' + utc().strftime("%Y-%m-%dT%H:%M:%SZ") + '"'
+  end
+end
+
+

 # these files must be required in order
 require './api/search'
@@ -158,7 +126,7 @@ if RACK_ENV.to_s == "development"
  end
 end

-error Moped::Errors::InvalidObjectId do
+error Mongo::Error::InvalidDocument do
  error 400, [t(:requested_object_not_found)].to_json
 end

@@ -170,10 +138,10 @@ error ArgumentError do
  error 400, [env['sinatra.error'].message].to_json
 end

-CommentService.blocked_hashes = Content.mongo_session[:blocked_hash].find.select(hash: 1).each.map {|d| d["hash"]}
+CommentService.blocked_hashes = Content.mongo_client[:blocked_hash].find(nil, projection: {hash: 1}).map {|d| d["hash"]}

 def get_db_is_master
-  Mongoid::Sessions.default.command(isMaster: 1)
+  Mongoid::Clients.default.command(isMaster: 1)
 end

 def get_es_status
@@ -186,7 +154,7 @@ get '/heartbeat' do
  db_ok = false
  begin
    res = get_db_is_master
-    db_ok = ( res["ismaster"] == true and Integer(res["ok"]) == 1 )
+    db_ok = res.ok? && res.documents.first['ismaster'] == true
  rescue
  end
  error 500, JSON.generate({"OK" => false, "check" => "db"}) unless db_ok
@@ -221,4 +189,4 @@ get '/selftest' do
      "#{ex.backtrace.first}: #{ex.message} (#{ex.class})\n\t#{ex.backtrace[1..-1].join("\n\t")}"
    ]
  end
-end
\ No newline at end of file
+end
--- a/bin/rake
+++ b/bin/rake
+#!/usr/bin/env ruby
+#
+# This file was generated by Bundler.
+#
+# The application 'rake' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require "pathname"
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("rake", "rake")
--- a/bin/rspec
+++ b/bin/rspec
+#!/usr/bin/env ruby
+#
+# This file was generated by Bundler.
+#
+# The application 'rspec' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require "pathname"
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("rspec-core", "rspec")
--- a/bin/unicorn
+++ b/bin/unicorn
+#!/usr/bin/env ruby
+#
+# This file was generated by Bundler.
+#
+# The application 'unicorn' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require "pathname"
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("unicorn", "unicorn")
--- a/config.ru
+++ b/config.ru
+# It is possible that the rack time out here is set to a different value than
+# on the edx-platform django_comment_client timeout. An attempt was made to
+# move these two values closer together (5s django_client_comment, 6s 
+# cs_comments_service from 20). This resulted in more reported timeout errors
+# on the cs_comments_service side which better reflected the timeout errors the
+# django_comment_client. On the downside, the shorter timeout lead to less time 
+# for processing longer queries in the background. The timeout has been set back
+# to 20s. Until these slow queries that benefit from being cached in the 
+# background are resolved, reducing the timeout is not suggested. 
+# More conversation at https://github.com/edx/cs_comments_service/pull/146
+# -Nov 18th, 2015
+
 require "rack-timeout"
 use Rack::Timeout           # Call as early as possible so rack-timeout runs before other middleware.
-Rack::Timeout.timeout = 20
+Rack::Timeout.timeout = 20 
+
+require "mongoid"
+use Mongoid::QueryCache::Middleware

 require './app'
 run Sinatra::Application
-
--- a/config/mongoid.yml
+++ b/config/mongoid.yml
+common: &default_client
+  options:
+    write:
+      w: 1
+    read:
+      mode: :primary
+    max_retries: <%= ENV['MONGOID_MAX_RETRIES'] || 1 %>
+    retry_interval: <%= ENV['MONGOID_RETRY_INTERVAL'] || 0 %>
+    timeout: <%= ENV['MONGOID_TIMEOUT'] || 0.5 %>
+    ssl: <%= ENV['MONGOID_USE_SSL'] || false %>
+
+common_uri: &default_uri
+  uri: <%= ENV['MONGOHQ_URL'] %>
+
 development:
-  sessions:
+  clients:
    default:
+      <<: *default_client
      database: cs_comments_service_development
      hosts:
        - localhost:27017
+
 test:
-  sessions:
+  clients:
    default:
+      <<: *default_client
      database: cs_comments_service_test
      hosts:
        - localhost:27017

-common: &default_session
-  uri: <%= ENV['MONGOHQ_URL'] %>
-  options:
-    skip_version_check: true
-    safe: true
-    consistency: strong
-    max_retries: <%= ENV['MONGOID_MAX_RETRIES'] || 1 %>
-    retry_interval: <%= ENV['MONGOID_RETRY_INTERVAL'] || 0 %>
-    timeout: <%= ENV['MONGOID_TIMEOUT'] || 0.5 %>
-    ssl: <%= ENV['MONGOID_USE_SSL'] || false %>
-
 production:
-  sessions:
+  clients:
    default:
-      <<: *default_session
+      <<: *default_uri
+      <<: *default_client

 edgeprod:
-  sessions:
+  clients:
    default:
-      <<: *default_session
+      <<: *default_uri
+      <<: *default_client

 edgestage:
-  sessions:
+  clients:
    default:
-      <<: *default_session
+      <<: *default_uri
+      <<: *default_client

 staging:
-  sessions:
+  clients:
    default:
-      <<: *default_session
+      <<: *default_uri
+      <<: *default_client

 loadtest:
-  sessions:
+  clients:
    default:
-      <<: *default_session
+      <<: *default_uri
+      <<: *default_client

 defaults: &defaults
  use_utc: false

--- a/config/unicorn.heroku.rb
+++ b/config/unicorn.heroku.rb
@@ -13,5 +13,5 @@ after_fork do |server, worker|
  Signal.trap 'TERM' do
    puts 'Unicorn worker intercepting TERM and doing nothing. Waiting for master to send QUIT'
  end
-  ::Mongoid.default_session.disconnect
+  ::Mongoid.default_client.close
 end
--- a/config/unicorn.rb
+++ b/config/unicorn.rb
@@ -7,5 +7,5 @@ listen "unix:#{data_dir}/forum.sock", :backlog => 512
 pid "#{data_dir}/forum_unicorn.pid"

 after_fork do |server, worker|
-  ::Mongoid.default_session.disconnect
+  ::Mongoid.default_client.close
 end
--- a/config/unicorn_tcp.rb
+++ b/config/unicorn_tcp.rb
@@ -11,5 +11,5 @@ data_dir = ENV['DATA_DIR'] || Dir.tmpdir
 pid "#{data_dir}/forum_unicorn.pid"

 after_fork do |server, worker|
-  ::Mongoid.default_session.disconnect
+  ::Mongoid.default_client.close
 end
--- a/lib/helpers.rb
+++ b/lib/helpers.rb
@@ -19,6 +19,16 @@ helpers do
    @comment ||= Comment.find(params[:comment_id])
  end

+  def verify_or_fix_cached_comment_count(comment, comment_hash)
+    # if child count cached value gets stale; re-calculate and update it
+    unless comment_hash["children"].nil?
+      if comment_hash["child_count"] != comment_hash["children"].length
+        comment.update_cached_child_count
+        comment_hash["child_count"] = comment.get_cached_child_count
+      end
+    end
+  end
+
  def source
    @source ||= case params["source_type"]
    when "user"
@@ -190,24 +200,21 @@ helpers do
        to_skip = (page - 1) * per_page
        has_more = false
        # batch_size is used to cap the number of documents we might load into memory at any given time
-        # TODO: starting with Mongoid 3.1, you can just do comment_threads.batch_size(size).each()
-        comment_threads.query.batch_size(CommentService.config["manual_pagination_batch_size"].to_i)
-        Mongoid.unit_of_work(disable: :current) do # this is to prevent Mongoid from memoizing every document we look at
-          comment_threads.each do |thread|
-            thread_key = thread._id.to_s
-            if !read_dates.has_key?(thread_key) || read_dates[thread_key] < thread.last_activity_at
-              if skipped >= to_skip
-                if threads.length == per_page
-                  has_more = true
-                  break
-                end
-                threads << thread
-              else
-                skipped += 1
+        comment_threads.batch_size(CommentService.config["manual_pagination_batch_size"].to_i).each do |thread|
+         thread_key = thread._id.to_s
+          if !read_dates.has_key?(thread_key) || read_dates[thread_key] < thread.last_activity_at
+            if skipped >= to_skip
+              if threads.length == per_page
+                has_more = true
+                break
              end
+              threads << thread
+            else
+              skipped += 1
            end
          end
        end
+
        # The following trick makes frontend pagers work without recalculating
        # the number of all unread threads per user on every request (since the number
        # of threads in a course could be tens or hundreds of thousands).  It has the 
@@ -219,7 +226,7 @@ helpers do
        # let the installed paginator library handle pagination
        num_pages = [1, (comment_threads.count / per_page.to_f).ceil].max
        page = [1, page].max
-        threads = comment_threads.page(page).per(per_page).to_a
+        threads = comment_threads.paginate(:page => page, :per_page => per_page).to_a
      end
      
      if threads.length == 0
@@ -228,7 +235,7 @@ helpers do
        pres_threads = ThreadListPresenter.new(threads, request_user, course_id)
        collection = pres_threads.to_hash
      end
-      {collection: collection, num_pages: num_pages, page: page}
+      {collection: collection, num_pages: num_pages, page: page, thread_count: comment_threads.count}
    end
  end

@@ -368,7 +375,7 @@ helpers do
    rescue
      # body was nil, or the hash function failed somehow - never mind
      return
-    end  
+    end
    if CommentService.blocked_hashes.include? hash then
      msg = t(:blocked_content_with_body_hash, :hash => hash) 
      logger.warn msg

--- a/lib/task_helpers.rb
+++ b/lib/task_helpers.rb
+module TaskHelpers
+  module ElasticsearchHelper
+    def self.create_index(name=nil)
+      name ||= "#{Content::ES_INDEX_NAME}_#{Time.now.strftime('%Y%m%d%H%M%S')}"
+      index = Tire.index(name)
+
+      LOG.info "Creating new index: #{name}..."
+      index.create
+
+      [CommentThread, Comment].each do |model|
+        LOG.info "Applying index mappings for #{model.name}"
+        model.put_search_index_mapping(index)
+      end
+      LOG.info '...done!'
+
+      index
+    end
+
+    def self.delete_index(name)
+      Tire.index(name).delete
+    end
+
+    def self.get_index
+      CommentThread.tire.index
+    end
+
+    def self.get_index_shard_count(name)
+      settings = Tire.index(name)
+      settings['index.number_of_shards']
+    end
+  end
+end
--- a/lib/tasks/db.rake
+++ b/lib/tasks/db.rake
+require 'factory_girl'
+
+namespace :db do
+  FactoryGirl.find_definitions
+
+  def create_test_user(id)
+    User.create!(external_id: id, username: "user#{id}")
+  end
+
+  task :init => :environment do
+    puts 'recreating indexes...'
+    [Comment, CommentThread, User, Notification, Subscription, Activity, Delayed::Backend::Mongoid::Job].each(&:remove_indexes).each(&:create_indexes)
+    puts 'finished'
+  end
+
+  task :clean => :environment do
+    Comment.delete_all
+    CommentThread.delete_all
+    User.delete_all
+    Notification.delete_all
+    Subscription.delete_all
+  end
+
+  THREADS_PER_COMMENTABLE = 20
+  TOP_COMMENTS_PER_THREAD = 3
+  ADDITIONAL_COMMENTS_PER_THREAD = 5
+
+  COURSE_ID = 'MITx/6.002x/2012_Fall'
+
+  def generate_comments_for(commentable_id, num_threads=THREADS_PER_COMMENTABLE, num_top_comments=TOP_COMMENTS_PER_THREAD, num_subcomments=ADDITIONAL_COMMENTS_PER_THREAD)
+    level_limit = CommentService.config['level_limit']
+
+
+    users = User.all.to_a
+
+    puts "Generating threads and comments for #{commentable_id}..."
+
+    threads = []
+    top_comments = []
+    additional_comments = []
+
+    num_threads.times do
+      inner_top_comments = []
+
+      # Create a new thread
+      comment_thread = FactoryGirl::create(:comment_thread, commentable_id: commentable_id, author: users.sample, course_id: COURSE_ID)
+      threads << comment_thread
+
+      # Subscribe a few users to the thread
+      users.sample(3).each { |user| user.subscribe(comment_thread) }
+
+      # Create a few top-level comments for the thread
+      (1 + rand(num_top_comments)).times do
+        endorsed = [true, false].sample
+        comment = FactoryGirl::create(:comment, author: users.sample, comment_thread: comment_thread, endorsed: endorsed, course_id: COURSE_ID)
+        top_comments << comment
+        inner_top_comments << comment
+      end
+
+      # Created additional nested comments
+      parent_comments = inner_top_comments
+      (level_limit-1).times do
+        current_level_comments = []
+        (1 + rand(num_subcomments)).times do
+          parent = parent_comments.sample
+          endorsed = [true, false].sample
+          child = FactoryGirl::create(:comment, author: users.sample, parent: parent, endorsed: endorsed)
+          current_level_comments << child
+        end
+        parent_comments = current_level_comments
+      end
+    end
+
+    puts 'voting'
+
+    (threads + top_comments + additional_comments).each do |c|
+      users.each do |user|
+        user.vote(c, [:up, :down].sample)
+      end
+    end
+    puts 'finished'
+  end
+
+
+  task :generate_comments, [:commentable_id, :num_threads, :num_top_comments, :num_subcomments] => :environment do |t, args|
+    args.with_defaults(num_threads: THREADS_PER_COMMENTABLE,
+                       num_top_comments: TOP_COMMENTS_PER_THREAD,
+                       num_subcomments: ADDITIONAL_COMMENTS_PER_THREAD)
+    generate_comments_for(args[:commentable_id], args[:num_threads], args[:num_top_comments], args[:num_subcomments])
+
+  end
+
+  task :seed => [:environment, :clean] do
+    Tire.index 'comment_threads' do
+      delete
+    end
+    CommentThread.create_elasticsearch_index
+
+    beginning_time = Time.now
+
+    (1..10).map { |id| create_test_user(id) }
+    generate_comments_for('video_1')
+    generate_comments_for('lab_1')
+    generate_comments_for('lab_2')
+
+    end_time = Time.now
+
+    puts "Number of comments generated: #{Comment.count}"
+    puts "Number of comment threads generated: #{CommentThread.count}"
+
+    puts "Time elapsed #{(end_time - beginning_time)*1000} milliseconds"
+
+  end
+
+  task :add_anonymous_to_peers => :environment do
+    Content.collection.find(:anonymous_to_peers => nil).update_all({'$set' => {anonymous_to_peers: false}})
+  end
+
+end
--- a/lib/tasks/i18n.rake
+++ b/lib/tasks/i18n.rake
+namespace :i18n do
+  desc "Push source strings to Transifex for translation"
+  task :push do
+    sh("tx push -s")
+  end
+
+  desc "Pull translated strings from Transifex"
+  task :pull do
+    sh("tx pull --mode=reviewed --all --minimum-perc=1")
+  end
+
+  desc "Clean the locale directory"
+  task :clean do
+    sh("git clean -f locale/")
+  end
+
+  desc "Commit translated strings to the repository"
+  task :commit => ["i18n:clean", "i18n:pull"] do
+    sh("git add locale")
+    sh("git commit -m 'Updated translations (autogenerated message)'")
+  end
+end
--- a/lib/tasks/jobs.rake
+++ b/lib/tasks/jobs.rake
+namespace :jobs do
+  desc "Clear the delayed_job queue."
+  task :clear => :environment do
+    Delayed::Job.delete_all
+  end
+
+  desc "Start a delayed_job worker."
+  task :work => :environment do
+    Delayed::Worker.new(:min_priority => ENV['MIN_PRIORITY'], :max_priority => ENV['MAX_PRIORITY'], :queues => (ENV['QUEUES'] || ENV['QUEUE'] || '').split(','), :quiet => false).start
+  end
+end
--- a/lib/tasks/search.rake
+++ b/lib/tasks/search.rake
+require 'task_helpers'
+
+namespace :search do
+  def import_from_cursor(cursor, index, opts)
+    tot = cursor.count
+    cnt = 0
+    t = Time.now
+    index.import cursor, {:method => :paginate, :per_page => opts[:batch_size]} do |documents|
+      if cnt % opts[:batch_size] == 0 then
+        elapsed_secs = (Time.now - t).round(2)
+        pct_complete = (100 * (cnt/tot.to_f)).round(2)
+        LOG.info "#{index.name}: imported #{cnt} of #{tot} (#{pct_complete}% complete after #{elapsed_secs} seconds)"
+      end
+      cnt += documents.length
+      sleep opts[:sleep_time]
+      documents
+    end
+    LOG.info "#{index.name}: finished importing #{cnt} documents"
+    cnt
+  end
+
+  def move_alias_to(name, index)
+    # if there was a previous index, switch over the alias to point to the new index
+    alias_ = Tire::Alias.find name
+    if alias_
+      # does the alias already point to this index?
+      if alias_.indices.include? index.name
+        return false
+      end
+      # remove the alias from wherever it points to now
+      LOG.info "alias already exists (will move): #{alias_.indices.to_ary.join(',')}"
+      alias_.indices.each do |old_index_name|
+        alias_.indices.delete old_index_name unless old_index_name == name
+      end
+    else
+      # create the alias
+      LOG.info "alias \"#{name}\" does not yet exist - creating."
+      alias_ = Tire::Alias.new :name => name
+    end
+    # point the alias at our new index
+    alias_.indices.add index.name
+    alias_.save
+    LOG.info "alias \"#{name}\" now points to index #{index.name}."
+    true
+  end
+
+  def do_reindex (opts, in_place=false)
+    start_time = Time.now
+
+    # create the new index with a unique name
+    new_index = TaskHelpers::ElasticsearchHelper.create_index
+
+    # unless the user is forcing a rebuild, or the index does not yet exist, we
+    # can do a Tire api reindex which is much faster than reimporting documents
+    # from mongo.
+    #
+    # Checking if the index exists is tricky.  Tire automatically created an index
+    # for the model class when the app loaded if one did not already exist.  However,
+    # it won't create an alias, which is what our app uses.  So if the index exists
+    # but not the alias, we know that it's auto-created.
+    old_index = TaskHelpers::ElasticsearchHelper.get_index
+    alias_name = old_index.name
+    alias_ = Tire::Alias.find alias_name
+    if alias_.nil?
+      # edge case.
+      # the alias doesn't exist, so we know the index was auto-created.
+      # We will delete it and replace it with an alias.
+      raise RuntimeError, 'Cannot reindex in-place, no valid source index' if in_place
+      LOG.warn 'deleting auto-created index to make room for the alias'
+      old_index.delete
+      # NOTE on the small chance that another process re-auto-creates the index
+      # we just deleted before we have a chance to create the alias, this next
+      # call will fail.
+      move_alias_to(Content::ES_INDEX_NAME, new_index_name)
+    end
+
+    op = in_place ? 'reindex' : '(re)build index'
+    LOG.info "preparing to #{op}"
+
+    content_types = %w(Comment CommentThread)
+    if in_place
+      # reindex, moving source documents directly from old index to new
+      LOG.info 'copying documents from original index (this may take a while!)'
+      old_index.reindex new_index.name
+      LOG.info 'done copying!'
+    else
+      # fetch all the documents ever, up til start_time
+      cursor = Content.where(:_type.in => content_types, :updated_at.lte => start_time)
+      # import them to the new index
+      import_from_cursor(cursor, new_index, opts)
+    end
+
+    # move the alias if necessary
+    did_alias_move = move_alias_to(Content::ES_INDEX_NAME, new_index)
+
+    if did_alias_move
+      #  Reimport any source documents that got updated since start_time,
+      #  while the alias still pointed to the old index.
+      #  Elasticsearch understands our document ids, so re-indexing the same
+      #  document won't create duplicates.
+      LOG.info "importing any documents that changed between #{start_time} and now"
+      cursor = Content.where(:_type.in => content_types, :updated_at.gte => start_time)
+      import_from_cursor(cursor, new_index, opts)
+    end
+  end
+
+  desc 'Copies contents of MongoDB into Elasticsearch if updated in the last N minutes.'
+  task :catchup, [:minutes, :batch_size, :sleep_time] => :environment do |t, args|
+    opts = batch_opts args
+    the_index = TaskHelpers::ElasticsearchHelper.get_index
+    alias_ = Tire::Alias.find the_index.name
+    # this check makes sure we are working with the index to which
+    # the desired model's alias presently points.
+    raise RuntimeError, "could not find live index" if alias_.nil?
+    start_time = Time.now - (args[:minutes].to_i * 60)
+    cursor = Content.where(:_type.in => %w(Comment CommentThread), :updated_at.gte => start_time)
+    import_from_cursor(cursor, the_index, opts)
+  end
+
+  def batch_opts(args)
+    args = args.to_hash
+    {:batch_size => args[:batch_size].nil? ? 500 : args[:batch_size].to_i,
+     :sleep_time => args[:sleep_time].nil? ? 0 : args[:sleep_time].to_i}
+  end
+
+  desc 'Removes any data from Elasticsearch that no longer exists in MongoDB.'
+  task :prune, [:batch_size, :sleep_time] => :environment do |t, args|
+    opts = batch_opts args
+    the_index = TaskHelpers::ElasticsearchHelper.get_index
+    puts "pruning #{the_index.name}"
+    alias_ = Tire::Alias.find the_index.name
+    raise RuntimeError, 'could not find live index' if alias_.nil?
+    scan_size = opts[:batch_size] / TaskHelpers::ElasticsearchHelper.get_index_shard_count(the_index.name)
+    cnt = 0
+    [CommentThread, Comment].each do |klass|
+      doc_type = klass.document_type
+      # this check makes sure we are working with the index to which
+      # the desired model's alias presently points.
+      search = Tire::Search::Scan.new the_index.name, {size: scan_size, type: doc_type}
+      search.each do |results|
+        es_ids = results.map(&:id)
+        mongo_ids = klass.where(:id.in => es_ids).map { |d| d.id.to_s }
+        to_delete = es_ids - mongo_ids
+        if to_delete.size > 0
+          cnt += to_delete.size
+          puts "deleting #{to_delete.size} orphaned #{doc_type} documents from elasticsearch"
+          the_index.bulk_delete (to_delete).map { |v| {"type" => doc_type, "id" => v} }
+        end
+        puts "#{the_index.name}/#{doc_type}: processed #{search.seen} of #{search.total}"
+        sleep opts[:sleep_time]
+      end
+    end
+    puts "done pruning #{the_index.name}, deleted a total of #{cnt} orphaned documents"
+  end
+
+  desc 'Rebuild the content index from MongoDB data.'
+  task :rebuild, [:batch_size, :sleep_time] => :environment do |t, args|
+    do_reindex(batch_opts(args))
+  end
+
+  desc 'Rebuild the content index from already-indexed data (in place).'
+  task :reindex, [:batch_size, :sleep_time] => :environment do |t, args|
+    do_reindex(batch_opts(args), true)
+  end
+
+  desc 'Generate a new, empty physical index, without bringing it online.'
+  task :create_index => :environment do
+    TaskHelpers::ElasticsearchHelper.create_index
+  end
+end
--- a/models/comment.rb
+++ b/models/comment.rb
@@ -5,7 +5,10 @@ class Comment < Content
  include Mongoid::Tree
  include Mongoid::Timestamps
  include Mongoid::MagicCounterCache
-  
+  include ActiveModel::MassAssignmentSecurity
+  include Tire::Model::Search
+  include Tire::Model::Callbacks
+
  voteable self, :up => +1, :down => -1

  field :course_id, type: String
@@ -14,22 +17,14 @@ class Comment < Content
  field :endorsement, type: Hash
  field :anonymous, type: Boolean, default: false
  field :anonymous_to_peers, type: Boolean, default: false
+  field :commentable_id, type: String
  field :at_position_list, type: Array, default: []
+  field :sk, type: String, default: nil
+  field :child_count, type: Integer

  index({author_id: 1, course_id: 1})
  index({_type: 1, comment_thread_id: 1, author_id: 1, updated_at: 1})

-  field :sk, type: String, default: nil
-  before_save :set_sk  
-  def set_sk()
-    # this attribute is explicitly write-once
-    if self.sk.nil?
-      self.sk = (self.parent_ids.dup << self.id).join("-") 
-    end
-  end
-
-  include Tire::Model::Search
-  include Tire::Model::Callbacks

  index_name Content::ES_INDEX_NAME

@@ -43,10 +38,10 @@ class Comment < Content
    indexes :created_at, type: :date, included_in_all: false
    indexes :updated_at, type: :date, included_in_all: false
  end
-  
+

  belongs_to :comment_thread, index: true
-  belongs_to :author, class_name: "User", index: true
+  belongs_to :author, class_name: 'User', index: true

  attr_accessible :body, :course_id, :anonymous, :anonymous_to_peers, :endorsed, :endorsement

@@ -57,13 +52,13 @@ class Comment < Content

  counter_cache :comment_thread

-  before_destroy :destroy_children # TODO async
-
+  before_destroy :destroy_children
  before_create :set_thread_last_activity_at
  before_update :set_thread_last_activity_at
+  before_save :set_sk

  def self.hash_tree(nodes)
-    nodes.map{|node, sub_nodes| node.to_hash.merge("children" => hash_tree(sub_nodes).compact)}
+    nodes.map { |node, sub_nodes| node.to_hash.merge('children' => hash_tree(sub_nodes).compact) }
  end

  # This should really go somewhere else, but sticking it here for now. This is
@@ -74,9 +69,9 @@ class Comment < Content
  # actually creates the subtree.
  def self.flatten_subtree(x)
    if x.is_a? Array
-      x.flatten.map{|y| self.flatten_subtree(y)}
+      x.flatten.map { |y| self.flatten_subtree(y) }
    elsif x.is_a? Hash
-      x.to_a.map{|y| self.flatten_subtree(y)}.flatten
+      x.to_a.map { |y| self.flatten_subtree(y) }.flatten
    else
      x
    end
@@ -97,20 +92,31 @@ class Comment < Content
      self.class.hash_tree(subtree_hash).first
    else
      as_document.slice(*%w[body course_id endorsed endorsement anonymous anonymous_to_peers created_at updated_at at_position_list])
-                 .merge("id" => _id)
-                 .merge("user_id" => author_id)
-                 .merge("username" => author_username) 
-                 .merge("depth" => depth)
-                 .merge("closed" => comment_thread.nil? ? false : comment_thread.closed) # ditto
-                 .merge("thread_id" => comment_thread_id)
-                 .merge("parent_id" => parent_ids[-1])
-                 .merge("commentable_id" => comment_thread.nil? ? nil : comment_thread.commentable_id) # ditto
-                 .merge("votes" => votes.slice(*%w[count up_count down_count point]))
-                 .merge("abuse_flaggers" => abuse_flaggers)
-                 .merge("type" => "comment")
+          .merge("id" => _id)
+          .merge("user_id" => author_id)
+          .merge("username" => author_username)
+          .merge("depth" => depth)
+          .merge("closed" => comment_thread.nil? ? false : comment_thread.closed)
+          .merge("thread_id" => comment_thread_id)
+          .merge("parent_id" => parent_ids[-1])
+          .merge("commentable_id" => comment_thread.nil? ? nil : comment_thread.commentable_id)
+          .merge("votes" => votes.slice(*%w[count up_count down_count point]))
+          .merge("abuse_flaggers" => abuse_flaggers)
+          .merge("type" => "comment")
+          .merge("child_count" => get_cached_child_count)
    end
  end
-  
+
+  def get_cached_child_count
+    update_cached_child_count if self.child_count.nil?
+    self.child_count
+  end
+
+  def update_cached_child_count
+    child_comments_count = Comment.where({"parent_id" => self._id}).count()
+    self.set(child_count: child_comments_count)
+  end
+
  def commentable_id
    #we need this to have a universal access point for the flag rake task
    if self.comment_thread_id
@@ -147,16 +153,22 @@ class Comment < Content
  end

  def self.by_date_range_and_thread_ids from_when, to_when, thread_ids
-     #return all content between from_when and to_when
+    #return all content between from_when and to_when

-     self.where(:created_at.gte => (from_when)).where(:created_at.lte => (to_when)).
-       where(:comment_thread_id.in => thread_ids)
+    self.where(:created_at.gte => (from_when)).where(:created_at.lte => (to_when)).
+        where(:comment_thread_id.in => thread_ids)
  end
-  
-private
+
+  private

  def set_thread_last_activity_at
-    self.comment_thread.update_attributes!(last_activity_at: Time.now.utc)
+    self.comment_thread.update_attribute(:last_activity_at, Time.now.utc)
  end

+  def set_sk
+    # this attribute is explicitly write-once
+    if self.sk.nil?
+      self.sk = (self.parent_ids.dup << self.id).join("-")
+    end
+  end
 end
--- a/models/comment_thread.rb
+++ b/models/comment_thread.rb
@@ -5,6 +5,10 @@ require_relative 'content'
 class CommentThread < Content

  include Mongoid::Timestamps
+  include Mongoid::Attributes::Dynamic
+  include ActiveModel::MassAssignmentSecurity
+  include Tire::Model::Search
+  include Tire::Model::Callbacks
  extend Enumerize

  voteable self, :up => +1, :down => -1
@@ -28,8 +32,6 @@ class CommentThread < Content

  index({author_id: 1, course_id: 1})

-  include Tire::Model::Search
-  include Tire::Model::Callbacks

  index_name Content::ES_INDEX_NAME

@@ -48,12 +50,12 @@ class CommentThread < Content
    indexes :commentable_id, type: :string, index: :not_analyzed, included_in_all: false
    indexes :author_id, type: :string, as: 'author_id', index: :not_analyzed, included_in_all: false
    indexes :group_id, type: :integer, as: 'group_id', index: :not_analyzed, included_in_all: false
-    indexes :id,         :index    => :not_analyzed
-    indexes :thread_id, :analyzer => :keyword, :as => "_id"
+    indexes :id, :index => :not_analyzed
+    indexes :thread_id, :analyzer => :keyword, :as => '_id'
  end

-  belongs_to :author, class_name: "User", inverse_of: :comment_threads, index: true#, autosave: true
-  has_many :comments, dependent: :destroy#, autosave: true# Use destroy to envoke callback on the top-level comments TODO async
+  belongs_to :author, class_name: 'User', inverse_of: :comment_threads, index: true
+  has_many :comments, dependent: :destroy # Use destroy to invoke callback on the top-level comments
  has_many :activities, autosave: true

  attr_accessible :title, :body, :course_id, :commentable_id, :anonymous, :anonymous_to_peers, :closed, :thread_type
@@ -69,24 +71,12 @@ class CommentThread < Content
  before_create :set_last_activity_at
  before_update :set_last_activity_at, :unless => lambda { closed_changed? }
  after_update :clear_endorsements
-
  before_destroy :destroy_subscriptions

  scope :active_since, ->(from_time) { where(:last_activity_at => {:$gte => from_time}) }
  scope :standalone_context, ->() { where(:context => :standalone) }
  scope :course_context, ->() { where(:context => :course) }

-  def self.new_dumb_thread(options={})
-    c = self.new
-    c.title = options[:title] || "title"
-    c.body = options[:body] || "body"
-    c.commentable_id = options[:commentable_id] || "commentable_id"
-    c.course_id = options[:course_id] || "course_id"
-    c.author = options[:author] || User.first
-    c.save!
-    c
-  end
-
  def activity_since(from_time=nil)
    if from_time
      activities.where(:created_at => {:$gte => from_time})
@@ -95,13 +85,21 @@ class CommentThread < Content
    end
  end

-  def activity_today; activity_since(Date.today.to_time); end
+  def activity_today
+    activity_since(Date.today.to_time)
+  end

-  def activity_this_week; activity_since(Date.today.to_time - 1.weeks); end
+  def activity_this_week
+    activity_since(Date.today.to_time - 1.weeks)
+  end

-  def activity_this_month; activity_since(Date.today.to_time - 1.months); end
+  def activity_this_month
+    activity_since(Date.today.to_time - 1.months)
+  end

-  def activity_overall; activity_since(nil); end
+  def activity_overall
+    activity_since(nil)
+  end

  def root_comments
    Comment.roots.where(comment_thread_id: self.id)
@@ -124,25 +122,26 @@ class CommentThread < Content
  end

  def to_hash(params={})
-    as_document.slice(*%w[thread_type title body course_id anonymous anonymous_to_peers commentable_id created_at updated_at at_position_list closed context])
-                     .merge("id" => _id, "user_id" => author_id,
-                            "username" => author_username,
-                            "votes" => votes.slice(*%w[count up_count down_count point]),
-                            "abuse_flaggers" => abuse_flaggers,
-                            "tags" => [],
-                            "type" => "thread",
-                            "group_id" => group_id,
-                            "pinned" => pinned?,
-                            "comments_count" => comment_count)
-    
+    as_document.slice(*%w[thread_type title body course_id anonymous anonymous_to_peers commentable_id created_at updated_at at_position_list closed context last_activity_at])
+        .merge('id' => _id,
+               'user_id' => author_id,
+               'username' => author_username,
+               'votes' => votes.slice(*%w[count up_count down_count point]),
+               'abuse_flaggers' => abuse_flaggers,
+               'tags' => [],
+               'type' => 'thread',
+               'group_id' => group_id,
+               'pinned' => pinned?,
+               'comments_count' => comment_count)
+
  end

  def comment_thread_id
    #so that we can use the comment thread id as a common attribute for flagging
    self.id
-  end  
-  
-private
+  end
+
+  private

  def set_last_activity_at
    self.last_activity_at = Time.now.utc unless last_activity_at_changed?
@@ -154,8 +153,8 @@ private
      # the last activity time on the thread. Therefore the callbacks would be mutually recursive and we end up with a
      # 'SystemStackError'. The 'set' method skips callbacks and therefore bypasses this issue.
      self.comments.each do |comment|
-        comment.set :endorsed, false
-        comment.set :endorsement, nil
+        comment.set(endorsed: false)
+        comment.set(endorsement: nil)
      end
    end
  end
@@ -163,5 +162,4 @@ private
  def destroy_subscriptions
    subscriptions.delete_all
  end
-
 end
--- a/models/content.rb
+++ b/models/content.rb
 class Content
-  
+
  include Mongoid::Document
  include Mongo::Voteable
-  
+
  field :visible, type: Boolean, default: true
  field :abuse_flaggers, type: Array, default: []
  field :historical_abuse_flaggers, type: Array, default: [] #preserve abuse flaggers after a moderator unflags
  field :author_username, type: String, default: nil
-  
-  index({_type: 1, course_id: 1, pinned: -1, created_at: -1 }, {background: true} )
-  index({_type: 1, course_id: 1, pinned: -1, comment_count: -1, created_at: -1}, {background: true})
-  index({_type: 1, course_id: 1, pinned: -1, "votes.point" => -1, created_at: -1}, {background: true})

+  index({_type: 1, course_id: 1, pinned: -1, created_at: -1}, {background: true})
+  index({_type: 1, course_id: 1, pinned: -1, comment_count: -1, created_at: -1}, {background: true})
+  index({_type: 1, course_id: 1, pinned: -1, 'votes.point' => -1, created_at: -1}, {background: true})
+  index({_type: 1, course_id: 1, pinned: -1, last_activity_at: -1, created_at: -1}, {background: true})
  index({comment_thread_id: 1, sk: 1}, {sparse: true})
  index({comment_thread_id: 1, endorsed: 1}, {sparse: true})
  index({commentable_id: 1}, {sparse: true, background: true})
@@ -27,10 +27,7 @@ class Content
  end

  before_save :set_username
-  def set_username
-    # avoid having to look this attribute up later, since it does not change
-    self.author_username = author.username
-  end
+

  def author_with_anonymity(attr=nil, attr_when_anonymous=nil)
    if not attr
@@ -43,7 +40,7 @@ class Content
  def self.flagged
    #return an array of flagged content
    holder = []
-    Content.where(:abuse_flaggers.ne => [],:abuse_flaggers.exists => true).each do |c|
+    Content.where(:abuse_flaggers.ne => [], :abuse_flaggers.exists => true).each do |c|
      holder << c
    end
    holder
@@ -53,61 +50,65 @@ class Content
    #take a hash of criteria (what) and return a hash of hashes
    #course => user => count

-    contributors = {}
-
-    map =  "function(){emit(this.author_id,1)}"
-    reduce  =  "function(k, vals) { var sum = 0; for(var i in vals) sum += vals[i]; return sum; }"
+    map = 'function(){emit(this.author_id,1)}'
+    reduce = 'function(k, vals) { var sum = 0; for(var i in vals) sum += vals[i]; return sum; }'

    contributors = []
-    self.where(what).map_reduce(map,reduce).out(replace: "results").each do |d|
+    self.where(what).map_reduce(map, reduce).out(replace: 'results').each do |d|
      contributors << d
    end

    #now sort and limit them

    #first sort destructively
-    contributors.sort! {|a,b| -a["value"] <=> -b["value"]}
+    contributors.sort! { |a, b| -a['value'] <=> -b['value'] }
    #then trim it
    contributors = contributors[0..(count - 1)]

    contributors

  end
-  
+
  def self.summary what
    #take a hash of criteria (what) and return a hash of hashes
    #of total users, votes, comments, endorsements, 
-    
+
    answer = {}
    vote_count = 0
    thread_count = 0
    comment_count = 0
    contributors = []
    content = self.where(what)
-    
+
    content.each do |c|
      contributors << c.author_id
-      contributors << c["votes"]["up"]
-      contributors << c["votes"]["down"]
-      vote_count += c["votes"]["count"]
-      if c._type == "CommentThread"
+      contributors << c['votes']['up']
+      contributors << c['votes']['down']
+      vote_count += c['votes']['count']
+      if c._type == 'CommentThread'
        thread_count += 1
-      elsif c._type == "Comment"
+      elsif c._type == 'Comment'
        comment_count += 1
      end
    end

    #uniquify contributors
    contributors = contributors.uniq
-    
+
    #assemble the answer and ship
-    
-    answer["vote_count"] = vote_count
-    answer["thread_count"] = thread_count
-    answer["comment_count"] = comment_count
-    answer["contributor_count"] = contributors.count
-    
+
+    answer['vote_count'] = vote_count
+    answer['thread_count'] = thread_count
+    answer['comment_count'] = comment_count
+    answer['contributor_count'] = contributors.count
+
    answer
  end

+  private
+
+  def set_username
+    # avoid having to look this attribute up later, since it does not change
+    self.author_username = author.username
+  end
 end
--- a/models/notification.rb
+++ b/models/notification.rb
 class Notification
  include Mongoid::Document
  include Mongoid::Timestamps
+  include ActiveModel::MassAssignmentSecurity

  field :notification_type, type: String
  field :info, type: Hash

--- a/models/subscription.rb
+++ b/models/subscription.rb
@@ -12,7 +12,7 @@ class Subscription
  index({source_id: 1, source_type: 1}, {background: true})

  def to_hash
-    as_document.slice(*%w[subscriber_id source_id source_type])
+    as_document.slice(*%w[subscriber_id source_id source_type]).merge("id" => _id)
  end

  def subscriber

--- a/models/user.rb
+++ b/models/user.rb
@@ -153,8 +153,9 @@ class ReadState
  field :last_read_times, type: Hash, default: {}
  embedded_in :user

-  validates :course_id, uniqueness: true, presence: true
-  
+  validates_presence_of :course_id
+  validates_uniqueness_of :course_id
+
  def to_hash
    to_json
  end

--- a/presenters/thread.rb
+++ b/presenters/thread.rb
@@ -23,7 +23,7 @@ class ThreadPresenter
    @is_endorsed = is_endorsed
  end

-  def to_hash with_responses=false, resp_skip=0, resp_limit=nil
+  def to_hash with_responses=false, resp_skip=0, resp_limit=nil, recursive=true
    raise ArgumentError unless resp_skip >= 0
    raise ArgumentError unless resp_limit.nil? or resp_limit >= 1
    h = @thread.to_hash
@@ -32,7 +32,11 @@ class ThreadPresenter
    h["endorsed"] = @is_endorsed || false
    if with_responses
      if @thread.thread_type.discussion? && resp_skip == 0 && resp_limit.nil?
-        content = Comment.where(comment_thread_id: @thread._id).order_by({"sk" => 1})
+        if recursive
+          content = Comment.where(comment_thread_id: @thread._id).order_by({"sk" => 1})
+        else
+          content = Comment.where(comment_thread_id: @thread._id, "parent_ids" => []).order_by({"sk" => 1})
+        end
        h["children"] = merge_response_content(content)
        h["resp_total"] = content.to_a.select{|d| d.depth == 0 }.length
      else
@@ -41,18 +45,20 @@ class ThreadPresenter
        when "question"
          endorsed_responses = responses.where(endorsed: true)
          non_endorsed_responses = responses.where(endorsed: false)
-          endorsed_response_info = get_paged_merged_responses(@thread._id, endorsed_responses, 0, nil)
+          endorsed_response_info = get_paged_merged_responses(@thread._id, endorsed_responses, 0, nil, recursive)
          non_endorsed_response_info = get_paged_merged_responses(
            @thread._id,
            non_endorsed_responses,
            resp_skip,
-            resp_limit
+            resp_limit,
+            recursive
          )
          h["endorsed_responses"] = endorsed_response_info["responses"]
          h["non_endorsed_responses"] = non_endorsed_response_info["responses"]
          h["non_endorsed_resp_total"] = non_endorsed_response_info["response_count"]
+          h["resp_total"] = non_endorsed_response_info["response_count"] + endorsed_response_info["response_count"]
        when "discussion"
-          response_info = get_paged_merged_responses(@thread._id, responses, resp_skip, resp_limit)
+          response_info = get_paged_merged_responses(@thread._id, responses, resp_skip, resp_limit, recursive)
          h["children"] = response_info["responses"]
          h["resp_total"] = response_info["response_count"]
        end
@@ -67,15 +73,20 @@ class ThreadPresenter
  # a hash containing the following:
  #   responses
  #     An array of hashes representing the page of responses (including
-  #     children)
+  #     children, if recursive is true)
  #   response_count
  #     The total number of responses
-  def get_paged_merged_responses(thread_id, responses, skip, limit)
+  def get_paged_merged_responses(thread_id, responses, skip, limit, recursive=false)
    response_ids = responses.only(:_id).sort({"sk" => 1}).to_a.map{|doc| doc["_id"]}
    paged_response_ids = limit.nil? ? response_ids.drop(skip) : response_ids.drop(skip).take(limit)
-    content = Comment.where(comment_thread_id: thread_id).
-      or({:parent_id => {"$in" => paged_response_ids}}, {:id => {"$in" => paged_response_ids}}).
-      sort({"sk" => 1})
+    if recursive
+      content = Comment.where(comment_thread_id: thread_id).
+        or({:parent_id => {"$in" => paged_response_ids}}, {:id => {"$in" => paged_response_ids}}).
+        sort({"sk" => 1})
+    else
+      content = Comment.where(comment_thread_id: thread_id, "parent_ids" => []).
+        where({:id => {"$in" => paged_response_ids}}).sort({"sk" => 1})
+    end
    {"responses" => merge_response_content(content), "response_count" => response_ids.length}
  end


--- a/presenters/thread_utils.rb
+++ b/presenters/thread_utils.rb
@@ -5,10 +5,10 @@ module ThreadUtils
    # only threads which are endorsed will have entries, value will always be true.
    endorsed_threads = {}
    thread_ids = threads.collect {|t| t._id}
-    Comment.collection.aggregate(
+    Comment.collection.aggregate([
      {"$match" => {"comment_thread_id" => {"$in" => thread_ids}, "endorsed" => true}},
      {"$group" => {"_id" => "$comment_thread_id"}}
-    ).each do |res| 
+    ]).each do |res|
      endorsed_threads[res["_id"].to_s] = true
    end
    endorsed_threads
@@ -26,7 +26,7 @@ module ThreadUtils
          thread_key = t._id.to_s
          if read_dates.has_key? thread_key
            is_read = read_dates[thread_key] >= t.updated_at
-            unread_comment_count = Comment.collection.where(
+            unread_comment_count = Comment.collection.find(
              :comment_thread_id => t._id,
              :author_id => {"$ne" => user.id},
              :updated_at => {"$gte" => read_dates[thread_key]}

--- a/spec/api/abuse_spec.rb
+++ b/spec/api/abuse_spec.rb
 require 'spec_helper'

-def create_comment_flag(comment_id, user_id)
-  create_flag("/api/v1/comments/" + comment_id + "/abuse_flag", user_id)
-end
+describe 'Abuse API' do
+  before(:each) { set_api_key_header }

-def create_thread_flag(thread_id, user_id)
-  create_flag("/api/v1/threads/" + thread_id + "/abuse_flag", user_id)
-end
+  shared_examples 'an abuse endpoint' do
+    let(:affected_entity_id) { affected_entity.id }
+    let(:user_id) { create(:user).id }

-def remove_thread_flag(thread_id, user_id)
-  remove_flag("/api/v1/threads/" + thread_id + "/abuse_unflag", user_id)
-end
+    it { should be_ok }

-def remove_comment_flag(comment_id, user_id)
-  remove_flag("/api/v1/comments/" + comment_id + "/abuse_unflag", user_id)
-end
+    it 'updates the abuse flaggers' do
+      subject

-def create_flag(put_command, user_id)
-  if user_id.nil?
-    put put_command
-  else
-    put put_command, user_id: user_id
-  end
-end
+      affected_entity.reload
+      expect(affected_entity.abuse_flaggers).to eq expected_abuse_flaggers
+      expect(non_affected_entity.abuse_flaggers).to have(0).items
+    end

-def remove_flag(put_command, user_id)
-  if user_id.nil?
-    put put_command
-  else
-    put put_command, user_id: user_id
+    context 'if the comment does not exist' do
+      let(:affected_entity_id) { 'does_not_exist' }
+      it { should be_bad_request }
+      its(:body) { should eq "[\"#{I18n.t(:requested_object_not_found)}\"]" }
+    end
+
+    context 'if no user_id is provided' do
+      let(:user_id) { nil }
+      it { should be_bad_request }
+      its(:body) { should eq "[\"#{I18n.t(:user_id_is_required)}\"]" }
+    end
  end
-end

-describe "app" do
-  describe "abuse" do
+  describe 'comment actions' do
+    let(:affected_entity) { create(:comment, abuse_flaggers: []) }
+    let(:non_affected_entity) { affected_entity.comment_thread }

-    before(:each) do
-      init_without_subscriptions
-      set_api_key_header
-    end
+    context 'when flagging a comment for abuse' do
+      let(:expected_abuse_flaggers) { [user_id] }
+      subject { put "/api/v1/comments/#{affected_entity_id}/abuse_flag", user_id: user_id }

-    describe "flag a comment as abusive" do
-      it "create or update the abuse_flags on the comment" do
-        comment = Comment.first
-        
-        # We get the count rather than just keeping the array, because the array
-        # will update as the Comment updates since the IdentityMap is enabled.
-        prev_abuse_flaggers_count = comment.abuse_flaggers.length
-        create_comment_flag("#{comment.id}", User.first.id)
-
-        comment = Comment.find(comment.id)
-        comment.abuse_flaggers.count.should == prev_abuse_flaggers_count + 1
-        # verify that the thread doesn't automatically get flagged
-        comment.comment_thread.abuse_flaggers.length.should == 0
-      end
-      it "returns 400 when the comment does not exist" do
-        create_comment_flag("does_not_exist", User.first.id)
-        last_response.status.should == 400
-        parse(last_response.body).first.should == I18n.t(:requested_object_not_found)
-      end
-      it "returns 400 when user_id is not provided" do
-        create_comment_flag("#{Comment.first.id}", nil)
-        last_response.status.should == 400
-        parse(last_response.body).first.should == I18n.t(:user_id_is_required)
-      end
-      #Would like to test the output of to_hash, but not sure how to deal with a Moped::BSON::Document object
-      #it "has a correct hash" do
-      #  create_flag("#{Comment.first.id}", User.first.id)
-      #  Comment.first.to_hash
-      #end
+      it_behaves_like 'an abuse endpoint'
    end
-    describe "flag a thread as abusive" do
-      it "create or update the abuse_flags on the comment" do
-        comment = Comment.first
-        thread = comment.comment_thread
-        prev_abuse_flaggers_count = thread.abuse_flaggers.count
-        create_thread_flag("#{thread.id}", User.first.id)
-
-        comment = Comment.find(comment.id)
-        comment.comment_thread.abuse_flaggers.count.should == prev_abuse_flaggers_count + 1
-        # verify that the comment doesn't automatically get flagged
-        comment.abuse_flaggers.length.should == 0
-      end
-      it "returns 400 when the thread does not exist" do
-        create_thread_flag("does_not_exist", User.first.id)
-        last_response.status.should == 400
-        parse(last_response.body).first.should == I18n.t(:requested_object_not_found)
-      end
-      it "returns 400 when user_id is not provided" do
-        create_thread_flag("#{Comment.first.comment_thread.id}", nil)
-        last_response.status.should == 400
-        parse(last_response.body).first.should == I18n.t(:user_id_is_required)
-      end
-      #Would like to test the output of to_hash, but not sure how to deal with a Moped::BSON::Document object
-      #it "has a correct hash" do
-      #  create_thread_flag("#{Comment.first.comment_thread.id}", User.first.id)
-      #  Comment.first.comment_thread.to_hash
-      #end
+
+    context 'when un-flagging a comment for abuse' do
+      let(:affected_entity) { create(:comment, abuse_flaggers: [user_id]) }
+      let(:expected_abuse_flaggers) { [] }
+      subject { put "/api/v1/comments/#{affected_entity_id}/abuse_unflag", user_id: user_id }
+
+      it_behaves_like 'an abuse endpoint'
    end
-    describe "unflag a comment as abusive" do
-      it "removes the user from the existing abuse_flaggers" do
-        comment = Comment.first
-        create_comment_flag("#{comment.id}", User.first.id)
-        
-        comment = Comment.first
-        prev_abuse_flaggers       = comment.abuse_flaggers
-        prev_abuse_flaggers_count = prev_abuse_flaggers.count
-        
-        prev_abuse_flaggers.should include User.first.id 
-        
-        remove_comment_flag("#{comment.id}", User.first.id)
-
-        comment = Comment.find(comment.id)
-        comment.abuse_flaggers.count.should == prev_abuse_flaggers_count - 1 
-        comment.abuse_flaggers.to_a.should_not include User.first.id
-      end
-      it "returns 400 when the comment does not exist" do
-        remove_comment_flag("does_not_exist", User.first.id)
-        last_response.status.should == 400
-        parse(last_response.body).first.should == I18n.t(:requested_object_not_found)
-      end
-      it "returns 400 when the thread does not exist" do
-        remove_thread_flag("does_not_exist", User.first.id)
-        last_response.status.should == 400
-        parse(last_response.body).first.should == I18n.t(:requested_object_not_found)
-      end
-      it "returns 400 when user_id is not provided" do
-        remove_thread_flag("#{Comment.first.comment_thread.id}", nil)
-        last_response.status.should == 400
-        parse(last_response.body).first.should == I18n.t(:user_id_is_required)
-      end
-      #Would like to test the output of to_hash, but not sure how to deal with a Moped::BSON::Document object
-      #it "has a correct hash" do
-      #  create_thread_flag("#{Comment.first.comment_thread.id}", User.first.id)
-      #  Comment.first.comment_thread.to_hash
-      #end
+  end
+
+  describe 'comment thread actions' do
+    let(:affected_entity) { create(:comment_thread, abuse_flaggers: []) }
+    let(:non_affected_entity) { create(:comment, comment_thread: affected_entity) }
+
+    context 'when flagging a comment thread for abuse' do
+      let(:expected_abuse_flaggers) { [user_id] }
+      subject { put "/api/v1/threads/#{affected_entity_id}/abuse_flag", user_id: user_id }
+
+      it_behaves_like 'an abuse endpoint'
    end
-    describe "unflag a thread as abusive" do
-      it "removes the user from the existing abuse_flaggers" do
-        thread = CommentThread.first
-        create_thread_flag("#{thread.id}", User.first.id)
-        
-        thread = CommentThread.first
-        prev_abuse_flaggers = thread.abuse_flaggers
-        prev_abuse_flaggers_count = prev_abuse_flaggers.count
-        
-        prev_abuse_flaggers.should include User.first.id 
-        
-        remove_thread_flag("#{thread.id}", User.first.id)
-
-        thread = CommentThread.find(thread.id)
-        thread.abuse_flaggers.count.should == prev_abuse_flaggers_count - 1 
-        thread.abuse_flaggers.to_a.should_not include User.first.id
-      end
-      it "returns 400 when the thread does not exist" do
-        remove_thread_flag("does_not_exist", User.first.id)
-        last_response.status.should == 400
-        parse(last_response.body).first.should == I18n.t(:requested_object_not_found)
-      end
-      it "returns 400 when user_id is not provided" do
-        remove_thread_flag("#{Comment.first.comment_thread.id}", nil)
-        last_response.status.should == 400
-        parse(last_response.body).first.should == I18n.t(:user_id_is_required)
-      end
-      #Would like to test the output of to_hash, but not sure how to deal with a Moped::BSON::Document object
-      #it "has a correct hash" do
-      #  create_thread_flag("#{Comment.first.comment_thread.id}", User.first.id)
-      #  Comment.first.comment_thread.to_hash
-      #end
+
+    context 'when un-flagging a comment thread for abuse' do
+      let(:affected_entity) { create(:comment_thread, abuse_flaggers: [user_id]) }
+      let(:expected_abuse_flaggers) { [] }
+      subject { put "/api/v1/threads/#{affected_entity_id}/abuse_unflag", user_id: user_id }
+
+      it_behaves_like 'an abuse endpoint'
    end
  end
 end
--- a/spec/api/comment_spec.rb
+++ b/spec/api/comment_spec.rb
--- a/spec/api/comment_thread_spec.rb
+++ b/spec/api/comment_thread_spec.rb
--- a/spec/api/commentable_spec.rb
+++ b/spec/api/commentable_spec.rb
--- a/spec/api/i18n_spec.rb
+++ b/spec/api/i18n_spec.rb
-require "spec_helper"
+require 'spec_helper'

-describe "i18n" do
+describe 'i18n' do

  before(:each) { set_api_key_header }

-  it "should respect the Accept-Language header" do
-    put "/api/v1/comments/does_not_exist/votes", {}, {"HTTP_ACCEPT_LANGUAGE" => "x-test"}
+  it 'should respect the Accept-Language header' do
+    put '/api/v1/comments/does_not_exist/votes', {}, {'HTTP_ACCEPT_LANGUAGE' => 'x-test'}
    last_response.status.should == 400
-    parse(last_response.body).first.should == "##x-test## requested object not found"
+    parse(last_response.body).first.should == '##x-test## requested object not found'
  end
 end
--- a/spec/api/notifications_spec.rb
+++ b/spec/api/notifications_spec.rb
@@ -17,7 +17,7 @@ describe "app" do
      random_string = (0...8).map{ ('a'..'z').to_a[rand(26)] }.join
      thread = CommentThread.new(
        title: "Test title", body: "elephant otter", course_id: "1",
-        commentable_id: commentable.id, comments_text_dummy: random_string
+        commentable_id: commentable.id, body: random_string
      )
      thread.thread_type = :discussion
      thread.author = user
@@ -94,7 +94,8 @@ describe "app" do

        subscription = Subscription.create({:subscriber_id => user._id.to_s, :source_id => thread._id.to_s})

-        comment = Comment.new(body: "dummy body text", course_id: "1", commentable_id: commentable.id)
+        comment = Comment.new(body: "dummy body text", course_id: "1")
+        comment.commentable_id = commentable.id
        comment.author = user
        comment.comment_thread = thread
        comment.save!

--- a/spec/api/query_spec.rb
+++ b/spec/api/query_spec.rb
 require 'spec_helper'
+require 'faker'

-describe "app" do

-  before (:each) { set_api_key_header }
+describe 'app' do
+  before(:each) { set_api_key_header }
+  let(:body) { Faker::Lorem.word }

-  let(:author) { create_test_user(1) }
-  describe "thread search" do
-    describe "GET /api/v1/search/threads" do
-      it "returns thread with query match" do
-        commentable = Commentable.new("question_1")
+  describe 'GET /api/v1/search/threads' do

-        random_string = (0...8).map{ ('a'..'z').to_a[rand(26)] }.join
+    shared_examples_for 'a search endpoint' do
+      subject do
+        refresh_es_index
+        get '/api/v1/search/threads', text: body
+      end

-        thread = CommentThread.new(title: "Test title", body: random_string, course_id: "1", commentable_id: commentable.id)
-        thread.thread_type = :discussion
-        thread.author = author
-        thread.save!
+      let(:matched_thread) { parse(subject.body)['collection'].select { |t| t['id'] == thread.id.to_s }.first }

-        sleep 3
+      it { should be_ok }

-        get "/api/v1/search/threads", text: random_string
-        last_response.should be_ok
-        threads = parse(last_response.body)['collection']
-        check_thread_result_json(nil, thread, threads.select{|t| t["id"] == thread.id.to_s}.first)
+      it 'returns thread with query match' do
+        expect(matched_thread).to_not be_nil
+        check_thread_result_json(nil, thread, matched_thread)
      end
-
    end
-  end
-
-  describe "comment search" do
-    describe "GET /api/v1/search/threads" do
-      it "returns thread with comment query match" do
-        commentable = Commentable.new("question_1")
-
-        random_string = (0...8).map{ ('a'..'z').to_a[rand(26)] }.join

-        thread = CommentThread.new(title: "Test title", body: "elephant otter", course_id: "1", commentable_id: commentable.id)
-        thread.thread_type = :discussion
-        thread.author = author
-        thread.save!
+    context 'when searching on thread content' do
+      let!(:thread) { create(:comment_thread, body: body) }

-        sleep 3
-
-        comment = Comment.new(body: random_string, course_id: "1", commentable_id: commentable.id)
-        comment.author = author
-        comment.comment_thread = thread
-        comment.save!
-
-        sleep 1
+      it_behaves_like 'a search endpoint'
+    end

-        get "/api/v1/search/threads", text: random_string
-        last_response.should be_ok
-        threads = parse(last_response.body)['collection']
-        check_thread_result_json(nil, thread, threads.select{|t| t["id"] == thread.id.to_s}.first)
+    context 'when searching on comment content' do
+      let!(:thread) do
+        comment = create(:comment, body: body)
+        thread = comment.comment_thread
      end
+
+      it_behaves_like 'a search endpoint'
    end
  end
 end
--- a/spec/api/user_spec.rb
+++ b/spec/api/user_spec.rb
@@ -361,5 +361,22 @@ describe "app" do

      include_examples "unicode data"
    end
+
+    describe "POST /api/v1/users/:user_id/read" do
+
+      before(:each) { setup_10_threads }
+
+      it "marks a thread as read for the user" do
+        thread = @threads["t0"]
+        user = create_test_user(42)
+        post "/api/v1/users/#{user.external_id}/read", source_type: "thread", source_id: thread.id
+        last_response.should be_ok
+        user.reload
+        read_states = user.read_states.where(course_id: thread.course_id).to_a
+        read_date = read_states.first.last_read_times[thread.id.to_s]
+        read_date.should >= thread.updated_at
+      end
+    end
+
  end
 end
--- a/spec/app_spec.rb
+++ b/spec/app_spec.rb
@@ -45,8 +45,11 @@ describe "app" do
    context "db check" do
      def test_db_check(response, is_success)
        db = double("db")
-        stub_const("Mongoid::Sessions", Class.new).stub(:default).and_return(db)
-        db.should_receive(:command).with({:isMaster => 1}).and_return(response)
+        stub_const("Mongoid::Clients", Class.new).stub(:default).and_return(db)
+        result = double('result')
+        result.stub(:ok?).and_return(response['ok'] == 1)
+        result.stub(:documents).and_return([response])
+        db.should_receive(:command).with({:isMaster => 1}).and_return(result)
        get "/heartbeat"
        if is_success
          last_response.status.should == 200
@@ -75,7 +78,7 @@ describe "app" do

      it "reports failure when db command raises an error" do
        db = double("db")
-        stub_const("Mongoid::Sessions", Class.new).stub(:default).and_return(db)
+        stub_const("Mongoid::Clients", Class.new).stub(:default).and_return(db)
        db.should_receive(:command).with({:isMaster => 1}).and_raise(StandardError)
        get "/heartbeat"
        last_response.status.should == 500
@@ -168,4 +171,4 @@ describe "app" do
    end

  end
-end
\ No newline at end of file
+end
--- a/spec/factories.rb
+++ b/spec/factories.rb
+require 'faker'
+
+# Reload i18n data for faker
+I18n.reload!
+
+FactoryGirl.define do
+  factory :user do
+    # Initialize the model with all attributes since we are using a custom _id field.
+    # See https://github.com/thoughtbot/factory_girl/issues/544.
+    initialize_with { new(attributes) }
+
+    sequence(:username) { |n| "#{Faker::Internet.user_name}_#{n}" }
+    sequence(:external_id) { username }
+  end
+
+  factory :comment_thread do
+    title { Faker::Lorem.sentence }
+    body { Faker::Lorem.paragraph }
+    course_id { Faker::Lorem.word }
+    thread_type :discussion
+    commentable_id { Faker::Lorem.word }
+    association :author, factory: :user
+    group_id nil
+    pinned false
+
+    trait :subscribe_author do
+      after(:create) do |thread|
+        thread.author.subscribe(thread)
+      end
+    end
+
+    trait :with_group_id do
+      group_id { Faker::Number.number(4) }
+    end
+  end
+
+  factory :comment do
+    association :author, factory: :user
+    comment_thread { parent ? parent.comment_thread : create(:comment_thread) }
+    body { Faker::Lorem.paragraph }
+    course_id { comment_thread.course_id }
+    commentable_id { comment_thread.commentable_id }
+    endorsed false
+  end
+end
--- a/spec/models/comment_spec.rb
+++ b/spec/models/comment_spec.rb
@@ -69,4 +69,32 @@ describe Comment do
      end
    end
  end
+
+  describe '#child_count' do
+    context 'with course_thread' do
+      it 'returns cached child count' do
+        comment = make_comment(author, course_thread, "comment")
+        child_comment = make_comment(author, comment, "comment")
+        expect(comment.get_cached_child_count).to eq(1)
+      end
+
+      it 'returns cached child count' do
+        comment = make_comment(author, course_thread, "comment")
+        child_comment = make_comment(author, comment, "comment")
+        comment.child_count = nil
+        expect(comment.get_cached_child_count).to eq(1)
+      end
+
+      it 'updates cached child count' do
+        comment = make_comment(author, course_thread, "comment")
+        expect(comment.get_cached_child_count).to eq(0)
+        comment.child_count = 2
+        expect(comment.get_cached_child_count).to eq(2)
+        comment.update_cached_child_count
+        expect(comment.get_cached_child_count).to eq(0)
+      end
+    end
+  end
+
+
 end
--- a/spec/presenters/thread_spec.rb
+++ b/spec/presenters/thread_spec.rb
@@ -77,19 +77,33 @@ describe ThreadPresenter do
        @reader = create_test_user('thread reader')
      end

-      it "handles with_responses=false" do
+      it "handles with_responses=false and recursive has no impact" do
        @threads_with_num_comments.each do |thread, num_comments|
          is_endorsed = num_comments > 0 && endorse_responses
-          hash = ThreadPresenter.new(thread, @reader, false, num_comments, is_endorsed).to_hash
+          # with response=false and recursive=false
+          hash = ThreadPresenter.new(thread, @reader, false, num_comments, is_endorsed).to_hash(false, 0, nil, false)
          check_thread_result(@reader, thread, hash)
          ['children', 'resp_skip', 'resp_limit', 'resp_total'].each {|k| (hash.has_key? k).should be_false }
+          # with response=false and recursive=true
+          hash = ThreadPresenter.new(thread, @reader, false, num_comments, is_endorsed).to_hash(false, 0, nil, true)
+          check_thread_result(@reader, thread, hash)
+          ['children', 'resp_skip', 'resp_limit', 'resp_total'].each {|k| (hash.has_key? k).should be_false }
+        end
+      end
+
+      it "handles with_responses=true and recursive=true" do
+        @threads_with_num_comments.each do |thread, num_comments|
+          is_endorsed = num_comments > 0 && endorse_responses
+          hash = ThreadPresenter.new(thread, @reader, false, num_comments, is_endorsed).to_hash(true, 0, nil, true)
+          check_thread_result(@reader, thread, hash)
+          check_thread_response_paging(thread, hash, 0, nil, false, true)
        end
      end

-      it "handles with_responses=true" do
+      it "handles with_responses=true and recursive=false" do
        @threads_with_num_comments.each do |thread, num_comments|
          is_endorsed = num_comments > 0 && endorse_responses
-          hash = ThreadPresenter.new(thread, @reader, false, num_comments, is_endorsed).to_hash true
+          hash = ThreadPresenter.new(thread, @reader, false, num_comments, is_endorsed).to_hash(true, 0, nil, false)
          check_thread_result(@reader, thread, hash)
          check_thread_response_paging(thread, hash)
        end
@@ -99,7 +113,7 @@ describe ThreadPresenter do
        @threads_with_num_comments.each do |thread, num_comments|
          is_endorsed = num_comments > 0 && endorse_responses
          [0, 1, 2, 9, 10, 11, 1000].each do |skip|
-            hash = ThreadPresenter.new(thread, @reader, false, num_comments, is_endorsed).to_hash true, skip
+            hash = ThreadPresenter.new(thread, @reader, false, num_comments, is_endorsed).to_hash(true, skip, nil, true)
            check_thread_result(@reader, thread, hash)
            check_thread_response_paging(thread, hash, skip)
          end
@@ -111,7 +125,7 @@ describe ThreadPresenter do
          is_endorsed = num_comments > 0 && endorse_responses
          [1, 2, 3, 9, 10, 11, 1000].each do |limit|
            [0, 1, 2, 9, 10, 11, 1000].each do |skip|
-              hash = ThreadPresenter.new(thread, @reader, false, num_comments, is_endorsed).to_hash true, skip, limit
+              hash = ThreadPresenter.new(thread, @reader, false, num_comments, is_endorsed).to_hash(true, skip, limit, true)
              check_thread_result(@reader, thread, hash)
              check_thread_response_paging(thread, hash, skip, limit)
            end
@@ -122,9 +136,9 @@ describe ThreadPresenter do
      it "fails with invalid arguments" do
        @threads_with_num_comments.each do |thread, num_comments|
          is_endorsed = num_comments > 0 && endorse_responses
-          expect{ThreadPresenter.new(thread, @reader, false, num_comments, is_endorsed).to_hash(true, -1, nil)}.to raise_error(ArgumentError)
+          expect{ThreadPresenter.new(thread, @reader, false, num_comments, is_endorsed).to_hash(true, -1, nil, true)}.to raise_error(ArgumentError)
          [-1, 0].each do |limit|
-            expect{ThreadPresenter.new(thread, @reader, false, num_comments, is_endorsed).to_hash(true, 0, limit)}.to raise_error(ArgumentError)
+            expect{ThreadPresenter.new(thread, @reader, false, num_comments, is_endorsed).to_hash(true, 0, limit, true)}.to raise_error(ArgumentError)
          end
        end
      end

--- a/spec/spec_helper.rb
+++ b/spec/spec_helper.rb
--- a/spec/support/database_cleaner.rb
+++ b/spec/support/database_cleaner.rb
+require 'database_cleaner'
+
+RSpec.configure do |config|
+  config.before(:suite) do
+    # Mongoid only supports truncation.
+    DatabaseCleaner.strategy = :truncation
+    DatabaseCleaner.clean_with(:truncation)
+  end
+
+  config.around(:each) do |example|
+    DatabaseCleaner.cleaning do
+      example.run
+    end
+  end
+end
--- a/spec/support/elasticsearch.rb
+++ b/spec/support/elasticsearch.rb
+def delete_es_index
+  Tire.index Content::ES_INDEX_NAME do
+    delete
+  end
+end
+
+def create_es_index
+  new_index = Tire.index Content::ES_INDEX_NAME
+  new_index.create
+  [CommentThread, Comment].each do |klass|
+    klass.put_search_index_mapping
+  end
+end
+
+def refresh_es_index
+  es_index_name = Content::ES_INDEX_NAME
+  Tire.index es_index_name do
+    refresh
+  end
+end
+
+RSpec.configure do |config|
+  config.before(:each) do
+    delete_es_index
+    create_es_index
+  end
+end
--- a/spec/support/factory_girl.rb
+++ b/spec/support/factory_girl.rb
+require 'factory_girl'
+
+RSpec.configure do |config|
+  config.include FactoryGirl::Syntax::Methods
+
+  FactoryGirl.find_definitions
+
+  config.before(:suite) do
+    begin
+      DatabaseCleaner.start
+      FactoryGirl.lint
+    ensure
+      DatabaseCleaner.clean
+    end
+  end
+end