Revert "Replaced Tire with elasticsearch-model"

This reverts commit 56b61a13.

Revert "Replaced Tire with elasticsearch-model"
This reverts commit 56b61a13.
488fb2e9 · Toby Lawrence · 05331999 · 488fb2e9 · 488fb2e9 · 488fb2e9
Commit 488fb2e9 authored Aug 09, 2016 by Toby Lawrence
22 changed files
--- a/.travis.yml
+++ b/.travis.yml
@@ -23,6 +23,4 @@ before_install:
  - mkdir -p ./mongo/log
  - mongod --fork --dbpath ./mongo/db --logpath ./mongo/log/mongodb.log
-script:
+script: bundle exec rspec
-  - bin/rake search:initialize
-  - bin/rspec
--- a/Gemfile
+++ b/Gemfile
@@ -31,8 +31,8 @@ gem 'will_paginate_mongoid', "~>2.0"
 gem 'rdiscount'
 gem 'nokogiri', "~>1.6.8"
-gem 'elasticsearch', '~> 0.4'
+gem 'tire', "0.6.2"
-gem 'elasticsearch-model', '~> 0.1.8'
+gem 'tire-contrib'
 gem 'dalli'
@@ -47,7 +47,6 @@ group :test do
  gem 'guard-unicorn'
  gem 'rack-test', :require => 'rack/test'
  gem 'rspec', '~> 2.11.0'
-  gem 'webmock', '~> 1.22'
 end
 gem 'newrelic_rpm'

--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -32,7 +32,7 @@ GEM
      minitest (~> 5.1)
      thread_safe (~> 0.3, >= 0.3.4)
      tzinfo (~> 1.1)
-    addressable (2.4.0)
+    ansi (1.5.0)
    bson (3.2.4)
    bson_ext (1.5.1)
    builder (3.2.2)
@@ -41,8 +41,6 @@ GEM
      simplecov
      url
    coderay (1.0.7)
-    crack (0.4.3)
-      safe_yaml (~> 1.0.0)
    dalli (2.1.0)
    database_cleaner (1.5.1)
    delayed_job (4.1.1)
@@ -55,33 +53,18 @@ GEM
    docile (1.1.5)
    domain_name (0.5.24)
      unf (>= 0.0.5, < 1.0.0)
-    elasticsearch (0.4.11)
-      elasticsearch-api (= 0.4.11)
-      elasticsearch-transport (= 0.4.11)
-    elasticsearch-api (0.4.11)
-      multi_json
-    elasticsearch-model (0.1.8)
-      activesupport (> 3)
-      elasticsearch (> 0.4)
-      hashie
-    elasticsearch-transport (0.4.11)
-      faraday
-      multi_json
    enumerize (0.11.0)
      activesupport (>= 3.2)
    factory_girl (4.5.0)
      activesupport (>= 3.0.0)
    faker (1.6.1)
      i18n (~> 0.5)
-    faraday (0.9.2)
-      multipart-post (>= 1.2, < 3)
    guard (1.3.2)
      listen (>= 0.4.2)
      thor (>= 0.14.6)
    guard-unicorn (0.0.7)
      guard (>= 1.1)
-    hashdiff (0.2.3)
+    hashr (0.0.22)
-    hashie (3.4.3)
    http-cookie (1.0.2)
      domain_name (~> 0.5)
    i18n (0.7.0)
@@ -106,7 +89,6 @@ GEM
      mongoid
      rake
    multi_json (1.11.2)
-    multipart-post (2.0.0)
    netrc (0.10.3)
    newrelic_rpm (3.16.0.318)
    nokogiri (1.6.8)
@@ -143,7 +125,6 @@ GEM
    rspec-expectations (2.11.2)
      diff-lcs (~> 1.1.3)
    rspec-mocks (2.11.2)
-    safe_yaml (1.0.4)
    simplecov (0.11.1)
      docile (~> 1.1.0)
      json (~> 1.8)
@@ -157,6 +138,16 @@ GEM
    thor (0.16.0)
    thread_safe (0.3.5)
    tilt (1.3.3)
+    tire (0.6.2)
+      activemodel (>= 3.0)
+      activesupport
+      ansi
+      hashr (~> 0.0.19)
+      multi_json (~> 1.3)
+      rake
+      rest-client (~> 1.6)
+    tire-contrib (0.1.1)
+      tire
    tzinfo (1.2.2)
      thread_safe (~> 0.1)
    unf (0.1.4)
@@ -167,10 +158,6 @@ GEM
      rack
      raindrops (~> 0.7)
    url (0.3.2)
-    webmock (1.22.3)
-      addressable (>= 2.3.6)
-      crack (>= 0.3.2)
-      hashdiff
    will_paginate (3.0.7)
    will_paginate_mongoid (2.0.1)
      mongoid
@@ -189,8 +176,6 @@ DEPENDENCIES
  database_cleaner (~> 1.5.1)
  delayed_job
  delayed_job_mongoid
-  elasticsearch (~> 0.4)
-  elasticsearch-model (~> 0.1.8)
  enumerize
  factory_girl (~> 4.0)
  faker (~> 1.6)
@@ -214,7 +199,8 @@ DEPENDENCIES
  rs_voteable_mongo!
  rspec (~> 2.11.0)
  sinatra
+  tire (= 0.6.2)
+  tire-contrib
  unicorn
-  webmock (~> 1.22)
  will_paginate_mongoid (~> 2.0)
  yajl-ruby
--- a/README.rst
+++ b/README.rst
@@ -14,8 +14,9 @@ An independent comment system which supports voting and nested comments. It
 also supports features including instructor endorsement for education-aimed
 discussion platforms.
-Getting Started
---------------
+Running the Server
+------------------
 If you are running cs_comments_service as part of edx-platform__ development under
 devstack, it is strongly recommended to read `those setup documents`__ first.  Note that
 devstack will take care of just about all of the installation, configuration, and
@@ -29,49 +30,15 @@ This service relies on Elasticsearch and MongoDB. By default the service will us
 however, if you wish to change these values, refer to `config/application.yml` and `config/mongoid.yml` for the
 environment variables that can be set to override the defaults.
-Install the requisite gems:
+Before the server is first run, ensure gems are installed by doing ``bundle install``.
-.. code-block:: bash
-    $ bundle install
-Setup the search index. Note that the command below creates an alias with a unique name (e.g. content_20160101), and
-assigns it a known alias: content. If you choose not to use the command below, you should still opt to reference your
-index by an alias rather than the actual index name. This will enable you to swap out indices (e.g. reindex) without
-having to take downtime or modify code with a new index name.
-.. code-block:: bash
-    $ bin/rake search:initialize
-Run ther server:
-.. code-block::
-    $ ruby app.rb
-By default Sinatra runs on port `4567`. If you'd like to use a different port pass the `-p` parameter:
+To run the server, do ``ruby app.rb [-p PORT]`` where PORT defaults to 4567.
-.. code-block::
-    $ ruby app.rb -p 5678
 Running Tests
 -------------
-Tests are built using the rspec__ framework, and can be run with the command below:
+To run tests, do ``bundle exec rspec``.  Append ``--help`` or see rspec documentation
+for additional options to this command.
-.. code-block::
-    $ bin/rspec
-If you'd like to view additional options for the command, append the `--help` option:
-.. code-block::
-    $ bin/rspec --help
-__ http://rspec.info/
 Internationalization (i18n) and Localization (l10n)
 ---------------------------------------------------
@@ -95,12 +62,12 @@ follow the instructions here__ to set up your ``.transifexrc`` file.
 __ http://support.transifex.com/customer/portal/articles/1000855-configuring-the-client
 To upload strings to Transifex for translation when you change the set
-of translatable strings: ``bin/rake i18n:push``
+of translatable strings: ``bundle exec rake i18n:push``
-To fetch the latest translations from Transifex: ``bin/rake i18n:pull``
+To fetch the latest translations from Transifex: ``bundle exec rake i18n:pull``
 The repository includes some translations so they will be available
-upon deployment. To commit an update to these: ``bin/rake i18n:commit``
+upon deployment. To commit an update to these: ``bundle exec rake i18n:commit``
 License
 -------

--- a/Rakefile
+++ b/Rakefile
@@ -17,23 +17,21 @@ rescue LoadError
  # no rspec available
 end
+Tire.configure do
+  url YAML.load(application_yaml)['elasticsearch_server']
+end
 LOG = Logger.new(STDERR)
 desc 'Load the environment'
 task :environment do
  environment = ENV['SINATRA_ENV'] || 'development'
  Sinatra::Base.environment = environment
  Mongoid.load!('config/mongoid.yml')
  Mongoid.logger.level = Logger::INFO
  module CommentService
    class << self;
-      attr_accessor :config
+      attr_accessor :config;
-      def search_enabled?
-        self.config[:enable_search]
-      end
    end
  end
@@ -43,8 +41,8 @@ task :environment do
  Dir[File.dirname(__FILE__) + '/models/*.rb'].each { |file| require file }
 end
+Dir.glob('lib/tasks/*.rake').each { |r| import r }
 task :console => :environment do
  binding.pry
 end
-Dir.glob('lib/tasks/*.rake').each { |r| import r }
--- a/api/comment_threads.rb
+++ b/api/comment_threads.rb
 get "#{APIPREFIX}/threads" do # retrieve threads by course
  threads = Content.where({"_type" => "CommentThread", "course_id" => params["course_id"]})
  if params[:commentable_ids]
    threads = threads.in({"commentable_id" => params[:commentable_ids].split(",")})
  end
  handle_threads_query(
-      threads,
+    threads,
-      params['user_id'],
+    params["user_id"],
-      params['course_id'],
+    params["course_id"],
-      get_group_ids_from_params(params),
+    get_group_ids_from_params(params),
-      value_to_boolean(params['flagged']),
+    value_to_boolean(params["flagged"]),
-      value_to_boolean(params['unread']),
+    value_to_boolean(params["unread"]),
-      value_to_boolean(params['unanswered']),
+    value_to_boolean(params["unanswered"]),
-      params['sort_key'],
+    params["sort_key"],
-      params['sort_order'],
+    params["sort_order"],
-      params['page'],
+    params["page"],
-      params['per_page']
+    params["per_page"]
  ).to_json
 end
@@ -83,12 +83,6 @@ post "#{APIPREFIX}/threads/:thread_id/comments" do |thread_id|
 end
 delete "#{APIPREFIX}/threads/:thread_id" do |thread_id|
-  begin
+  thread.destroy
-    thread.destroy
-  rescue Elasticsearch::Transport::Transport::Errors::NotFound
-    # If the thread is not in the index, that's actually a good thing given that we just removed it.
-    # Note that this exception will probably only be encountered for tests that don't wait for the index
-    # to be refreshed before attempting to destroy a newly-recreated thread.
-  end
  thread.to_hash.to_json
 end
--- a/api/comments.rb
+++ b/api/comments.rb
@@ -60,4 +60,5 @@ delete "#{APIPREFIX}/comments/:comment_id" do |comment_id|
      pass
    end
  end
+  comment.to_hash.to_json
 end
--- a/api/search.rb
+++ b/api/search.rb
-def get_thread_ids(context, group_ids, local_params, search_text)
+get "#{APIPREFIX}/search/threads" do
-  filters = []
+  local_params = params # Necessary for params to be available inside blocks
-  filters.push({term: {commentable_id: local_params['commentable_id']}}) if local_params['commentable_id']
+  group_ids = get_group_ids_from_params(local_params)
-  filters.push({terms: {commentable_id: local_params['commentable_ids'].split(',')}}) if local_params['commentable_ids']
+  context = local_params["context"] ? local_params["context"] : "course"
-  filters.push({term: {course_id: local_params['course_id']}}) if local_params['course_id']
+  search_text = local_params["text"]
+  if !search_text
-  filters.push({or: [
+    {}.to_json
-      {not: {exists: {field: :context}}},
+  else
-      {term: {context: context}}
+    # Because threads and comments are currently separate unrelated documents in
-  ]})
+    # Elasticsearch, we must first query for all matching documents, then
+    # extract the set of thread ids, and then sort the threads by the specified
+    # criteria and paginate. For performance reasons, we currently limit the
+    # number of documents considered (ordered by update recency), which means
+    # that matching threads can be missed if the search terms are very common.
-  unless group_ids.empty?
+    get_matching_thread_ids = lambda do |search_text|
-    filters.push(
+      self.class.trace_execution_scoped(["Custom/get_search_threads/es_search"]) do
-        {
+        search = Tire.search Content::ES_INDEX_NAME do
-            or: [
+          query do
-                {:not => {:exists => {:field => :group_id}}},
+            match [:title, :body], search_text, :operator => "AND"
-                {:terms => {:group_id => group_ids}}
+            filtered do
-            ]
+              filter :term, :commentable_id => local_params["commentable_id"] if local_params["commentable_id"]
-        }
+              filter :terms, :commentable_id => local_params["commentable_ids"].split(",") if local_params["commentable_ids"]
-    )
+              filter :term, :course_id => local_params["course_id"] if local_params["course_id"]
-  end
+              filter :or, [
+                {:not => {:exists => {:field => :context}}},
+                {:term => {:context => context}}
+              ]
-  self.class.trace_execution_scoped(['Custom/get_search_threads/es_search']) do
+              if not group_ids.empty?
-    body = {
+                if group_ids.length > 1
-        size: CommentService.config['max_deep_search_comment_count'].to_i,
+                  group_id_criteria = {:terms => {:group_id => group_ids}}
-        sort: [
+                else
-            {updated_at: :desc}
+                  group_id_criteria = {:term => {:group_id => group_ids[0]}}
-        ],
+                end
-        query: {
-            multi_match: {
-                query: search_text,
-                fields: [:title, :body],
-                operator: :AND
-            },
-            filtered: {
-                filter: {
-                    and: filters
-                }
-            }
-        }
-    }
-    response = Elasticsearch::Model.client.search(index: Content::ES_INDEX_NAME, body: body)
+                filter :or, [
+                  {:not => {:exists => {:field => :group_id}}},
+                  group_id_criteria
+                ]
+              end
-    thread_ids = Set.new
+            end
-    response['hits']['hits'].each do |hit|
+          end
-      case hit['_type']
+          sort do
-        when CommentThread.document_type
+            by "updated_at", "desc"
-          thread_ids.add(hit['_id'])
+          end
-        when Comment.document_type
+          size CommentService.config["max_deep_search_comment_count"].to_i
-          thread_ids.add(hit['_source']['comment_thread_id'])
+        end
-        else
+        thread_ids = Set.new
-          # There shouldn't be any other document types. Nevertheless, ignore them, if they are present.
+        search.results.each do |content|
-          next
+          case content.type
+          when "comment_thread"
+            thread_ids.add(content.id)
+          when "comment"
+            thread_ids.add(content.comment_thread_id)
+          end
+        end
+        thread_ids
      end
    end
-    thread_ids
-  end
-end
-def get_suggested_text(search_text)
-  body = {
-      suggestions: {
-          text: search_text,
-          phrase: {
-              field: :_all
-          }
-      }
-  }
-  response = Elasticsearch::Model.client.suggest(index: Content::ES_INDEX_NAME, body: body)
-  suggestions = response.fetch('suggestions', [])
-  if suggestions.length > 0
-    options = suggestions[0]['options']
-    if options.length > 0
-      return options[0]['text']
-    end
-  end
-  nil
-end
-def get_threads(context, group_ids, local_params, search_text)
-  # Because threads and comments are currently separate unrelated documents in
-  # Elasticsearch, we must first query for all matching documents, then
-  # extract the set of thread ids, and then sort the threads by the specified
-  # criteria and paginate. For performance reasons, we currently limit the
-  # number of documents considered (ordered by update recency), which means
-  # that matching threads can be missed if the search terms are very common.
-  thread_ids = get_thread_ids(context, group_ids, local_params, search_text)
-  corrected_text = nil
-  if thread_ids.empty?
    # Sadly, Elasticsearch does not have a facility for computing suggestions
    # with respect to a filter. It would be expensive to determine the best
    # suggestion with respect to our filter parameters, so we simply re-query
    # with the top suggestion. If that has no results, then we return no results
    # and no correction.
-    corrected_text = get_suggested_text(search_text)
+    thread_ids = get_matching_thread_ids.call(search_text)
-    thread_ids = get_thread_ids(context, group_ids, local_params, corrected_text) if corrected_text
+    corrected_text = nil
-    corrected_text = nil if thread_ids.empty?
+    if thread_ids.empty?
-  end
+      suggest = Tire.suggest Content::ES_INDEX_NAME do
+        suggestion "" do
+          text search_text
+          phrase :_all
+        end
+      end
+      corrected_text = suggest.results.texts.first
+      thread_ids = get_matching_thread_ids.call(corrected_text) if corrected_text
+      corrected_text = nil if thread_ids.empty?
+    end
-  result_obj = handle_threads_query(
+    result_obj = handle_threads_query(
-      CommentThread.in({_id: thread_ids.to_a}),
+      CommentThread.in({"_id" => thread_ids.to_a}),
-      local_params['user_id'],
+      local_params["user_id"],
-      local_params['course_id'],
+      local_params["course_id"],
      group_ids,
-      value_to_boolean(local_params['flagged']),
+      value_to_boolean(local_params["flagged"]),
-      value_to_boolean(local_params['unread']),
+      value_to_boolean(local_params["unread"]),
-      value_to_boolean(local_params['unanswered']),
+      value_to_boolean(local_params["unanswered"]),
-      local_params['sort_key'],
+      local_params["sort_key"],
-      local_params['sort_order'],
+      local_params["sort_order"],
-      local_params['page'],
+      local_params["page"],
-      local_params['per_page'],
+      local_params["per_page"],
      context
-  )
+    )
+    if !result_obj.empty?
-  unless result_obj.empty?
+      result_obj[:corrected_text] = corrected_text
-    result_obj[:corrected_text] = corrected_text
+      # NOTE this reflects the total results from ES, but does not consider
-    # NOTE this reflects the total results from ES, but does not consider
+      # any post-filtering that might happen (e.g. unread, flagged...) before
-    # any post-filtering that might happen (e.g. unread, flagged...) before
+      # results are shown to the user.
-    # results are shown to the user.
+      result_obj[:total_results] = thread_ids.size
-    result_obj[:total_results] = thread_ids.size
+    end
-  end
+    result_obj.to_json
-  result_obj.to_json
-end
-get "#{APIPREFIX}/search/threads" do
-  local_params = params # Necessary for params to be available inside blocks
-  group_ids = get_group_ids_from_params(local_params)
-  context = local_params["context"] ? local_params["context"] : "course"
-  search_text = local_params["text"]
-  if !search_text
-    '{}'
-  else
-    get_threads(context, group_ids, local_params, search_text)
  end
 end
--- a/app.rb
+++ b/app.rb
@@ -14,10 +14,6 @@ module CommentService
  class << self
    attr_accessor :config
    attr_accessor :blocked_hashes
-    def search_enabled?
-      self.config[:enable_search]
-    end
  end
  API_VERSION = 'v1'
  API_PREFIX = "/api/#{API_VERSION}"
@@ -30,6 +26,11 @@ end
 application_yaml = ERB.new(File.read("config/application.yml")).result()
 CommentService.config = YAML.load(application_yaml).with_indifferent_access
+Tire.configure do
+  url CommentService.config[:elasticsearch_server]
+  logger STDERR if ENV["ENABLE_ELASTICSEARCH_DEBUGGING"]
+end
 Mongoid.load!("config/mongoid.yml", environment)
 Mongoid.logger.level = Logger::INFO
 Mongo::Logger.logger.level = ENV["ENABLE_MONGO_DEBUGGING"] ? Logger::DEBUG : Logger::INFO
@@ -47,13 +48,11 @@ helpers do
  end
 end
-Dir[File.dirname(__FILE__) + '/lib/**/*.rb'].each { |file| require file }
+Dir[File.dirname(__FILE__) + '/lib/**/*.rb'].each {|file| require file}
-Dir[File.dirname(__FILE__) + '/models/*.rb'].each { |file| require file }
+Dir[File.dirname(__FILE__) + '/models/*.rb'].each {|file| require file}
-Dir[File.dirname(__FILE__) + '/presenters/*.rb'].each { |file| require file }
+Dir[File.dirname(__FILE__) + '/presenters/*.rb'].each {|file| require file}
-Elasticsearch::Model.client = Elasticsearch::Client.new(host: CommentService.config[:elasticsearch_server], log: false)
+# Ensure elasticsearch index mappings exist.
-# Ensure Elasticsearch index mappings exist.
 Comment.put_search_index_mapping
 CommentThread.put_search_index_mapping
@@ -107,6 +106,7 @@ class Time
 end
 # these files must be required in order
 require './api/search'
 require './api/commentables'
@@ -138,61 +138,55 @@ error ArgumentError do
  error 400, [env['sinatra.error'].message].to_json
 end
-CommentService.blocked_hashes = Content.mongo_client[:blocked_hash].find(nil, projection: {hash: 1}).map { |d| d["hash"] }
+CommentService.blocked_hashes = Content.mongo_client[:blocked_hash].find(nil, projection: {hash: 1}).map {|d| d["hash"]}
 def get_db_is_master
  Mongoid::Clients.default.command(isMaster: 1)
 end
-def elasticsearch_health
+def get_es_status
-  Elasticsearch::Model.client.cluster.health
+  res = Tire::Configuration.client.get Tire::Configuration.url
+  JSON.parse res.body
 end
+get '/heartbeat' do
-def is_mongo_available?
+  # mongo is reachable and ready to handle requests
+  db_ok = false
  begin
-    response = get_db_is_master
+    res = get_db_is_master
-    return response.ok? && (response.documents.first['ismaster'] == true)
+    db_ok = res.ok? && res.documents.first['ismaster'] == true
  rescue
-    # ignored
  end
+  error 500, JSON.generate({"OK" => false, "check" => "db"}) unless db_ok
-  false
+  # E_S is reachable and ready to handle requests
-end
+  es_ok = false
-def is_elasticsearch_available?
  begin
-    health = elasticsearch_health
+    es_status = get_es_status
-    return !health['timed_out'] && %w(yellow green).include?(health['status'])
+    es_ok = es_status["status"] == 200
  rescue
-    # ignored
  end
+  error 500, JSON.generate({"OK" => false, "check" => "es"}) unless es_ok
-  false
+  JSON.generate({"OK" => true})
-end
-get '/heartbeat' do
-  error 500, JSON.generate({OK: false, check: :db}) unless is_mongo_available?
-  error 500, JSON.generate({OK: false, check: :es}) unless is_elasticsearch_available?
-  JSON.generate({OK: true})
 end
 get '/selftest' do
  begin
    t1 = Time.now
    status = {
-        db: get_db_is_master,
+      "db" => get_db_is_master,
-        es: elasticsearch_health,
+      "es" => get_es_status,
-        last_post_created: (Content.last.created_at rescue nil),
+      "last_post_created" => (Content.last.created_at rescue nil),
-        total_posts: Content.count,
+      "total_posts" => Content.count,
-        total_users: User.count,
+      "total_users" => User.count,
-        elapsed_time: Time.now - t1
+      "elapsed_time" => Time.now - t1
    }
    JSON.generate(status)
  rescue => ex
-    [500,
+    [ 500,
-     {'Content-Type' => 'text/plain'},
+      {'Content-Type' => 'text/plain'},
-     "#{ex.backtrace.first}: #{ex.message} (#{ex.class})\n\t#{ex.backtrace[1..-1].join("\n\t")}"
+      "#{ex.backtrace.first}: #{ex.message} (#{ex.class})\n\t#{ex.backtrace[1..-1].join("\n\t")}"
    ]
  end
 end
--- a/config/application.yml
+++ b/config/application.yml
@@ -4,4 +4,3 @@ elasticsearch_server: <%= ENV['SEARCH_SERVER'] || 'http://localhost:9200' %>
 max_deep_search_comment_count: 5000
 default_locale: <%= ENV['SERVICE_LANGUAGE'] || 'en-US' %>
 manual_pagination_batch_size: <%= ENV['MANUAL_PAGINATION_BATCH_SIZE'] || 500 %>
-enable_search: true
--- a/lib/task_helpers.rb
+++ b/lib/task_helpers.rb
-require 'elasticsearch'
 module TaskHelpers
  module ElasticsearchHelper
-    LOG = Logger.new(STDERR)
    def self.create_index(name=nil)
      name ||= "#{Content::ES_INDEX_NAME}_#{Time.now.strftime('%Y%m%d%H%M%S')}"
+      index = Tire.index(name)
+      LOG.info "Creating new index: #{name}..."
+      index.create
-      mappings = {}
+      [CommentThread, Comment].each do |model|
-      [Comment, CommentThread].each do |model|
+        LOG.info "Applying index mappings for #{model.name}"
-        mappings.merge! model.mappings.to_hash
+        model.put_search_index_mapping(index)
      end
+      LOG.info '...done!'
-      Elasticsearch::Model.client.indices.create(index: name, body: {mappings: mappings})
+      index
-      LOG.info "Created new index: #{name}."
-      name
    end
    def self.delete_index(name)
-      begin
+      Tire.index(name).delete
-        Elasticsearch::Model.client.indices.delete(index: name)
-        LOG.info "Deleted index: #{name}."
-      rescue Elasticsearch::Transport::Transport::Errors::NotFound
-        # NOTE (CCB): Future versions of the Elasticsearch client support the ignore parameter,
-        # that can be used to ignore 404 errors.
-        LOG.info "Unable to delete non-existent index: #{name}."
-      end
    end
-    def self.get_index_shard_count(name)
+    def self.get_index
-      settings = Elasticsearch::Model.client.indices.get_settings(index: name)
+      CommentThread.tire.index
-      settings[name]['settings']['index']['number_of_shards']
    end
-    def self.move_alias(alias_name, index_name)
+    def self.get_index_shard_count(name)
-      actions = [
+      settings = Tire.index(name)
-          {add: {index: index_name, alias: alias_name}}
+      settings['index.number_of_shards']
-      ]
-      begin
-        response = Elasticsearch::Model.client.indices.get_alias(name: alias_name)
-        if response.length
-          actions.unshift({remove: {index: response.keys.join(','), alias: alias_name}})
-        end
-      rescue Elasticsearch::Transport::Transport::Errors::NotFound
-        # NOTE (CCB): Future versions of the Elasticsearch client support the ignore parameter,
-        # that can be used to ignore 404 errors.
-      end
-      body = {actions: actions}
-      Elasticsearch::Model.client.indices.update_aliases(body: body)
-      LOG.info "Alias [#{alias_name}] now points to index [#{index_name}]."
-    end
-    def self.refresh_index(name)
-      Elasticsearch::Model.client.indices.refresh(index: name)
    end
  end
 end
--- a/lib/tasks/db.rake
+++ b/lib/tasks/db.rake
@@ -91,6 +91,11 @@ namespace :db do
  end
  task :seed => [:environment, :clean] do
+    Tire.index 'comment_threads' do
+      delete
+    end
+    CommentThread.create_elasticsearch_index
    beginning_time = Time.now
    (1..10).map { |id| create_test_user(id) }

--- a/lib/tasks/search.rake
+++ b/lib/tasks/search.rake
 require 'task_helpers'
 namespace :search do
-  desc 'Indexes content updated in the last N minutes.'
+  def import_from_cursor(cursor, index, opts)
-  task :catchup, [:minutes] => :environment do |t, args|
+    tot = cursor.count
-    start_time = Time.now - (args[:minutes].to_i * 60)
+    cnt = 0
+    t = Time.now
+    index.import cursor, {:method => :paginate, :per_page => opts[:batch_size]} do |documents|
+      if cnt % opts[:batch_size] == 0 then
+        elapsed_secs = (Time.now - t).round(2)
+        pct_complete = (100 * (cnt/tot.to_f)).round(2)
+        LOG.info "#{index.name}: imported #{cnt} of #{tot} (#{pct_complete}% complete after #{elapsed_secs} seconds)"
+      end
+      cnt += documents.length
+      sleep opts[:sleep_time]
+      documents
+    end
+    LOG.info "#{index.name}: finished importing #{cnt} documents"
+    cnt
+  end
-    [Comment, CommentThread].each do |model|
+  def move_alias_to(name, index)
-      model.where(:updated_at.gte => start_time).import(index: Content::ES_INDEX_NAME)
+    # if there was a previous index, switch over the alias to point to the new index
+    alias_ = Tire::Alias.find name
+    if alias_
+      # does the alias already point to this index?
+      if alias_.indices.include? index.name
+        return false
+      end
+      # remove the alias from wherever it points to now
+      LOG.info "alias already exists (will move): #{alias_.indices.to_ary.join(',')}"
+      alias_.indices.each do |old_index_name|
+        alias_.indices.delete old_index_name unless old_index_name == name
+      end
+    else
+      # create the alias
+      LOG.info "alias \"#{name}\" does not yet exist - creating."
+      alias_ = Tire::Alias.new :name => name
    end
+    # point the alias at our new index
+    alias_.indices.add index.name
+    alias_.save
+    LOG.info "alias \"#{name}\" now points to index #{index.name}."
+    true
  end
-  desc 'Reindex all data from the database'
+  def do_reindex (opts, in_place=false)
-  task :reindex, [:index] => :environment do |t, args|
+    start_time = Time.now
-    args.with_defaults(:index => Content::ES_INDEX_NAME)
-    [Comment, CommentThread].each do |model|
+    # create the new index with a unique name
-      model.import(index: args[:index])
+    new_index = TaskHelpers::ElasticsearchHelper.create_index
+    # unless the user is forcing a rebuild, or the index does not yet exist, we
+    # can do a Tire api reindex which is much faster than reimporting documents
+    # from mongo.
+    #
+    # Checking if the index exists is tricky.  Tire automatically created an index
+    # for the model class when the app loaded if one did not already exist.  However,
+    # it won't create an alias, which is what our app uses.  So if the index exists
+    # but not the alias, we know that it's auto-created.
+    old_index = TaskHelpers::ElasticsearchHelper.get_index
+    alias_name = old_index.name
+    alias_ = Tire::Alias.find alias_name
+    if alias_.nil?
+      # edge case.
+      # the alias doesn't exist, so we know the index was auto-created.
+      # We will delete it and replace it with an alias.
+      raise RuntimeError, 'Cannot reindex in-place, no valid source index' if in_place
+      LOG.warn 'deleting auto-created index to make room for the alias'
+      old_index.delete
+      # NOTE on the small chance that another process re-auto-creates the index
+      # we just deleted before we have a chance to create the alias, this next
+      # call will fail.
+      move_alias_to(Content::ES_INDEX_NAME, new_index_name)
+    end
+    op = in_place ? 'reindex' : '(re)build index'
+    LOG.info "preparing to #{op}"
+    content_types = %w(Comment CommentThread)
+    if in_place
+      # reindex, moving source documents directly from old index to new
+      LOG.info 'copying documents from original index (this may take a while!)'
+      old_index.reindex new_index.name
+      LOG.info 'done copying!'
+    else
+      # fetch all the documents ever, up til start_time
+      cursor = Content.where(:_type.in => content_types, :updated_at.lte => start_time)
+      # import them to the new index
+      import_from_cursor(cursor, new_index, opts)
+    end
+    # move the alias if necessary
+    did_alias_move = move_alias_to(Content::ES_INDEX_NAME, new_index)
+    if did_alias_move
+      #  Reimport any source documents that got updated since start_time,
+      #  while the alias still pointed to the old index.
+      #  Elasticsearch understands our document ids, so re-indexing the same
+      #  document won't create duplicates.
+      LOG.info "importing any documents that changed between #{start_time} and now"
+      cursor = Content.where(:_type.in => content_types, :updated_at.gte => start_time)
+      import_from_cursor(cursor, new_index, opts)
    end
  end
-  desc 'Generate a new, empty physical index, without bringing it online.'
+  desc 'Copies contents of MongoDB into Elasticsearch if updated in the last N minutes.'
-  task :create_index => :environment do
+  task :catchup, [:minutes, :batch_size, :sleep_time] => :environment do |t, args|
-    TaskHelpers::ElasticsearchHelper.create_index
+    opts = batch_opts args
+    the_index = TaskHelpers::ElasticsearchHelper.get_index
+    alias_ = Tire::Alias.find the_index.name
+    # this check makes sure we are working with the index to which
+    # the desired model's alias presently points.
+    raise RuntimeError, "could not find live index" if alias_.nil?
+    start_time = Time.now - (args[:minutes].to_i * 60)
+    cursor = Content.where(:_type.in => %w(Comment CommentThread), :updated_at.gte => start_time)
+    import_from_cursor(cursor, the_index, opts)
+  end
+  def batch_opts(args)
+    args = args.to_hash
+    {:batch_size => args[:batch_size].nil? ? 500 : args[:batch_size].to_i,
+     :sleep_time => args[:sleep_time].nil? ? 0 : args[:sleep_time].to_i}
+  end
+  desc 'Removes any data from Elasticsearch that no longer exists in MongoDB.'
+  task :prune, [:batch_size, :sleep_time] => :environment do |t, args|
+    opts = batch_opts args
+    the_index = TaskHelpers::ElasticsearchHelper.get_index
+    puts "pruning #{the_index.name}"
+    alias_ = Tire::Alias.find the_index.name
+    raise RuntimeError, 'could not find live index' if alias_.nil?
+    scan_size = opts[:batch_size] / TaskHelpers::ElasticsearchHelper.get_index_shard_count(the_index.name)
+    cnt = 0
+    [CommentThread, Comment].each do |klass|
+      doc_type = klass.document_type
+      # this check makes sure we are working with the index to which
+      # the desired model's alias presently points.
+      search = Tire::Search::Scan.new the_index.name, {size: scan_size, type: doc_type}
+      search.each do |results|
+        es_ids = results.map(&:id)
+        mongo_ids = klass.where(:id.in => es_ids).map { |d| d.id.to_s }
+        to_delete = es_ids - mongo_ids
+        if to_delete.size > 0
+          cnt += to_delete.size
+          puts "deleting #{to_delete.size} orphaned #{doc_type} documents from elasticsearch"
+          the_index.bulk_delete (to_delete).map { |v| {"type" => doc_type, "id" => v} }
+        end
+        puts "#{the_index.name}/#{doc_type}: processed #{search.seen} of #{search.total}"
+        sleep opts[:sleep_time]
+      end
+    end
+    puts "done pruning #{the_index.name}, deleted a total of #{cnt} orphaned documents"
+  end
+  desc 'Rebuild the content index from MongoDB data.'
+  task :rebuild, [:batch_size, :sleep_time] => :environment do |t, args|
+    do_reindex(batch_opts(args))
  end
-  desc 'Creates a new search index and points the "content" alias to it'
+  desc 'Rebuild the content index from already-indexed data (in place).'
-  task :initialize => :environment do
+  task :reindex, [:batch_size, :sleep_time] => :environment do |t, args|
-    index = TaskHelpers::ElasticsearchHelper.create_index
+    do_reindex(batch_opts(args), true)
-    TaskHelpers::ElasticsearchHelper.move_alias(Content::ES_INDEX_NAME, index)
  end
-  desc 'Sets/moves an alias to the specified index'
+  desc 'Generate a new, empty physical index, without bringing it online.'
-  task :move_alias, [:alias, :index] => :environment do |t, args|
+  task :create_index => :environment do
-    TaskHelpers::ElasticsearchHelper.move_alias(args[:alias], args[:index])
+    TaskHelpers::ElasticsearchHelper.create_index
  end
 end
--- a/models/comment.rb
+++ b/models/comment.rb
-require_relative 'concerns/searchable'
 require_relative 'content'
 require_relative 'constants'
@@ -7,7 +6,8 @@ class Comment < Content
  include Mongoid::Timestamps
  include Mongoid::MagicCounterCache
  include ActiveModel::MassAssignmentSecurity
-  include Searchable
+  include Tire::Model::Search
+  include Tire::Model::Callbacks
  voteable self, :up => +1, :down => -1

--- a/models/comment_thread.rb
+++ b/models/comment_thread.rb
 require 'new_relic/agent/method_tracer'
-require_relative 'concerns/searchable'
 require_relative 'content'
 require_relative 'constants'
@@ -7,7 +6,8 @@ class CommentThread < Content
  include Mongoid::Timestamps
  include Mongoid::Attributes::Dynamic
  include ActiveModel::MassAssignmentSecurity
-  include Searchable
+  include Tire::Model::Search
+  include Tire::Model::Callbacks
  extend Enumerize
  voteable self, :up => +1, :down => -1
@@ -31,6 +31,7 @@ class CommentThread < Content
  index({author_id: 1, course_id: 1})
  index_name Content::ES_INDEX_NAME
  mapping do
@@ -39,8 +40,10 @@ class CommentThread < Content
    indexes :created_at, type: :date, included_in_all: false
    indexes :updated_at, type: :date, included_in_all: false
    indexes :last_activity_at, type: :date, included_in_all: false
    indexes :comment_count, type: :integer, included_in_all: false
    indexes :votes_point, type: :integer, as: 'votes_point', included_in_all: false
    indexes :context, type: :string, index: :not_analyzed, included_in_all: false
    indexes :course_id, type: :string, index: :not_analyzed, included_in_all: false
    indexes :commentable_id, type: :string, index: :not_analyzed, included_in_all: false

--- a/models/concerns/searchable.rb
+++ b/models/concerns/searchable.rb
-module Searchable
-  extend ActiveSupport::Concern
-  included do
-    include Elasticsearch::Model
-    # We specify our own callbacks, instead of using Elasticsearch::Model::Callbacks, so that we can disable
-    # indexing for tests where search functionality is not needed. This should improve test execution times.
-    after_create :index_document
-    after_update :update_indexed_document
-    after_destroy :delete_document
-    def self.put_search_index_mapping(index=nil)
-      index ||= self.index_name
-      success = self.__elasticsearch__.client.indices.put_mapping(index: index, type: self.document_type, body: self.mappings.to_hash)
-      unless success
-        logger.warn "WARNING! could not apply search index mapping for #{self.name}"
-      end
-    end
-    def as_indexed_json(options={})
-      # TODO: Play with the `MyModel.indexes` method -- reject non-mapped attributes, `:as` options, etc
-      self.as_json(options.merge root: false)
-    end
-    private
-    def index_document
-      __elasticsearch__.index_document if CommentService.search_enabled?
-    end
-    # This is named in this manner to prevent collisions with Mongoid's update_document method.
-    def update_indexed_document
-      __elasticsearch__.update_document if CommentService.search_enabled?
-    end
-    def delete_document
-      __elasticsearch__.delete_document if CommentService.search_enabled?
-    end
-  end
-end
--- a/models/content.rb
+++ b/models/content.rb
 class Content
  include Mongoid::Document
  include Mongo::Voteable
-  ES_INDEX_NAME = 'content'
  field :visible, type: Boolean, default: true
  field :abuse_flaggers, type: Array, default: []
  field :historical_abuse_flaggers, type: Array, default: [] #preserve abuse flaggers after a moderator unflags
@@ -18,6 +16,16 @@ class Content
  index({comment_thread_id: 1, endorsed: 1}, {sparse: true})
  index({commentable_id: 1}, {sparse: true, background: true})
+  ES_INDEX_NAME = 'content'
+  def self.put_search_index_mapping(idx=nil)
+    idx ||= self.tire.index
+    success = idx.mapping(self.tire.document_type, {:properties => self.tire.mapping})
+    unless success
+      logger.warn "WARNING! could not apply search index mapping for #{self.name}"
+    end
+  end
  before_save :set_username

--- a/spec/api/query_spec.rb
+++ b/spec/api/query_spec.rb
@@ -3,7 +3,6 @@ require 'faker'
 describe 'app' do
-  include_context 'search_enabled'
  before(:each) { set_api_key_header }
  let(:body) { Faker::Lorem.word }

--- a/spec/api/search_spec.rb
+++ b/spec/api/search_spec.rb
@@ -3,14 +3,15 @@ require 'unicode_shared_examples'
 describe "app" do
  describe "search" do
-    include_context 'search_enabled'
    before (:each) { set_api_key_header }
    let(:author) { create_test_user(42) }
    let(:course_id) { "test/course/id" }
    def get_result_ids(result)
-      result["collection"].map { |t| t["id"] }
+      result["collection"].map {|t| t["id"]}
    end
    describe "GET /api/v1/search/threads" do
@@ -20,17 +21,17 @@ describe "app" do
        result.should == {}
      end
-      it "returns an empty result if text parameter is missing" do
+      it "returns an empty reuslt if text parameter is missing" do
        get "/api/v1/search/threads", course_id: course_id
        assert_empty_response
      end
-      it "returns an empty result if sort key is invalid" do
+      it "returns an empty reuslt if sort key is invalid" do
        get "/api/v1/search/threads", course_id: course_id, text: "foobar", sort_key: "invalid", sort_order: "desc"
        assert_empty_response
      end
-      it "returns an empty result if sort order is invalid" do
+      it "returns an empty reuslt if sort order is invalid" do
        get "/api/v1/search/threads", course_id: course_id, text: "foobar", sort_key: "date", sort_order: "invalid"
        assert_empty_response
      end
@@ -68,13 +69,13 @@ describe "app" do
          last_response.should be_ok
          result = parse(last_response.body)
          actual_ids = Set.new get_result_ids(result)
-          expected_ids = Set.new expected_thread_indexes.map { |i| threads[i].id.to_s }
+          expected_ids = Set.new expected_thread_indexes.map {|i| threads[i].id.to_s}
          actual_ids.should == expected_ids
        end
        it "by course_id" do
          get "/api/v1/search/threads", text: "text", course_id: "test/course/id0"
-          assert_response_contains((0..29).find_all { |i| i % 2 == 0 })
+          assert_response_contains((0..29).find_all {|i| i % 2 == 0})
        end
        it "by context" do
@@ -86,7 +87,7 @@ describe "app" do
          user = create_test_user(Random.new)
          user.mark_as_read(threads[0])
          get "/api/v1/search/threads", text: "text", course_id: "test/course/id0", user_id: user.id, unread: true
-          assert_response_contains((1..29).find_all { |i| i % 2 == 0 })
+          assert_response_contains((1..29).find_all {|i| i % 2 == 0})
        end
        it "with flagged filter" do
@@ -120,22 +121,22 @@ describe "app" do
        it "by commentable_id" do
          get "/api/v1/search/threads", text: "text", commentable_id: "commentable0"
-          assert_response_contains((0..29).find_all { |i| i % 3 == 0 })
+          assert_response_contains((0..29).find_all {|i| i % 3 == 0})
        end
        it "by commentable_ids" do
          get "/api/v1/search/threads", text: "text", commentable_ids: "commentable0,commentable1"
-          assert_response_contains((0..29).find_all { |i| i % 3 == 0 || i % 3 == 1 })
+          assert_response_contains((0..29).find_all {|i| i % 3 == 0 || i % 3 == 1})
        end
        it "by group_id" do
          get "/api/v1/search/threads", text: "text", group_id: "1"
-          assert_response_contains((0..29).find_all { |i| i % 5 == 0 || i % 5 == 1 })
+          assert_response_contains((0..29).find_all {|i| i % 5 == 0 || i % 5 == 1})
        end
        it "by group_ids" do
          get "/api/v1/search/threads", text: "text", group_ids: "1,2"
-          expected_ids = (0..29).find_all { |i| i % 5 == 0 || i % 5 == 1 || i % 5 == 2 }
+          expected_ids = (0..29).find_all {|i| i % 5 == 0 || i % 5 == 1 || i % 5 == 2}
          assert_response_contains(expected_ids)
        end
@@ -147,8 +148,8 @@ describe "app" do
      describe "sorting works" do
        let!(:threads) do
-          threads = (0..5).map { |i| make_thread(author, "text", course_id, "dummy") }
+          threads = (0..5).map {|i| make_thread(author, "text", course_id, "dummy")}
-          [1, 2].map { |i| author.vote(threads[i], :up) }
+          [1, 2].map {|i| author.vote(threads[i], :up)}
          [1, 3].map do |i|
            threads[i].comment_count = 5
            threads[i].save!
@@ -163,7 +164,7 @@ describe "app" do
          last_response.should be_ok
          result = parse(last_response.body)
          actual_ids = get_result_ids(result)
-          expected_ids = expected_thread_indexes.map { |i| threads[i].id.to_s }
+          expected_ids = expected_thread_indexes.map {|i| threads[i].id.to_s}
          actual_ids.should == expected_ids
        end
@@ -196,7 +197,7 @@ describe "app" do
      describe "pagination" do
        let!(:threads) do
-          threads = (1..50).map { |i| make_thread(author, "text", course_id, "dummy") }
+          threads = (1..50).map {|i| make_thread(author, "text", course_id, "dummy")}
          refresh_es_index
          threads
        end
@@ -209,7 +210,7 @@ describe "app" do
            result = parse(last_response.body)
            result_ids += get_result_ids(result)
          end
-          result_ids.should == threads.reverse.map { |t| t.id.to_s }
+          result_ids.should == threads.reverse.map {|t| t.id.to_s}
        end
        it "works correctly with page size 1" do
@@ -226,7 +227,7 @@ describe "app" do
      end
      describe "spelling correction" do
-        let(:commentable_id) { "test_commentable" }
+        let(:commentable_id) {"test_commentable"}
        def check_correction(original_text, corrected_text)
          get "/api/v1/search/threads", text: original_text
@@ -291,8 +292,8 @@ describe "app" do
        end
      end
-      it 'returns the correct values for total_results and num_pages' do
+      it "returns the correct values for total_results and num_pages" do
-        course_id = 'test/course/id'
+        course_id = "test/course/id"
        for i in 1..100 do
          text = "all"
          text += " half" if i % 2 == 0
@@ -301,14 +302,15 @@ describe "app" do
          text += " one" if i == 100
          # There is currently a bug that causes only 10 threads with matching
          # titles/bodies to be considered, so this test case uses comments.
-          create(:comment, course_id: course_id, body: text)
+          thread = make_thread(author, "dummy text", course_id, "dummy_commentable")
+          make_comment(author, thread, text)
        end
        # Elasticsearch does not necessarily make newly indexed content
        # available immediately, so we must explicitly refresh the index
        refresh_es_index
        test_text = lambda do |text, expected_total_results, expected_num_pages|
-          get '/api/v1/search/threads', course_id: course_id, text: text, per_page: '10'
+          get "/api/v1/search/threads", course_id: course_id, text: text, per_page: "10"
          last_response.should be_ok
          result = parse(last_response.body)
          result["total_results"].should == expected_total_results

--- a/spec/app_spec.rb
+++ b/spec/app_spec.rb
--- a/spec/spec_helper.rb
+++ b/spec/spec_helper.rb
@@ -16,9 +16,6 @@ require 'yajl'
 require 'support/database_cleaner'
 require 'support/elasticsearch'
 require 'support/factory_girl'
-require 'webmock/rspec'
-WebMock.allow_net_connect!
 # setup test environment
 set :environment, :test

--- a/spec/support/elasticsearch.rb
+++ b/spec/support/elasticsearch.rb
-require 'task_helpers'
+def delete_es_index
+  Tire.index Content::ES_INDEX_NAME do
-def refresh_es_index
+    delete
-  TaskHelpers::ElasticsearchHelper.refresh_index(Content::ES_INDEX_NAME)
-end
-RSpec.shared_context 'search_enabled' do
-  before(:all) do
-    CommentService.config[:enable_search] = true
  end
+end
-  before(:each) do
+def create_es_index
-    index = TaskHelpers::ElasticsearchHelper.create_index
+  new_index = Tire.index Content::ES_INDEX_NAME
-    TaskHelpers::ElasticsearchHelper.move_alias(Content::ES_INDEX_NAME, index)
+  new_index.create
+  [CommentThread, Comment].each do |klass|
+    klass.put_search_index_mapping
  end
+end
-  after(:each) do
+def refresh_es_index
-    TaskHelpers::ElasticsearchHelper.delete_index(Content::ES_INDEX_NAME)
+  es_index_name = Content::ES_INDEX_NAME
+  Tire.index es_index_name do
+    refresh
  end
 end
 RSpec.configure do |config|
-  config.before(:suite) do
+  config.before(:each) do
-    CommentService.config[:enable_search] = false
+    delete_es_index
+    create_es_index
  end
 end