Replaced Tire with elasticsearch-model

- Replaced Tire with elasticsearch-model - Updated README - Added task to initialize a new index

Replaced Tire with elasticsearch-model
- Replaced Tire with elasticsearch-model - Updated README - Added task to initialize a new index
56b61a13 · Clinton Blackburn · 26adedc2 · 56b61a13 · 56b61a13 · 56b61a13
Commit 56b61a13 authored Jan 04, 2016 by Clinton Blackburn
22 changed files
--- a/.travis.yml
+++ b/.travis.yml
@@ -23,4 +23,6 @@ before_install:
  - mkdir -p ./mongo/log
  - mongod --fork --dbpath ./mongo/db --logpath ./mongo/log/mongodb.log
-script: bundle exec rspec
+script:
+  - bin/rake search:initialize
+  - bin/rspec
--- a/Gemfile
+++ b/Gemfile
@@ -31,8 +31,8 @@ gem 'will_paginate_mongoid', "~>2.0"
 gem 'rdiscount'
 gem 'nokogiri', "~>1.6.7.2"
-gem 'tire', "0.6.2"
+gem 'elasticsearch', '~> 0.4'
-gem 'tire-contrib'
+gem 'elasticsearch-model', '~> 0.1.8'
 gem 'dalli'
@@ -47,6 +47,7 @@ group :test do
  gem 'guard-unicorn'
  gem 'rack-test', :require => 'rack/test'
  gem 'rspec', '~> 2.11.0'
+  gem 'webmock', '~> 1.22'
 end
 gem 'newrelic_rpm'

--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -32,7 +32,7 @@ GEM
      minitest (~> 5.1)
      thread_safe (~> 0.3, >= 0.3.4)
      tzinfo (~> 1.1)
-    ansi (1.5.0)
+    addressable (2.4.0)
    bson (3.2.4)
    bson_ext (1.5.1)
    builder (3.2.2)
@@ -41,6 +41,8 @@ GEM
      simplecov
      url
    coderay (1.0.7)
+    crack (0.4.3)
+      safe_yaml (~> 1.0.0)
    dalli (2.1.0)
    database_cleaner (1.5.1)
    delayed_job (4.1.1)
@@ -53,18 +55,33 @@ GEM
    docile (1.1.5)
    domain_name (0.5.24)
      unf (>= 0.0.5, < 1.0.0)
+    elasticsearch (0.4.11)
+      elasticsearch-api (= 0.4.11)
+      elasticsearch-transport (= 0.4.11)
+    elasticsearch-api (0.4.11)
+      multi_json
+    elasticsearch-model (0.1.8)
+      activesupport (> 3)
+      elasticsearch (> 0.4)
+      hashie
+    elasticsearch-transport (0.4.11)
+      faraday
+      multi_json
    enumerize (0.11.0)
      activesupport (>= 3.2)
    factory_girl (4.5.0)
      activesupport (>= 3.0.0)
    faker (1.6.1)
      i18n (~> 0.5)
+    faraday (0.9.2)
+      multipart-post (>= 1.2, < 3)
    guard (1.3.2)
      listen (>= 0.4.2)
      thor (>= 0.14.6)
    guard-unicorn (0.0.7)
      guard (>= 1.1)
-    hashr (0.0.22)
+    hashdiff (0.2.3)
+    hashie (3.4.3)
    http-cookie (1.0.2)
      domain_name (~> 0.5)
    i18n (0.7.0)
@@ -89,6 +106,7 @@ GEM
      mongoid
      rake
    multi_json (1.11.2)
+    multipart-post (2.0.0)
    netrc (0.10.3)
    newrelic_rpm (3.15.0.314)
    nokogiri (1.6.7.2)
@@ -123,6 +141,7 @@ GEM
    rspec-expectations (2.11.2)
      diff-lcs (~> 1.1.3)
    rspec-mocks (2.11.2)
+    safe_yaml (1.0.4)
    simplecov (0.11.1)
      docile (~> 1.1.0)
      json (~> 1.8)
@@ -136,16 +155,6 @@ GEM
    thor (0.16.0)
    thread_safe (0.3.5)
    tilt (1.3.3)
-    tire (0.6.2)
-      activemodel (>= 3.0)
-      activesupport
-      ansi
-      hashr (~> 0.0.19)
-      multi_json (~> 1.3)
-      rake
-      rest-client (~> 1.6)
-    tire-contrib (0.1.1)
-      tire
    tzinfo (1.2.2)
      thread_safe (~> 0.1)
    unf (0.1.4)
@@ -156,6 +165,10 @@ GEM
      rack
      raindrops (~> 0.7)
    url (0.3.2)
+    webmock (1.22.3)
+      addressable (>= 2.3.6)
+      crack (>= 0.3.2)
+      hashdiff
    will_paginate (3.0.7)
    will_paginate_mongoid (2.0.1)
      mongoid
@@ -174,6 +187,8 @@ DEPENDENCIES
  database_cleaner (~> 1.5.1)
  delayed_job
  delayed_job_mongoid
+  elasticsearch (~> 0.4)
+  elasticsearch-model (~> 0.1.8)
  enumerize
  factory_girl (~> 4.0)
  faker (~> 1.6)
@@ -197,9 +212,8 @@ DEPENDENCIES
  rs_voteable_mongo!
  rspec (~> 2.11.0)
  sinatra
-  tire (= 0.6.2)
-  tire-contrib
  unicorn
+  webmock (~> 1.22)
  will_paginate_mongoid (~> 2.0)
  yajl-ruby

--- a/README.rst
+++ b/README.rst
@@ -14,9 +14,8 @@ An independent comment system which supports voting and nested comments. It
 also supports features including instructor endorsement for education-aimed
 discussion platforms.
+Getting Started
-Running the Server
+---------------
------------------
 If you are running cs_comments_service as part of edx-platform__ development under
 devstack, it is strongly recommended to read `those setup documents`__ first.  Note that
 devstack will take care of just about all of the installation, configuration, and
@@ -30,15 +29,49 @@ This service relies on Elasticsearch and MongoDB. By default the service will us
 however, if you wish to change these values, refer to `config/application.yml` and `config/mongoid.yml` for the
 environment variables that can be set to override the defaults.
-Before the server is first run, ensure gems are installed by doing ``bundle install``.
+Install the requisite gems:
+.. code-block:: bash
+    $ bundle install
+Setup the search index. Note that the command below creates an alias with a unique name (e.g. content_20160101), and
+assigns it a known alias: content. If you choose not to use the command below, you should still opt to reference your
+index by an alias rather than the actual index name. This will enable you to swap out indices (e.g. reindex) without
+having to take downtime or modify code with a new index name.
+.. code-block:: bash
+    $ bin/rake search:initialize
+Run ther server:
+.. code-block::
+    $ ruby app.rb
-To run the server, do ``ruby app.rb [-p PORT]`` where PORT defaults to 4567.
+By default Sinatra runs on port `4567`. If you'd like to use a different port pass the `-p` parameter:
+.. code-block::
+    $ ruby app.rb -p 5678
 Running Tests
 -------------
-To run tests, do ``bundle exec rspec``.  Append ``--help`` or see rspec documentation
+Tests are built using the rspec__ framework, and can be run with the command below:
-for additional options to this command.
+.. code-block::
+    $ bin/rspec
+If you'd like to view additional options for the command, append the `--help` option:
+.. code-block::
+    $ bin/rspec --help
+__ http://rspec.info/
 Internationalization (i18n) and Localization (l10n)
 ---------------------------------------------------
@@ -62,12 +95,12 @@ follow the instructions here__ to set up your ``.transifexrc`` file.
 __ http://support.transifex.com/customer/portal/articles/1000855-configuring-the-client
 To upload strings to Transifex for translation when you change the set
-of translatable strings: ``bundle exec rake i18n:push``
+of translatable strings: ``bin/rake i18n:push``
-To fetch the latest translations from Transifex: ``bundle exec rake i18n:pull``
+To fetch the latest translations from Transifex: ``bin/rake i18n:pull``
 The repository includes some translations so they will be available
-upon deployment. To commit an update to these: ``bundle exec rake i18n:commit``
+upon deployment. To commit an update to these: ``bin/rake i18n:commit``
 License
 -------

--- a/Rakefile
+++ b/Rakefile
@@ -17,21 +17,23 @@ rescue LoadError
  # no rspec available
 end
-Tire.configure do
-  url YAML.load(application_yaml)['elasticsearch_server']
-end
 LOG = Logger.new(STDERR)
 desc 'Load the environment'
 task :environment do
  environment = ENV['SINATRA_ENV'] || 'development'
  Sinatra::Base.environment = environment
  Mongoid.load!('config/mongoid.yml')
  Mongoid.logger.level = Logger::INFO
  module CommentService
    class << self;
-      attr_accessor :config;
+      attr_accessor :config
+      def search_enabled?
+        self.config[:enable_search]
+      end
    end
  end
@@ -41,8 +43,8 @@ task :environment do
  Dir[File.dirname(__FILE__) + '/models/*.rb'].each { |file| require file }
 end
-Dir.glob('lib/tasks/*.rake').each { |r| import r }
 task :console => :environment do
  binding.pry
 end
+Dir.glob('lib/tasks/*.rake').each { |r| import r }
--- a/api/comment_threads.rb
+++ b/api/comment_threads.rb
 get "#{APIPREFIX}/threads" do # retrieve threads by course
  threads = Content.where({"_type" => "CommentThread", "course_id" => params["course_id"]})
  if params[:commentable_ids]
    threads = threads.in({"commentable_id" => params[:commentable_ids].split(",")})
  end
  handle_threads_query(
-    threads,
+      threads,
-    params["user_id"],
+      params['user_id'],
-    params["course_id"],
+      params['course_id'],
-    get_group_ids_from_params(params),
+      get_group_ids_from_params(params),
-    value_to_boolean(params["flagged"]),
+      value_to_boolean(params['flagged']),
-    value_to_boolean(params["unread"]),
+      value_to_boolean(params['unread']),
-    value_to_boolean(params["unanswered"]),
+      value_to_boolean(params['unanswered']),
-    params["sort_key"],
+      params['sort_key'],
-    params["sort_order"],
+      params['sort_order'],
-    params["page"],
+      params['page'],
-    params["per_page"]
+      params['per_page']
  ).to_json
 end
@@ -89,6 +89,12 @@ post "#{APIPREFIX}/threads/:thread_id/comments" do |thread_id|
 end
 delete "#{APIPREFIX}/threads/:thread_id" do |thread_id|
-  thread.destroy
+  begin
+    thread.destroy
+  rescue Elasticsearch::Transport::Transport::Errors::NotFound
+    # If the thread is not in the index, that's actually a good thing given that we just removed it.
+    # Note that this exception will probably only be encountered for tests that don't wait for the index
+    # to be refreshed before attempting to destroy a newly-recreated thread.
+  end
  thread.to_hash.to_json
 end
--- a/api/comments.rb
+++ b/api/comments.rb
@@ -60,5 +60,4 @@ delete "#{APIPREFIX}/comments/:comment_id" do |comment_id|
      pass
    end
  end
-  comment.to_hash.to_json
 end
--- a/api/search.rb
+++ b/api/search.rb
-get "#{APIPREFIX}/search/threads" do
+def get_thread_ids(context, group_ids, local_params, search_text)
-  local_params = params # Necessary for params to be available inside blocks
+  filters = []
-  group_ids = get_group_ids_from_params(local_params)
+  filters.push({term: {commentable_id: local_params['commentable_id']}}) if local_params['commentable_id']
-  context = local_params["context"] ? local_params["context"] : "course"
+  filters.push({terms: {commentable_id: local_params['commentable_ids'].split(',')}}) if local_params['commentable_ids']
-  search_text = local_params["text"]
+  filters.push({term: {course_id: local_params['course_id']}}) if local_params['course_id']
-  if !search_text
-    {}.to_json
+  filters.push({or: [
-  else
+      {not: {exists: {field: :context}}},
-    # Because threads and comments are currently separate unrelated documents in
+      {term: {context: context}}
-    # Elasticsearch, we must first query for all matching documents, then
+  ]})
-    # extract the set of thread ids, and then sort the threads by the specified
-    # criteria and paginate. For performance reasons, we currently limit the
-    # number of documents considered (ordered by update recency), which means
-    # that matching threads can be missed if the search terms are very common.
-    get_matching_thread_ids = lambda do |search_text|
+  unless group_ids.empty?
-      self.class.trace_execution_scoped(["Custom/get_search_threads/es_search"]) do
+    filters.push(
-        search = Tire.search Content::ES_INDEX_NAME do
+        {
-          query do
+            or: [
-            match [:title, :body], search_text, :operator => "AND"
+                {:not => {:exists => {:field => :group_id}}},
-            filtered do
+                {:terms => {:group_id => group_ids}}
-              filter :term, :commentable_id => local_params["commentable_id"] if local_params["commentable_id"]
+            ]
-              filter :terms, :commentable_id => local_params["commentable_ids"].split(",") if local_params["commentable_ids"]
+        }
-              filter :term, :course_id => local_params["course_id"] if local_params["course_id"]
+    )
-              filter :or, [
+  end
-                {:not => {:exists => {:field => :context}}},
-                {:term => {:context => context}}
-              ]
-              if not group_ids.empty?
+  self.class.trace_execution_scoped(['Custom/get_search_threads/es_search']) do
-                if group_ids.length > 1
+    body = {
-                  group_id_criteria = {:terms => {:group_id => group_ids}}
+        size: CommentService.config['max_deep_search_comment_count'].to_i,
-                else
+        sort: [
-                  group_id_criteria = {:term => {:group_id => group_ids[0]}}
+            {updated_at: :desc}
-                end
+        ],
+        query: {
+            multi_match: {
+                query: search_text,
+                fields: [:title, :body],
+                operator: :AND
+            },
+            filtered: {
+                filter: {
+                    and: filters
+                }
+            }
+        }
+    }
-                filter :or, [
+    response = Elasticsearch::Model.client.search(index: Content::ES_INDEX_NAME, body: body)
-                  {:not => {:exists => {:field => :group_id}}},
-                  group_id_criteria
-                ]
-              end
-            end
+    thread_ids = Set.new
-          end
+    response['hits']['hits'].each do |hit|
-          sort do
+      case hit['_type']
-            by "updated_at", "desc"
+        when CommentThread.document_type
-          end
+          thread_ids.add(hit['_id'])
-          size CommentService.config["max_deep_search_comment_count"].to_i
+        when Comment.document_type
-        end
+          thread_ids.add(hit['_source']['comment_thread_id'])
-        thread_ids = Set.new
+        else
-        search.results.each do |content|
+          # There shouldn't be any other document types. Nevertheless, ignore them, if they are present.
-          case content.type
+          next
-          when "comment_thread"
-            thread_ids.add(content.id)
-          when "comment"
-            thread_ids.add(content.comment_thread_id)
-          end
-        end
-        thread_ids
      end
    end
+    thread_ids
+  end
+end
+def get_suggested_text(search_text)
+  body = {
+      suggestions: {
+          text: search_text,
+          phrase: {
+              field: :_all
+          }
+      }
+  }
+  response = Elasticsearch::Model.client.suggest(index: Content::ES_INDEX_NAME, body: body)
+  suggestions = response.fetch('suggestions', [])
+  if suggestions.length > 0
+    options = suggestions[0]['options']
+    if options.length > 0
+      return options[0]['text']
+    end
+  end
+  nil
+end
+def get_threads(context, group_ids, local_params, search_text)
+  # Because threads and comments are currently separate unrelated documents in
+  # Elasticsearch, we must first query for all matching documents, then
+  # extract the set of thread ids, and then sort the threads by the specified
+  # criteria and paginate. For performance reasons, we currently limit the
+  # number of documents considered (ordered by update recency), which means
+  # that matching threads can be missed if the search terms are very common.
+  thread_ids = get_thread_ids(context, group_ids, local_params, search_text)
+  corrected_text = nil
+  if thread_ids.empty?
    # Sadly, Elasticsearch does not have a facility for computing suggestions
    # with respect to a filter. It would be expensive to determine the best
    # suggestion with respect to our filter parameters, so we simply re-query
    # with the top suggestion. If that has no results, then we return no results
    # and no correction.
-    thread_ids = get_matching_thread_ids.call(search_text)
+    corrected_text = get_suggested_text(search_text)
-    corrected_text = nil
+    thread_ids = get_thread_ids(context, group_ids, local_params, corrected_text) if corrected_text
-    if thread_ids.empty?
+    corrected_text = nil if thread_ids.empty?
-      suggest = Tire.suggest Content::ES_INDEX_NAME do
+  end
-        suggestion "" do
-          text search_text
-          phrase :_all
-        end
-      end
-      corrected_text = suggest.results.texts.first
-      thread_ids = get_matching_thread_ids.call(corrected_text) if corrected_text
-      corrected_text = nil if thread_ids.empty?
-    end
-    result_obj = handle_threads_query(
+  result_obj = handle_threads_query(
-      CommentThread.in({"_id" => thread_ids.to_a}),
+      CommentThread.in({_id: thread_ids.to_a}),
-      local_params["user_id"],
+      local_params['user_id'],
-      local_params["course_id"],
+      local_params['course_id'],
      group_ids,
-      value_to_boolean(local_params["flagged"]),
+      value_to_boolean(local_params['flagged']),
-      value_to_boolean(local_params["unread"]),
+      value_to_boolean(local_params['unread']),
-      value_to_boolean(local_params["unanswered"]),
+      value_to_boolean(local_params['unanswered']),
-      local_params["sort_key"],
+      local_params['sort_key'],
-      local_params["sort_order"],
+      local_params['sort_order'],
-      local_params["page"],
+      local_params['page'],
-      local_params["per_page"],
+      local_params['per_page'],
      context
-    )
+  )
-    if !result_obj.empty?
-      result_obj[:corrected_text] = corrected_text
+  unless result_obj.empty?
-      # NOTE this reflects the total results from ES, but does not consider
+    result_obj[:corrected_text] = corrected_text
-      # any post-filtering that might happen (e.g. unread, flagged...) before
+    # NOTE this reflects the total results from ES, but does not consider
-      # results are shown to the user.
+    # any post-filtering that might happen (e.g. unread, flagged...) before
-      result_obj[:total_results] = thread_ids.size
+    # results are shown to the user.
-    end
+    result_obj[:total_results] = thread_ids.size
-    result_obj.to_json
+  end
+  result_obj.to_json
+end
+get "#{APIPREFIX}/search/threads" do
+  local_params = params # Necessary for params to be available inside blocks
+  group_ids = get_group_ids_from_params(local_params)
+  context = local_params["context"] ? local_params["context"] : "course"
+  search_text = local_params["text"]
+  if !search_text
+    '{}'
+  else
+    get_threads(context, group_ids, local_params, search_text)
  end
 end
--- a/app.rb
+++ b/app.rb
@@ -14,6 +14,10 @@ module CommentService
  class << self
    attr_accessor :config
    attr_accessor :blocked_hashes
+    def search_enabled?
+      self.config[:enable_search]
+    end
  end
  API_VERSION = 'v1'
  API_PREFIX = "/api/#{API_VERSION}"
@@ -26,11 +30,6 @@ end
 application_yaml = ERB.new(File.read("config/application.yml")).result()
 CommentService.config = YAML.load(application_yaml).with_indifferent_access
-Tire.configure do
-  url CommentService.config[:elasticsearch_server]
-  logger STDERR if ENV["ENABLE_ELASTICSEARCH_DEBUGGING"]
-end
 Mongoid.load!("config/mongoid.yml", environment)
 Mongoid.logger.level = Logger::INFO
 Mongo::Logger.logger.level = ENV["ENABLE_MONGO_DEBUGGING"] ? Logger::DEBUG : Logger::INFO
@@ -48,11 +47,13 @@ helpers do
  end
 end
-Dir[File.dirname(__FILE__) + '/lib/**/*.rb'].each {|file| require file}
+Dir[File.dirname(__FILE__) + '/lib/**/*.rb'].each { |file| require file }
-Dir[File.dirname(__FILE__) + '/models/*.rb'].each {|file| require file}
+Dir[File.dirname(__FILE__) + '/models/*.rb'].each { |file| require file }
-Dir[File.dirname(__FILE__) + '/presenters/*.rb'].each {|file| require file}
+Dir[File.dirname(__FILE__) + '/presenters/*.rb'].each { |file| require file }
-# Ensure elasticsearch index mappings exist.
+Elasticsearch::Model.client = Elasticsearch::Client.new(host: CommentService.config[:elasticsearch_server], log: false)
+# Ensure Elasticsearch index mappings exist.
 Comment.put_search_index_mapping
 CommentThread.put_search_index_mapping
@@ -106,7 +107,6 @@ class Time
 end
 # these files must be required in order
 require './api/search'
 require './api/commentables'
@@ -138,55 +138,61 @@ error ArgumentError do
  error 400, [env['sinatra.error'].message].to_json
 end
-CommentService.blocked_hashes = Content.mongo_client[:blocked_hash].find(nil, projection: {hash: 1}).map {|d| d["hash"]}
+CommentService.blocked_hashes = Content.mongo_client[:blocked_hash].find(nil, projection: {hash: 1}).map { |d| d["hash"] }
 def get_db_is_master
  Mongoid::Clients.default.command(isMaster: 1)
 end
-def get_es_status
+def elasticsearch_health
-  res = Tire::Configuration.client.get Tire::Configuration.url
+  Elasticsearch::Model.client.cluster.health
-  JSON.parse res.body
 end
-get '/heartbeat' do
-  # mongo is reachable and ready to handle requests
+def is_mongo_available?
-  db_ok = false
  begin
-    res = get_db_is_master
+    response = get_db_is_master
-    db_ok = res.ok? && res.documents.first['ismaster'] == true
+    return response.ok? && (response.documents.first['ismaster'] == true)
  rescue
+    # ignored
  end
-  error 500, JSON.generate({"OK" => false, "check" => "db"}) unless db_ok
-  # E_S is reachable and ready to handle requests
+  false
-  es_ok = false
+end
+def is_elasticsearch_available?
  begin
-    es_status = get_es_status
+    health = elasticsearch_health
-    es_ok = es_status["status"] == 200
+    return !health['timed_out'] && %w(yellow green).include?(health['status'])
  rescue
+    # ignored
  end
-  error 500, JSON.generate({"OK" => false, "check" => "es"}) unless es_ok
-  JSON.generate({"OK" => true})
+  false
+end
+get '/heartbeat' do
+  error 500, JSON.generate({OK: false, check: :db}) unless is_mongo_available?
+  error 500, JSON.generate({OK: false, check: :es}) unless is_elasticsearch_available?
+  JSON.generate({OK: true})
 end
 get '/selftest' do
  begin
    t1 = Time.now
    status = {
-      "db" => get_db_is_master,
+        db: get_db_is_master,
-      "es" => get_es_status,
+        es: elasticsearch_health,
-      "last_post_created" => (Content.last.created_at rescue nil),
+        last_post_created: (Content.last.created_at rescue nil),
-      "total_posts" => Content.count,
+        total_posts: Content.count,
-      "total_users" => User.count,
+        total_users: User.count,
-      "elapsed_time" => Time.now - t1
+        elapsed_time: Time.now - t1
    }
    JSON.generate(status)
  rescue => ex
-    [ 500,
+    [500,
-      {'Content-Type' => 'text/plain'},
+     {'Content-Type' => 'text/plain'},
-      "#{ex.backtrace.first}: #{ex.message} (#{ex.class})\n\t#{ex.backtrace[1..-1].join("\n\t")}"
+     "#{ex.backtrace.first}: #{ex.message} (#{ex.class})\n\t#{ex.backtrace[1..-1].join("\n\t")}"
    ]
  end
 end
--- a/config/application.yml
+++ b/config/application.yml
@@ -4,3 +4,4 @@ elasticsearch_server: <%= ENV['SEARCH_SERVER'] || 'http://localhost:9200' %>
 max_deep_search_comment_count: 5000
 default_locale: <%= ENV['SERVICE_LANGUAGE'] || 'en-US' %>
 manual_pagination_batch_size: <%= ENV['MANUAL_PAGINATION_BATCH_SIZE'] || 500 %>
+enable_search: true
--- a/lib/task_helpers.rb
+++ b/lib/task_helpers.rb
+require 'elasticsearch'
 module TaskHelpers
  module ElasticsearchHelper
+    LOG = Logger.new(STDERR)
    def self.create_index(name=nil)
      name ||= "#{Content::ES_INDEX_NAME}_#{Time.now.strftime('%Y%m%d%H%M%S')}"
-      index = Tire.index(name)
-      LOG.info "Creating new index: #{name}..."
-      index.create
-      [CommentThread, Comment].each do |model|
+      mappings = {}
-        LOG.info "Applying index mappings for #{model.name}"
+      [Comment, CommentThread].each do |model|
-        model.put_search_index_mapping(index)
+        mappings.merge! model.mappings.to_hash
      end
-      LOG.info '...done!'
-      index
+      Elasticsearch::Model.client.indices.create(index: name, body: {mappings: mappings})
+      LOG.info "Created new index: #{name}."
+      name
    end
    def self.delete_index(name)
-      Tire.index(name).delete
+      begin
+        Elasticsearch::Model.client.indices.delete(index: name)
+        LOG.info "Deleted index: #{name}."
+      rescue Elasticsearch::Transport::Transport::Errors::NotFound
+        # NOTE (CCB): Future versions of the Elasticsearch client support the ignore parameter,
+        # that can be used to ignore 404 errors.
+        LOG.info "Unable to delete non-existent index: #{name}."
+      end
    end
-    def self.get_index
+    def self.get_index_shard_count(name)
-      CommentThread.tire.index
+      settings = Elasticsearch::Model.client.indices.get_settings(index: name)
+      settings[name]['settings']['index']['number_of_shards']
    end
-    def self.get_index_shard_count(name)
+    def self.move_alias(alias_name, index_name)
-      settings = Tire.index(name)
+      actions = [
-      settings['index.number_of_shards']
+          {add: {index: index_name, alias: alias_name}}
+      ]
+      begin
+        response = Elasticsearch::Model.client.indices.get_alias(name: alias_name)
+        if response.length
+          actions.unshift({remove: {index: response.keys.join(','), alias: alias_name}})
+        end
+      rescue Elasticsearch::Transport::Transport::Errors::NotFound
+        # NOTE (CCB): Future versions of the Elasticsearch client support the ignore parameter,
+        # that can be used to ignore 404 errors.
+      end
+      body = {actions: actions}
+      Elasticsearch::Model.client.indices.update_aliases(body: body)
+      LOG.info "Alias [#{alias_name}] now points to index [#{index_name}]."
+    end
+    def self.refresh_index(name)
+      Elasticsearch::Model.client.indices.refresh(index: name)
    end
  end
 end
--- a/lib/tasks/db.rake
+++ b/lib/tasks/db.rake
@@ -91,11 +91,6 @@ namespace :db do
  end
  task :seed => [:environment, :clean] do
-    Tire.index 'comment_threads' do
-      delete
-    end
-    CommentThread.create_elasticsearch_index
    beginning_time = Time.now
    (1..10).map { |id| create_test_user(id) }

--- a/lib/tasks/search.rake
+++ b/lib/tasks/search.rake
 require 'task_helpers'
 namespace :search do
-  def import_from_cursor(cursor, index, opts)
+  desc 'Indexes content updated in the last N minutes.'
-    tot = cursor.count
+  task :catchup, [:minutes] => :environment do |t, args|
-    cnt = 0
-    t = Time.now
-    index.import cursor, {:method => :paginate, :per_page => opts[:batch_size]} do |documents|
-      if cnt % opts[:batch_size] == 0 then
-        elapsed_secs = (Time.now - t).round(2)
-        pct_complete = (100 * (cnt/tot.to_f)).round(2)
-        LOG.info "#{index.name}: imported #{cnt} of #{tot} (#{pct_complete}% complete after #{elapsed_secs} seconds)"
-      end
-      cnt += documents.length
-      sleep opts[:sleep_time]
-      documents
-    end
-    LOG.info "#{index.name}: finished importing #{cnt} documents"
-    cnt
-  end
-  def move_alias_to(name, index)
-    # if there was a previous index, switch over the alias to point to the new index
-    alias_ = Tire::Alias.find name
-    if alias_
-      # does the alias already point to this index?
-      if alias_.indices.include? index.name
-        return false
-      end
-      # remove the alias from wherever it points to now
-      LOG.info "alias already exists (will move): #{alias_.indices.to_ary.join(',')}"
-      alias_.indices.each do |old_index_name|
-        alias_.indices.delete old_index_name unless old_index_name == name
-      end
-    else
-      # create the alias
-      LOG.info "alias \"#{name}\" does not yet exist - creating."
-      alias_ = Tire::Alias.new :name => name
-    end
-    # point the alias at our new index
-    alias_.indices.add index.name
-    alias_.save
-    LOG.info "alias \"#{name}\" now points to index #{index.name}."
-    true
-  end
-  def do_reindex (opts, in_place=false)
-    start_time = Time.now
-    # create the new index with a unique name
-    new_index = TaskHelpers::ElasticsearchHelper.create_index
-    # unless the user is forcing a rebuild, or the index does not yet exist, we
-    # can do a Tire api reindex which is much faster than reimporting documents
-    # from mongo.
-    #
-    # Checking if the index exists is tricky.  Tire automatically created an index
-    # for the model class when the app loaded if one did not already exist.  However,
-    # it won't create an alias, which is what our app uses.  So if the index exists
-    # but not the alias, we know that it's auto-created.
-    old_index = TaskHelpers::ElasticsearchHelper.get_index
-    alias_name = old_index.name
-    alias_ = Tire::Alias.find alias_name
-    if alias_.nil?
-      # edge case.
-      # the alias doesn't exist, so we know the index was auto-created.
-      # We will delete it and replace it with an alias.
-      raise RuntimeError, 'Cannot reindex in-place, no valid source index' if in_place
-      LOG.warn 'deleting auto-created index to make room for the alias'
-      old_index.delete
-      # NOTE on the small chance that another process re-auto-creates the index
-      # we just deleted before we have a chance to create the alias, this next
-      # call will fail.
-      move_alias_to(Content::ES_INDEX_NAME, new_index_name)
-    end
-    op = in_place ? 'reindex' : '(re)build index'
-    LOG.info "preparing to #{op}"
-    content_types = %w(Comment CommentThread)
-    if in_place
-      # reindex, moving source documents directly from old index to new
-      LOG.info 'copying documents from original index (this may take a while!)'
-      old_index.reindex new_index.name
-      LOG.info 'done copying!'
-    else
-      # fetch all the documents ever, up til start_time
-      cursor = Content.where(:_type.in => content_types, :updated_at.lte => start_time)
-      # import them to the new index
-      import_from_cursor(cursor, new_index, opts)
-    end
-    # move the alias if necessary
-    did_alias_move = move_alias_to(Content::ES_INDEX_NAME, new_index)
-    if did_alias_move
-      #  Reimport any source documents that got updated since start_time,
-      #  while the alias still pointed to the old index.
-      #  Elasticsearch understands our document ids, so re-indexing the same
-      #  document won't create duplicates.
-      LOG.info "importing any documents that changed between #{start_time} and now"
-      cursor = Content.where(:_type.in => content_types, :updated_at.gte => start_time)
-      import_from_cursor(cursor, new_index, opts)
-    end
-  end
-  desc 'Copies contents of MongoDB into Elasticsearch if updated in the last N minutes.'
-  task :catchup, [:minutes, :batch_size, :sleep_time] => :environment do |t, args|
-    opts = batch_opts args
-    the_index = TaskHelpers::ElasticsearchHelper.get_index
-    alias_ = Tire::Alias.find the_index.name
-    # this check makes sure we are working with the index to which
-    # the desired model's alias presently points.
-    raise RuntimeError, "could not find live index" if alias_.nil?
    start_time = Time.now - (args[:minutes].to_i * 60)
-    cursor = Content.where(:_type.in => %w(Comment CommentThread), :updated_at.gte => start_time)
-    import_from_cursor(cursor, the_index, opts)
-  end
-  def batch_opts(args)
+    [Comment, CommentThread].each do |model|
-    args = args.to_hash
+      model.where(:updated_at.gte => start_time).import(index: Content::ES_INDEX_NAME)
-    {:batch_size => args[:batch_size].nil? ? 500 : args[:batch_size].to_i,
+    end
-     :sleep_time => args[:sleep_time].nil? ? 0 : args[:sleep_time].to_i}
  end
-  desc 'Removes any data from Elasticsearch that no longer exists in MongoDB.'
+  desc 'Reindex all data from the database'
-  task :prune, [:batch_size, :sleep_time] => :environment do |t, args|
+  task :reindex, [:index] => :environment do |t, args|
-    opts = batch_opts args
+    args.with_defaults(:index => Content::ES_INDEX_NAME)
-    the_index = TaskHelpers::ElasticsearchHelper.get_index
+    [Comment, CommentThread].each do |model|
-    puts "pruning #{the_index.name}"
+      model.import(index: args[:index])
-    alias_ = Tire::Alias.find the_index.name
-    raise RuntimeError, 'could not find live index' if alias_.nil?
-    scan_size = opts[:batch_size] / TaskHelpers::ElasticsearchHelper.get_index_shard_count(the_index.name)
-    cnt = 0
-    [CommentThread, Comment].each do |klass|
-      doc_type = klass.document_type
-      # this check makes sure we are working with the index to which
-      # the desired model's alias presently points.
-      search = Tire::Search::Scan.new the_index.name, {size: scan_size, type: doc_type}
-      search.each do |results|
-        es_ids = results.map(&:id)
-        mongo_ids = klass.where(:id.in => es_ids).map { |d| d.id.to_s }
-        to_delete = es_ids - mongo_ids
-        if to_delete.size > 0
-          cnt += to_delete.size
-          puts "deleting #{to_delete.size} orphaned #{doc_type} documents from elasticsearch"
-          the_index.bulk_delete (to_delete).map { |v| {"type" => doc_type, "id" => v} }
-        end
-        puts "#{the_index.name}/#{doc_type}: processed #{search.seen} of #{search.total}"
-        sleep opts[:sleep_time]
-      end
    end
-    puts "done pruning #{the_index.name}, deleted a total of #{cnt} orphaned documents"
  end
-  desc 'Rebuild the content index from MongoDB data.'
+  desc 'Generate a new, empty physical index, without bringing it online.'
-  task :rebuild, [:batch_size, :sleep_time] => :environment do |t, args|
+  task :create_index => :environment do
-    do_reindex(batch_opts(args))
+    TaskHelpers::ElasticsearchHelper.create_index
  end
-  desc 'Rebuild the content index from already-indexed data (in place).'
+  desc 'Creates a new search index and points the "content" alias to it'
-  task :reindex, [:batch_size, :sleep_time] => :environment do |t, args|
+  task :initialize => :environment do
-    do_reindex(batch_opts(args), true)
+    index = TaskHelpers::ElasticsearchHelper.create_index
+    TaskHelpers::ElasticsearchHelper.move_alias(Content::ES_INDEX_NAME, index)
  end
-  desc 'Generate a new, empty physical index, without bringing it online.'
+  desc 'Sets/moves an alias to the specified index'
-  task :create_index => :environment do
+  task :move_alias, [:alias, :index] => :environment do |t, args|
-    TaskHelpers::ElasticsearchHelper.create_index
+    TaskHelpers::ElasticsearchHelper.move_alias(args[:alias], args[:index])
  end
 end
--- a/models/comment.rb
+++ b/models/comment.rb
+require_relative 'concerns/searchable'
 require_relative 'content'
 class Comment < Content
@@ -6,8 +7,7 @@ class Comment < Content
  include Mongoid::Timestamps
  include Mongoid::MagicCounterCache
  include ActiveModel::MassAssignmentSecurity
-  include Tire::Model::Search
+  include Searchable
-  include Tire::Model::Callbacks
  voteable self, :up => +1, :down => -1

--- a/models/comment_thread.rb
+++ b/models/comment_thread.rb
 # -*- coding: utf-8 -*-
 require 'new_relic/agent/method_tracer'
+require_relative 'concerns/searchable'
 require_relative 'content'
 class CommentThread < Content
@@ -7,8 +8,7 @@ class CommentThread < Content
  include Mongoid::Timestamps
  include Mongoid::Attributes::Dynamic
  include ActiveModel::MassAssignmentSecurity
-  include Tire::Model::Search
+  include Searchable
-  include Tire::Model::Callbacks
  extend Enumerize
  voteable self, :up => +1, :down => -1
@@ -32,7 +32,6 @@ class CommentThread < Content
  index({author_id: 1, course_id: 1})
  index_name Content::ES_INDEX_NAME
  mapping do
@@ -41,10 +40,8 @@ class CommentThread < Content
    indexes :created_at, type: :date, included_in_all: false
    indexes :updated_at, type: :date, included_in_all: false
    indexes :last_activity_at, type: :date, included_in_all: false
    indexes :comment_count, type: :integer, included_in_all: false
    indexes :votes_point, type: :integer, as: 'votes_point', included_in_all: false
    indexes :context, type: :string, index: :not_analyzed, included_in_all: false
    indexes :course_id, type: :string, index: :not_analyzed, included_in_all: false
    indexes :commentable_id, type: :string, index: :not_analyzed, included_in_all: false

--- a/models/concerns/searchable.rb
+++ b/models/concerns/searchable.rb
+module Searchable
+  extend ActiveSupport::Concern
+  included do
+    include Elasticsearch::Model
+    # We specify our own callbacks, instead of using Elasticsearch::Model::Callbacks, so that we can disable
+    # indexing for tests where search functionality is not needed. This should improve test execution times.
+    after_create :index_document
+    after_update :update_indexed_document
+    after_destroy :delete_document
+    def self.put_search_index_mapping(index=nil)
+      index ||= self.index_name
+      success = self.__elasticsearch__.client.indices.put_mapping(index: index, type: self.document_type, body: self.mappings.to_hash)
+      unless success
+        logger.warn "WARNING! could not apply search index mapping for #{self.name}"
+      end
+    end
+    def as_indexed_json(options={})
+      # TODO: Play with the `MyModel.indexes` method -- reject non-mapped attributes, `:as` options, etc
+      self.as_json(options.merge root: false)
+    end
+    private
+    def index_document
+      __elasticsearch__.index_document if CommentService.search_enabled?
+    end
+    # This is named in this manner to prevent collisions with Mongoid's update_document method.
+    def update_indexed_document
+      __elasticsearch__.update_document if CommentService.search_enabled?
+    end
+    def delete_document
+      __elasticsearch__.delete_document if CommentService.search_enabled?
+    end
+  end
+end
--- a/models/content.rb
+++ b/models/content.rb
 class Content
  include Mongoid::Document
  include Mongo::Voteable
+  ES_INDEX_NAME = 'content'
  field :visible, type: Boolean, default: true
  field :abuse_flaggers, type: Array, default: []
  field :historical_abuse_flaggers, type: Array, default: [] #preserve abuse flaggers after a moderator unflags
@@ -16,16 +18,6 @@ class Content
  index({comment_thread_id: 1, endorsed: 1}, {sparse: true})
  index({commentable_id: 1}, {sparse: true, background: true})
-  ES_INDEX_NAME = 'content'
-  def self.put_search_index_mapping(idx=nil)
-    idx ||= self.tire.index
-    success = idx.mapping(self.tire.document_type, {:properties => self.tire.mapping})
-    unless success
-      logger.warn "WARNING! could not apply search index mapping for #{self.name}"
-    end
-  end
  before_save :set_username

--- a/spec/api/query_spec.rb
+++ b/spec/api/query_spec.rb
@@ -3,6 +3,7 @@ require 'faker'
 describe 'app' do
+  include_context 'search_enabled'
  before(:each) { set_api_key_header }
  let(:body) { Faker::Lorem.word }

--- a/spec/api/search_spec.rb
+++ b/spec/api/search_spec.rb
@@ -3,15 +3,14 @@ require 'unicode_shared_examples'
 describe "app" do
  describe "search" do
+    include_context 'search_enabled'
    before (:each) { set_api_key_header }
    let(:author) { create_test_user(42) }
    let(:course_id) { "test/course/id" }
    def get_result_ids(result)
-      result["collection"].map {|t| t["id"]}
+      result["collection"].map { |t| t["id"] }
    end
    describe "GET /api/v1/search/threads" do
@@ -21,17 +20,17 @@ describe "app" do
        result.should == {}
      end
-      it "returns an empty reuslt if text parameter is missing" do
+      it "returns an empty result if text parameter is missing" do
        get "/api/v1/search/threads", course_id: course_id
        assert_empty_response
      end
-      it "returns an empty reuslt if sort key is invalid" do
+      it "returns an empty result if sort key is invalid" do
        get "/api/v1/search/threads", course_id: course_id, text: "foobar", sort_key: "invalid", sort_order: "desc"
        assert_empty_response
      end
-      it "returns an empty reuslt if sort order is invalid" do
+      it "returns an empty result if sort order is invalid" do
        get "/api/v1/search/threads", course_id: course_id, text: "foobar", sort_key: "date", sort_order: "invalid"
        assert_empty_response
      end
@@ -69,13 +68,13 @@ describe "app" do
          last_response.should be_ok
          result = parse(last_response.body)
          actual_ids = Set.new get_result_ids(result)
-          expected_ids = Set.new expected_thread_indexes.map {|i| threads[i].id.to_s}
+          expected_ids = Set.new expected_thread_indexes.map { |i| threads[i].id.to_s }
          actual_ids.should == expected_ids
        end
        it "by course_id" do
          get "/api/v1/search/threads", text: "text", course_id: "test/course/id0"
-          assert_response_contains((0..29).find_all {|i| i % 2 == 0})
+          assert_response_contains((0..29).find_all { |i| i % 2 == 0 })
        end
        it "by context" do
@@ -87,7 +86,7 @@ describe "app" do
          user = create_test_user(Random.new)
          user.mark_as_read(threads[0])
          get "/api/v1/search/threads", text: "text", course_id: "test/course/id0", user_id: user.id, unread: true
-          assert_response_contains((1..29).find_all {|i| i % 2 == 0})
+          assert_response_contains((1..29).find_all { |i| i % 2 == 0 })
        end
        it "with flagged filter" do
@@ -121,22 +120,22 @@ describe "app" do
        it "by commentable_id" do
          get "/api/v1/search/threads", text: "text", commentable_id: "commentable0"
-          assert_response_contains((0..29).find_all {|i| i % 3 == 0})
+          assert_response_contains((0..29).find_all { |i| i % 3 == 0 })
        end
        it "by commentable_ids" do
          get "/api/v1/search/threads", text: "text", commentable_ids: "commentable0,commentable1"
-          assert_response_contains((0..29).find_all {|i| i % 3 == 0 || i % 3 == 1})
+          assert_response_contains((0..29).find_all { |i| i % 3 == 0 || i % 3 == 1 })
        end
        it "by group_id" do
          get "/api/v1/search/threads", text: "text", group_id: "1"
-          assert_response_contains((0..29).find_all {|i| i % 5 == 0 || i % 5 == 1})
+          assert_response_contains((0..29).find_all { |i| i % 5 == 0 || i % 5 == 1 })
        end
        it "by group_ids" do
          get "/api/v1/search/threads", text: "text", group_ids: "1,2"
-          expected_ids = (0..29).find_all {|i| i % 5 == 0 || i % 5 == 1 || i % 5 == 2}
+          expected_ids = (0..29).find_all { |i| i % 5 == 0 || i % 5 == 1 || i % 5 == 2 }
          assert_response_contains(expected_ids)
        end
@@ -148,8 +147,8 @@ describe "app" do
      describe "sorting works" do
        let!(:threads) do
-          threads = (0..5).map {|i| make_thread(author, "text", course_id, "dummy")}
+          threads = (0..5).map { |i| make_thread(author, "text", course_id, "dummy") }
-          [1, 2].map {|i| author.vote(threads[i], :up)}
+          [1, 2].map { |i| author.vote(threads[i], :up) }
          [1, 3].map do |i|
            threads[i].comment_count = 5
            threads[i].save!
@@ -164,7 +163,7 @@ describe "app" do
          last_response.should be_ok
          result = parse(last_response.body)
          actual_ids = get_result_ids(result)
-          expected_ids = expected_thread_indexes.map {|i| threads[i].id.to_s}
+          expected_ids = expected_thread_indexes.map { |i| threads[i].id.to_s }
          actual_ids.should == expected_ids
        end
@@ -197,7 +196,7 @@ describe "app" do
      describe "pagination" do
        let!(:threads) do
-          threads = (1..50).map {|i| make_thread(author, "text", course_id, "dummy")}
+          threads = (1..50).map { |i| make_thread(author, "text", course_id, "dummy") }
          refresh_es_index
          threads
        end
@@ -210,7 +209,7 @@ describe "app" do
            result = parse(last_response.body)
            result_ids += get_result_ids(result)
          end
-          result_ids.should == threads.reverse.map {|t| t.id.to_s}
+          result_ids.should == threads.reverse.map { |t| t.id.to_s }
        end
        it "works correctly with page size 1" do
@@ -227,7 +226,7 @@ describe "app" do
      end
      describe "spelling correction" do
-        let(:commentable_id) {"test_commentable"}
+        let(:commentable_id) { "test_commentable" }
        def check_correction(original_text, corrected_text)
          get "/api/v1/search/threads", text: original_text
@@ -292,8 +291,8 @@ describe "app" do
        end
      end
-      it "returns the correct values for total_results and num_pages" do
+      it 'returns the correct values for total_results and num_pages' do
-        course_id = "test/course/id"
+        course_id = 'test/course/id'
        for i in 1..100 do
          text = "all"
          text += " half" if i % 2 == 0
@@ -302,15 +301,14 @@ describe "app" do
          text += " one" if i == 100
          # There is currently a bug that causes only 10 threads with matching
          # titles/bodies to be considered, so this test case uses comments.
-          thread = make_thread(author, "dummy text", course_id, "dummy_commentable")
+          create(:comment, course_id: course_id, body: text)
-          make_comment(author, thread, text)
        end
        # Elasticsearch does not necessarily make newly indexed content
        # available immediately, so we must explicitly refresh the index
        refresh_es_index
        test_text = lambda do |text, expected_total_results, expected_num_pages|
-          get "/api/v1/search/threads", course_id: course_id, text: text, per_page: "10"
+          get '/api/v1/search/threads', course_id: course_id, text: text, per_page: '10'
          last_response.should be_ok
          result = parse(last_response.body)
          result["total_results"].should == expected_total_results

--- a/spec/app_spec.rb
+++ b/spec/app_spec.rb
--- a/spec/spec_helper.rb
+++ b/spec/spec_helper.rb
@@ -16,6 +16,9 @@ require 'yajl'
 require 'support/database_cleaner'
 require 'support/elasticsearch'
 require 'support/factory_girl'
+require 'webmock/rspec'
+WebMock.allow_net_connect!
 # setup test environment
 set :environment, :test

--- a/spec/support/elasticsearch.rb
+++ b/spec/support/elasticsearch.rb
-def delete_es_index
+require 'task_helpers'
-  Tire.index Content::ES_INDEX_NAME do
-    delete
+def refresh_es_index
-  end
+  TaskHelpers::ElasticsearchHelper.refresh_index(Content::ES_INDEX_NAME)
 end
-def create_es_index
-  new_index = Tire.index Content::ES_INDEX_NAME
+RSpec.shared_context 'search_enabled' do
-  new_index.create
+  before(:all) do
-  [CommentThread, Comment].each do |klass|
+    CommentService.config[:enable_search] = true
-    klass.put_search_index_mapping
  end
-end
-def refresh_es_index
+  before(:each) do
-  es_index_name = Content::ES_INDEX_NAME
+    index = TaskHelpers::ElasticsearchHelper.create_index
-  Tire.index es_index_name do
+    TaskHelpers::ElasticsearchHelper.move_alias(Content::ES_INDEX_NAME, index)
-    refresh
+  end
+  after(:each) do
+    TaskHelpers::ElasticsearchHelper.delete_index(Content::ES_INDEX_NAME)
  end
 end
 RSpec.configure do |config|
-  config.before(:each) do
+  config.before(:suite) do
-    delete_es_index
+    CommentService.config[:enable_search] = false
-    create_es_index
  end
 end