Merge pull request #230 from edx/robrap/improve-rebuild-index

TNL-6464: Minor changes to rebuild_index.

lib/task_helpers.rb

@@ -14,7 +14,8 @@ module TaskHelpers
     # +alias_name+:: (optional) The alias to point to the new index.
     # +batch_size+:: (optional) The number of elements to index at a time. Defaults to 500.
     # +sleep_time+:: (optional) The number of seconds to sleep between batches. Defaults to 0.
-    def self.rebuild_index(alias_name=nil, batch_size=500, sleep_time=0)
+    # +extra_catchup_minutes+:: (optional) The number of extra minutes to catchup. Defaults to 5.
+    def self.rebuild_index(alias_name=nil, batch_size=500, sleep_time=0, extra_catchup_minutes=5)
       initial_start_time = Time.now
       index_name = create_index()
 
@@ -30,10 +31,12 @@ module TaskHelpers
         # Just in case initial rebuild took days and first catch up takes hours,
         # we catch up once before the alias move and once afterwards.
         first_catchup_start_time = Time.now
-        catchup_index(initial_start_time, index_name, batch_size, sleep_time)
+        adjusted_start_time = initial_start_time - (extra_catchup_minutes * 60)
+        catchup_index(adjusted_start_time, index_name, batch_size, sleep_time)
 
         move_alias(alias_name, index_name, force_delete: true)
-        catchup_index(first_catchup_start_time, alias_name, batch_size, sleep_time)
+        adjusted_start_time = first_catchup_start_time - (extra_catchup_minutes * 60)
+        catchup_index(adjusted_start_time, alias_name, batch_size, sleep_time)
       end
 
       LOG.info "Rebuild index complete."

lib/tasks/search.rake

@@ -11,12 +11,18 @@ namespace :search do
   end
 
   desc 'Rebuilds a new index of all data from the database and then updates alias.'
-  task :rebuild_index, [:call_move_alias, :batch_size, :sleep_time] => :environment do |t, args|
-    args.with_defaults(:call_move_alias => false)
+  task :rebuild_index, [:call_move_alias, :batch_size, :sleep_time, :extra_catchup_minutes] => :environment do |t, args|
+    args.with_defaults(:call_move_alias => true)
     args.with_defaults(:batch_size => 500)
-    args.with_defaults(:sleep_time => 0)
-    alias_name = args[:call_move_alias] ? Content::ES_INDEX_NAME : nil
-    TaskHelpers::ElasticsearchHelper.rebuild_index(alias_name, args[:batch_size].to_i, args[:sleep_time].to_i)
+    args.with_defaults(:sleep_time => 0)  # sleep time between batches in seconds
+    args.with_defaults(:extra_catchup_minutes => 5) # additional catchup time in minutes
+    alias_name = args[:call_move_alias] === true ? Content::ES_INDEX_NAME : nil
+    TaskHelpers::ElasticsearchHelper.rebuild_index(
+        alias_name,
+        args[:batch_size].to_i,
+        args[:sleep_time].to_i,
+        args[:extra_catchup_minutes].to_i
+    )
   end
 
   desc 'Generate a new, empty physical index, without bringing it online.'

spec/lib/tasks/search_rake_spec.rb

@@ -11,21 +11,22 @@ describe "search:rebuild_index" do
   its(:prerequisites) { should include("environment") }
 
   it "calls rebuild_index with defaults" do
-    TaskHelpers::ElasticsearchHelper.should_receive(:rebuild_index).with(nil, 500, 0)
+    TaskHelpers::ElasticsearchHelper.should_receive(:rebuild_index).with(Content::ES_INDEX_NAME, 500, 0, 5)
 
     subject.invoke
   end
 
   it "calls rebuild_index with arguments" do
     # Rake calls receive arguments as strings.
-    call_move_alias = 'true'
+    call_move_alias = 'false'
     batch_size = '100'
     sleep_time = '2'
+    extra_catchup_minutes = '10'
     TaskHelpers::ElasticsearchHelper.should_receive(:rebuild_index).with(
-          Content::ES_INDEX_NAME, batch_size.to_i, sleep_time.to_i
+          nil, batch_size.to_i, sleep_time.to_i, extra_catchup_minutes.to_i
     )
 
-    subject.invoke(call_move_alias, batch_size, sleep_time)
+    subject.invoke(call_move_alias, batch_size, sleep_time, extra_catchup_minutes)
   end
 end