Sleep before attempting to restart forums

If it fails with this exit code, rake search:validate failed, meaning
the elasticsearch mappings or index isn't in sync with the code. To
avoid bouncing the app too quickly when this happens, sleep for a bit

playbooks/roles/forum/defaults/main.yml

@@ -40,6 +40,10 @@ FORUM_LISTEN_HOST: "0.0.0.0"
 FORUM_LISTEN_PORT: "4567"
 FORUM_USE_TCP: false
 
+# If forums fails to start because rake search:validate_index failed,
+# wait this long before attempting to restart it
+FORUM_RESTART_DELAY: 60
+
 forum_environment:
   RBENV_ROOT: "{{ forum_rbenv_root }}"
   GEM_HOME: "{{ forum_gem_root }}"

playbooks/roles/forum/templates/forum-supervisor.sh.j2

@@ -6,7 +6,13 @@ cd {{ forum_code_dir }}
 {% if devstack %}
 {{ forum_rbenv_shims }}/ruby app.rb
 {% elif FORUM_USE_TCP %}
-{{ forum_binstubs_dir }}/unicorn -c config/unicorn_tcp.rb
+{{ forum_binstubs_dir }}/unicorn -c config/unicorn_tcp.rb -I '.'
 {% else %}
-{{ forum_binstubs_dir }}/unicorn -c config/unicorn.rb
+{{ forum_binstubs_dir }}/unicorn -c config/unicorn.rb -I '.'
 {% endif %}
+
+# If forums fails to start because elasticsearch isn't migrated, sleep so supervisord
+# doesn't attempt to restart it immediately.
+# 101 is the magic exit code forums uses to mean "rake search:validate_index failed"
+exit_code="$?"
+[ "$exit_code" -eq 101 ] && sleep {{ FORUM_RESTART_DELAY }} && exit "$exit_code"