Merge pull request #2489 from edx/ned/close-all-the-caches

Close all caches in gunicorn prefork

playbooks/roles/analytics_api/templates/edx/app/analytics_api/analytics_api_gunicorn.py.j2

@@ -9,15 +9,9 @@ bind = "{{ analytics_api_gunicorn_host }}:{{ analytics_api_gunicorn_port }}"
 pythonpath = "{{ analytics_api_code_dir }}"
 workers = {{ ANALYTICS_API_GUNICORN_WORKERS }}
 
+{{ common_close_all_caches }}
+
 def post_fork(server, worker):
-    """
-    Close the cache so that newly forked workers cannot accidentally share the
-    socket with the processes they were forked from. This prevents a race
-    condition in which one worker could get a cache response intended for
-    another worker.
-    """
-    from django.core.cache import cache
-    if hasattr(cache, 'close'):
-        cache.close()
+    close_all_caches()
 
 {{ ANALYTICS_API_GUNICORN_EXTRA_CONF }}

playbooks/roles/common_vars/defaults/main.yml

@@ -140,3 +140,24 @@ common_redhat_variants:
   - CentOS
   - Red Hat Enterprise Linux
   - Amazon
+
+# Code used in gunicorn post_fork functions to be sure we aren't sharing cache
+# connections among forked children.
+common_close_all_caches: |
+    def close_all_caches():
+        # Close the cache so that newly forked workers cannot accidentally share
+        # the socket with the processes they were forked from. This prevents a race
+        # condition in which one worker could get a cache response intended for
+        # another worker.
+        # We do this in a way that is safe for 1.4 and 1.8 while we still have some
+        # 1.4 installations.
+        from django.conf import settings
+        from django.core import cache as django_cache
+        if hasattr(django_cache, 'caches'):
+            get_cache = django_cache.caches.__getitem__
+        else:
+            get_cache = django_cache.get_cache
+        for cache_name in settings.CACHES:
+            cache = get_cache(cache_name)
+            if hasattr(cache, 'close'):
+                cache.close()

playbooks/roles/edx_notes_api/templates/edx/app/edx_notes_api/edx_notes_api_gunicorn.py.j2

@@ -16,15 +16,9 @@ workers = {{ EDX_NOTES_API_WORKERS }}
 workers = (multiprocessing.cpu_count()-1) * 2 + 2
 {% endif %}
 
+{{ common_close_all_caches }}
+
 def post_fork(server, worker):
-    """
-    Close the cache so that newly forked workers cannot accidentally share the
-    socket with the processes they were forked from. This prevents a race
-    condition in which one worker could get a cache response intended for
-    another worker.
-    """
-    from django.core.cache import cache
-    if hasattr(cache, 'close'):
-        cache.close()
+    close_all_caches()
 
 {{ EDX_NOTES_API_WORKERS_EXTRA_CONF }}

playbooks/roles/edxapp/templates/cms_gunicorn.py.j2

@@ -20,15 +20,9 @@ workers = {{ EDXAPP_WORKERS.cms }}
 workers = (multiprocessing.cpu_count()-1) * {{ worker_core_mult.cms }} + {{ worker_core_mult.cms }}
 {% endif %}
 
+{{ common_close_all_caches }}
+
 def post_fork(server, worker):
-    """
-    Close the cache so that newly forked workers cannot accidentally share the
-    socket with the processes they were forked from. This prevents a race
-    condition in which one worker could get a cache response intended for
-    another worker.
-    """
-    from django.core.cache import cache
-    if hasattr(cache, 'close'):
-        cache.close()
+    close_all_caches()
 
 {{ EDXAPP_CMS_GUNICORN_EXTRA_CONF }}

playbooks/roles/edxapp/templates/lms_gunicorn.py.j2

@@ -20,15 +20,9 @@ workers = {{ EDXAPP_WORKERS.lms }}
 workers = (multiprocessing.cpu_count()-1) * {{ worker_core_mult.lms }} + {{ worker_core_mult.lms }}
 {% endif %}
 
+{{ common_close_all_caches }}
+
 def post_fork(server, worker):
-    """
-    Close the cache so that newly forked workers cannot accidentally share the
-    socket with the processes they were forked from. This prevents a race
-    condition in which one worker could get a cache response intended for
-    another worker.
-    """
-    from django.core.cache import cache
-    if hasattr(cache, 'close'):
-        cache.close()
+    close_all_caches()
 
 {{ EDXAPP_LMS_GUNICORN_EXTRA_CONF }}

playbooks/roles/gitreload/templates/edx/app/gitreload/gitreload_gunicorn.py.j2

@@ -11,15 +11,9 @@ bind = "{{ gitreload_gunicorn_host }}:{{ gitreload_gunicorn_port }}"
 
 workers = {{ gitreload_gunicorn_workers }}
 
+{{ common_close_all_caches }}
+
 def post_fork(server, worker):
-    """
-    Close the cache so that newly forked workers cannot accidentally share the
-    socket with the processes they were forked from. This prevents a race
-    condition in which one worker could get a cache response intended for
-    another worker.
-    """
-    from django.core.cache import cache
-    if hasattr(cache, 'close'):
-        cache.close()
+    close_all_caches()
 
 {{ GITRELOAD_GUNICORN_EXTRA_CONF }}

playbooks/roles/ora/templates/ora_gunicorn.py.j2

@@ -10,15 +10,9 @@ bind = "{{ ora_gunicorn_host }}:{{ ora_gunicorn_port }}"
 pythonpath = "{{ ora_code_dir }}"
 workers = {{ ora_gunicorn_workers }}
 
+{{ common_close_all_caches }}
+
 def post_fork(server, worker):
-    """
-    Close the cache so that newly forked workers cannot accidentally share the
-    socket with the processes they were forked from. This prevents a race
-    condition in which one worker could get a cache response intended for
-    another worker.
-    """
-    from django.core.cache import cache
-    if hasattr(cache, 'close'):
-        cache.close()
+    close_all_caches()
 
 {{ ORA_GUNICORN_EXTRA_CONF }}

playbooks/roles/xqueue/templates/xqueue_gunicorn.py.j2

@@ -16,15 +16,9 @@ workers = {{ XQUEUE_WORKERS }}
 workers = (multiprocessing.cpu_count()-1) * 2 + 2
 {% endif %}
 
+{{ common_close_all_caches }}
+
 def post_fork(server, worker):
-    """
-    Close the cache so that newly forked workers cannot accidentally share the
-    socket with the processes they were forked from. This prevents a race
-    condition in which one worker could get a cache response intended for
-    another worker.
-    """
-    from django.core.cache import cache
-    if hasattr(cache, 'close'):
-        cache.close()
+    close_all_caches()
 
 {{ XQUEUE_GUNICORN_WORKERS_EXTRA_CONF }}