Merge pull request #2388 from edx/ormsbee/gunicorn_post_fork_cache

Add post_fork to gunicorn config to prevent cache race condition.

playbooks/roles/analytics_api/templates/edx/app/analytics_api/analytics_api_gunicorn.py.j2

@@ -9,4 +9,14 @@ bind = "{{ analytics_api_gunicorn_host }}:{{ analytics_api_gunicorn_port }}"
 pythonpath = "{{ analytics_api_code_dir }}"
 workers = {{ ANALYTICS_API_GUNICORN_WORKERS }}
 
+def post_fork(server, worker):
+    """
+    Close the cache so that newly forked workers cannot accidentally share the
+    socket with the processes they were forked from. This prevents a race
+    condition in which one worker could get a cache response intended for
+    another worker.
+    """
+    from django.core.cache import cache
+    cache.close()
+
 {{ ANALYTICS_API_GUNICORN_EXTRA_CONF }}

playbooks/roles/ansible-role-django-ida/templates/templates/edx/app/ROLE_NAME/ROLE_NAME_gunicorn.py.j2

@@ -9,4 +9,14 @@ pythonpath = "{{ '{{' }} {{ role_name }}_code_dir }}"
 workers = {{ '{{' }} {{ role_name|upper }}_GUNICORN_WORKERS }}
 worker_class = "{{ '{{' }} {{ role_name|upper }}_GUNICORN_WORKER_CLASS }}"
 
+def post_fork(server, worker):
+    """
+    Close the cache so that newly forked workers cannot accidentally share the
+    socket with the processes they were forked from. This prevents a race
+    condition in which one worker could get a cache response intended for
+    another worker.
+    """
+    from django.core.cache import cache
+    cache.close()
+
 {{ '{{' }} {{ role_name|upper }}_GUNICORN_EXTRA_CONF }}

playbooks/roles/ecommerce/templates/edx/app/ecommerce/ecommerce_gunicorn.py.j2

@@ -9,4 +9,14 @@ pythonpath = "{{ ecommerce_code_dir }}"
 workers = {{ ECOMMERCE_GUNICORN_WORKERS }}
 worker_class = "{{ ECOMMERCE_GUNICORN_WORKER_CLASS }}"
 
+def post_fork(server, worker):
+    """
+    Close the cache so that newly forked workers cannot accidentally share the
+    socket with the processes they were forked from. This prevents a race
+    condition in which one worker could get a cache response intended for
+    another worker.
+    """
+    from django.core.cache import cache
+    cache.close()
+
 {{ ECOMMERCE_GUNICORN_EXTRA_CONF }}

playbooks/roles/edx_notes_api/templates/edx/app/edx_notes_api/edx_notes_api_gunicorn.py.j2

@@ -16,4 +16,14 @@ workers = {{ EDX_NOTES_API_WORKERS }}
 workers = (multiprocessing.cpu_count()-1) * 2 + 2
 {% endif %}
 
+def post_fork(server, worker):
+    """
+    Close the cache so that newly forked workers cannot accidentally share the
+    socket with the processes they were forked from. This prevents a race
+    condition in which one worker could get a cache response intended for
+    another worker.
+    """
+    from django.core.cache import cache
+    cache.close()
+
 {{ EDX_NOTES_API_WORKERS_EXTRA_CONF }}

playbooks/roles/edxapp/templates/cms_gunicorn.py.j2

@@ -20,5 +20,14 @@ workers = {{ EDXAPP_WORKERS.cms }}
 workers = (multiprocessing.cpu_count()-1) * {{ worker_core_mult.cms }} + {{ worker_core_mult.cms }}
 {% endif %}
 
+def post_fork(server, worker):
+    """
+    Close the cache so that newly forked workers cannot accidentally share the
+    socket with the processes they were forked from. This prevents a race
+    condition in which one worker could get a cache response intended for
+    another worker.
+    """
+    from django.core.cache import cache
+    cache.close()
 
 {{ EDXAPP_CMS_GUNICORN_EXTRA_CONF }}

playbooks/roles/edxapp/templates/lms_gunicorn.py.j2

@@ -20,5 +20,14 @@ workers = {{ EDXAPP_WORKERS.lms }}
 workers = (multiprocessing.cpu_count()-1) * {{ worker_core_mult.lms }} + {{ worker_core_mult.lms }}
 {% endif %}
 
+def post_fork(server, worker):
+    """
+    Close the cache so that newly forked workers cannot accidentally share the
+    socket with the processes they were forked from. This prevents a race
+    condition in which one worker could get a cache response intended for
+    another worker.
+    """
+    from django.core.cache import cache
+    cache.close()
 
 {{ EDXAPP_LMS_GUNICORN_EXTRA_CONF }}

playbooks/roles/gitreload/templates/edx/app/gitreload/gitreload_gunicorn.py.j2

@@ -11,4 +11,14 @@ bind = "{{ gitreload_gunicorn_host }}:{{ gitreload_gunicorn_port }}"
 
 workers = {{ gitreload_gunicorn_workers }}
 
+def post_fork(server, worker):
+    """
+    Close the cache so that newly forked workers cannot accidentally share the
+    socket with the processes they were forked from. This prevents a race
+    condition in which one worker could get a cache response intended for
+    another worker.
+    """
+    from django.core.cache import cache
+    cache.close()
+
 {{ GITRELOAD_GUNICORN_EXTRA_CONF }}

playbooks/roles/ora/templates/ora_gunicorn.py.j2

@@ -10,4 +10,14 @@ bind = "{{ ora_gunicorn_host }}:{{ ora_gunicorn_port }}"
 pythonpath = "{{ ora_code_dir }}"
 workers = {{ ora_gunicorn_workers }}
 
+def post_fork(server, worker):
+    """
+    Close the cache so that newly forked workers cannot accidentally share the
+    socket with the processes they were forked from. This prevents a race
+    condition in which one worker could get a cache response intended for
+    another worker.
+    """
+    from django.core.cache import cache
+    cache.close()
+
 {{ ORA_GUNICORN_EXTRA_CONF }}

playbooks/roles/programs/templates/edx/app/programs/programs_gunicorn.py.j2

@@ -9,4 +9,14 @@ pythonpath = "{{ programs_code_dir }}"
 workers = {{ PROGRAMS_GUNICORN_WORKERS }}
 worker_class = "{{ PROGRAMS_GUNICORN_WORKER_CLASS }}"
 
+def post_fork(server, worker):
+    """
+    Close the cache so that newly forked workers cannot accidentally share the
+    socket with the processes they were forked from. This prevents a race
+    condition in which one worker could get a cache response intended for
+    another worker.
+    """
+    from django.core.cache import cache
+    cache.close()
+
 {{ PROGRAMS_GUNICORN_EXTRA_CONF }}

playbooks/roles/xqueue/templates/xqueue_gunicorn.py.j2

@@ -16,4 +16,14 @@ workers = {{ XQUEUE_WORKERS }}
 workers = (multiprocessing.cpu_count()-1) * 2 + 2
 {% endif %}
 
+def post_fork(server, worker):
+    """
+    Close the cache so that newly forked workers cannot accidentally share the
+    socket with the processes they were forked from. This prevents a race
+    condition in which one worker could get a cache response intended for
+    another worker.
+    """
+    from django.core.cache import cache
+    cache.close()
+
 {{ XQUEUE_GUNICORN_WORKERS_EXTRA_CONF }}

playbooks/roles/xserver/templates/xserver_gunicorn.py.j2

@@ -10,4 +10,14 @@ bind = "{{ xserver_gunicorn_host }}:{{ xserver_gunicorn_port }}"
 pythonpath = "{{ xserver_code_dir }}"
 workers = {{ xserver_gunicorn_workers }}
 
+def post_fork(server, worker):
+    """
+    Close the cache so that newly forked workers cannot accidentally share the
+    socket with the processes they were forked from. This prevents a race
+    condition in which one worker could get a cache response intended for
+    another worker.
+    """
+    from django.core.cache import cache
+    cache.close()
+
 {{ XSERVER_GUNICORN_EXTRA_CONF }}