Improve 502 maintenance page OPS-3164

- Fake heartbeat to stay in ELB by checking for a file
- Return empty body with json content-type for api endpoints

playbooks/roles/nginx/templates/edx/app/nginx/sites-available/cms.j2

@@ -30,6 +30,8 @@ server {
 error_page {{ k }} {{ v }};
   {% endfor %}
 
+{% include "empty_json.j2" %}
+
   listen {{ EDXAPP_CMS_NGINX_PORT }} {{ default_site }};
 
   {% if NGINX_ENABLE_SSL %}
@@ -66,34 +68,15 @@ error_page {{ k }} {{ v }};
   {% include "common-settings.j2" %}
 
   location @proxy_to_cms_app {
-    {% if NGINX_SET_X_FORWARDED_HEADERS %}
-    proxy_set_header X-Forwarded-Proto $scheme;
-    proxy_set_header X-Forwarded-Port $server_port;
-    proxy_set_header X-Forwarded-For $remote_addr;
-    {% else %}
-    proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto;
-    proxy_set_header X-Forwarded-Port $http_x_forwarded_port;
-    proxy_set_header X-Forwarded-For $http_x_forwarded_for;
-    {% endif %}
-
-    # newrelic-specific header records the time when nginx handles a request.
-    proxy_set_header X-Queue-Start "t=${msec}";
+{% include "cms_proxy.j2" %}
+  }
 
-    proxy_set_header Host $http_host;
+  location @proxy_to_cms_app_api {
+    error_page 504 @empty_json;
+    error_page 502 @empty_json;
+    error_page 500 @empty_json;
 
-    proxy_redirect off;
-    proxy_pass http://cms-backend;
-    {% if NGINX_CMS_PROXY_CONNECT_TIMEOUT %}
-    proxy_connect_timeout {{ NGINX_CMS_PROXY_CONNECT_TIMEOUT }};
-    {% endif %}
-    {% if NGINX_CMS_PROXY_SEND_TIMEOUT %}
-    proxy_send_timeout {{ NGINX_CMS_PROXY_SEND_TIMEOUT }};
-    {% endif %}
-    {% if NGINX_CMS_PROXY_READ_TIMEOUT %}
-    proxy_read_timeout {{ NGINX_CMS_PROXY_READ_TIMEOUT }};
-    {% endif %}
-
-    {{ NGINX_EDXAPP_CMS_APP_EXTRA }}
+{% include "cms_proxy.j2" %}
   }
 
   location / {
@@ -117,7 +100,7 @@ error_page {{ k }} {{ v }};
   # uses the authorization header so we can't have
   # basic auth on it as well.
   location /api {
-    try_files $uri @proxy_to_cms_app;
+    try_files $uri @proxy_to_cms_app_api;
   }
 
 {% if NGINX_DJANGO_ADMIN_ACCESS_CIDRS and EDXAPP_ENABLE_DJANGO_ADMIN_RESTRICTION %}

playbooks/roles/nginx/templates/edx/app/nginx/sites-available/cms_proxy.j2

+{% if NGINX_SET_X_FORWARDED_HEADERS %}
+    proxy_set_header X-Forwarded-Proto $scheme;
+    proxy_set_header X-Forwarded-Port $server_port;
+    proxy_set_header X-Forwarded-For $remote_addr;
+{% else %}
+    proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto;
+    proxy_set_header X-Forwarded-Port $http_x_forwarded_port;
+    proxy_set_header X-Forwarded-For $http_x_forwarded_for;
+{% endif %}
+
+    # newrelic-specific header records the time when nginx handles a request.
+    proxy_set_header X-Queue-Start "t=${msec}";
+
+    proxy_set_header Host $http_host;
+
+    proxy_redirect off;
+    proxy_pass http://cms-backend;
+{% if NGINX_CMS_PROXY_CONNECT_TIMEOUT %}
+    proxy_connect_timeout {{ NGINX_CMS_PROXY_CONNECT_TIMEOUT }};
+{% endif %}
+{% if NGINX_CMS_PROXY_SEND_TIMEOUT %}
+    proxy_send_timeout {{ NGINX_CMS_PROXY_SEND_TIMEOUT }};
+{% endif %}
+{% if NGINX_CMS_PROXY_READ_TIMEOUT %}
+    proxy_read_timeout {{ NGINX_CMS_PROXY_READ_TIMEOUT }};
+{% endif %}
+
+{{ NGINX_EDXAPP_CMS_APP_EXTRA }}

playbooks/roles/nginx/templates/edx/app/nginx/sites-available/empty_json.j2

+  location @empty_json {
+    # This location will return an empty body with content-type application/json
+    # If this location is referenced by the error_page directive the
+    # response code will be the error response code (i.e. 502), not 200
+    # despite the "return 200" directive
+    default_type application/json;
+    return 200;
+  }
+

playbooks/roles/nginx/templates/edx/app/nginx/sites-available/lms.j2

@@ -79,6 +79,8 @@ server {
 error_page {{ k }} {{ v }};
   {% endfor %}
 
+{% include "empty_json.j2" %}
+
   listen {{ EDXAPP_LMS_NGINX_PORT }} {{ default_site }};
 
   {% if NGINX_ENABLE_SSL %}
@@ -128,25 +130,16 @@ error_page {{ k }} {{ v }};
   {% endif -%}
 
   location @proxy_to_lms_app {
-    {% if NGINX_SET_X_FORWARDED_HEADERS %}
-    proxy_set_header X-Forwarded-Proto $scheme;
-    proxy_set_header X-Forwarded-Port $server_port;
-    proxy_set_header X-Forwarded-For $remote_addr;
-    {% else %}
-    proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto;
-    proxy_set_header X-Forwarded-Port $http_x_forwarded_port;
-    proxy_set_header X-Forwarded-For $http_x_forwarded_for;
-    {% endif %}
-
-    # newrelic-specific header records the time when nginx handles a request.
-    proxy_set_header X-Queue-Start "t=${msec}";
+{% include "lms_proxy.j2" %}
 
-    proxy_set_header Host $http_host;
+  }
 
-    proxy_redirect off;
-    proxy_pass http://lms-backend;
+  location @proxy_to_lms_app_api {
+    error_page 504 @empty_json;
+    error_page 502 @empty_json;
+    error_page 500 @empty_json;
 
-    {{ NGINX_EDXAPP_LMS_APP_EXTRA }}
+{% include "lms_proxy.j2" %}
   }
 
   location / {
@@ -188,31 +181,31 @@ error_page {{ k }} {{ v }};
   # uses the authorization header so we can't have
   # basic auth on it as well.
   location /api {
-    try_files $uri @proxy_to_lms_app;
+    try_files $uri @proxy_to_lms_app_api;
   }
 
   #enterprise API
   location /enterprise/api {
-    try_files $uri @proxy_to_lms_app;
+    try_files $uri @proxy_to_lms_app_api;
   }
 
   # Consent API
   location /consent/api {
-    try_files $uri @proxy_to_lms_app;
+    try_files $uri @proxy_to_lms_app_api;
   }
 
   # Need a separate location for the image uploads endpoint to limit upload sizes
   location ~ ^/api/profile_images/[^/]*/[^/]*/upload$ {
-    try_files $uri @proxy_to_lms_app;
+    try_files $uri @proxy_to_lms_app_api;
     client_max_body_size {{ EDXAPP_PROFILE_IMAGE_MAX_BYTES + 1000 }};
   }
 
   location /notifier_api {
-    try_files $uri @proxy_to_lms_app;
+    try_files $uri @proxy_to_lms_app_api;
   }
 
   location /user_api {
-    try_files $uri @proxy_to_lms_app;
+    try_files $uri @proxy_to_lms_app_api;
   }
 
   # No basic auth security on the github_service_hook url, so that github can use it for cms
@@ -241,6 +234,12 @@ error_page {{ k }} {{ v }};
 
   # No basic auth security on the heartbeat url, so that ELB can use it
   location /heartbeat {
+    # If /edx/var/nginx/server-static/maintenance_heartbeat.txt exists serve an
+    # empty 200 so the instance stays in the load balancer to serve the
+    # maintenance page
+    if (-f /edx/var/nginx/server-static/maintenance_heartbeat.txt) {
+       return 200;
+    }
     try_files $uri @proxy_to_lms_app;
   }
 

playbooks/roles/nginx/templates/edx/app/nginx/sites-available/lms_proxy.j2

+{% if NGINX_SET_X_FORWARDED_HEADERS %}
+    proxy_set_header X-Forwarded-Proto $scheme;
+    proxy_set_header X-Forwarded-Port $server_port;
+    proxy_set_header X-Forwarded-For $remote_addr;
+{% else %}
+    proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto;
+    proxy_set_header X-Forwarded-Port $http_x_forwarded_port;
+    proxy_set_header X-Forwarded-For $http_x_forwarded_for;
+{% endif %}
+
+    # newrelic-specific header records the time when nginx handles a request.
+    proxy_set_header X-Queue-Start "t=${msec}";
+
+    proxy_set_header Host $http_host;
+
+    proxy_redirect off;
+    proxy_pass http://lms-backend;
+
+    {{ NGINX_EDXAPP_LMS_APP_EXTRA }}