Add the robots section if a robots file exists.

playbooks/roles/nginx/templates/cms.j2

@@ -66,6 +66,8 @@ server {
     try_files $uri @proxy_to_cms_app;
   }
 
+  {% include "robots.j2" %}
+
   # Check security on this
   location ~ /static/(?P<file>.*) {
     root {{ edxapp_data_dir }};

playbooks/roles/nginx/templates/devpi.j2

@@ -12,5 +12,7 @@ server {
         proxy_set_header  X-outside-url $scheme://$host;
         proxy_set_header  X-Real-IP $remote_addr;
     }
+
+    {% include robots.j2 %}
 }
 

playbooks/roles/nginx/templates/forum.j2

@@ -40,6 +40,8 @@ server {
     try_files $uri @proxy_to_app;
   }
 
+  {% include "robots.j2" %}
+
 location @proxy_to_app {
     proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto;
     proxy_set_header X-Forwarded-Port $http_x_forwarded_port;

playbooks/roles/nginx/templates/lms-preview.j2

@@ -46,6 +46,8 @@ server {
     try_files $uri @proxy_to_lms-preview_app;
   }
 
+  {% include "robots.j2" %}
+
   # Check security on this
   location ~ /static/(?P<file>.*) {
     root {{ edxapp_data_dir}};

playbooks/roles/nginx/templates/lms.j2

@@ -62,6 +62,8 @@ server {
     try_files $uri @proxy_to_lms_app;
   }
 
+  {% include "robots.j2" %}
+
   # Check security on this
   location ~ /static/(?P<file>.*) {
     root {{ edxapp_data_dir }};

playbooks/roles/nginx/templates/ora.j2

@@ -31,6 +31,8 @@ server {
     expires epoch;
   }
 
+  {% include "robots.j2" %}
+
   location @proxy_to_app {
     client_max_body_size 75K;
     proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto;

playbooks/roles/nginx/templates/robots.j2

+{% if NGINX_ROBOT_RULES|length > 0 %}
+  location /robots.txt {
+    root {{ nginx_app_dir }};
+    try_files $uri /robots.txt =404;
+  }
+{% endif %}

playbooks/roles/nginx/templates/xqueue.j2

@@ -17,6 +17,8 @@ server {
     try_files $uri @proxy_to_app;
   }
 
+  {% include "robots.j2" %}
+
 location @proxy_to_app {
     proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto;
     proxy_set_header X-Forwarded-Port $http_x_forwarded_port;