Merge pull request #936 from edx/feanil/nginx_robots

Feanil/nginx robots

playbooks/roles/nginx/defaults/main.yml

@@ -52,3 +52,5 @@ nginx_cfg:
   # nginx configuration
   version_html: "{{ nginx_app_dir }}/versions.html"
   version_json: "{{ nginx_app_dir }}/versions.json"
+
+NGINX_ROBOT_RULES: [ ]

playbooks/roles/nginx/tasks/main.yml

@@ -31,16 +31,24 @@
 
 - name: Server configuration file
   template: >
-    src=nginx.conf.j2 dest=/etc/nginx/nginx.conf
+    src=etc/nginx/nginx.conf.j2 dest=/etc/nginx/nginx.conf
     owner=root group={{ common_web_user }} mode=0644
   notify: reload nginx
 
 - name: Creating common nginx configuration
   template: >
-    src=edx-release.j2 dest={{ nginx_sites_available_dir }}/edx-release
+    src=edx/app/nginx/sites-available/edx-release.j2
+    dest={{ nginx_sites_available_dir }}/edx-release
     owner=root group=root mode=0600
   notify: reload nginx
 
+- name: Create robot rules
+  template: >
+    src=edx/app/nginx/robots.txt.j2 dest={{ nginx_app_dir }}/robots.txt
+    owner=root group={{ common_web_user }} mode=0644
+  notify: reload nginx
+  when: NGINX_ROBOT_RULES|length > 0
+
 - name: Creating link for common nginx configuration
   file: >
     src={{ nginx_sites_available_dir }}/edx-release
@@ -50,7 +58,8 @@
 
 - name: Copying nginx configs for {{ nginx_sites }}
   template: >
-    src={{ item }}.j2 dest={{ nginx_sites_available_dir }}/{{ item }}
+    src=edx/app/nginx/sites-available/{{ item }}.j2
+    dest={{ nginx_sites_available_dir }}/{{ item }}
     owner=root group={{ common_web_user }} mode=0640
   notify: reload nginx
   with_items: nginx_sites
@@ -113,24 +122,14 @@
 
 - name: Set up nginx access log rotation
   template: >
-    dest=/etc/logrotate.d/nginx-access src=edx_logrotate_nginx_access.j2
-    owner=root group=root mode=644
-
-# removing default link
-- name: Removing default nginx config and restart (enabled)
-  file: path={{ nginx_sites_enabled_dir }}/default state=absent
-  notify: reload nginx
-
-# Note that nginx logs to /var/log until it reads its configuration, so /etc/logrotate.d/nginx is still good
-
-- name: Set up nginx access log rotation
-  template: >
-    dest=/etc/logrotate.d/nginx-access src=edx_logrotate_nginx_access.j2
+    src=etc/logrotate.d/edx_logrotate_nginx_access.j2
+    dest=/etc/logrotate.d/nginx-access
     owner=root group=root mode=644
 
 - name: Set up nginx access log rotation
   template: >
-    dest=/etc/logrotate.d/nginx-error src=edx_logrotate_nginx_error.j2
+    src=etc/logrotate.d/edx_logrotate_nginx_error.j2
+    dest=/etc/logrotate.d/nginx-error
     owner=root group=root mode=644
 
 # If tasks that notify restart nginx don't change the state of the remote system

playbooks/roles/nginx/templates/edx/app/nginx/robots.txt.j2

+{% for item in NGINX_ROBOT_RULES %}
+User-agent: {{ item.agent }}
+Disallow: {{ item.disallow }}
+
+{% endfor %}

playbooks/roles/nginx/templates/cms.j2 → playbooks/roles/nginx/templates/edx/app/nginx/sites-available/cms.j2

@@ -66,6 +66,8 @@ server {
     try_files $uri @proxy_to_cms_app;
   }
 
+  {% include "robots.j2" %}
+
   # Check security on this
   location ~ /static/(?P<file>.*) {
     root {{ edxapp_data_dir }};

playbooks/roles/nginx/templates/devpi.j2 → playbooks/roles/nginx/templates/edx/app/nginx/sites-available/devpi.j2

@@ -12,5 +12,7 @@ server {
         proxy_set_header  X-outside-url $scheme://$host;
         proxy_set_header  X-Real-IP $remote_addr;
     }
+
+    {% include robots.j2 %}
 }
 

playbooks/roles/nginx/templates/forum.j2 → playbooks/roles/nginx/templates/edx/app/nginx/sites-available/forum.j2

@@ -40,6 +40,8 @@ server {
     try_files $uri @proxy_to_app;
   }
 
+  {% include "robots.j2" %}
+
 location @proxy_to_app {
     proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto;
     proxy_set_header X-Forwarded-Port $http_x_forwarded_port;

playbooks/roles/nginx/templates/lms-preview.j2 → playbooks/roles/nginx/templates/edx/app/nginx/sites-available/lms-preview.j2

@@ -46,6 +46,8 @@ server {
     try_files $uri @proxy_to_lms-preview_app;
   }
 
+  {% include "robots.j2" %}
+
   # Check security on this
   location ~ /static/(?P<file>.*) {
     root {{ edxapp_data_dir}};

playbooks/roles/nginx/templates/lms.j2 → playbooks/roles/nginx/templates/edx/app/nginx/sites-available/lms.j2

@@ -62,6 +62,8 @@ server {
     try_files $uri @proxy_to_lms_app;
   }
 
+  {% include "robots.j2" %}
+
   # Check security on this
   location ~ /static/(?P<file>.*) {
     root {{ edxapp_data_dir }};

playbooks/roles/nginx/templates/ora.j2 → playbooks/roles/nginx/templates/edx/app/nginx/sites-available/ora.j2

@@ -31,6 +31,8 @@ server {
     expires epoch;
   }
 
+  {% include "robots.j2" %}
+
   location @proxy_to_app {
     client_max_body_size 75K;
     proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto;

playbooks/roles/nginx/templates/edx/app/nginx/sites-available/robots.j2

+{% if NGINX_ROBOT_RULES|length > 0 %}
+  location /robots.txt {
+    root {{ nginx_app_dir }};
+    try_files $uri /robots.txt =404;
+  }
+{% endif %}

playbooks/roles/nginx/templates/xqueue.j2 → playbooks/roles/nginx/templates/edx/app/nginx/sites-available/xqueue.j2

@@ -17,6 +17,8 @@ server {
     try_files $uri @proxy_to_app;
   }
 
+  {% include "robots.j2" %}
+
 location @proxy_to_app {
     proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto;
     proxy_set_header X-Forwarded-Port $http_x_forwarded_port;