Add a robots.txt file if robot rules are defined.

playbooks/roles/nginx/defaults/main.yml

@@ -52,3 +52,5 @@ nginx_cfg:
   # nginx configuration
   version_html: "{{ nginx_app_dir }}/versions.html"
   version_json: "{{ nginx_app_dir }}/versions.json"
+
+NGINX_ROBOT_RULES: [ ]

playbooks/roles/nginx/tasks/main.yml

@@ -41,6 +41,13 @@
     owner=root group=root mode=0600
   notify: reload nginx
 
+- name: Create robot rules
+  template: >
+    src=robots.txt.j2 dest={{ nginx_app_dir }}/robots.txt
+    owner=root group={{ common_web_user }} mode=0644
+  notify: reload nginx
+  when: NGINX_ROBOT_RULES|length > 0
+
 - name: Creating link for common nginx configuration
   file: >
     src={{ nginx_sites_available_dir }}/edx-release

playbooks/roles/nginx/templates/robots.txt.j2

+{% for item in NGINX_ROBOT_RULES %}
+User-agent: {{ item.agent }}
+Disallow: {{ item.disallow }}
+
+{% endfor %}