Merge pull request #559 from edx/jarv/nltk-download

switch to s3 download for nltk

playbooks/roles/discern/defaults/main.yml

@@ -31,6 +31,8 @@ discern_ease_pre_requirements_file: "{{ discern_ease_code_dir }}/pre-requirement
 discern_ease_post_requirements_file: "{{ discern_ease_code_dir }}/requirements.txt"
 
 discern_nltk_data_dir: "{{ discern_data_dir}}/nltk_data"
+discern_nltk_download_url: http://edx-static.s3.amazonaws.com/nltk/nltk-data-20131113.tar.gz
+discern_nltk_tmp_file: "{{ discern_nltk_data_dir }}/nltk.tmp.tar.tz"
 
 discern_source_repo: https://github.com/edx/discern.git
 discern_settings: discern.aws

playbooks/roles/discern/tasks/deploy.yml

@@ -71,17 +71,22 @@
   tags:
     - deploy
 
-#Needed for the ease package to work
-- name: discern | install nltk data using rendered shell script
-  shell: >
-    {{ discern_venv_dir }}/bin/python -m nltk.downloader -d {{ discern_nltk_data_dir }} all
-      creates={{ discern_nltk_data_dir }}/taggers/maxent_treebank_pos_tagger/english.pickle
+- name: discern | download and install nltk
+  shell: |
+      set -e
+      curl -o {{ discern_nltk_tmp_file }} {{ discern_nltk_download_url }}
+      tar zxf {{ discern_nltk_tmp_file }}
+      rm -f {{ discern_nltk_tmp_file }}
+      touch {{ discern_nltk_download_url|basename }}-installed
+        creates={{ discern_data_dir }}/{{ discern_nltk_download_url|basename }}-installed
+        chdir={{ discern_data_dir }}
   sudo_user: "{{ discern_user }}"
   notify:
     - discern | restart discern
   tags:
     - deploy
 
+
 #Run this instead of using the ansible module because the ansible module only support syncdb of these three, and does not
 #support virtualenvs as of this comment
 - name: discern | django syncdb migrate and collectstatic for discern

playbooks/roles/ora/defaults/main.yml

@@ -12,6 +12,9 @@ ora_user: "ora"
 ora_deploy_path: "{{ ora_venv_bin }}:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
 ora_nltk_data_dir: "{{ ora_data_dir}}/nltk_data"
 
+ora_nltk_download_url: http://edx-static.s3.amazonaws.com/nltk/nltk-data-20131113.tar.gz
+ora_nltk_tmp_file: "{{ ora_nltk_data_dir }}/nltk.tmp.tar.tz"
+
 ora_source_repo: https://github.com/edx/edx-ora.git
 ora_version: 'master'
 ora_pre_requirements_file: "{{ ora_code_dir }}/pre-requirements.txt"

playbooks/roles/ora/tasks/ease.yml

@@ -48,10 +48,15 @@
   tags:
   - deploy
 
-- name: ora | install nltk data using rendered shell script
-  command: >
-    "{{ora_ease_venv_dir}}/bin/python -m nltk.downloader -d {{ ora_nltk_data_dir }} all"
-      creates={{ ora_nltk_data_dir }}/taggers/maxent_treebank_pos_tagger/english.pickle
+- name: ora | download and install nltk
+  shell: |
+      set -e
+      curl -o {{ ora_nltk_tmp_file }} {{ ora_nltk_download_url }}
+      tar zxf {{ ora_nltk_tmp_file }}
+      rm -f {{ ora_nltk_tmp_file }}
+      touch {{ ora_nltk_download_url|basename }}-installed
+        creates={{ ora_data_dir }}/{{ ora_nltk_download_url|basename }}-installed
+        chdir={{ ora_data_dir }}
   sudo_user: "{{ common_web_user }}"
   notify:
     - ora | restart ora