Merge pull request #392 from antoviaque/supervisor-ora

Move ora to supervisor

playbooks/roles/ora/defaults/main.yml

@@ -24,6 +24,50 @@ ora_ease_version: 'HEAD'
 ora_ease_pre_requirements_file: "{{ ora_ease_code_dir }}/pre-requirements.txt"
 ora_ease_post_requirements_file: "{{ ora_ease_code_dir }}/requirements.txt"
 
+ORA_LOGGING_ENV: "prod-grader"
+ORA_REQUESTS_TIMEOUT: 5
+ORA_QUEUES_TO_PULL_FROM: "open-ended"
+ORA_TIME_BETWEEN_XQUEUE_PULLS: 5
+ORA_TIME_BETWEEN_EXPIRED_CHECKS: 1800
+ORA_GRADER_SETTINGS_DIRECTORY: "grader_settings/"
+ORA_MAX_NUMBER_OF_TIMES_TO_RETRY_GRADING: 10
+ORA_MIN_TO_USE_ML: 100
+ORA_ML_PATH: "machine-learning/"
+ORA_ML_MODEL_PATH: "ml_models/"
+ORA_TIME_BETWEEN_ML_CREATOR_CHECKS: 300
+ORA_TIME_BETWEEN_ML_GRADER_CHECKS: 5
+ORA_MIN_TO_USE_PEER: 10
+ORA_PEER_GRADER_COUNT: 3
+ORA_PEER_GRADER_MINIMUM_TO_CALIBRATE: 3
+ORA_PEER_GRADER_MAXIMUM_TO_CALIBRATE: 6
+ORA_PEER_GRADER_MIN_NORMALIZED_CALIBRATION_ERROR: 0.5
+ORA_EXPIRE_SUBMISSIONS_AFTER: 1296000
+ORA_RESET_SUBMISSIONS_AFTER: 600
+ORA_LOCAL_LOGLEVEL: "DEBUG"
+ORA_DEBUG: false
+ORA_SYSLOG_SERVER: "localhost"
+ORA_USE_S3_TO_STORE_MODELS: false
+ORA_S3_BUCKETNAME: "openended-prod"
+ORA_LANG: "en_US.UTF-8"
+
+ORA_USERS:
+  "xqueue_pull": "password"
+  "lms": "password"
+
+ORA_XQUEUE_URL: "http://localhost:18040"
+
+ORA_DJANGO_USER: "lms"
+ORA_DJANGO_PASSWORD: "password"
+ORA_URL: "http://localhost:18060"
+
+ORA_MYSQL_DB_NAME: 'ora'
+ORA_MYSQL_USER: 'root'
+ORA_MYSQL_PASSWORD: ''
+ORA_MYSQL_HOST: 'localhost'
+ORA_MYSQL_PORT: '3306'
+
+ORA_AWS_ACCESS_KEY_ID: ''
+ORA_AWS_SECRET_ACCESS_KEY: ''
 
 # Default nginx listen port
 # These should be overrided if you want
@@ -38,51 +82,55 @@ ora_gunicorn_host: 127.0.0.1
 # appropriate for running all edX
 # services on a single server.
 ora_env_config:
-  "LOGGING_ENV": "prod-grader"
-  "LOG_DIR": "/logs"
-  "REQUESTS_TIMEOUT": "5"
-  "QUEUES_TO_PULL_FROM": "open-ended"
-  "TIME_BETWEEN_XQUEUE_PULLS": "5"
-  "TIME_BETWEEN_EXPIRED_CHECKS": "1800"
-  "GRADER_SETTINGS_DIRECTORY": "grader_settings/"
-  "MAX_NUMBER_OF_TIMES_TO_RETRY_GRADING": "10"
-  "MIN_TO_USE_ML": "100"
-  "ML_PATH": "machine-learning/"
-  "ML_MODEL_PATH": "ml_models/"
-  "TIME_BETWEEN_ML_CREATOR_CHECKS": "300"
-  "TIME_BETWEEN_ML_GRADER_CHECKS": "5"
-  "MIN_TO_USE_PEER": "10"
-  "PEER_GRADER_COUNT": "3"
-  "PEER_GRADER_MINIMUM_TO_CALIBRATE": "3"
-  "PEER_GRADER_MAXIMUM_TO_CALIBRATE": "6"
-  "PEER_GRADER_MIN_NORMALIZED_CALIBRATION_ERROR": ".5"
-  "EXPIRE_SUBMISSIONS_AFTER": "1296000"
-  "RESET_SUBMISSIONS_AFTER": "600"
-  "LOCAL_LOGLEVEL": "DEBUG"
-  "DEBUG": false
-  "SYSLOG_SERVER": "localhost"
-  "USE_S3_TO_STORE_MODELS": false
-  "S3_BUCKETNAME": "openended-prod"
+  LOGGING_ENV: $ORA_LOGGING_ENV
+  LOG_DIR: "{{ COMMON_DATA_DIR }}/logs/xqueue"
+  REQUESTS_TIMEOUT: $ORA_REQUESTS_TIMEOUT
+  QUEUES_TO_PULL_FROM: $ORA_QUEUES_TO_PULL_FROM
+  TIME_BETWEEN_XQUEUE_PULLS: $ORA_TIME_BETWEEN_XQUEUE_PULLS
+  TIME_BETWEEN_EXPIRED_CHECKS: $ORA_TIME_BETWEEN_EXPIRED_CHECKS
+  GRADER_SETTINGS_DIRECTORY: $ORA_GRADER_SETTINGS_DIRECTORY
+  MAX_NUMBER_OF_TIMES_TO_RETRY_GRADING: $ORA_MAX_NUMBER_OF_TIMES_TO_RETRY_GRADING
+  MIN_TO_USE_ML: $ORA_MIN_TO_USE_ML
+  ML_PATH: $ORA_ML_PATH
+  ML_MODEL_PATH: $ORA_ML_MODEL_PATH
+  TIME_BETWEEN_ML_CREATOR_CHECKS: $ORA_TIME_BETWEEN_ML_CREATOR_CHECKS
+  TIME_BETWEEN_ML_GRADER_CHECKS: $ORA_TIME_BETWEEN_ML_GRADER_CHECKS
+  MIN_TO_USE_PEER: $ORA_MIN_TO_USE_PEER
+  PEER_GRADER_COUNT: $ORA_PEER_GRADER_COUNT
+  PEER_GRADER_MINIMUM_TO_CALIBRATE: $ORA_PEER_GRADER_MINIMUM_TO_CALIBRATE
+  PEER_GRADER_MAXIMUM_TO_CALIBRATE: $ORA_PEER_GRADER_MAXIMUM_TO_CALIBRATE
+  PEER_GRADER_MIN_NORMALIZED_CALIBRATION_ERROR: $ORA_PEER_GRADER_MIN_NORMALIZED_CALIBRATION_ERROR
+  EXPIRE_SUBMISSIONS_AFTER: $ORA_EXPIRE_SUBMISSIONS_AFTER
+  RESET_SUBMISSIONS_AFTER: $ORA_RESET_SUBMISSIONS_AFTER
+  LOCAL_LOGLEVEL: $ORA_LOCAL_LOGLEVEL
+  DEBUG: $ORA_DEBUG
+  SYSLOG_SERVER: ORA_SYSLOG_SERVER
+  USE_S3_TO_STORE_MODELS: ORA_USE_S3_TO_STORE_MODELS
+  S3_BUCKETNAME: $ORA_S3_BUCKETNAME
 
 ora_auth_config:
-  "USERS":
-    "xqueue_pull": "password"
-    "lms": "password"
-  "XQUEUE_INTERFACE":
-    "django_auth":
-      "password": "password"
-      "username": "lms"
-    "basic_auth": [ "edx", "edx"]
-    "url": "http://localhost:18040"
-  "GRADING_CONTROLLER_INTERFACE":
-    "django_auth":
-      "password": "password"
-      "username": "lms"
-    "url": "http://localhost:18060"
-  'DATABASES':
-    'default': { 'ENGINE': 'django.db.backends.mysql', 'NAME': 'ora', 'USER': 'root', 'PASSWORD': '', 'HOST': 'localhost', 'PORT': '3306' }
-  'AWS_ACCESS_KEY_ID' : ''
-  'AWS_SECRET_ACCESS_KEY' : ''
+  USERS: $ORA_USERS
+  XQUEUE_INTERFACE:
+    django_auth:
+      password: $XQUEUE_DJANGO_USER
+      username: $XQUEUE_DJANGO_PASSWORD
+    basic_auth: [$XQUEUE_BASIC_AUTH_USER, $XQUEUE_BASIC_AUTH_PASSWORD]
+    url: $ORA_XQUEUE_URL
+  GRADING_CONTROLLER_INTERFACE:
+    django_auth:
+      password: $ORA_DJANGO_PASSWORD
+      username: $ORA_DJANGO_USER
+    url: $ORA_URL
+  DATABASES:
+    default: 
+      ENGINE: 'django.db.backends.mysql'
+      NAME: $ORA_MYSQL_DB_NAME
+      USER: $ORA_MYSQL_USER
+      PASSWORD: $ORA_MYSQL_PASSWORD
+      HOST: $ORA_MYSQL_HOST
+      PORT: $ORA_MYSQL_PORT
+  AWS_ACCESS_KEY_ID: $ORA_AWS_ACCESS_KEY_ID
+  AWS_SECRET_ACCESS_KEY: $ORA_AWS_SECRET_ACCESS_KEY
 
 ora_debian_pkgs:
   - python-software-properties

playbooks/roles/ora/handlers/main.yml

 ---
-- name: ora | restart edx-ora
-  service: name=edx-ora state=restarted
-- name: ora | restart edx-ora-celery
-  service: name=edx-ora-celery state=restarted
+- name: ora | restart ora
+  supervisorctl: >
+    name=ora
+    supervisorctl_path={{ supervisor_ctl }}
+    config={{ supervisor_cfg }}
+    state=restarted
+
+- name: ora | restart ora_celery
+  supervisorctl: >
+    name=ora_celery
+    supervisorctl_path={{ supervisor_ctl }}
+    config={{ supervisor_cfg }}
+    state=restarted

playbooks/roles/ora/tasks/deploy.yml

 
-- include: ease.yml
-
-- name: ora | See if edx-ora.conf is installed
-  shell: "[ -f /etc/init/edx-ora.conf ] && echo 'Found' || echo ''"
-  register: edx_ora_installed
+- name: ora | See if supervisord config file `ora.conf` is installed
+  shell: "[ -f {{ supervisor_cfg_dir }}/ora.conf ] && echo 'Found' || echo ''"
+  register: ora_installed
   tags:
   - deploy
 
-- name: ora | See if edx-ora-celery.conf is installed
-  shell: "[ -f /etc/init/edx-ora-celery.conf ] && echo 'Found' || echo ''"
-  register: edx_ora_celery_installed
+- name: ora | See if supervisord `ora_celery.conf` is installed
+  shell: "[ -f {{ supervisor_cfg_dir }}/ora_celery.conf ] && echo 'Found' || echo ''"
+  register: ora_celery_installed
   tags:
   - deploy
 
 # Stop ora service.
-- name: ora | stop edx-ora service
-  service: name=edx-ora state=stopped
-  when: edx_ora_installed.stdout == 'Found'
-  notify: ora | restart edx-ora
+- name: ora | stop ora service
+  supervisorctl: >
+    name=ora
+    supervisorctl_path={{ supervisor_ctl }}
+    config={{ supervisor_cfg }}
+    state=stopped
+  when: ora_installed.stdout == 'Found'
+  notify: ora | restart ora
   tags:
   - deploy
 
-- name: ora | stop edx-ora-celery service
-  service: name=edx-ora-celery state=stopped
-  when: edx_ora_celery_installed.stdout == 'Found'
-  notify: ora | restart edx-ora-celery
+- name: ora | stop ora_celery service
+  supervisorctl: >
+    name=ora_celery
+    supervisorctl_path={{ supervisor_ctl }}
+    config={{ supervisor_cfg }}
+    state=stopped
+  when: ora_celery_installed.stdout == 'Found'
+  notify: ora | restart ora_celery
   tags:
   - deploy
 
+- include: ease.yml
+
 - name: ora | create ora application config
   template: src=ora.env.json.j2 dest={{ora_app_dir}}/ora.env.json
   sudo_user: "{{ ora_user }}"
@@ -40,23 +48,24 @@
   tags:
   - deploy
 
-- name: ora | create ora upstart script
-  template: src=edx-ora.conf.j2 dest=/etc/init/edx-ora.conf
+- name: ora | create supervisor scripts - ora, ora_celery
+  template: >
+    src={{ item }}.conf.j2 dest={{ supervisor_cfg_dir }}/{{ item }}.conf
+    owner={{ supervisor_user }} group={{ common_web_user }} mode=0644
+  with_items: ['ora', 'ora_celery']
   tags:
   - deploy
 
-- name: ora | create ora-celery upstart script
-  template: src=edx-ora-celery.conf.j2 dest=/etc/init/edx-ora-celery.conf
-  tags:
-  - deploy
+- name: ora | reload supervisor to allow to start the new services
+  shell: "{{ supervisor_ctl }} -c {{ supervisor_cfg }} reload"
 
 # Do A Checkout
 - name: ora | git checkout ora repo into {{ ora_app_dir }}
   git: dest={{ ora_code_dir }} repo={{ ora_source_repo }} version={{ ora_version }}
   sudo_user: "{{ ora_user }}"
   notify:
-    - ora | restart edx-ora
-    - ora | restart edx-ora-celery
+    - ora | restart ora
+    - ora | restart ora_celery
   tags:
   - deploy
 
@@ -69,8 +78,8 @@
   pip: requirements="{{ ora_pre_requirements_file }}" virtualenv="{{ ora_venv_dir }}" state=present
   sudo_user: "{{ ora_user }}"
   notify:
-    - ora | restart edx-ora
-    - ora | restart edx-ora-celery
+    - ora | restart ora
+    - ora | restart ora_celery
   tags:
   - deploy
 
@@ -79,8 +88,8 @@
   pip: requirements="{{ ora_post_requirements_file }}" virtualenv="{{ ora_venv_dir }}" state=present
   sudo_user: "{{ ora_user }}"
   notify:
-    - ora | restart edx-ora
-    - ora | restart edx-ora-celery
+    - ora | restart ora
+    - ora | restart ora_celery
   tags:
   - deploy
 
@@ -88,8 +97,8 @@
 - name: ora | change memory commit settings -- needed for redis
   command: sysctl vm.overcommit_memory=1
   notify:
-    - ora | restart edx-ora
-    - ora | restart edx-ora-celery
+    - ora | restart ora
+    - ora | restart ora_celery
   tags:
   - deploy
 
@@ -98,8 +107,8 @@
   when: migrate_db is defined and migrate_db|lower == "yes"
   sudo_user: "{{ ora_user }}"
   notify:
-    - ora | restart edx-ora
-    - ora | restart edx-ora-celery
+    - ora | restart ora
+    - ora | restart ora_celery
   tags:
   - deploy
 
@@ -107,7 +116,7 @@
   shell: SERVICE_VARIANT=ora {{ora_venv_dir}}/bin/django-admin.py update_users --settings=edx_ora.aws --pythonpath={{ora_code_dir}}
   sudo_user: "{{ ora_user }}"
   notify:
-    - ora | restart edx-ora
-    - ora | restart edx-ora-celery
+    - ora | restart ora
+    - ora | restart ora_celery
   tags:
   - deploy

playbooks/roles/ora/tasks/ease.yml

-- name: ora | See if edx-ora.conf is installed
-  shell: "[ -f /etc/init/edx-ora.conf ] && echo 'Found' || echo ''"
-  register: edx_ora_installed
-  tags:
-  - deploy
-
-- name: ora | See if edx-ora-celery.conf is installed
-  shell: "[ -f /etc/init/edx-ora-celery.conf ] && echo 'Found' || echo ''"
-  register: edx_ora_celery_installed
-  tags:
-  - deploy
-
-# Stop ora service.
-- name: ora | stop edx-ora service
-  service: name=edx-ora state=stopped
-  when: edx_ora_installed.stdout == 'Found'
-  tags:
-  - deploy
-
-- name: ora | stop edx-ora-celery service
-  service: name=edx-ora-celery state=stopped
-  when: edx_ora_celery_installed.stdout == 'Found'
-  tags:
-  - deploy
-
-
 # Do A Checkout
 - name: ora | git checkout ease repo into its base dir
   git: dest={{ora_ease_code_dir}} repo={{ora_ease_source_repo}} version={{ora_ease_version}}
@@ -70,6 +44,6 @@
 - name: ora | install nltk data using rendered shell script
   command: "{{ora_ease_venv_dir}}/bin/python -m nltk.downloader -d {{ ora_nltk_data_dir }} all"
   when: ora_nltk_data_installed.stdout  != "Found"
-  sudo_user: "{{ ora_user }}"
+  sudo_user: "{{ common_web_user }}"
   tags:
   - deploy

playbooks/roles/ora/tasks/main.yml

@@ -8,13 +8,19 @@
     name="{{ ora_user }}" home="{{ ora_app_dir }}"
     createhome=no shell=/bin/false
 
-- name: ora | create ora app and data dir
+- name: ora | create ora app dir
   file: >
     path="{{ item }}" state=directory
     owner="{{ ora_user }}" group="{{ common_web_group }}"
   with_items:
     - "{{ ora_venvs_dir }}"
     - "{{ ora_app_dir }}"
+
+- name: ora | create ora data dir, owned by {{ common_web_user }}
+  file: >
+    path="{{ item }}" state=directory
+    owner="{{ common_web_user }}" group="{{ common_web_group }}"
+  with_items:
     - "{{ ora_data_dir }}"
     - "{{ ora_app_dir }}/ml_models"
 

playbooks/roles/ora/templates/edx-ora-celery.conf.j2

-#Celery task for edx-ora
-
-description "Celery grading controller"
-author "Vik Paruchuri <vik@edx.org>"
-
-start on runlevel [2345]
-stop on runlevel [!2345]
-
-respawn
-respawn limit 3 30
-
-env DJANGO_SETTINGS_MODULE=edx_ora.aws
-env SERVICE_VARIANT=ora
-
-chdir {{ ora_code_dir }}
-setuid {{ ora_user }}
-
-exec {{ ora_venv_dir }}/bin/python {{ ora_code_dir }}/manage.py celeryd --loglevel=info --settings=edx_ora.aws --pythonpath={{ ora_code_dir}} -B --autoscale=4,1

playbooks/roles/ora/templates/edx-ora.conf.j2

-# /etc/init/edx-ora.conf
-
-description "edX ORA server"
-author "edx <info@edx.org>"
-
-start on runlevel [2345]
-stop on runlevel [!2345]
-
-respawn
-respawn limit 3 30
-
-env PID=/var/run/gunicorn/edx-ora.pid
-env WORKERS={{ ora_gunicorn_workers }}
-env PORT={{ ora_gunicorn_port }}
-env ADDRESS={{ ora_gunicorn_host }}
-env LANG=en_US.UTF-8
-env DJANGO_SETTINGS_MODULE=edx_ora.aws
-env SERVICE_VARIANT=ora
-env NLTK_DATA={{ ora_nltk_data_dir }}
-
-pre-start script
-end script
-
-chdir {{ ora_code_dir }}
-setuid {{ ora_user }}
-
-exec {{ ora_venv_dir}}/bin/gunicorn --preload -b $ADDRESS:$PORT -w $WORKERS --timeout=90 --pythonpath={{ ora_code_dir}} edx_ora.wsgi

playbooks/roles/ora/templates/ora.conf.j2

+[program:ora]
+
+command={{ ora_venv_bin }}/gunicorn --preload -b {{ ora_gunicorn_host }}:{{ ora_gunicorn_port }} -w {{ ora_gunicorn_workers }} --timeout=90 --pythonpath={{ ora_code_dir}} edx_ora.wsgi
+
+user={{ common_web_user }}
+directory={{ ora_code_dir }}
+
+environment=PID=/var/run/gunicorn/edx-ora.pid,WORKERS={{ ora_gunicorn_workers }},PORT={{ ora_gunicorn_port }},ADDRESS={{ ora_gunicorn_host }},LANG={{ ORA_LANG }},DJANGO_SETTINGS_MODULE=edx_ora.aws,SERVICE_VARIANT=ora,NLTK_DATA={{ ora_nltk_data_dir }}
+
+stdout_logfile={{ supervisor_log_dir }}/%(program_name)-stdout.log
+stderr_logfile={{ supervisor_log_dir }}/%(program_name)-stderr.log
+

playbooks/roles/ora/templates/ora_celery.conf.j2

+[program:ora_celery]
+
+command={{ ora_venv_bin }}/python {{ ora_code_dir }}/manage.py celeryd --loglevel=info --settings=edx_ora.aws --pythonpath={{ ora_code_dir }} -B --autoscale=4,1 --schedule={{ ora_data_dir }}/celerybeat-schedule
+
+user={{ common_web_user }}
+directory={{ ora_code_dir }}
+
+environment=DJANGO_SETTINGS_MODULE=edx_ora.aws,SERVICE_VARIANT=ora
+
+stdout_logfile={{ supervisor_log_dir }}/%(program_name)-stdout.log
+stderr_logfile={{ supervisor_log_dir }}/%(program_name)-stderr.log
+