Merge pull request #1892 from edx/feanil/edx_ecommerce

Feanil/edx ecommerce

playbooks/ansible.cfg

@@ -7,3 +7,4 @@
 jinja2_extensions=jinja2.ext.do
 host_key_checking = False
 roles_path=../../ansible-roles/roles:../../ansible-private/roles:../../ansible-roles/
+ansible_managed=This file is created and updated by ansible, edit at your peril
\ No newline at end of file

playbooks/edx-east/ansible.cfg

@@ -7,3 +7,4 @@
 jinja2_extensions=jinja2.ext.do
 host_key_checking=False
 roles_path=../../../ansible-roles/roles:../../../ansible-private/roles:../../../ansible-roles/
+ansible_managed=This file is created and updated by ansible, edit at your peril

playbooks/edx-east/edx_ecommerce.yml

+- name: Deploy edX Ecommerce
+  hosts: all
+  sudo: True
+  gather_facts: True
+  vars:
+    ENABLE_DATADOG: False
+    ENABLE_SPLUNKFORWARDER: False
+    ENABLE_NEWRELIC: False
+  roles:
+    - role: nginx
+      nginx_sites:
+        - edx_ecommerce
+    - aws
+    - edx_ecommerce
+    - role: datadog
+      when: COMMON_ENABLE_DATADOG
+    - role: splunkforwarder
+      when: COMMON_ENABLE_SPLUNKFORWARDER
+    - role: newrelic
+      when: COMMON_ENABLE_NEWRELIC

playbooks/roles/edx_ecommerce/defaults/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Defaults for role edx_ecommerce
+# 
+
+EDX_ECOMMERCE_GIT_IDENTITY: !!null
+
+# depends upon Newrelic being enabled via COMMON_ENABLE_NEWRELIC
+# and a key being provided via NEWRELIC_LICENSE_KEY
+EDX_ECOMMERCE_NEWRELIC_APPNAME: "{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-{{ edx_ecommerce_service_name }}"
+EDX_ECOMMERCE_PIP_EXTRA_ARGS: "-i {{ COMMON_PYPI_MIRROR_URL }}"
+EDX_ECOMMERCE_NGINX_PORT: "18130"
+
+EDX_ECOMMERCE_DEFAULT_DB_NAME: 'edx_ecommerce'
+
+EDX_ECOMMERCE_DATABASES:
+  # rw user
+  default:
+    ENGINE: 'django.db.backends.mysql'
+    NAME: '{{ EDX_ECOMMERCE_DEFAULT_DB_NAME }}'
+    USER: 'ecomm001'
+    PASSWORD: 'password'
+    HOST: 'localhost'
+    PORT: '3306'
+
+EDX_ECOMMERCE_VERSION: "master"
+EDX_DJANGO_OSCAR_VERSION: "master"
+EDX_DJANGO_OSCAR_EXTENSIONS_VERSION: "master"
+
+# Default dummy user, override this!!
+EDX_ECOMMERCE_USERS:
+  "dummy-api-user": "changeme"
+
+EDX_ECOMMERCE_SECRET_KEY: 'Your secret key here'
+EDX_ECOMMERCE_TIME_ZONE: 'UTC'
+EDX_ECOMMERCE_LANGUAGE_CODE: 'en-us'
+EDX_ECOMMERCE_EMAIL_HOST: 'localhost'
+EDX_ECOMMERCE_EMAIL_HOST_USER: 'mail_user'
+EDX_ECOMMERCE_EMAIL_HOST_PASSWORD: 'mail_password'
+EDX_ECOMMERCE_EMAIL_PORT: 587
+EDX_ECOMMERCE_AUTH_TOKEN: 'put-your-api-token-here'
+
+EDX_ECOMMERCE_SERVICE_CONFIG:
+  EDX_ECOMMERCE_DATABASE: 'reports'
+  SECRET_KEY: '{{ EDX_ECOMMERCE_SECRET_KEY }}'
+  TIME_ZONE: '{{ EDX_ECOMMERCE_TIME_ZONE }}'
+  LANGUAGE_CODE: '{{EDX_ECOMMERCE_LANGUAGE_CODE }}'
+  # email config
+  EMAIL_HOST: '{{ EDX_ECOMMERCE_EMAIL_HOST }}'
+  EMAIL_HOST_PASSWORD: '{{ EDX_ECOMMERCE_EMAIL_HOST_PASSWORD }}'
+  EMAIL_HOST_USER: '{{ EDX_ECOMMERCE_EMAIL_HOST_USER }}'
+  EMAIL_PORT: $EDX_ECOMMERCE_EMAIL_PORT
+  API_AUTH_TOKEN: '{{ EDX_ECOMMERCE_AUTH_TOKEN }}'
+  STATICFILES_DIRS: []
+#  STATICFILES_DIRS: ['{{ edx_ecommerce_static_path }}']
+  STATIC_ROOT: "{{ COMMON_DATA_DIR }}/{{ edx_ecommerce_service_name }}/staticfiles"
+  # db config
+  DATABASE_OPTIONS:
+    connect_timeout: 10
+  DATABASES: '{{ EDX_ECOMMERCE_DATABASES }}'
+
+EDX_ECOMMERCE_REPOS:
+  - PROTOCOL: "{{ COMMON_GIT_PROTOCOL }}"
+    DOMAIN: "{{ COMMON_GIT_MIRROR }}"
+    PATH: "{{ COMMON_GIT_PATH }}"
+    REPO: edx-ecommerce.git
+    VERSION: "{{ EDX_ECOMMERCE_VERSION }}"
+    DESTINATION: "{{ edx_ecommerce_code_dir }}"
+    SSH_KEY: "{{ EDX_ECOMMERCE_GIT_IDENTITY }}"
+  - PROTOCOL: "{{ COMMON_GIT_PROTOCOL }}"
+    DOMAIN: "{{ COMMON_GIT_MIRROR }}"
+    PATH: "{{ COMMON_GIT_PATH }}"
+    REPO: django-oscar.git
+    VERSION: "{{ EDX_DJANGO_OSCAR_VERSION }}"
+    DESTINATION: "{{ edx_ecommerce_home }}/depends/django-oscar"
+    SSH_KEY: "{{ EDX_ECOMMERCE_GIT_IDENTITY }}"
+  - PROTOCOL: "{{ COMMON_GIT_PROTOCOL }}"
+    DOMAIN: "{{ COMMON_GIT_MIRROR }}"
+    PATH: "{{ COMMON_GIT_PATH }}"
+    REPO: django-oscar-extensions.git
+    VERSION: "{{ EDX_DJANGO_OSCAR_EXTENSIONS_VERSION }}"
+    DESTINATION: "{{ edx_ecommerce_home }}/depends/django-oscar-extensions"
+    SSH_KEY: "{{ EDX_ECOMMERCE_GIT_IDENTITY }}"
+
+    
+EDX_ECOMMERCE_GUNICORN_WORKERS: "2"
+EDX_ECOMMERCE_GUNICORN_EXTRA: ""
+EDX_ECOMMERCE_GUNICORN_EXTRA_CONF: ""
+
+#
+# vars are namespace with the module name.
+#
+edx_ecommerce_environment:
+  DJANGO_SETTINGS_MODULE: "ecommerce.settings.production"
+  EDX_ECOMMERCE_CFG: "{{ COMMON_CFG_DIR  }}/{{ edx_ecommerce_service_name }}.yml"
+
+edx_ecommerce_service_name: "edx_ecommerce"
+edx_ecommerce_user: "{{ edx_ecommerce_service_name }}"
+edx_ecommerce_home: "{{ COMMON_APP_DIR }}/{{ edx_ecommerce_service_name }}"
+edx_ecommerce_code_dir: "{{ edx_ecommerce_home }}/{{ edx_ecommerce_service_name }}"
+edx_ecommerce_static_path: "{{ edx_ecommerce_code_dir }}/ecommerce/static"
+
+edx_ecommerce_gunicorn_host: "127.0.0.1"
+edx_ecommerce_gunicorn_port: "8130"
+edx_ecommerce_gunicorn_timeout: "300"
+
+edx_ecommerce_log_dir: "{{ COMMON_LOG_DIR }}/{{ edx_ecommerce_service_name }}"
+
+edx_ecommerce_requirements_base: "{{ edx_ecommerce_code_dir }}/requirements"
+edx_ecommerce_requirements:
+  - production.txt
+  #- optional.txt
+
+#
+# OS packages
+#
+
+edx_ecommerce_debian_pkgs:
+  - libmysqlclient-dev
+  - libjpeg-dev
+
+edx_ecommerce_redhat_pkgs: []

playbooks/roles/edx_ecommerce/meta/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Role includes for role edx_ecommerce
+# 
+dependencies:
+  - role: edx_service
+    edx_service_name: "{{ edx_ecommerce_service_name }}"
+    edx_service_config: "{{ EDX_ECOMMERCE_SERVICE_CONFIG }}"
+    edx_service_repos: "{{ EDX_ECOMMERCE_REPOS }}"
+    edx_service_user: "{{ edx_ecommerce_user }}"
+    edx_service_home: "{{ edx_ecommerce_home }}"
+    edx_service_packages:
+      debian: "{{ edx_ecommerce_debian_pkgs }}"
+      redhat: "{{ edx_ecommerce_redhat_pkgs }}"
+  - supervisor

playbooks/roles/edx_ecommerce/tasks/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+#
+# Tasks for role edx_ecommerce
+# 
+# Overview:
+# 
+#
+# Dependencies:
+#
+# 
+# Example play:
+#
+#
+
+- name: "add gunicorn configuration file"
+  template: >
+    src=edx/app/edx_ecommerce/edx_ecommerce_gunicorn.py.j2
+    dest={{ edx_ecommerce_home }}/edx_ecommerce_gunicorn.py
+  sudo_user: "{{ edx_ecommerce_user }}"
+
+- name: install application requirements
+  pip: >
+    requirements="{{ edx_ecommerce_requirements_base }}/{{ item }}"
+    virtualenv="{{ edx_ecommerce_home }}/venvs/{{ edx_ecommerce_service_name }}"
+    state=present
+  sudo_user: "{{ edx_ecommerce_user }}"
+  with_items: edx_ecommerce_requirements
+
+- name: migrate
+  shell: >
+    chdir={{ edx_ecommerce_code_dir }}
+    DB_MIGRATION_USER={{ COMMON_MYSQL_MIGRATE_USER }}
+    DB_MIGRATION_PASS={{ COMMON_MYSQL_MIGRATE_PASS }}
+    {{ edx_ecommerce_home }}/venvs/{{ edx_ecommerce_service_name }}/bin/python ./manage.py migrate --noinput
+  sudo_user: "{{ edx_ecommerce_user }}"
+  environment: "{{ edx_ecommerce_environment }}"
+  when: migrate_db is defined and migrate_db|lower == "yes"
+
+- name: run collectstatic
+  shell: >
+    chdir={{ edx_ecommerce_code_dir }}
+    {{ edx_ecommerce_home }}/venvs/{{ edx_ecommerce_service_name }}/bin/python manage.py collectstatic  --noinput
+  sudo_user: "{{ edx_ecommerce_user }}"
+  environment: "{{ edx_ecommerce_environment }}"
+
+- name: write out the supervisor wrapper
+  template: >
+    src=edx/app/edx_ecommerce/edx_ecommerce.sh.j2
+    dest={{ edx_ecommerce_home }}/{{ edx_ecommerce_service_name }}.sh
+    mode=0650 owner={{ supervisor_user }} group={{ common_web_user }}
+
+- name: write supervisord config
+  template: >
+    src=edx/app/supervisor/conf.d.available/edx_ecommerce.conf.j2
+    dest="{{ supervisor_available_dir }}/{{ edx_ecommerce_service_name }}.conf"
+    owner={{ supervisor_user }} group={{ common_web_user }} mode=0644
+
+- name: setup the ecommence env file
+  template: >
+    src="./{{ edx_ecommerce_home }}/{{ edx_ecommerce_service_name }}_env.j2"
+    dest="{{ edx_ecommerce_home }}/edx_ecommerce_env"
+    owner={{ edx_ecommerce_user }}
+    group={{ edx_ecommerce_user }}
+    mode=0644
+
+- name: enable supervisor script
+  file: >
+    src={{ supervisor_available_dir }}/{{ edx_ecommerce_service_name }}.conf
+    dest={{ supervisor_cfg_dir }}/{{ edx_ecommerce_service_name }}.conf
+    state=link
+    force=yes
+  when: not disable_edx_services
+
+- name: update supervisor configuration
+  shell: "{{ supervisor_ctl }} -c {{ supervisor_cfg }} update"
+  when: not disable_edx_services
+
+- name: create symlinks from the-er venv bin dir
+  file: >
+    src="{{ edx_ecommerce_home }}/venvs/{{ edx_ecommerce_service_name }}/bin/{{ item }}"
+    dest="{{ COMMON_BIN_DIR }}/{{ item.split('.')[0] }}.edx_ecommerce"
+    state=link
+  with_items:
+  - python
+  - pip
+  - django-admin.py
+
+- name: create symlinks from the repo dir
+  file: >
+    src="{{ edx_ecommerce_code_dir }}/{{ item }}"
+    dest="{{ COMMON_BIN_DIR }}/{{ item.split('.')[0] }}.edx_ecommerce"
+    state=link
+  with_items:
+  - manage.py
+
+- name: restart the applicaton
+  supervisorctl_local: >
+    state=restarted
+    supervisorctl_path={{ supervisor_ctl }}
+    config={{ supervisor_cfg }}
+    name={{ edx_ecommerce_service_name }}
+  when: not disable_edx_services
+  sudo_user: "{{ supervisor_service_user }}"

playbooks/roles/edx_ecommerce/templates/edx/app/edx_ecommerce/edx_ecommerce.sh.j2

+#!/usr/bin/env bash
+
+{{ ansible_managed }}
+
+{% set edx_ecommerce_venv_bin = edx_ecommerce_home + "/venvs/" + edx_ecommerce_service_name + "/bin" %}
+{% if COMMON_ENABLE_NEWRELIC_APP %}
+{% set executable = edx_ecommerce_venv_bin + '/newrelic-admin run-program ' + edx_ecommerce_venv_bin + '/gunicorn' %}
+{% else %}
+{% set executable = edx_ecommerce_venv_bin + '/gunicorn' %}
+{% endif %}
+
+{% if COMMON_ENABLE_NEWRELIC_APP %}
+export NEW_RELIC_APP_NAME="{{ EDX_ECOMMERCE_NEWRELIC_APPNAME }}"
+export NEW_RELIC_LICENSE_KEY="{{ NEWRELIC_LICENSE_KEY }}"
+{% endif -%}
+
+source {{ edx_ecommerce_home }}/edx_ecommerce_env
+# TODO fix application
+{{ executable }} -c {{ edx_ecommerce_home }}/edx_ecommerce_gunicorn.py {{ EDX_ECOMMERCE_GUNICORN_EXTRA }} ecommerce.wsgi:application

playbooks/roles/edx_ecommerce/templates/edx/app/edx_ecommerce/edx_ecommerce_env.j2

+# {{ ansible_managed }}
+
+{% for name,value in edx_ecommerce_environment.items() -%}
+{%- if value -%}
+export {{ name }}="{{ value }}"
+{% endif %}
+{%- endfor %}

playbooks/roles/edx_ecommerce/templates/edx/app/edx_ecommerce/edx_ecommerce_gunicorn.py.j2

+"""
+gunicorn configuration file: http://docs.gunicorn.org/en/develop/configure.html
+{{ ansible_managed }}
+"""
+
+timeout = {{ edx_ecommerce_gunicorn_timeout }}
+bind = "{{ edx_ecommerce_gunicorn_host }}:{{ edx_ecommerce_gunicorn_port }}"
+pythonpath = "{{ edx_ecommerce_code_dir }}"
+workers = {{ EDX_ECOMMERCE_GUNICORN_WORKERS }}
+
+{{ EDX_ECOMMERCE_GUNICORN_EXTRA_CONF }}

playbooks/roles/edx_ecommerce/templates/edx/app/supervisor/conf.d.available/edx_ecommerce.conf.j2

+#
+# {{ ansible_managed }}
+# 
+[program:{{ edx_ecommerce_service_name }}]
+
+command={{ edx_ecommerce_home }}/{{ edx_ecommerce_service_name }}.sh
+user={{ common_web_user }}
+directory={{ edx_ecommerce_code_dir }}
+stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log
+stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log
+killasgroup=true
+stopasgroup=true

playbooks/roles/edxlocal/tasks/main.yml

@@ -102,18 +102,32 @@
     encoding=utf8
   when: INSIGHTS_DATABASES is defined
 
+- name: create database for edx_ecommerce
+  mysql_db: >
+    db="{{ EDX_ECOMMERCE_DEFAULT_DB_NAME }}"
+    state=present
+    encoding=utf8
+  when: EDX_ECOMMERCE_DEFAULT_DB_NAME is defined
+
+- name: setup users for edx_ecommerce
+  mysql_user: >
+    name="{{ EDX_ECOMMERCE_DEFAULT_DB_NAME }}"
+    password="{{ EDX_ECOMMERCE_DATABASES.default.PASSWORD }}"
+    priv='{{ EDX_ECOMMERCE_DEFAULT_DB_NAME }}.*:SELECT,INSERT,UPDATE,DELETE'
+  when: EDX_ECOMMERCE_DEFAULT_DB_NAME is defined
+
 - name: create api user for the analytics api
   mysql_user: >
     name=api001
-    password=password
-    priv='{{ ANALYTICS_API_SERVICE_CONFIG['DATABASES']['default']['NAME'] }}.*:ALL/reports.*:SELECT'
+    password="{{ ANALYTICS_API_DATABASES.default.PASSWORD }}"
+    priv='{{ ANALYTICS_API_DATABASES.default.NAME }}.*:ALL/reports.*:SELECT'
   when: ANALYTICS_API_SERVICE_CONFIG is defined
 
 - name: create read-only reports user for the analytics-api
   mysql_user: >
     name=reports001
-    password=password
-    priv='{{ ANALYTICS_API_SERVICE_CONFIG['DATABASES']['reports']['NAME'] }}.*:SELECT'
+    password="{{ ANALYTICS_API_DATABASES.reports.PASSWORD }}"
+    priv='{{ ANALYTICS_API_DATABASES.reports.NAME }}.*:SELECT'
   when: ANALYTICS_API_SERVICE_CONFIG is defined
 
 - name: setup the edx-notes-api db user
@@ -145,6 +159,7 @@
     - "{{ ANALYTICS_API_DEFAULT_DB_NAME|default('None') }}"
     - "{{ ANALYTICS_API_REPORTS_DB_NAME|default('None') }}"
     - "{{ INSIGHTS_DATABASE_NAME|default('None') }}"
+    - "{{ EDX_ECOMMERCE_DEFAULT_DB_NAME|default('None') }}"
 
 - name: setup the read-only db user
   mysql_user: >

playbooks/roles/nginx/defaults/main.yml

@@ -93,6 +93,8 @@ nginx_gitreload_gunicorn_hosts:
   - 127.0.0.1
 nginx_edx_notes_api_gunicorn_hosts:
   - 127.0.0.1
+nginx_edx_ecommerce_gunicorn_hosts:
+  - 127.0.0.1
 
 nginx_cfg:
   #   - link - turn on

playbooks/roles/nginx/templates/edx/app/nginx/sites-available/edx_ecommerce.j2

+#
+# {{ ansible_managed }}
+#
+upstream edx_ecommerce_app_server {
+    {% for host in nginx_edx_ecommerce_gunicorn_hosts %}
+        server {{ host }}:{{ edx_ecommerce_gunicorn_port }} fail_timeout=0;
+    {% endfor %}
+}
+
+server {
+  listen {{ EDX_ECOMMERCE_NGINX_PORT }} default_server;
+
+  location ~ ^/static/(?P<file>.*) {
+    root {{ COMMON_DATA_DIR }}/{{ edx_ecommerce_service_name }};
+    try_files /staticfiles/$file =404;
+  }
+
+  location / {
+    try_files $uri @proxy_to_app;
+  }
+
+  {% include "robots.j2" %}
+
+location @proxy_to_app {
+    proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto;
+    proxy_set_header X-Forwarded-Port $http_x_forwarded_port;
+    proxy_set_header X-Forwarded-For $http_x_forwarded_for;
+    proxy_set_header Host $http_host;
+
+    proxy_redirect off;
+    proxy_pass http://edx_ecommerce_app_server;
+  }
+}
+

playbooks/security.sh

+#!/bin/bash
+
+set -ex
+
+# https://alas.aws.amazon.com/ALAS-2015-473.html
+
+check_vulnerability() {
+  cat > glibc_check.c << EOF
+#include <netdb.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+
+#define CANARY "in_the_coal_mine"
+
+struct {
+  char buffer[1024];
+  char canary[sizeof(CANARY)];
+} temp = { "buffer", CANARY };
+
+int main(void) {
+  struct hostent resbuf;
+  struct hostent *result;
+  int herrno;
+  int retval;
+
+  /*** strlen (name) = size_needed - sizeof (*host_addr) - sizeof (*h_addr_ptrs) - 1; ***/
+  size_t len = sizeof(temp.buffer) - 16*sizeof(unsigned char) - 2*sizeof(char *) - 1;
+  char name[sizeof(temp.buffer)];
+  memset(name, '0', len);
+  name[len] = '\0';
+
+  retval = gethostbyname_r(name, &resbuf, temp.buffer, sizeof(temp.buffer), &result, &herrno);
+
+  if (strcmp(temp.canary, CANARY) != 0) {
+    puts("vulnerable");
+    exit(EXIT_SUCCESS);
+  }
+  if (retval == ERANGE) {
+    puts("not vulnerable");
+    exit(EXIT_SUCCESS);
+  }
+  puts("should not happen");
+  exit(EXIT_FAILURE);
+}
+/* from http://www.openwall.com/lists/oss-security/2015/01/27/9 */
+EOF
+  gcc glibc_check.c -o glibc_check
+  ./glibc_check
+}
+
+upgrade_packages() {
+  sudo apt-get clean
+  sudo mv /etc/apt/sources.list /tmp/sources.list.bk
+  sudo sh -c 'echo "deb http://http.us.debian.org/debian wheezy main contrib non-free" >>  /etc/apt/sources.list'
+  sudo sh -c 'echo "deb http://security.debian.org wheezy/updates main contrib non-free" >>  /etc/apt/sources.list'
+  sudo apt-get update -y
+  sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --force-yes --only-upgrade libgcc1 bash
+  sudo mv /tmp/sources.list.bk /etc/apt/sources.list
+  sudo apt-get clean
+  sudo /etc/init.d/ssh restart
+  [ "$(check_vulnerability)" == "not vulnerable" ]
+}
+
+upgrade_packages