Merge pull request #1242 from edx/jarv/migrate-changes

Jarv/migrate changes

playbooks/edx-east/edx_continuous_integration.yml

@@ -19,12 +19,13 @@
       - analytics-api
       nginx_default_sites:
       - lms
-    - edxlocal
+    - role: edxlocal
+      tags: edxlocal
     - mongo
     - { role: 'edxapp', celery_worker: True }
     - edxapp
     - role: demo
-      tags: ['demo']
+      tags: demo
     - { role: 'rabbitmq', rabbitmq_ip: '127.0.0.1' }
     - oraclejdk
     - elasticsearch

playbooks/roles/analytics-api/defaults/main.yml

@@ -20,37 +20,10 @@ ANALYTICS_API_PIP_EXTRA_ARGS: "-i {{ COMMON_PYPI_MIRROR_URL }}"
 ANALYTICS_API_NGINX_PORT: "18100"
 
 ANALYTICS_API_VERSION: "master"
-#
-# vars are namespace with the module name.
-#
-analytics_api_role_name: "analytics-api"
-analytics_api_service_name: "analytics-api"
-analytics_api_user: "analytics-api"
 
-analytics_api_app_dir: "{{ COMMON_APP_DIR }}/{{ analytics_api_service_name }}"
-analytics_api_home: "{{ COMMON_APP_DIR }}/{{ analytics_api_service_name }}"
-analytics_api_venv_base: "{{ analytics_api_home }}/venvs"
-analytics_api_venv_dir: "{{ analytics_api_venv_base }}/{{ analytics_api_service_name }}"
-analytics_api_venv_bin: "{{ analytics_api_venv_dir }}/bin"
-analytics_api_code_dir: "{{ analytics_api_app_dir }}/edx-analytics-data-api"
-analytics_api_conf_dir: "{{ analytics_api_home }}"
-analytics_api_gunicorn_host: "127.0.0.1"
-analytics_api_gunicorn_port: "8100"
-analytics_api_gunicorn_workers: "8"
-analytics_api_gunicorn_timeout: "300"
-
-analytics_api_django_settings: "production"
-analytics_api_source_repo: "git@{{ COMMON_GIT_MIRROR }}:edx/edx-analytics-data-api"
-analytics_api_git_ssh_opts: "-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i {{ analytics_api_git_identity_file }}"
-analytics_api_git_identity_file: "{{ analytics_api_app_dir }}/git-identity"
-
-analytics_api_log_dir: "{{ COMMON_LOG_DIR }}/{{ analytics_api_service_name }}"
-
-analytics_api_requirements_base: "{{ analytics_api_code_dir }}/requirements"
-analytics_api_requirements:
-  - base.txt
-  - production.txt
-  - optional.txt
+# Default dummy user, override this!!
+ANALYTICS_API_USERS:
+  "dummy-api-user": "changeme"
 
 ANALYTICS_API_CONFIG:
   SECRET_KEY: 'Your secret key here'
@@ -84,6 +57,44 @@ ANALYTICS_API_CONFIG:
       PASSWORD: 'password'
       HOST: 'localhost'
       PORT: '3306'
+
+#
+# vars are namespace with the module name.
+#
+
+analytics_api_environment:
+  DJANGO_SETTINGS_MODULE: "analyticsdataserver.settings.production"
+  ANALYTICS_API_CFG: "{{ COMMON_CFG_DIR  }}/{{ analytics_api_service_name }}.yaml"
+
+analytics_api_role_name: "analytics-api"
+analytics_api_service_name: "analytics-api"
+analytics_api_user: "analytics-api"
+
+analytics_api_app_dir: "{{ COMMON_APP_DIR }}/{{ analytics_api_service_name }}"
+analytics_api_home: "{{ COMMON_APP_DIR }}/{{ analytics_api_service_name }}"
+analytics_api_venv_base: "{{ analytics_api_home }}/venvs"
+analytics_api_venv_dir: "{{ analytics_api_venv_base }}/{{ analytics_api_service_name }}"
+analytics_api_venv_bin: "{{ analytics_api_venv_dir }}/bin"
+analytics_api_code_dir: "{{ analytics_api_app_dir }}/edx-analytics-data-api"
+analytics_api_conf_dir: "{{ analytics_api_home }}"
+analytics_api_gunicorn_host: "127.0.0.1"
+analytics_api_gunicorn_port: "8100"
+analytics_api_gunicorn_workers: "8"
+analytics_api_gunicorn_timeout: "300"
+
+analytics_api_django_settings: "production"
+analytics_api_source_repo: "git@{{ COMMON_GIT_MIRROR }}:edx/edx-analytics-data-api"
+analytics_api_git_ssh_opts: "-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i {{ analytics_api_git_identity_file }}"
+analytics_api_git_identity_file: "{{ analytics_api_app_dir }}/git-identity"
+
+analytics_api_log_dir: "{{ COMMON_LOG_DIR }}/{{ analytics_api_service_name }}"
+
+analytics_api_requirements_base: "{{ analytics_api_code_dir }}/requirements"
+analytics_api_requirements:
+  - base.txt
+  - production.txt
+  - optional.txt
+
 #
 # OS packages
 #

playbooks/roles/analytics-api/tasks/deploy.yml

@@ -5,6 +5,14 @@
     content="{{ ANALYTICS_API_GIT_IDENTITY }}" dest={{ analytics_api_git_identity_file }}
     owner={{ analytics_api_user }} group={{ analytics_api_user }} mode=0600
 
+- name: setup the analytics-api env file
+  template: >
+    src="edx/app/analytics-api/analytics_api_env.j2"
+    dest="{{ analytics_api_app_dir }}/analytics_api_env"
+    owner={{ analytics_api_user }}
+    group={{ analytics_api_user }}
+    mode=0644
+
 - name: checkout code
   git: >
     dest={{ analytics_api_code_dir }} repo={{ analytics_api_source_repo }} version={{ ANALYTICS_API_VERSION }}
@@ -21,14 +29,6 @@
     mode=0644 owner={{ analytics_api_user }} group={{ analytics_api_user }}
   notify: restart the analytics service
 
-- name: run collectstatic
-  shell: >
-    chdir={{ analytics_api_code_dir }}
-    {{ analytics_api_venv_bin }}/python manage.py collectstatic --settings analyticsdataserver.settings.production --noinput --settings analyticsdataserver.settings.production
-  sudo_user: "{{ analytics_api_user }}"
-  environment:
-    ANALYTICS_API_CFG: "{{ COMMON_CFG_DIR  }}/{{ analytics_api_service_name }}.yaml"
-
 - name: install application requirements
   pip: >
     requirements="{{ analytics_api_requirements_base }}/{{ item }}"
@@ -37,6 +37,31 @@
   notify: restart the analytics service
   with_items: analytics_api_requirements
 
+- name: syncdb and migrate
+  shell: >
+    chdir={{ analytics_api_code_dir }}
+    DB_MIGRATION_USER={{ COMMON_MYSQL_MIGRATE_USER }}
+    DB_MIGRATION_PASS={{ COMMON_MYSQL_MIGRATE_PASS }}
+    {{ analytics_api_venv_bin }}/python ./manage.py syncdb --migrate --noinput
+  sudo_user: "{{ analytics_api_user }}"
+  environment: "{{ analytics_api_environment }}"
+  when: migrate_db is defined and migrate_db|lower == "yes"
+
+- name: run collectstatic
+  shell: >
+    chdir={{ analytics_api_code_dir }}
+    {{ analytics_api_venv_bin }}/python manage.py collectstatic  --noinput
+  sudo_user: "{{ analytics_api_user }}"
+  environment: "{{ analytics_api_environment }}"
+
+- name: create api users
+  shell: >
+    chdir={{ analytics_api_code_dir }}
+    {{ analytics_api_venv_bin }}/python manage.py set_api_key {{ item.key }} {{ item.value }} --create-user
+  sudo_user: "{{ analytics_api_user }}"
+  environment: "{{ analytics_api_environment }}"
+  with_dict: ANALYTICS_API_USERS
+
 - name: write out the supervisior wrapper
   template: >
     src=edx/app/analytics-api/analytics-api.sh.j2
@@ -64,5 +89,23 @@
   shell: "{{ supervisor_ctl }} -c {{ supervisor_cfg }} update"
   when: not disable_edx_services
 
+- name: create symlinks from the venv bin dir
+  file: >
+    src="{{ analytics_api_venv_bin }}/{{ item }}"
+    dest="{{ COMMON_BIN_DIR }}/{{ item.split('.')[0] }}.analytics-api"
+    state=link
+  with_items:
+  - python
+  - pip
+  - django-admin.py
+
+- name: create symlinks from the repo dir
+  file: >
+    src="{{ analytics_api_code_dir }}/{{ item }}"
+    dest="{{ COMMON_BIN_DIR }}/{{ item.split('.')[0] }}.analytics-api"
+    state=link
+  with_items:
+  - manage.py
+
 - name: remove read-only ssh key for the content repo
   file: path={{ analytics_api_git_identity_file }} state=absent

playbooks/roles/analytics-api/templates/edx/app/analytics-api/analytics-api.sh.j2

@@ -13,8 +13,6 @@ export NEW_RELIC_APP_NAME="{{ ANALYTICS_API_NEWRELIC_APPNAME }}"
 export NEW_RELIC_LICENSE_KEY="{{ NEWRELIC_LICENSE_KEY }}"
 {% endif -%}
 
-export SERVICE_VARIANT={{ analytics_api_service_name }}
-export DJANGO_SETTINGS_MODULE="analyticsdataserver.settings.production"
-export ANALYTICS_API_CFG="{{ COMMON_CFG_DIR  }}/{{ analytics_api_service_name }}.yaml"
+source {{ analytics_api_app_dir }}/analytics_api_env
 
 {{ executable }} --pythonpath={{ analytics_api_code_dir }} -b {{ analytics_api_gunicorn_host }}:{{ analytics_api_gunicorn_port }} -w {{ analytics_api_gunicorn_workers }} --timeout={{ analytics_api_gunicorn_timeout }}  analyticsdataserver.wsgi:application

playbooks/roles/analytics-api/templates/edx/app/analytics-api/analytics_api_env.j2

+# {{ ansible_managed }}
+
+{% for name,value in analytics_api_environment.items() -%}
+{%- if value -%}
+export {{ name }}="{{ value }}"
+{% endif %}
+{%- endfor %}

playbooks/roles/common/defaults/main.yml

@@ -44,11 +44,11 @@ COMMON_SSH_PASSWORD_AUTH: "no"
 # the migrate user is granted table alter privs on all dbs
 
 COMMON_MYSQL_READ_ONLY_USER: 'read_only'
-COMMON_MYSQL_READ_ONLY_PASS: !!null
+COMMON_MYSQL_READ_ONLY_PASS: 'password'
 COMMON_MYSQL_ADMIN_USER: 'admin'
-COMMON_MYSQL_ADMIN_PASS: !!null
+COMMON_MYSQL_ADMIN_PASS: 'password'
 COMMON_MYSQL_MIGRATE_USER: 'migrate'
-COMMON_MYSQL_MIGRATE_PASS: !!null
+COMMON_MYSQL_MIGRATE_PASS: 'password'
 
 COMMON_MONGO_READ_ONLY_USER: 'read_only'
 COMMON_MONGO_READ_ONLY_PASS: !!null

playbooks/roles/edxapp/defaults/main.yml

@@ -32,11 +32,9 @@ EDXAPP_MONGO_DB_NAME: 'edxapp'
 EDXAPP_MYSQL_DB_NAME: 'edxapp'
 EDXAPP_MYSQL_USER: 'edxapp001'
 EDXAPP_MYSQL_USER_ADMIN: 'root'
-EDXAPP_MYSQL_USER_MIGRATE: 'migrate'
 EDXAPP_MYSQL_PASSWORD: 'password'
 EDXAPP_MYSQL_PASSWORD_READ_ONLY: 'password'
 EDXAPP_MYSQL_PASSWORD_ADMIN: 'password'
-EDXAPP_MYSQL_PASSWORD_MIGRATE: 'password'
 EDXAPP_MYSQL_REPLICA_DB_NAME: "{{ EDXAPP_MYSQL_DB_NAME }}"
 EDXAPP_MYSQL_REPLICA_USER: "{{ EDXAPP_MYSQL_USER }}"
 EDXAPP_MYSQL_REPLICA_PASSWORD: "{{ EDXAPP_MYSQL_PASSWORD }}"
@@ -185,9 +183,6 @@ EDXAPP_USE_GIT_IDENTITY: false
 # into this var
 EDXAPP_GIT_IDENTITY: !!null
 
-# Configuration for database migration
-EDXAPP_TEST_MIGRATE_DB_NAME: "{{ COMMON_ENVIRONMENT }}_{{ COMMON_DEPLOYMENT }}_test_{{ EDXAPP_MYSQL_DB_NAME }}"
-
 EDXAPP_UPDATE_STATIC_FILES_KEY: false
 # Set this to true if you want to install the private pip
 # requirements in the edx-platform repo.

playbooks/roles/edxapp/tasks/service_variant_config.yml

@@ -75,24 +75,6 @@
   when: celery_worker is defined and not disable_edx_services
   sudo_user: "{{ supervisor_user }}"
 
-# Fake syncdb with migrate, only when fake_migrations is defined
-# This overrides the database name to be the test database which
-# the default application user has full write access to.
-#
-# This is run in cases where you want to test to see if migrations
-# work without actually runnning them (when creating AMIs for example).
-- name: syncdb and migrate
-  shell: >
-    chdir={{ edxapp_code_dir }}
-    {{ edxapp_venv_bin}}/python manage.py lms syncdb --migrate --noinput --settings=aws_migrate
-  when: fake_migrations is defined and migrate_db is defined and migrate_db|lower == "yes"
-  sudo_user: "{{ edxapp_user }}"
-  environment:
-    DB_MIGRATION_NAME: "{{ EDXAPP_TEST_MIGRATE_DB_NAME }}"
-  notify:
-  - "restart edxapp"
-  - "restart edxapp_workers"
-
 # Syncdb with migrate when the migrate user is overridden in extra vars
 - name: syncdb and migrate
   shell: >
@@ -107,54 +89,6 @@
   - "restart edxapp"
   - "restart edxapp_workers"
 
-# Syncdb with migrate when the default migrate user is not set,
-# in this case use the EDXAPP_MYSQL_USER_MIGRATE user to run migrations
-- name: syncdb and migrate
-  shell: >
-    chdir={{ edxapp_code_dir }}
-    {{ edxapp_venv_bin}}/python manage.py lms syncdb --migrate --noinput --settings=aws_migrate
-  when: fake_migrations is not defined and migrate_db is defined and migrate_db|lower == "yes" and not COMMON_MYSQL_MIGRATE_PASS
-  environment:
-    DB_MIGRATION_USER: "{{ EDXAPP_MYSQL_USER_MIGRATE }}"
-    DB_MIGRATION_PASS: "{{ EDXAPP_MYSQL_PASSWORD_MIGRATE }}"
-  sudo_user: "{{ edxapp_user }}"
-  notify:
-  - "restart edxapp"
-  - "restart edxapp_workers"
-
-
-# Fake migrate, only when fake_migrations is defined
-# This overrides the database name to be the test database which
-# the default application user has full write access to
-- name: db migrate
-  shell: >
-    chdir={{ edxapp_code_dir }}
-    {{ edxapp_venv_bin}}/python manage.py lms migrate --noinput --settings=aws_migrate
-  when: fake_migrations is defined and migrate_only is defined and migrate_only|lower == "yes"
-  sudo_user: "{{ edxapp_user }}"
-  environment:
-    DB_MIGRATION_NAME: "{{ EDXAPP_TEST_MIGRATE_DB_NAME }}"
-  notify:
-  - "restart edxapp"
-  - "restart edxapp_workers"
-
-
-
-# Regular migrations
-- name: db migrate
-  shell: >
-    chdir={{ edxapp_code_dir }}
-    {{ edxapp_venv_bin}}/python manage.py lms migrate --noinput --settings=aws_migrate
-  when: fake_migrations is not defined and migrate_only is defined and migrate_only|lower == "yes"
-  environment:
-    DB_MIGRATION_USER: "{{ EDXAPP_MYSQL_USER_MIGRATE }}"
-    DB_MIGRATION_PASS: "{{ EDXAPP_MYSQL_PASSWORD_MIGRATE }}"
-  sudo_user: "{{ edxapp_user }}"
-  notify:
-  - "restart edxapp"
-  - "restart edxapp_workers"
-
-
 # Gather assets using rake if possible
 
 - name: gather {{ item }} static assets with rake

playbooks/roles/edxlocal/tasks/main.yml

@@ -13,13 +13,6 @@
 - name: install packages needed for single server
   apt: pkg={{','.join(edxlocal_debian_pkgs)}} install_recommends=yes state=present
 
-- name: setup the migration db user
-  mysql_user: >
-    name={{ EDXAPP_MYSQL_USER_MIGRATE }}
-    password={{ EDXAPP_MYSQL_PASSWORD_MIGRATE}}
-    priv='{{EDXAPP_MYSQL_DB_NAME}}.*:ALL'
-  when: EDXAPP_MYSQL_USER_MIGRATE is defined
-
 - name: setup the edxapp db user
   mysql_user: >
     name={{ EDXAPP_MYSQL_USER }}
@@ -39,14 +32,14 @@
     name={{ XQUEUE_MYSQL_USER }}
     password={{ XQUEUE_MYSQL_PASSWORD }}
     priv='{{XQUEUE_MYSQL_DB_NAME}}.*:ALL'
-  when: XQUEUE_MYSQL_USER is defined and not disable_edx_services
+  when: XQUEUE_MYSQL_USER is defined
 
 - name: create a database for xqueue
   mysql_db: >
     db=xqueue
     state=present
     encoding=utf8
-  when: XQUEUE_MYSQL_USER is defined and not disable_edx_services
+  when: XQUEUE_MYSQL_USER is defined
 
 - name: setup the ora db user
   mysql_user: >
@@ -62,44 +55,57 @@
     encoding=utf8
   when: ORA_MYSQL_USER is defined
 
-- name: setup the discern db user
-  mysql_user: >
-    name={{ DISCERN_MYSQL_USER }}
-    password={{ DISCERN_MYSQL_PASSWORD }}
-    priv='{{DISCERN_MYSQL_DB_NAME}}.*:ALL'
-  when: DISCERN_MYSQL_USER is defined and not disable_edx_services
-
-
-- name: create a database for discern
-  mysql_db: >
-    db=discern
-    state=present
-    encoding=utf8
-  when: DISCERN_MYSQL_USER is defined and not disable_edx_services
-
 - name: create databases for analytics api
   mysql_db: >
     db={{ item }}
     state=present
     encoding=utf8
+  when: ANALYTICS_API_CONFIG is defined
   with_items:
-    - 'reports'
-    - 'analytics-api'
-  when: analytics_api_role_name is defined
+    - "{{ ANALYTICS_API_CONFIG['DATABASES']['default']['NAME'] }}"
+    - "{{ ANALYTICS_API_CONFIG['DATABASES']['reports']['NAME'] }}"
+
 
 - name: create api user for the analytics api
   mysql_user: >
     name=api001
     password=password
-    priv='analytics-api.*:ALL/reports.*:SELECT'
-  when: analytics_api_role_name is defined
+    priv='{{ ANALYTICS_API_CONFIG['DATABASES']['default']['NAME'] }}.*:ALL/reports.*:SELECT'
+  when: ANALYTICS_API_CONFIG is defined
 
 - name: create read-only reports user for the analytics-api
   mysql_user: >
     name=reports001
     password=password
-    priv='reports.*:SELECT'
-  when: analytics_api_role_name is defined
+    priv='{{ ANALYTICS_API_CONFIG['DATABASES']['reports']['NAME'] }}.*:SELECT'
+  when: ANALYTICS_API_CONFIG is defined
+
+- name: setup the migration db user
+  mysql_user: >
+    name={{ COMMON_MYSQL_MIGRATE_USER }}
+    password={{ COMMON_MYSQL_MIGRATE_PASS }}
+    priv='{{ item }}.*:ALL'
+    append_privs=yes
+  when: item != 'None'
+  with_items:
+    - "{{ EDXAPP_MYSQL_DB_NAME|default('None') }}"
+    - "{{ XQUEUE_MYSQL_DB_NAME|default('None') }}"
+    - "{{ ORA_MYSQL_DB_NAME|default('None') }}"
+    - "{{ ANALYTICS_API_CONFIG['DATABASES']['default']['NAME']|default('None') }}"
+    - "{{ ANALYTICS_API_CONFIG['DATABASES']['reports']['NAME']|default('None') }}"
+
+- name: setup the read-only db user
+  mysql_user: >
+    name={{ COMMON_MYSQL_READ_ONLY_USER }}
+    password={{ COMMON_MYSQL_READ_ONLY_PASS }}
+    priv='*.*:ALL'
+
+- name: setup the admin db user
+  mysql_user: >
+    name={{ COMMON_MYSQL_ADMIN_USER }}
+    password={{ COMMON_MYSQL_ADMIN_PASS }}
+    priv='*.*:CREATE USER'
+
 
 - name: install memcached
   apt: pkg=memcached state=present