updates to the migrate user, removing legacy tasks in edxapp

playbooks/edx-east/edx_continuous_integration.yml

@@ -19,12 +19,13 @@
       - analytics-api
       nginx_default_sites:
       - lms
-    - edxlocal
+    - role: edxlocal
+      tags: edxlocal
     - mongo
     - { role: 'edxapp', celery_worker: True }
     - edxapp
     - role: demo
-      tags: ['demo']
+      tags: demo
     - { role: 'rabbitmq', rabbitmq_ip: '127.0.0.1' }
     - oraclejdk
     - elasticsearch

playbooks/roles/analytics-api/defaults/main.yml

@@ -23,6 +23,11 @@ ANALYTICS_API_VERSION: "master"
 #
 # vars are namespace with the module name.
 #
+
+analytics_api_environment:
+  DJANGO_SETTINGS_MODULE: "analyticsdataserver.settings.production"
+  ANALYTICS_API_CFG: "{{ COMMON_CFG_DIR  }}/{{ analytics_api_service_name }}.yaml"
+
 analytics_api_role_name: "analytics-api"
 analytics_api_service_name: "analytics-api"
 analytics_api_user: "analytics-api"

playbooks/roles/analytics-api/tasks/deploy.yml

@@ -5,6 +5,14 @@
     content="{{ ANALYTICS_API_GIT_IDENTITY }}" dest={{ analytics_api_git_identity_file }}
     owner={{ analytics_api_user }} group={{ analytics_api_user }} mode=0600
 
+- name: setup the analytics-api env file
+  template: >
+    src="edx/app/analytics-api/analytics_api_env.j2"
+    dest="{{ analytics_api_app_dir }}/analytics_api_env"
+    owner={{ analytics_api_user }}
+    group={{ analytics_api_user }}
+    mode=0644
+
 - name: checkout code
   git: >
     dest={{ analytics_api_code_dir }} repo={{ analytics_api_source_repo }} version={{ ANALYTICS_API_VERSION }}
@@ -26,8 +34,7 @@
     chdir={{ analytics_api_code_dir }}
     {{ analytics_api_venv_bin }}/python manage.py collectstatic --settings analyticsdataserver.settings.production --noinput --settings analyticsdataserver.settings.production
   sudo_user: "{{ analytics_api_user }}"
-  environment:
-    ANALYTICS_API_CFG: "{{ COMMON_CFG_DIR  }}/{{ analytics_api_service_name }}.yaml"
+  environment: "{{ analytics_api_environment }}"
 
 - name: install application requirements
   pip: >
@@ -37,6 +44,16 @@
   notify: restart the analytics service
   with_items: analytics_api_requirements
 
+- name: syncdb and migrate
+  shell: >
+    chdir={{ analytics_api_code_dir }}
+    DB_MIGRATION_USER={{ COMMON_MYSQL_MIGRATE_USER }}
+    DB_MIGRATION_PASS={{ COMMON_MYSQL_MIGRATE_PASS }}
+    {{ analytics_api_venv_bin }}/python ./manage.py syncdb --migrate --noinput
+  sudo_user: "{{ analytics_api_user }}"
+  environment: "{{ analytics_api_environment }}"
+  when: migrate_db is defined and migrate_db|lower == "yes"
+
 - name: write out the supervisior wrapper
   template: >
     src=edx/app/analytics-api/analytics-api.sh.j2
@@ -64,5 +81,23 @@
   shell: "{{ supervisor_ctl }} -c {{ supervisor_cfg }} update"
   when: not disable_edx_services
 
+- name: create symlinks from the venv bin dir
+  file: >
+    src="{{ analytics_api_venv_bin }}/{{ item }}"
+    dest="{{ COMMON_BIN_DIR }}/{{ item.split('.')[0] }}.analytics-api"
+    state=link
+  with_items:
+  - python
+  - pip
+  - django-admin.py
+
+- name: create symlinks from the repo dir
+  file: >
+    src="{{ analytics_api_code_dir }}/{{ item }}"
+    dest="{{ COMMON_BIN_DIR }}/{{ item.split('.')[0] }}.analytics-api"
+    state=link
+  with_items:
+  - manage.py
+
 - name: remove read-only ssh key for the content repo
   file: path={{ analytics_api_git_identity_file }} state=absent

playbooks/roles/analytics-api/templates/edx/app/analytics-api/analytics-api.sh.j2

@@ -13,8 +13,6 @@ export NEW_RELIC_APP_NAME="{{ ANALYTICS_API_NEWRELIC_APPNAME }}"
 export NEW_RELIC_LICENSE_KEY="{{ NEWRELIC_LICENSE_KEY }}"
 {% endif -%}
 
-export SERVICE_VARIANT={{ analytics_api_service_name }}
-export DJANGO_SETTINGS_MODULE="analyticsdataserver.settings.production"
-export ANALYTICS_API_CFG="{{ COMMON_CFG_DIR  }}/{{ analytics_api_service_name }}.yaml"
+source {{ analytics_api_app_dir }}/analytics_api_env
 
 {{ executable }} --pythonpath={{ analytics_api_code_dir }} -b {{ analytics_api_gunicorn_host }}:{{ analytics_api_gunicorn_port }} -w {{ analytics_api_gunicorn_workers }} --timeout={{ analytics_api_gunicorn_timeout }}  analyticsdataserver.wsgi:application

playbooks/roles/common/defaults/main.yml

@@ -44,11 +44,11 @@ COMMON_SSH_PASSWORD_AUTH: "no"
 # the migrate user is granted table alter privs on all dbs
 
 COMMON_MYSQL_READ_ONLY_USER: 'read_only'
-COMMON_MYSQL_READ_ONLY_PASS: !!null
+COMMON_MYSQL_READ_ONLY_PASS: 'password'
 COMMON_MYSQL_ADMIN_USER: 'admin'
-COMMON_MYSQL_ADMIN_PASS: !!null
+COMMON_MYSQL_ADMIN_PASS: 'password'
 COMMON_MYSQL_MIGRATE_USER: 'migrate'
-COMMON_MYSQL_MIGRATE_PASS: !!null
+COMMON_MYSQL_MIGRATE_PASS: 'password'
 
 COMMON_MONGO_READ_ONLY_USER: 'read_only'
 COMMON_MONGO_READ_ONLY_PASS: !!null

playbooks/roles/edxapp/tasks/service_variant_config.yml

@@ -75,24 +75,6 @@
   when: celery_worker is defined and not disable_edx_services
   sudo_user: "{{ supervisor_user }}"
 
-# Fake syncdb with migrate, only when fake_migrations is defined
-# This overrides the database name to be the test database which
-# the default application user has full write access to.
-#
-# This is run in cases where you want to test to see if migrations
-# work without actually runnning them (when creating AMIs for example).
-- name: syncdb and migrate
-  shell: >
-    chdir={{ edxapp_code_dir }}
-    {{ edxapp_venv_bin}}/python manage.py lms syncdb --migrate --noinput --settings=aws_migrate
-  when: fake_migrations is defined and migrate_db is defined and migrate_db|lower == "yes"
-  sudo_user: "{{ edxapp_user }}"
-  environment:
-    DB_MIGRATION_NAME: "{{ EDXAPP_TEST_MIGRATE_DB_NAME }}"
-  notify:
-  - "restart edxapp"
-  - "restart edxapp_workers"
-
 # Syncdb with migrate when the migrate user is overridden in extra vars
 - name: syncdb and migrate
   shell: >
@@ -107,54 +89,6 @@
   - "restart edxapp"
   - "restart edxapp_workers"
 
-# Syncdb with migrate when the default migrate user is not set,
-# in this case use the EDXAPP_MYSQL_USER_MIGRATE user to run migrations
-- name: syncdb and migrate
-  shell: >
-    chdir={{ edxapp_code_dir }}
-    {{ edxapp_venv_bin}}/python manage.py lms syncdb --migrate --noinput --settings=aws_migrate
-  when: fake_migrations is not defined and migrate_db is defined and migrate_db|lower == "yes" and not COMMON_MYSQL_MIGRATE_PASS
-  environment:
-    DB_MIGRATION_USER: "{{ EDXAPP_MYSQL_USER_MIGRATE }}"
-    DB_MIGRATION_PASS: "{{ EDXAPP_MYSQL_PASSWORD_MIGRATE }}"
-  sudo_user: "{{ edxapp_user }}"
-  notify:
-  - "restart edxapp"
-  - "restart edxapp_workers"
-
-
-# Fake migrate, only when fake_migrations is defined
-# This overrides the database name to be the test database which
-# the default application user has full write access to
-- name: db migrate
-  shell: >
-    chdir={{ edxapp_code_dir }}
-    {{ edxapp_venv_bin}}/python manage.py lms migrate --noinput --settings=aws_migrate
-  when: fake_migrations is defined and migrate_only is defined and migrate_only|lower == "yes"
-  sudo_user: "{{ edxapp_user }}"
-  environment:
-    DB_MIGRATION_NAME: "{{ EDXAPP_TEST_MIGRATE_DB_NAME }}"
-  notify:
-  - "restart edxapp"
-  - "restart edxapp_workers"
-
-
-
-# Regular migrations
-- name: db migrate
-  shell: >
-    chdir={{ edxapp_code_dir }}
-    {{ edxapp_venv_bin}}/python manage.py lms migrate --noinput --settings=aws_migrate
-  when: fake_migrations is not defined and migrate_only is defined and migrate_only|lower == "yes"
-  environment:
-    DB_MIGRATION_USER: "{{ EDXAPP_MYSQL_USER_MIGRATE }}"
-    DB_MIGRATION_PASS: "{{ EDXAPP_MYSQL_PASSWORD_MIGRATE }}"
-  sudo_user: "{{ edxapp_user }}"
-  notify:
-  - "restart edxapp"
-  - "restart edxapp_workers"
-
-
 # Gather assets using rake if possible
 
 - name: gather {{ item }} static assets with rake

playbooks/roles/edxlocal/tasks/main.yml

@@ -13,13 +13,6 @@
 - name: install packages needed for single server
   apt: pkg={{','.join(edxlocal_debian_pkgs)}} install_recommends=yes state=present
 
-- name: setup the migration db user
-  mysql_user: >
-    name={{ EDXAPP_MYSQL_USER_MIGRATE }}
-    password={{ EDXAPP_MYSQL_PASSWORD_MIGRATE}}
-    priv='{{EDXAPP_MYSQL_DB_NAME}}.*:ALL'
-  when: EDXAPP_MYSQL_USER_MIGRATE is defined
-
 - name: setup the edxapp db user
   mysql_user: >
     name={{ EDXAPP_MYSQL_USER }}
@@ -39,14 +32,14 @@
     name={{ XQUEUE_MYSQL_USER }}
     password={{ XQUEUE_MYSQL_PASSWORD }}
     priv='{{XQUEUE_MYSQL_DB_NAME}}.*:ALL'
-  when: XQUEUE_MYSQL_USER is defined and not disable_edx_services
+  when: XQUEUE_MYSQL_USER is defined
 
 - name: create a database for xqueue
   mysql_db: >
     db=xqueue
     state=present
     encoding=utf8
-  when: XQUEUE_MYSQL_USER is defined and not disable_edx_services
+  when: XQUEUE_MYSQL_USER is defined
 
 - name: setup the ora db user
   mysql_user: >
@@ -62,44 +55,57 @@
     encoding=utf8
   when: ORA_MYSQL_USER is defined
 
-- name: setup the discern db user
-  mysql_user: >
-    name={{ DISCERN_MYSQL_USER }}
-    password={{ DISCERN_MYSQL_PASSWORD }}
-    priv='{{DISCERN_MYSQL_DB_NAME}}.*:ALL'
-  when: DISCERN_MYSQL_USER is defined and not disable_edx_services
-
-
-- name: create a database for discern
-  mysql_db: >
-    db=discern
-    state=present
-    encoding=utf8
-  when: DISCERN_MYSQL_USER is defined and not disable_edx_services
-
 - name: create databases for analytics api
   mysql_db: >
     db={{ item }}
     state=present
     encoding=utf8
+  when: ANALYTICS_API_CONFIG is defined
   with_items:
-    - 'reports'
-    - 'analytics-api'
-  when: analytics_api_role_name is defined
+    - "{{ ANALYTICS_API_CONFIG['DATABASES']['default']['NAME'] }}"
+    - "{{ ANALYTICS_API_CONFIG['DATABASES']['reports']['NAME'] }}"
+
 
 - name: create api user for the analytics api
   mysql_user: >
     name=api001
     password=password
-    priv='analytics-api.*:ALL/reports.*:SELECT'
-  when: analytics_api_role_name is defined
+    priv='{{ ANALYTICS_API_CONFIG['DATABASES']['default']['NAME'] }}.*:ALL/reports.*:SELECT'
+  when: ANALYTICS_API_CONFIG is defined
 
 - name: create read-only reports user for the analytics-api
   mysql_user: >
     name=reports001
     password=password
-    priv='reports.*:SELECT'
-  when: analytics_api_role_name is defined
+    priv='{{ ANALYTICS_API_CONFIG['DATABASES']['reports']['NAME'] }}.*:SELECT'
+  when: ANALYTICS_API_CONFIG is defined
+
+- name: setup the migration db user
+  mysql_user: >
+    name={{ COMMON_MYSQL_MIGRATE_USER }}
+    password={{ COMMON_MYSQL_MIGRATE_PASS }}
+    priv='{{ item }}.*:ALL'
+    append_privs=yes
+  when: item != 'None'
+  with_items:
+    - "{{ EDXAPP_MYSQL_DB_NAME|default('None') }}"
+    - "{{ XQUEUE_MYSQL_DB_NAME|default('None') }}"
+    - "{{ ORA_MYSQL_DB_NAME|default('None') }}"
+    - "{{ ANALYTICS_API_CONFIG['DATABASES']['default']['NAME']|default('None') }}"
+    - "{{ ANALYTICS_API_CONFIG['DATABASES']['reports']['NAME']|default('None') }}"
+
+- name: setup the read-only db user
+  mysql_user: >
+    name={{ COMMON_MYSQL_READ_ONLY_USER }}
+    password={{ COMMON_MYSQL_READ_ONLY_PASS }}
+    priv='*.*:ALL'
+
+- name: setup the admin db user
+  mysql_user: >
+    name={{ COMMON_MYSQL_ADMIN_USER }}
+    password={{ COMMON_MYSQL_ADMIN_PASS }}
+    priv='*.*:CREATE USER'
+
 
 - name: install memcached
   apt: pkg=memcached state=present