Merge pull request #1835 from edx/max/git-refactor

Refactor insights to use edx_service

playbooks/roles/insights/defaults/main.yml

@@ -11,8 +11,6 @@
 # Defaults for role insights
 # 
 
-INSIGHTS_GIT_IDENTITY: !!null
-
 INSIGHTS_MEMCACHE: [ 'localhost:11211' ]
 INSIGHTS_FEEDBACK_EMAIL: 'dashboard@example.com'
 INSIGHTS_MKTG_BASE: 'http://example.com'
@@ -103,29 +101,36 @@ INSIGHTS_CONFIG:
     connect_timeout: 10
   DATABASES: "{{ INSIGHTS_DATABASES }}"
 
-
-INSIGHTS_VERSION: "master"
 INSIGHTS_NEWRELIC_APPNAME: "{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-analytics-api"
 INSIGHTS_PIP_EXTRA_ARGS: "-i {{ COMMON_PYPI_MIRROR_URL }}"
 INSIGHTS_NGINX_PORT: "18110"
 INSIGHTS_GUNICORN_WORKERS: "2"
 INSIGHTS_GUNICORN_EXTRA: ""
+
+INSIGHTS_VERSION: "master"
+INSIGHTS_GIT_IDENTITY: !!null
+
+INSIGHTS_REPOS:
+  - PROTOCOL: "{{ COMMON_GIT_PROTOCOL }}"
+    DOMAIN: "{{ COMMON_GIT_MIRROR }}"
+    PATH: "{{ COMMON_GIT_PATH }}"
+    REPO: edx-analytics-dashboard.git
+    VERSION: "{{ INSIGHTS_VERSION }}"
+    DESTINATION: "{{ insights_code_dir }}"
+    SSH_KEY: "{{ INSIGHTS_GIT_IDENTITY }}"
+
 #
 # vars are namespace with the module name.
 #
 insights_environment:
   DJANGO_SETTINGS_MODULE: "analytics_dashboard.settings.production"
-  ANALYTICS_DASHBOARD_CFG: "{{ COMMON_CFG_DIR  }}/{{ insights_service_name }}.yaml"
+  ANALYTICS_DASHBOARD_CFG: "{{ COMMON_CFG_DIR  }}/{{ insights_service_name }}.yml"
 
-insights_role_name: "insights"
-insights_service_name: "{{ insights_role_name }}"
-insights_user: "{{ insights_role_name }}"
+insights_service_name: insights
+insights_user: "{{ insights_service_name }}"
 insights_app_dir: "{{ COMMON_APP_DIR }}/{{ insights_service_name }}"
 insights_home: "{{ COMMON_APP_DIR }}/{{ insights_service_name }}"
-insights_venv_base: "{{ insights_home }}/venvs"
-insights_venv_dir: "{{ insights_venv_base }}/{{ insights_service_name }}"
-insights_venv_bin: "{{ insights_venv_dir }}/bin"
-insights_code_dir: "{{ insights_app_dir }}/edx-analytics-dashboard"
+insights_code_dir: "{{ insights_app_dir }}/edx_analytics_dashboard"
 insights_python_path: "{{ insights_code_dir }}/analytics_dashboard"
 insights_conf_dir: "{{ insights_home }}"
 insights_log_dir: "{{ COMMON_LOG_DIR }}/{{ insights_service_name }}"
@@ -141,9 +146,6 @@ insights_gunicorn_timeout: "300"
 insights_wsgi: "analytics_dashboard.wsgi:application"
 
 insights_django_settings: "analytics_dashboard.settings.production"
-insights_source_repo: "git@{{ COMMON_GIT_MIRROR }}:/edx/edx-analytics-dashboard"
-insights_git_ssh_opts: "-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i {{ insights_git_identity_file }}"
-insights_git_identity_file: "{{ insights_home }}/git-identity"
 insights_manage: "{{ insights_code_dir }}/analytics_dashboard/manage.py"
 
 insights_requirements_base: "{{ insights_code_dir }}/requirements"

playbooks/roles/insights/handlers/main.yml

----
-#
-# edX Configuration
-#
-# github:     https://github.com/edx/configuration
-# wiki:       https://github.com/edx/configuration/wiki
-# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
-# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
-#
-#
-#
-# Handlers for role insights
-# 
-# Overview:
-# 
-#
-
-- name: "restart insights"
-  supervisorctl_local: >
-    name={{ insights_service_name }}
-    supervisorctl_path={{ supervisor_ctl }}
-    config={{ supervisor_cfg }}
-    state=restarted
-  when: not disable_edx_services

playbooks/roles/insights/meta/main.yml

@@ -12,6 +12,9 @@
 # 
 dependencies:
   - role: edx_service
-    edx_role_name: "{{ insights_role_name }}"
     edx_service_name: "{{ insights_service_name }}"
+    edx_service_config: "{{ INSIGHTS_CONFIG }}"
+    edx_service_repos: "{{ INSIGHTS_REPOS }}"
+    edx_service_user: "{{ insights_user }}"
+    edx_service_home: "{{ insights_home }}"
   - supervisor

playbooks/roles/insights/tasks/deploy.yml

----
-
-- name: install read-only ssh key
-  copy: >
-    content="{{ INSIGHTS_GIT_IDENTITY }}" dest={{ insights_git_identity_file }}
-    owner={{ insights_user }} group={{ insights_user }} mode=0600
-
-- name: setup the insights env file
-  template: >
-    src="edx/app/insights/insights_env.j2"
-    dest="{{ insights_app_dir }}/insights_env"
-    owner={{ insights_user }}
-    group={{ insights_user }}
-    mode=0644
-
-- name: checkout code
-  git: >
-    dest={{ insights_code_dir }} repo={{ insights_source_repo }} version={{ INSIGHTS_VERSION }}
-    accept_hostkey=yes
-    ssh_opts="{{ insights_git_ssh_opts }}"
-  register: insights_code_checkout
-  notify: restart insights
-  sudo_user: "{{ insights_user }}"
-
-- name: write out app config file
-  template: >
-    src=edx/app/insights/insights.yaml.j2
-    dest={{ COMMON_CFG_DIR  }}/{{ insights_service_name }}.yaml
-    mode=0644 owner={{ insights_user }} group={{ insights_user }}
-  notify: restart insights
-
-- name: install application requirements
-  pip: >
-    requirements="{{ insights_requirements_base }}/{{ item }}"
-    virtualenv="{{ insights_venv_dir }}" state=present
-    extra_args="--exists-action w"
-  sudo_user: "{{ insights_user }}"
-  notify: restart insights
-  with_items: insights_requirements
-
-- name: create nodeenv
-  shell: >
-    creates={{ insights_nodeenv_dir }}
-    {{ insights_venv_bin }}/nodeenv {{ insights_nodeenv_dir }}
-  sudo_user: "{{ insights_user }}"
-
-- name: install node dependencies
-  npm: executable={{ insights_nodeenv_bin }}/npm path={{ insights_code_dir }} production=yes
-  sudo_user: "{{ insights_user }}"
-
-- name: install bower dependencies
-  shell: >
-    chdir={{ insights_code_dir }}
-    . {{ insights_nodeenv_bin }}/activate && {{ insights_node_bin }}/bower install --production --config.interactive=false
-  sudo_user: "{{ insights_user }}"
-
-- name: migrate
-  shell: >
-    chdir={{ insights_code_dir }}
-    DB_MIGRATION_USER={{ COMMON_MYSQL_MIGRATE_USER }}
-    DB_MIGRATION_PASS={{ COMMON_MYSQL_MIGRATE_PASS }}
-    {{ insights_venv_bin }}/python {{ insights_manage }} migrate --noinput
-  sudo_user: "{{ insights_user }}"
-  environment: "{{ insights_environment }}"
-  when: migrate_db is defined and migrate_db|lower == "yes"
-
-- name: run r.js optimizer
-  shell: >
-    chdir={{ insights_code_dir }}
-    . {{ insights_nodeenv_bin }}/activate && {{ insights_node_bin }}/r.js -o build.js
-  sudo_user: "{{ insights_user }}"
-
-- name: run collectstatic
-  shell: >
-    chdir={{ insights_code_dir }}
-    {{ insights_venv_bin }}/python {{ insights_manage }} {{ item }}
-  sudo_user: "{{ insights_user }}"
-  environment: "{{ insights_environment }}"
-  with_items:
-    - "collectstatic --noinput"
-    - "compress"
-
-- name: compile translations
-  shell: >
-    chdir={{ insights_code_dir }}/analytics_dashboard
-    . {{ insights_venv_bin }}/activate && i18n_tool generate -v
-  sudo_user: "{{ insights_user }}"
-
-- name: write out the supervisior wrapper
-  template: >
-    src=edx/app/insights/insights.sh.j2
-    dest={{ insights_app_dir }}/{{ insights_service_name }}.sh
-    mode=0650 owner={{ supervisor_user }} group={{ common_web_user }}
-  notify: restart insights
-
-- name: write supervisord config
-  template: >
-    src=edx/app/supervisor/conf.d.available/insights.conf.j2
-    dest="{{ supervisor_available_dir }}/{{ insights_service_name }}.conf"
-    owner={{ supervisor_user }} group={{ common_web_user }} mode=0644
-  notify: restart insights
-
-- name: enable supervisor script
-  file: >
-    src={{ supervisor_available_dir }}/{{ insights_service_name }}.conf
-    dest={{ supervisor_cfg_dir }}/{{ insights_service_name }}.conf
-    state=link
-    force=yes
-  notify: restart insights
-  when: not disable_edx_services
-
-- name: update supervisor configuration
-  shell: "{{ supervisor_ctl }} -c {{ supervisor_cfg }} update"
-  when: not disable_edx_services
-
-- name: create symlinks from the venv bin dir
-  file: >
-    src="{{ insights_venv_bin }}/{{ item }}"
-    dest="{{ COMMON_BIN_DIR }}/{{ item.split('.')[0] }}.{{ insights_role_name }}"
-    state=link
-  with_items:
-  - python
-  - pip
-  - django-admin.py
-
-- name: create manage.py symlink
-  file: >
-    src="{{ insights_manage }}"
-    dest="{{ COMMON_BIN_DIR }}/manage.{{ insights_role_name }}"
-    state=link
-
-- name: remove read-only ssh key for the content repo
-  file: path={{ insights_git_identity_file }} state=absent
-
-- include: tag_ec2.yml tags=deploy
-  when: COMMON_TAG_EC2_INSTANCE

playbooks/roles/insights/tasks/main.yml

@@ -21,7 +21,115 @@
 #
 #
 
-- fail: msg="You must provide a private key for the Insights repo"
-  when: not INSIGHTS_GIT_IDENTITY
+- name: setup the insights env file
+  template: >
+    src="edx/app/insights/insights_env.j2"
+    dest="{{ insights_app_dir }}/insights_env"
+    owner={{ insights_user }}
+    group={{ insights_user }}
+    mode=0644
 
-- include: deploy.yml tags=deploy
+- name: install application requirements
+  pip: >
+    requirements="{{ insights_requirements_base }}/{{ item }}"
+    virtualenv="{{ insights_home }}/venvs/{{ insights_service_name }}" 
+    state=present extra_args="--exists-action w"
+  sudo_user: "{{ insights_user }}"
+  with_items: insights_requirements
+
+- name: create nodeenv
+  shell: >
+    creates={{ insights_nodeenv_dir }}
+    {{ insights_home }}/venvs/{{ insights_service_name }}/bin/nodeenv {{ insights_nodeenv_dir }}
+  sudo_user: "{{ insights_user }}"
+
+- name: install node dependencies
+  npm: executable={{ insights_nodeenv_bin }}/npm path={{ insights_code_dir }} production=yes
+  sudo_user: "{{ insights_user }}"
+
+- name: install bower dependencies
+  shell: >
+    chdir={{ insights_code_dir }}
+    . {{ insights_nodeenv_bin }}/activate && {{ insights_node_bin }}/bower install --production --config.interactive=false
+  sudo_user: "{{ insights_user }}"
+
+- name: migrate
+  shell: >
+    chdir={{ insights_code_dir }}
+    DB_MIGRATION_USER={{ COMMON_MYSQL_MIGRATE_USER }}
+    DB_MIGRATION_PASS={{ COMMON_MYSQL_MIGRATE_PASS }}
+    {{ insights_home }}/venvs/{{ insights_service_name }}/bin/python {{ insights_manage }} migrate --noinput
+  sudo_user: "{{ insights_user }}"
+  environment: "{{ insights_environment }}"
+  when: migrate_db is defined and migrate_db|lower == "yes"
+
+- name: run r.js optimizer
+  shell: >
+    chdir={{ insights_code_dir }}
+    . {{ insights_nodeenv_bin }}/activate && {{ insights_node_bin }}/r.js -o build.js
+  sudo_user: "{{ insights_user }}"
+
+- name: run collectstatic
+  shell: >
+    chdir={{ insights_code_dir }}
+    {{ insights_home }}/venvs/{{ insights_service_name }}/bin/python {{ insights_manage }} {{ item }}
+  sudo_user: "{{ insights_user }}"
+  environment: "{{ insights_environment }}"
+  with_items:
+    - "collectstatic --noinput"
+    - "compress"
+
+- name: compile translations
+  shell: >
+    chdir={{ insights_code_dir }}/analytics_dashboard
+    . {{ insights_home }}/venvs/{{ insights_service_name }}/bin/activate && i18n_tool generate -v
+  sudo_user: "{{ insights_user }}"
+
+- name: write out the supervisior wrapper
+  template: >
+    src=edx/app/insights/insights.sh.j2
+    dest={{ insights_app_dir }}/{{ insights_service_name }}.sh
+    mode=0650 owner={{ supervisor_user }} group={{ common_web_user }}
+
+- name: write supervisord config
+  template: >
+    src=edx/app/supervisor/conf.d.available/insights.conf.j2
+    dest="{{ supervisor_available_dir }}/{{ insights_service_name }}.conf"
+    owner={{ supervisor_user }} group={{ common_web_user }} mode=0644
+
+- name: enable supervisor script
+  file: >
+    src={{ supervisor_available_dir }}/{{ insights_service_name }}.conf
+    dest={{ supervisor_cfg_dir }}/{{ insights_service_name }}.conf
+    state=link
+    force=yes
+  when: not disable_edx_services
+
+- name: update supervisor configuration
+  shell: "{{ supervisor_ctl }} -c {{ supervisor_cfg }} update"
+  when: not disable_edx_services
+
+- name: create symlinks from the venv bin dir
+  file: >
+    src="{{ insights_home }}/venvs/{{ insights_service_name }}/bin/{{ item }}"
+    dest="{{ COMMON_BIN_DIR }}/{{ item.split('.')[0] }}.{{ insights_service_name }}"
+    state=link
+  with_items:
+  - python
+  - pip
+  - django-admin.py
+
+- name: create manage.py symlink
+  file: >
+    src="{{ insights_manage }}"
+    dest="{{ COMMON_BIN_DIR }}/manage.{{ insights_service_name }}"
+    state=link
+
+- name: restart insights
+  supervisorctl_local: >
+    state=restarted
+    supervisorctl_path={{ supervisor_ctl }}
+    config={{ supervisor_cfg }}
+    name={{ insights_service_name }}
+  when: not disable_edx_services
+  sudo_user: "{{ supervisor_service_user }}"
\ No newline at end of file

playbooks/roles/insights/tasks/tag_ec2.yml

----
-
-- name: get instance information
-  action: ec2_facts
-
-- name: tag instance
-  ec2_tag: resource={{ ansible_ec2_instance_id }} region={{ ansible_ec2_placement_region }}
-  args:
-    tags:
-      "version:insights" : "{{ insights_source_repo }} {{ insights_code_checkout.after |truncate(7,True,'')}}"
-  when: insights_code_checkout.after is defined

playbooks/roles/insights/templates/edx/app/insights/insights.sh.j2

@@ -2,6 +2,7 @@
 
 # {{ ansible_managed }}
 
+{% set insights_venv_bin = insights_home + '/venvs/' + insights_service_name + '/bin' %}
 {% if COMMON_ENABLE_NEWRELIC_APP %}
 {% set executable = insights_venv_bin + '/newrelic-admin run-program ' + insights_venv_bin + '/gunicorn' %}
 {% else %}

playbooks/roles/insights/templates/edx/app/insights/insights.yaml.j2

----
-# {{ ansible_managed }}
-
-{{ INSIGHTS_CONFIG | to_nice_yaml }}