Merge pull request #1789 from edx/max/git-refactor

Refactor git checkout into edx_service role

playbooks/roles/common/defaults/main.yml

@@ -31,7 +31,9 @@ COMMON_PYPI_MIRROR_URL: 'https://pypi.python.org/simple'
 COMMON_NPM_MIRROR_URL: 'http://registry.npmjs.org'
 COMMON_UBUNTU_APT_KEYSERVER: "http://keyserver.ubuntu.com/pks/lookup?op=get&fingerprint=on&search="
 # do not include http/https
+COMMON_GIT_PROTOCOL: 'https'
 COMMON_GIT_MIRROR: 'github.com'
+COMMON_GIT_PATH: 'edx'
 # override this var to set a different hostname
 COMMON_HOSTNAME: ""
 

playbooks/roles/edx-notes-api/defaults/main.yml

@@ -11,35 +11,42 @@
 # Defaults for role edx-notes-api
 # 
 
-EDX_NOTES_API_VERSION: "master"
 EDX_NOTES_API_GUNICORN_WORKERS_EXTRA: ""
 EDX_NOTES_API_WORKERS_EXTRA_CONF: ""
-EDX_NOTES_API_LANG: "en-us"
+EDX_NOTES_API_LANG: en-us
 EDX_NOTES_API_WORKERS: !!null
-EDX_NOTES_API_DATASTORE_NAME: "edx-notes-api"
-EDX_NOTES_API_MYSQL_DB_USER: "notes001"
+EDX_NOTES_API_DATASTORE_NAME: edx-notes-api
+EDX_NOTES_API_MYSQL_DB_USER: notes001
 EDX_NOTES_API_MYSQL_DB_NAME: "{{ EDX_NOTES_API_DATASTORE_NAME }}"
-EDX_NOTES_API_MYSQL_DB_PASS: "secret"
-EDX_NOTES_API_MYSQL_HOST: "localhost"
-EDX_NOTES_API_ELASTICSEARCH_URL: "http://127.0.0.1:9200"
+EDX_NOTES_API_MYSQL_DB_PASS: secret
+EDX_NOTES_API_MYSQL_HOST: localhost
+EDX_NOTES_API_ELASTICSEARCH_URL: http://127.0.0.1:9200
 EDX_NOTES_API_NEWRELIC_APPNAME: "{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-notes"
 # Change these values!!
 EDX_NOTES_API_SECRET_KEY: "i^,9%i=e=y/Nlpb=Mkx!j&,HD9d/17F][9P,FLdwM2+G6|]BEU"
-EDX_NOTES_API_CLIENT_ID: "edx-notes-id"
-EDX_NOTES_API_CLIENT_SECRET: "edx-notes-secret"
+EDX_NOTES_API_CLIENT_ID: edx-notes-id
+EDX_NOTES_API_CLIENT_SECRET: edx-notes-secret
 
 EDX_NOTES_API_DATABASES:
   # rw user
   default:
-    ENGINE: "django.db.backends.mysql"
+    ENGINE: django.db.backends.mysql
     NAME: "{{ EDX_NOTES_API_MYSQL_DB_NAME }}"
     USER: "{{ EDX_NOTES_API_MYSQL_DB_USER }}"
     PASSWORD: "{{ EDX_NOTES_API_MYSQL_DB_PASS }}"
     HOST: "{{ EDX_NOTES_API_MYSQL_HOST }}"
     PORT: 3306
 EDX_NOTES_API_ALLOWED_HOSTS:
-  - "localhost"
-    
+  - localhost
+
+EDX_NOTES_API_REPOS:
+  - PROTOCOL: "{{COMMON_GIT_PROTOCOL}}"
+    DOMAIN: "{{COMMON_GIT_MIRROR}}"
+    PATH: "{{COMMON_GIT_PATH}}"
+    REPO: edx-notes-api.git
+    VERSION: e0d/migration-overrides
+    DESTINATION: "{{edx_notes_api_code_dir}}"
+
 #
 # This data structure will be written out to yaml configuration file
 # in /edx/etc
@@ -65,16 +72,18 @@ edx_notes_api_service_config:
       ENGINE: 'notesserver.highlight.ElasticsearchSearchEngine'
       URL: "{{ EDX_NOTES_API_ELASTICSEARCH_URL }}"
       INDEX_NAME: '{{ EDX_NOTES_API_DATASTORE_NAME }}'
-  DISABLE_TOKEN_CHECK: True  
+  DISABLE_TOKEN_CHECK: True
+
 #
 # vars are namespace with the module name.
 #
-edx_notes_api_role_name: "edx-notes-api"
+edx_notes_api_code_dir: "{{ edx_notes_api_app_dir }}/edx-notes-api"
+edx_notes_api_role_name: edx-notes-api
 edx_notes_api_service_name: "{{ edx_notes_api_role_name }}"
 # underscore name
 edx_notes_api_role_name_safe: "{{ edx_notes_api_role_name.replace('-', '_') }}"
 
-edx_notes_api_source_repo: "https://{{ COMMON_GIT_MIRROR }}/edx/edx-notes-api.git"
+edx_notes_api_source_repo: https://{{ COMMON_GIT_MIRROR }}/edx/edx-notes-api.git
 
 edx_notes_api_user: "{{ edx_notes_api_role_name }}"
 edx_notes_api_app_dir: "{{ COMMON_APP_DIR }}/{{ edx_notes_api_service_name }}"
@@ -82,7 +91,6 @@ edx_notes_api_home: "{{ COMMON_APP_DIR }}/{{ edx_notes_api_service_name }}"
 edx_notes_api_venv_base: "{{ edx_notes_api_home }}/venvs"
 edx_notes_api_venv_dir: "{{ edx_notes_api_venv_base }}/{{ edx_notes_api_service_name }}"
 edx_notes_api_venv_bin: "{{ edx_notes_api_venv_dir }}/bin"
-edx_notes_api_code_dir: "{{ edx_notes_api_app_dir }}/edx-notes-api"
 edx_notes_api_python_path: "{{ edx_notes_api_code_dir }}/edx-notes-api"
 edx_notes_api_conf_dir: "{{ edx_notes_api_home }}"
 edx_notes_api_log_dir: "{{ COMMON_LOG_DIR }}/{{ edx_notes_api_service_name }}"
@@ -90,7 +98,7 @@ edx_notes_api_log_dir: "{{ COMMON_LOG_DIR }}/{{ edx_notes_api_service_name }}"
 edx_notes_api_gunicorn_host: "127.0.0.1"
 edx_notes_api_gunicorn_port: "8120"
 edx_notes_api_gunicorn_timeout: "300"
-edx_notes_api_wsgi: "notesserver.wsgi:application"
+edx_notes_api_wsgi: notesserver.wsgi:application
 edx_notes_api_nginx_port: "18120"
 edx_notes_api_manage: "{{ edx_notes_api_code_dir }}/manage.py" 
 edx_notes_api_requirements_base: "{{ edx_notes_api_code_dir }}/requirements"
@@ -103,7 +111,7 @@ edx_notes_api_requirements:
 # OS packages
 #
 edx_notes_api_debian_pkgs:
-  - 'libmysqlclient-dev'
-  - 'python-mysqldb'
+  - libmysqlclient-dev
+  - python-mysqldb
 
 edx_notes_api_redhat_pkgs: []

playbooks/roles/edx-notes-api/meta/main.yml

@@ -24,4 +24,5 @@ dependencies:
     edx_role_name: "{{ edx_notes_api_role_name }}"
     edx_service_name: "{{ edx_notes_api_service_name }}"
     edx_service_config: "{{ edx_notes_api_service_config }}"
+    edx_service_repos: "{{ edx_notes_api_repos }}"
   - supervisor

playbooks/roles/edx-notes-api/tasks/main.yml

@@ -44,13 +44,6 @@
 #   - role: newrelic
 #     when: COMMON_ENABLE_NEWRELIC
 
-- name: checkout code
-  git: >
-    dest={{ edx_notes_api_code_dir }} repo={{ edx_notes_api_source_repo }} version={{ EDX_NOTES_API_VERSION }}
-    accept_hostkey=yes key_file="{{ edx_notes_api_home }}/.ssh/id_rsa"
-  register: edx_notes_api_code_checkout
-  sudo_user: "{{ edx_notes_api_role_name }}"
-  
 - name: install application requirements
   pip: >
     requirements="{{ edx_notes_api_requirements_base }}/{{ item }}"
@@ -58,7 +51,7 @@
     extra_args="--exists-action w"
   sudo_user: "{{ edx_notes_api_user }}"
   with_items: edx_notes_api_requirements
-  
+
 - name: migrate
   shell: >
     chdir={{ edx_notes_api_code_dir }}
@@ -77,7 +70,7 @@
     mode=0650 owner={{ supervisor_user }} group={{ common_web_user }}
   tags:
     - deploy
-    
+
 - name: write supervisord config
   template: >
     src=edx/app/supervisor/conf.d.available/edx_notes_api.conf.j2
@@ -111,7 +104,7 @@
   when: not disable_edx_services
   tags:
     - deploy
-    
+
 - name: create symlinks from the venv bin dir
   file: >
     src="{{ edx_notes_api_venv_bin }}/{{ item }}"
@@ -127,6 +120,3 @@
     src="{{ edx_notes_api_manage }}"
     dest="{{ COMMON_BIN_DIR }}/manage.{{ edx_notes_api_role_name }}"
     state=link
-
-- include: tag_ec2.yml tags=deploy
-  when: COMMON_TAG_EC2_INSTANCE

playbooks/roles/edx-notes-api/tasks/tag_ec2.yml

----
-
-- name: get instance information
-  action: ec2_facts
-
-- name: tag instance
-  ec2_tag: resource={{ ansible_ec2_instance_id }} region={{ ansible_ec2_placement_region }}
-  args:
-    tags:
-      "version:{{ edx_notes_api_service_name }}" : "{{ edx_notes_api_source_repo }} {{ edx_notes_api_code_checkout.after |truncate(7,True,'')}}"
-  when: edx_notes_api_code_checkout.after is defined

playbooks/roles/edx_service/tasks/main.yml

@@ -76,7 +76,7 @@
   tags:
     - deploy
   when: edx_service_config is defined
-    
+
 # Replace dashes with underscores to support roles that use
 # dashes (the role vars will contain underscores)
 - name: install a bunch of system packages on which edx_service relies
@@ -88,3 +88,62 @@
   yum: pkg={{ item }} state=present
   with_items: "{{ edx_service_name.replace('-', '_') }}_redhat_pkgs"
   when: ansible_distribution in common_redhat_variants
+
+
+- name: check if git repo exists before pruning
+  stat: path={{ item.DESTINATION }}/.git
+  register: git_dir_exists
+  with_items: edx_service_repos
+
+- name: git prune before checking out
+  shell: cd {{ item.0.DESTINATION }} && git remote prune origin
+  sudo_user: "{{ edx_service_name }}"
+  when: >
+    (item.1.stat.exists) and 
+    (item.1.stat.isdir is defined) and
+    item.1.stat.isdir
+  with_together:
+    - edx_service_repos
+    - git_dir_exists.results
+
+
+- name: validate GIT.PROTOCOL
+  fail: msg='GIT.PROTOCOL must be "https" or "ssh"'
+  when: (item.PROTOCOL != "https") and (item.PROTOCOL != "ssh")
+  with_items: edx_service_repos
+
+- name: checkout code over ssh
+  git: >
+    repo=git@{{ item.DOMAIN }}:{{ item.PATH }}/{{ item.REPO }}
+    dest={{ item.DESTINATION }}  version={{ item.VERSION }}
+    accept_hostkey=yes key_file={{ edx_notes_api_home }}/.ssh/id_rsa
+  sudo_user: "{{ edx_role_name }}"
+  register: code_checkout
+  when: item.PROTOCOL == "ssh"
+  with_items: edx_service_repos
+
+- name: checkout code over https
+  git: >
+    repo=https://{{ item.DOMAIN }}/{{ item.PATH }}/{{ item.REPO }}
+    dest={{ item.DESTINATION }}  version={{ item.VERSION }}
+  sudo_user: "{{ edx_role_name }}"
+  register: code_checkout
+  when: item.PROTOCOL == "https"
+  with_items: edx_service_repos
+
+
+- name: get instance information
+  action: ec2_facts
+  #old syntax - should be fixed
+
+- name: tag instance
+  ec2_tag: resource={{ ansible_ec2_instance_id }} region={{ ansible_ec2_placement_region }}
+  args:
+    tags:
+      "version:{{edx_service_name}}" : "{{ item.0.DOMAIN }}/{{ item.0.PATH }}/{{ item.0.REPO }} {{ item.1.after |truncate(7,True,'') }}"
+  when: item.1.after is defined
+  with_together: 
+    - edx_service_repos
+    - code_checkout.results
+
+#TODO: restart supervisor- depends on supervisor being refactored into this role