Merge pull request #342 from edx/jarv/mirror

Jarv/mirror

playbooks/edx-east/edx_mirror.yml

+# ansible-playbook --limit tag_Name_mirror edx_mirror.yml --user ubuntu -i ec2.py
+- name: Configure instance(s)
+  hosts: all
+  sudo: True
+  gather_facts: False
+  roles:
+    - common
+    - role: nginx
+      nginx_sites:
+      - devpi
+      - gh_mirror
+      tags: ['r_nginx']
+    - role: supervisor
+      supervisor_servers:
+      - devpi
+    - role: devpi
+      tags: ['r_devpi']
+    - role: gh_mirror
+      tags: ['r_gh_mirror']
+ 

playbooks/group_vars/all

@@ -18,3 +18,7 @@ secure_dir: 'secure_example'
 # this indicates the path to site-specific (with precedence)
 # things like nginx template files
 local_dir:  '../../ansible_local'
+# include http/https
+PYPI_MIRROR_URL: 'https://pypi.python.org/simple'
+# do not include http/https
+GIT_MIRROR: 'github.com'

playbooks/roles/common/tasks/create_venv.yml

@@ -11,13 +11,19 @@
   - install
 
 - name: common | pip install virtualenv
-  pip: name=virtualenv state=present
+  pip: >
+    name=virtualenv 
+    state=present
+    extra_args="-i {{ PYPI_MIRROR_URL }}"
   tags:
   - venv_base
   - install
 
 - name: common | pip install virtualenvwrapper
-  pip: name=virtualenvwrapper state=present
+  pip: >
+    name=virtualenvwrapper 
+    state=present
+    extra_args="-i {{ PYPI_MIRROR_URL }}"
   tags:
   - venv_base
   - install
@@ -35,7 +41,11 @@
   - install
 
 - name: common |  pip install gunicorn
-  pip: name=gunicorn virtualenv="{{venv_dir}}" state=present
+  pip: >
+    name=gunicorn 
+    virtualenv="{{venv_dir}}" 
+    state=present
+    extra_args="-i {{ PYPI_MIRROR_URL }}"
   tags:
   - gunicorn
   - install

playbooks/roles/devpi/defaults/main.yml

+---
+devpi_venv_dir: "{{ app_base_dir }}/devpi/venvs/devpi"
+devpi_pip_pkgs:
+  - devpi-server
+  - eventlet
+devpi_nginx_port: 80
+devpi_port: 4040
+devpi_data_dir: /var/devpi/data
+devpi_user: devpi
+devpi_group: devpi
+devpi_server_name: 'pypy.*'

playbooks/roles/devpi/handlers/main.yml

+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+#
+# Defaults for role devpi
+#
+---
+- name: devpi | restart devpi
+  supervisorctl: >
+    state=restarted 
+    config={{ supervisor_cfg }}
+    name=devpi-server
+
+- name: devpi | start devpi
+  supervisorctl: >
+    state=started
+    config={{ supervisor_cfg }}
+    name=devpi-server
+

playbooks/roles/devpi/tasks/main.yml

+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+#
+# Tasks for role devpi
+#
+# Overview:
+#   Creates a pypi caching server
+#
+# Dependencies:
+#   - common
+#   - nginx
+#   - supervisor
+#
+# Example play:
+#   roles:
+#     - common
+#     - role: nginx
+#       nginx_sites:
+#       - devpi
+#     - role: supervisor
+#       supervisor_servers:
+#       - devpi
+#     - devpi
+
+---
+- name: devpi | create devpi user
+  user: >
+    name={{ devpi_user }}
+    state=present
+
+- name: devpi | create virtualenv directory
+  file: >
+    path={{ devpi_venv_dir }}
+    state=directory
+    owner={{ devpi_user }}
+    group={{ devpi_group }}
+  notify: devpi | restart devpi
+
+- name: devpi | create the devpi data directory
+  file: >
+    path={{ devpi_data_dir }}
+    state=directory
+    owner={{ devpi_user }}
+    group={{ devpi_group }}
+
+- name: devpi | install devpi pip pkgs
+  pip: >
+    name={{ item }}
+    state=present 
+    virtualenv={{ devpi_venv_dir }}
+  with_items: devpi_pip_pkgs
+  notify: devpi | restart devpi
+
+- name: supervisor | ensure supervisor is started
+  service: name=supervisor state=started
+
+- name: devpi | ensure devpi is running
+  supervisorctl: >
+    state=started
+    config={{ supervisor_cfg }}
+    name=devpi-server

playbooks/roles/edxapp/defaults/main.yml

@@ -270,11 +270,11 @@ worker_core_mult:
 #To turn off theming, specify edxapp_theme_name: ''
 #Stanford, for example, uses edxapp_theme_name: 'stanford'
 edxapp_theme_name: ''
-edxapp_theme_source_repo: 'https://github.com/Stanford-Online/edx-theme.git'
+edxapp_theme_source_repo: 'https://{{ GIT_MIRROR }}/Stanford-Online/edx-theme.git'
 edxapp_theme_version: 'HEAD'
 
 # make this the public URL instead of writable
-edx_platform_repo: https://github.com/edx/edx-platform.git
+edx_platform_repo: "https://{{ GIT_MIRROR }}/edx/edx-platform.git"
 # `edx_platform_commit` can be anything that git recognizes as a commit
 # reference, including a tag, a branch name, or a commit hash
 edx_platform_commit: 'release'

playbooks/roles/edxapp/tasks/deploy.yml

@@ -106,9 +106,33 @@
 
 # Python plays that need to be run after platform updates.
 
+# Substitute github mirror in all requirements files
+#
+- name: Updating requirement files for git mirror
+  command: |
+    /bin/sed -i -e 's/github\.com/{{ GIT_MIRROR }}/g' {{ item }}
+  with_items:
+    - "{{ pre_requirements_file }}"
+    - "{{ post_requirements_file }}"
+    - "{{ repo_requirements_file }}"
+    - "{{ github_requirements_file }}"
+    - "{{ local_requirements_file }}"
+    - "{{ sandbox_base_requirements }}"
+    - "{{ sandbox_local_requirements }}"
+    - "{{ sandbox_post_requirements }}"
+  tags:
+    - lms
+    - cms
+    - install
+    - deploy
+
 # Install the python pre requirements into {{ venv_dir }}
 - name : install python pre-requirements
-  pip: requirements="{{pre_requirements_file}}" virtualenv="{{venv_dir}}" state=present
+  pip: >
+    requirements="{{pre_requirements_file}}" 
+    virtualenv="{{venv_dir}}" 
+    state=present
+    extra_args="-i {{ PYPI_MIRROR_URL }}"
   tags:
   - lms
   - cms
@@ -120,7 +144,7 @@
   # Need to use shell rather than pip so that we can maintain the context of our current working directory; some
   # requirements are pathed relative to the edx-platform repo. Using the pip from inside the virtual environment implicitly
   # installs everything into that virtual environment.
-  shell: cd {{ edx_platform_code_dir }} && {{ venv_dir }}/bin/pip install --exists-action w --use-mirrors -r {{ base_requirements_file }}
+  shell: cd {{ edx_platform_code_dir }} && {{ venv_dir }}/bin/pip install -i {{ PYPI_MIRROR_URL }} --exists-action w --use-mirrors -r {{ base_requirements_file }}
   tags:
   - lms
   - cms
@@ -129,7 +153,11 @@
 
 # Install the python post requirements into {{ venv_dir }}
 - name : install python post-requirements
-  pip: requirements="{{post_requirements_file}}" virtualenv="{{venv_dir}}" state=present
+  pip: >
+    requirements="{{post_requirements_file}}" 
+    virtualenv="{{venv_dir}}" 
+    state=present
+    extra_args="-i {{ PYPI_MIRROR_URL }}"
   tags:
   - lms
   - cms
@@ -141,7 +169,7 @@
   # Need to use shell rather than pip so that we can maintain the context of our current working directory; some
   # requirements are pathed relative to the edx-platform repo. Using the pip from inside the virtual environment implicitly
   # installs everything into that virtual environment.
-  shell: cd {{ edx_platform_code_dir }} && {{ venv_dir }}/bin/pip install --exists-action w --use-mirrors -r {{ item }}
+  shell: cd {{ edx_platform_code_dir }} && {{ venv_dir }}/bin/pip install -i {{ PYPI_MIRROR_URL }} --exists-action w --use-mirrors -r {{ item }}
   with_items:
   - "{{ repo_requirements_file }}"
   - "{{ github_requirements_file }}"
@@ -158,7 +186,7 @@
   # Need to use shell rather than pip so that we can maintain the context of our current working directory; some
   # requirements are pathed relative to the edx-platform repo. Using the pip from inside the virtual environment implicitly
   # installs everything into that virtual environment.
-  shell: cd {{ edx_platform_code_dir }} && {{ venv_dir }}/bin/pip install --exists-action w --use-mirrors -r {{ item }}
+  shell: cd {{ edx_platform_code_dir }} && {{ venv_dir }}/bin/pip install -i {{ PYPI_MIRROR_URL }} --exists-action w --use-mirrors -r {{ item }}
   with_items:
   - "{{ sandbox_base_requirements }}"
   - "{{ sandbox_local_requirements }}"

playbooks/roles/edxlocal/tasks/main.yml

@@ -14,7 +14,11 @@
   apt: pkg={{','.join(edxlocal_debian_pkgs)}} install_recommends=yes state=present
 
 - name: edxlocal | install python pymongo for mongo_user ansible module
-  pip: name=pymongo state=present version=2.6.3
+  pip: >
+    name=pymongo 
+    state=present 
+    version=2.6.3
+    extra_args="-i {{ PYPI_MIRROR_URL }}"
 
 - name: edxlocal | create a database for edxapp
   mysql_db: >

playbooks/roles/gh_mirror/defaults/main.yml

+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+#
+# Defaults for role gh_mirror
+#
+---
+gh_mirror_nginx_port: 80
+gh_mirror_server_name: 'git.*'
+gh_mirror_data_dir: /var/git/mirrors
+gh_mirror_app_dir: /opt/gh_mirror
+gh_mirror_user: git-mirror
+gh_mirror_group: git-mirror
+gh_mirror_orgs:
+  - edX
+  - MITx
+  - eventbrite
+  - dementrock
+  - mfogel
+  - mitocw
+  - Stanford-Online
+gh_mirror_debian_pkgs:
+  - fcgiwrap
+gh_mirror_pip_pkgs:
+  - pyyaml
+  - requests
+gh_mirror_app_files:
+  - repos_from_orgs.py

playbooks/roles/gh_mirror/files/repos_from_orgs.py

+#!/usr/bin/python
+#   Given a list of repos in a yaml
+#   file will create or update mirrors
+#
+#   Generates /var/tmp/repos.json from
+#   a yaml file containing a list of
+#   github organizations
+
+import yaml
+import sys
+import requests
+import json
+import subprocess
+import os
+import logging
+import fcntl
+from os.path import dirname, abspath, join
+from argparse import ArgumentParser
+
+def check_running(run_type=''):
+
+    pid_file = '{}-{}.pid'.format(
+        os.path.basename(__file__),run_type)
+    fp = open(pid_file, 'w')
+    try:
+        fcntl.lockf(fp, fcntl.LOCK_EX | fcntl.LOCK_NB)
+    except IOError:
+        # another instance is running
+        sys.exit(0)
+
+def run_cmd(cmd):
+    logging.debug('running: {}\n'.format(cmd))
+    process = subprocess.Popen(
+        cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+        shell=True)
+    for line in iter(process.stdout.readline, ""):
+        logging.debug(line)
+
+
+def parse_args():
+    parser = ArgumentParser()
+    parser.add_argument('-r', '--refresh', action='store_true',
+                        help="Refresh the list of repos", default=False)
+    parser.add_argument('-d', '--datadir', help="repo directory")
+    return parser.parse_args()
+
+
+def refresh_cache():
+    path = dirname(abspath(__file__))
+    try:
+        with open(join(path, 'orgs.yml')) as f:
+            orgs = yaml.load(f)
+    except IOError:
+        print "Unable to read {}/orgs.yml, does it exist?".format(path)
+        sys.exit(1)
+
+    repos = []
+
+    for org in orgs:
+        page = 1
+        while True:
+            r = requests.get('https://api.github.com/users/{}/repos?page={}'.format(org, page))
+            org_data = r.json()
+            # request pages until we get zero results
+            if not isinstance(org_data, list) or len(org_data) == 0:
+                break
+            for repo_data in org_data:
+                if 'html_url' in repo_data:
+                    repos.append({'html_url': repo_data['html_url'],
+                                  'name': repo_data['name'],
+                                  'org': repo_data['owner']['login']})
+            page += 1
+    with open('/var/tmp/repos.json', 'wb') as f:
+        f.write(json.dumps(repos))
+
+
+def update_repos():
+    with open('/var/tmp/repos.json') as f:
+        repos = json.load(f)
+    for repo in repos:
+        repo_path = os.path.join(args.datadir, repo['org'], repo['name'] + '.git')
+        if not os.path.exists(repo_path):
+            run_cmd('mkdir -p {}'.format(repo_path))
+            run_cmd('git clone --mirror {} {}'.format(repo['html_url'], repo_path))
+            run_cmd('cd {} && git update-server-info'.format(repo_path))
+        else:
+            run_cmd('cd {} && git remote-update'.format(repo_path))
+            run_cmd('cd {} && git update-server-info'.format(repo_path))
+
+if __name__ == '__main__':
+    args = parse_args()
+    logging.basicConfig(filename='/var/log/repos-from-orgs.log',
+                        level=logging.DEBUG)
+    if args.refresh:
+        check_running('refresh')
+        refresh_cache()
+    else:
+        check_running()
+        if not args.datadir:
+            print "Please specificy a repository directory"
+            sys.exit(1)
+        if not os.path.exists('/var/tmp/repos.json'):
+            refresh_cache()
+        update_repos()

playbooks/roles/gh_mirror/tasks/main.yml

+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+#
+# Tasks for role gh_mirror
+#
+# Overview:
+#   Creates a github read-only mirror server
+#   Will sync all public repos in gh_mirror_orgs
+#
+# Dependencies:
+#   - common
+#   - nginx
+# 
+# Example play:
+#   roles:
+#     - common
+#     - role: nginx
+#       nginx_sites:
+#       - gh_mirror
+#     - gh_mirror
+
+
+---
+- name: gh_mirror | install pip packages
+  pip: name={{ item }} state=present
+  with_items: gh_mirror_pip_pkgs
+
+- name: gh_mirror | install debian packages
+  apt: >
+    pkg={{ ",".join(gh_mirror_debian_pkgs) }}
+    state=present
+    update_cache=yes
+
+- name: gh_mirror | create gh_mirror user
+  user: >
+    name={{ gh_mirror_user }}
+    state=present
+
+- name: gh_mirror | create the gh_mirror data directory
+  file: >
+    path={{ gh_mirror_data_dir }}
+    state=directory
+    owner={{ gh_mirror_user }}
+    group={{ gh_mirror_group }}
+
+- name: gh_mirror | create the gh_mirror app directory
+  file: >
+    path={{ gh_mirror_app_dir }}
+    state=directory
+
+- name: gh_mirror | create org config
+  template: src=orgs.yml.j2 dest={{ gh_mirror_app_dir }}/orgs.yml
+
+- name: copying sync scripts
+  copy: src={{ item }} dest={{ gh_mirror_app_dir }}/{{ item }}
+  with_items: "{{ gh_mirror_app_files }}"
+
+- name: creating cron job to update repos
+  cron:
+    name: "update repos from github"
+    job: "/usr/bin/python {{ gh_mirror_app_dir }}/repos_from_orgs.py -d {{ gh_mirror_data_dir }}"
+    
+- name: creating cron to update github repo list
+  cron: 
+    name: "refresh repo list from github"
+    job: "/usr/bin/python {{ gh_mirror_app_dir}}/repos_from_orgs.py -r"
+    minute: 0
+      

playbooks/roles/gh_mirror/templates/orgs.yml.j2

+# {{ ansible_managed }}
+{{ gh_mirror_orgs | to_nice_yaml }}

playbooks/roles/nginx/templates/devpi.j2

+server {
+    listen {{ devpi_nginx_port }};
+    server_name {{ devpi_server_name }};
+    gzip             on;
+    gzip_min_length  2000;
+    gzip_proxied     any;
+    gzip_types       text/html application/json;
+
+    location / {
+        root {{ devpi_data_dir }};
+        proxy_pass http://localhost:{{ devpi_port }};
+        proxy_set_header  X-outside-url $scheme://$host;
+        proxy_set_header  X-Real-IP $remote_addr;
+    }
+}
+

playbooks/roles/nginx/templates/gh_mirror.j2

+server {
+    listen       {{ gh_mirror_nginx_port }};
+    server_name  {{ gh_mirror_server_name }};
+    location ~ (/.*) {
+        root {{ gh_mirror_data_dir }};
+        fastcgi_pass  unix:/var/run/fcgiwrap.socket;
+        fastcgi_param SCRIPT_FILENAME     /usr/lib/git-core/git-http-backend;
+        # This won't work if the include is put before
+        # SCRIPT_FILENAME
+        include       fastcgi_params;
+        # export all repositories under GIT_PROJECT_ROOT
+        fastcgi_param GIT_HTTP_EXPORT_ALL "";
+        fastcgi_param GIT_PROJECT_ROOT    {{ gh_mirror_data_dir }};
+        fastcgi_param PATH_INFO           $1;
+    }
+}
+

playbooks/roles/supervisor/defaults/main.yml

+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+#
+# Defaults for role supervisor
+#
+---
+supervisor_log_dir: /var/log/supervisor
+supervisor_cfg: /etc/supervisord.conf

playbooks/roles/supervisor/tasks/main.yml

+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+#
+# Tasks for role supervisor
+#
+# Overview:
+#   Parameterized role for supervisord
+#   Supervisor templates must exist in the
+#   templates/ dir for each server
+# 
+# Dependencies:
+#   - common
+#
+# Example play:
+#   roles:
+#     - common
+#     - role: supervisor
+#       supervisor_servers:
+#       - ...
+
+---
+- fail: supervisor_servers is a required parameter for this role
+  when: supervisor_servers is not defined
+
+- name: supervisor | install supervisor at the system level
+  pip: name=supervisor state=present
+
+- name: supervisor | create supervisor directories
+  file: name={{ item }} state=directory
+  with_items:
+    - /etc/supervisor
+    - /etc/supervisor/conf.d
+    - "{{ supervisor_log_dir }}"
+
+- name: supervisor | create supervisor upstart job
+  template: src=supervisor-upstart.conf.j2 dest=/etc/init/supervisor.conf
+
+- name: supervisor | create supervisor master config
+  template: src=supervisord.conf.j2 dest={{ supervisor_cfg }}
+
+- name: supervisor | create supervisor configs
+  template: src={{ item }}.conf.j2 dest=/etc/supervisor/conf.d/{{ item }}.conf
+  with_items: supervisor_servers
+
+- name: supervisor | ensure supervisor is started
+  service: name=supervisor state=started

playbooks/roles/supervisor/templates/devpi.conf.j2

+[program:devpi-server]
+command={{ devpi_venv_dir }}/bin/devpi-server --port {{ devpi_port }} --serverdir {{ devpi_data_dir }}
+priority=999
+startsecs = 5
+redirect_stderr = True
+autostart=True
+user={{ devpi_user }}

playbooks/roles/supervisor/templates/supervisor-upstart.conf.j2

+description     "supervisord"
+
+start on runlevel [2345]
+stop on runlevel [!2345]
+
+respawn
+
+exec /usr/local/bin/supervisord --nodaemon --configuration {{ supervisor_cfg }}

playbooks/roles/supervisor/templates/supervisord.conf.j2

+; supervisor config file
+
+[unix_http_server]
+file=/var/run//supervisor.sock   ; (the path to the socket file)
+chmod=0700                       ; sockef file mode (default 0700)
+
+[supervisord]
+logfile=/var/log/supervisor/supervisord.log ; (main log file;default $CWD/supervisord.log)
+pidfile=/var/run/supervisord.pid ; (supervisord pidfile;default supervisord.pid)
+childlogdir=/var/log/supervisor            ; ('AUTO' child log dir, default $TEMP)
+
+; the below section must remain in the config file for RPC
+; (supervisorctl/web interface) to work, additional interfaces may be
+; added by defining them in separate rpcinterface: sections
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
+
+[supervisorctl]
+serverurl=unix:///var/run//supervisor.sock ; use a unix:// URL  for a unix socket
+
+; The [include] section can just contain the "files" setting.  This
+; setting can list multiple files (separated by whitespace or
+; newlines).  It can also contain wildcards.  The filenames are
+; interpreted as relative to this file.  Included files *cannot*
+; include files themselves.
+
+[include]
+files = /etc/supervisor/conf.d/*.conf
+

util/jenkins/ansible-deploy.sh

@@ -48,7 +48,8 @@ ora_version: $ora_version
 ease_version: $ease_version
 
 ansible_ssh_private_key_file: /var/lib/jenkins/${keypair}.pem
-
+PYPI_MIRROR_URL: 'https://pypi.edx.org/root/pypi/+simple/'
+GIT_MIRROR: 'git.edx.org'
 EOF
 
 cat $extra_vars

util/jenkins/ansible-provision.sh

@@ -79,6 +79,8 @@ region: $region
 instance_tags: '{"environment": "$environment", "github_username": "$github_username", "Name": "$name_tag", "source": "jenkins", "owner": "$BUILD_USER"}'
 root_ebs_size: $root_ebs_size
 name_tag: $name_tag
+PYPI_MIRROR_URL: 'https://pypi.edx.org/root/pypi/+simple/'
+GIT_MIRROR: 'git.edx.org'
 gh_users:
   - user: jarv
     groups: