adding the github mirror server role

playbooks/edx-east/edx_mirror.yml

+# ansible-playbook --limit tag_Name_mirror edx_mirror.yml --user ubuntu -i ec2.py
 - name: Configure instance(s)
   hosts: all
   sudo: True
@@ -7,8 +8,13 @@
     - role: nginx
       nginx_sites:
       - devpi
+      - gh_mirror
+      tags: ['r_nginx']
     - role: supervisor
       supervisor_servers:
       - devpi
-    - devpi
+    - role: devpi
+      tags: ['r_devpi']
+    - role: gh_mirror
+      tags: ['r_gh_mirror']
  

playbooks/roles/devpi/defaults/main.yml

@@ -8,3 +8,4 @@ devpi_port: 4040
 devpi_data_dir: /var/devpi/data
 devpi_user: devpi
 devpi_group: devpi
+devpi_server_name: 'pypy.*'

playbooks/roles/devpi/tasks/main.yml

@@ -12,7 +12,7 @@
     group={{ devpi_group }}
   notify: devpi | restart devpi
 
-  name: devpi | create the devpi data directory
+- name: devpi | create the devpi data directory
   file: >
     path={{ devpi_data_dir }}
     state=directory

playbooks/roles/gh_mirror/defaults/main.yml

+---
+gh_mirror_nginx_port: 80
+gh_mirror_server_name: 'git.*'
+gh_mirror_data_dir: /var/git/mirrors
+gh_mirror_app_dir: /opt/gh_mirror
+gh_mirror_user: git-mirror
+gh_mirror_group: git-mirror
+gh_mirror_orgs:
+  - edX
+  - MITx
+  - eventbrite
+  - dementrock
+  - mfogel
+gh_mirror_debian_pkgs:
+  - python-yaml
+  - python-requests
+gh_app_files:
+  - gh_mirror_cron.sh
+  - repo_from_orgs.py

playbooks/roles/gh_mirror/files/gh_mirror_cron.sh

+#!/usr/bin/env bash
+
+if [[ -z $1 ]]; then
+    echo "Path to data directory required"
+    exit 1
+fi
+
+data_dir=$1
+dir=$(dirname $0)
+
+if [[ ! -f /var/tmp/repos.txt ]]; then
+    python $dir/repos_from_orgs.py
+fi
+
+for repo_url in $(cat /var/tmp/repos.txt); do 
+
+    repo_name=${repo_url##*/}
+    if [[ ! -d $data_dir/$repo_name ]]; then
+        git clone --mirror $repo_url $data_dir/$repo_name
+    else
+        cd $data_dir/$repo_name
+        git remote update
+    fi
+done

playbooks/roles/gh_mirror/files/repos_from_orgs.py

+#!/usr/bin/python
+# Generates /var/tmp/repos.txt from
+# a yaml file containing a list of
+# github organizations
+
+import yaml
+import sys
+import requests
+from os.path import dirname, abspath, join
+
+path = dirname(abspath(__file__))
+
+try:
+    with open(join(path, 'orgs.yml')) as f:
+        orgs = yaml.load(f)
+except IOError:
+    print "Unable to read {}/orgs.yml, does it exist?".format(path)
+    sys.exit(1)
+
+repos = []
+
+for org in orgs:
+    r = requests.get('https://api.github.com/orgs/{}/repos'.format(org))
+    org_data = r.json
+    for repo_data in org_data:
+        repos.append(repo_data['html_url'])
+
+with open('/var/tmp/repos.txt', 'wb') as f:
+    f.write('\n'.join(repos))

playbooks/roles/gh_mirror/tasks/main.yml

+---
+- name: gh_mirror | create gh_mirror user
+  user: >
+    name={{ gh_mirror_user }}
+    state=present
+
+- name: gh_mirror | create the gh_mirror data directory
+  file: >
+    path={{ gh_mirror_data_dir }}
+    state=directory
+    owner={{ gh_mirror_user }}
+    group={{ gh_mirror_group }}
+
+- name: gh_mirror | create the gh_mirror app directory
+  file: >
+    path={{ gh_mirror_app_dir }}
+    state=directory
+
+- name: gh_mirror | create org config
+  template: src=orgs.yml.j2 dest={{ gh_mirror_app_dir }}/orgs.yml
+
+- name: copying sync scripts
+  copy: src={{ item }} dest={{ gh_mirror_app_dir }}/{{ item }}
+  with_items: "{{ gh_mirror_app_files }}"

playbooks/roles/gh_mirror/templates/orgs.yml.j2

+# {{ ansible_managed }}
+{{ gh_mirror_orgs | to_nice_yaml }}

playbooks/roles/nginx/templates/devpi.j2

 server {
     listen {{ devpi_nginx_port }};
+    server_name {{ devpi_server_name }};
     gzip             on;
     gzip_min_length  2000;
     gzip_proxied     any;
     gzip_types       text/html application/json;
 
-    root {{ devpi_venv_dir }}/test/data;  # arbitrary for now
     location / {
+        root {{ devpi_data_dir }};
         proxy_pass http://localhost:{{ devpi_port }};
         proxy_set_header  X-outside-url $scheme://$host;
         proxy_set_header  X-Real-IP $remote_addr;

playbooks/roles/nginx/templates/gh_mirror.j2

+server {
+    listen       {{ gh_mirror_nginx_port }};
+    server_name  {{ gh_mirror_server_name }};
+    location / {
+        root {{ gh_mirror_data_dir }};
+    }
+}
+